Skip to content

Commit

Permalink
review changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@attiasas
attiasas committed Nov 1, 2023
1 parent 1772407 commit 3a02b25
Show file tree
Hide file tree
Showing 4 changed files with 432 additions and 8 deletions.
19 changes: 16 additions & 3 deletions utils/coreutils/techutils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,10 +188,10 @@ func DetectedTechnologiesList() (technologies []string) {
if errorutils.CheckError(err) != nil {
return
}
return DetectedTechnologiesListInPath(wd, false)
return detectedTechnologiesListInPath(wd, false)
}

func DetectedTechnologiesListInPath(path string, recursive bool) (technologies []string) {
func detectedTechnologiesListInPath(path string, recursive bool) (technologies []string) {
detectedTechnologies, err := DetectTechnologies(path, false, recursive)
if err != nil {
return
Expand All @@ -214,7 +214,7 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [
}
workingDirectoryToIndicators, excludedTechAtWorkingDir := mapFilesToRelevantWorkingDirectories(filesList, requestedDescriptors)
strJson, err := json.MarshalIndent(workingDirectoryToIndicators, "", " ")
if err == nil && len(workingDirectoryToIndicators) > 0 {
if errorutils.CheckError(err) == nil && len(workingDirectoryToIndicators) > 0 {
log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson))
}
technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors)
Expand All @@ -225,6 +225,15 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [
}

// Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors.
// files: The file paths to map.
// requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology.
// Returns:
// 1. workingDirectoryToIndicators: A map of working directories to the files that are relevant to the technologies.
// wd1: [wd1/indicator, wd1/descriptor]
// wd/wd2: [wd/wd2/indicator]
// 2. excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory.
// wd1: [tech1, tech2]
// wd/wd2: [tech1]
func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors map[Technology][]string) (workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology) {
workingDirectoryToIndicatorsSet := make(map[string]*datastructures.Set[string])
excludedTechAtWorkingDir = make(map[string][]Technology)
Expand Down Expand Up @@ -290,6 +299,10 @@ func isExclude(path string, techData TechData) bool {
}

// Map working directories to technologies according to the given workingDirectoryToIndicators map files.
// workingDirectoryToIndicators: A map of working directories to the files inside the directory that are relevant to the technologies.
// excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory.
// requestedTechs: The technologies to check, if empty all technologies will be checked.
// requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology to detect.
func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology, requestedDescriptors map[Technology][]string) (technologiesDetected map[Technology]map[string][]string) {
// Get the relevant technologies to check
technologies := requestedTechs
Expand Down
234 changes: 234 additions & 0 deletions utils/coreutils/techutils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,240 @@ func TestMapWorkingDirectoriesToTechnologies(t *testing.T) {
}
}

func TestGetExistingRootDir(t *testing.T) {
tests := []struct {
name string
path string
workingDirectoryToIndicators map[string][]string
expected string
}{
{
name: "empty",
path: "",
workingDirectoryToIndicators: map[string][]string{},
expected: "",
},
{
name: "no match",
path: "dir",
workingDirectoryToIndicators: map[string][]string{
filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")},
"dir2": {filepath.Join("dir2", "go.mod")},
"dir3": {},
filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")},
},
expected: "dir",
},
{
name: "match root",
path: filepath.Join("directory", "dir2"),
workingDirectoryToIndicators: map[string][]string{
filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")},
"dir2": {filepath.Join("dir2", "go.mod")},
"dir3": {},
filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")},
},
expected: filepath.Join("directory", "dir2"),
},
{
name: "match sub",
path: filepath.Join("directory", "dir2"),
workingDirectoryToIndicators: map[string][]string{
filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")},
"dir2": {filepath.Join("dir2", "go.mod")},
"directory": {},
filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")},
},
expected: "directory",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, getExistingRootDir(test.path, test.workingDirectoryToIndicators))
})
}
}

func TestCleanSubDirectories(t *testing.T) {
tests := []struct {
name string
workingDirectoryToFiles map[string][]string
expected map[string][]string
}{
{
name: "empty",
workingDirectoryToFiles: map[string][]string{},
expected: map[string][]string{},
},
{
name: "no sub directories",
workingDirectoryToFiles: map[string][]string{
"directory": {filepath.Join("directory", "file")},
filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")},
filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")},
},
expected: map[string][]string{
"directory": {filepath.Join("directory", "file")},
filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")},
filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")},
},
},
{
name: "sub directories",
workingDirectoryToFiles: map[string][]string{
filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")},
filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")},
"dir": {filepath.Join("dir", "file")},
"directory": {filepath.Join("directory", "file")},
filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "file")},
filepath.Join("dir", "dir2", "dir3"): {filepath.Join("dir", "dir2", "dir3", "file")},
filepath.Join("dir", "dir2", "dir3", "dir4"): {filepath.Join("dir", "dir2", "dir3", "dir4", "file")},
},
expected: map[string][]string{
"directory": {filepath.Join("directory", "file")},
"dir": {
filepath.Join("dir", "file"),
filepath.Join("dir", "dir", "file"),
filepath.Join("dir", "directory", "file"),
filepath.Join("dir", "dir2", "file"),
filepath.Join("dir", "dir2", "dir3", "file"),
filepath.Join("dir", "dir2", "dir3", "dir4", "file"),
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cleaned := cleanSubDirectories(test.workingDirectoryToFiles)
cleanedKeys := maps.Keys(cleaned)
expectedKeys := maps.Keys(test.expected)
assert.ElementsMatch(t, expectedKeys, cleanedKeys, "expected: %s, actual: %s", expectedKeys, cleanedKeys)
for key, value := range test.expected {
assert.ElementsMatch(t, value, cleaned[key], "expected: %s, actual: %s", value, cleaned[key])
}
})
}
}

func TestGetTechInformationFromWorkingDir(t *testing.T) {
workingDirectoryToIndicators := map[string][]string{
"folder": {filepath.Join("folder", "pom.xml")},
filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")},
filepath.Join("folder", "sub2"): {filepath.Join("folder", "sub2", "pom.xml")},
"dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json"), filepath.Join("dir", "build.gradle.kts"), filepath.Join("dir", "project.sln"), filepath.Join("dir", "blabla.txt")},
"directory": {filepath.Join("directory", "npm-shrinkwrap.json")},
"dir3": {filepath.Join("dir3", "package.json"), filepath.Join("dir3", ".yarn")},
filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")},
filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")},
filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")},
filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile"), filepath.Join("users", "test", "package", "build.gradle")},
filepath.Join("dir", "sub1"): {filepath.Join("dir", "sub1", "project.csproj")},
}
excludedTechAtWorkingDir := map[string][]Technology{
filepath.Join("users", "test", "package"): {Pip},
"dir3": {Npm},
}

tests := []struct {
name string
tech Technology
requestedDescriptors map[Technology][]string
expected map[string][]string
}{
{
name: "mavenTest",
tech: Maven,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{
"folder": {
filepath.Join("folder", "pom.xml"),
filepath.Join("folder", "sub1", "pom.xml"),
filepath.Join("folder", "sub2", "pom.xml"),
},
},
},
{
name: "npmTest",
tech: Npm,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{
"dir": {filepath.Join("dir", "package.json")},
"directory": {},
},
},
{
name: "yarnTest",
tech: Yarn,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{"dir3": {filepath.Join("dir3", "package.json")}},
},
{
name: "golangTest",
tech: Go,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}},
},
{
name: "pipTest",
tech: Pip,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{
filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")},
filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")},
},
},
{
name: "pipRequestedDescriptorTest",
tech: Pip,
requestedDescriptors: map[Technology][]string{Pip: {"blabla.txt"}},
expected: map[string][]string{
"dir": {filepath.Join("dir", "blabla.txt")},
filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")},
filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")},
},
},
{
name: "pipenvTest",
tech: Pipenv,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile")}},
},
{
name: "gradleTest",
tech: Gradle,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{
filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "build.gradle")},
"dir": {filepath.Join("dir", "build.gradle.kts")},
},
},
{
name: "nugetTest",
tech: Nuget,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}},
},
{
name: "dotnetTest",
tech: Dotnet,
requestedDescriptors: map[Technology][]string{},
expected: map[string][]string{"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
techInformation := getTechInformationFromWorkingDir(test.tech, workingDirectoryToIndicators, excludedTechAtWorkingDir, test.requestedDescriptors)
expectedKeys := maps.Keys(test.expected)
actualKeys := maps.Keys(techInformation)
assert.ElementsMatch(t, expectedKeys, actualKeys, "expected: %s, actual: %s", expectedKeys, actualKeys)
for key, value := range test.expected {
assert.ElementsMatch(t, value, techInformation[key], "expected: %s, actual: %s", value, techInformation[key])
}
})
}
}

func TestContainsApplicabilityScannableTech(t *testing.T) {
tests := []struct {
name string
Expand Down
8 changes: 4 additions & 4 deletions xray/commands/audit/scarunner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) {
log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...")
return
}
printScansInformation(scans)
logScanInfo(scans)

defer func() {
// Make sure to return to the original working directory, executeScaScan may change it
Expand Down Expand Up @@ -106,7 +106,7 @@ func getExcludePattern(params *AuditParams, recursive bool) string {
return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, recursive)
}

func printScansInformation(scans []*xrayutils.ScaScanResult) {
func logScanInfo(scans []*xrayutils.ScaScanResult) {
scansJson, err := json.MarshalIndent(scans, "", " ")
if err == nil {
log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), string(scansJson)))
Expand All @@ -129,14 +129,14 @@ func getRequestedDirectoriesToScan(currentWorkingDir string, params *AuditParams
func executeScaScan(serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) (err error) {
// Get the dependency tree for the technology in the working directory.
if err = os.Chdir(scan.WorkingDirectory); err != nil {
return
return errorutils.CheckError(err)
}
flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology)
if techErr != nil {
return fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error())
}
if flattenTree == nil || len(flattenTree.Nodes) == 0 {
return errors.New("no dependencies were found. Please try to build your project and re-run the audit command")
return errorutils.CheckErrorf("no dependencies were found. Please try to build your project and re-run the audit command")
}
// Scan the dependency tree.
scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees)
Expand Down
Loading

0 comments on commit 3a02b25

Please sign in to comment.