From 47cfa674a562d0da486688f52ca28fdfb02e7bf9 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:16:18 +0300 Subject: [PATCH 01/50] refactor: Remove threading on domains processing --- README.md | 3 - cmd/xsubfind3r/main.go | 141 ++++++++++++++++++----------------------- 2 files changed, 61 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 0be42e0..7ee3a04 100644 --- a/README.md +++ b/README.md @@ -180,9 +180,6 @@ SOURCES: -u, --sources-to-use string[] comma(,) separeted sources to use -e, --sources-to-exclude string[] comma(,) separeted sources to exclude -OPTIMIZATION: - -t, --threads int number of threads (default: 50) - OUTPUT: --no-color bool disable colored output -o, --output string output subdomains' file path diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 1253c2e..6245432 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -8,7 +8,6 @@ import ( "reflect" "strconv" "strings" - "sync" "github.com/hueristiq/hqgolog" "github.com/hueristiq/hqgolog/formatter" @@ -23,12 +22,11 @@ import ( var ( au aurora.Aurora - domainsSlice []string + domains []string domainsListFilePath string listSources bool sourcesToUse []string sourcesToExclude []string - threads int monochrome bool output string outputDirectory string @@ -38,16 +36,14 @@ var ( func init() { // defaults - defaultThreads := 50 defaultYAMLConfigFile := fmt.Sprintf("~/.hueristiq/%s/config.yaml", configuration.NAME) // Handle CLI arguments, flags & help message (pflag) - pflag.StringSliceVarP(&domainsSlice, "domain", "d", []string{}, "") + pflag.StringSliceVarP(&domains, "domain", "d", []string{}, "") pflag.StringVarP(&domainsListFilePath, "list", "l", "", "") pflag.BoolVar(&listSources, "sources", false, "") pflag.StringSliceVarP(&sourcesToUse, "use-sources", "u", []string{}, "") pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "") - pflag.IntVarP(&threads, "threads", "t", defaultThreads, "") pflag.BoolVar(&monochrome, "no-color", false, "") pflag.StringVarP(&output, "output", "o", "", "") pflag.StringVarP(&outputDirectory, "outputDirectory", "O", "", "") @@ -70,9 +66,6 @@ func init() { h += " -u, --sources-to-use string[] comma(,) separeted sources to use\n" h += " -e, --sources-to-exclude string[] comma(,) separeted sources to exclude\n" - h += "\nOPTIMIZATION:\n" - h += fmt.Sprintf(" -t, --threads int number of threads (default: %d)\n", defaultThreads) - h += "\nOUTPUT:\n" h += " --no-color bool disable colored output\n" h += " -o, --output string output subdomains file path\n" @@ -116,16 +109,20 @@ func main() { fmt.Fprintln(os.Stderr, configuration.BANNER) } + var err error + + var config configuration.Configuration + // Read in configuration. - config, err := configuration.Read(YAMLConfigFile) + config, err = configuration.Read(YAMLConfigFile) if err != nil { hqgolog.Fatal().Msg(err.Error()) } - // List suported sources. + // If --sources: List suported sources & exit. if listSources { hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(config.Sources))).Bold()) - hqgolog.Info().Msgf("sources marked with %v need key(s) or token(s) to work.", au.Underline("*").Bold()) + hqgolog.Info().Msgf("sources marked with %v take in key(s) or token(s).", au.Underline("*").Bold()) hqgolog.Print().Msg("") needsKey := make(map[string]interface{}) @@ -148,56 +145,48 @@ func main() { os.Exit(0) } - domains := make(chan string, threads) + // Load input domains. - // Load input domains - go func() { - defer close(domains) + // input domains: file + if domainsListFilePath != "" { + var file *os.File - // input domains: slice - for _, domain := range domainsSlice { - domains <- domain + file, err = os.Open(domainsListFilePath) + if err != nil { + hqgolog.Error().Msg(err.Error()) } - // input domains: file - if domainsListFilePath != "" { - file, err := os.Open(domainsListFilePath) - if err != nil { - hqgolog.Error().Msg(err.Error()) - } - - scanner := bufio.NewScanner(file) + scanner := bufio.NewScanner(file) - for scanner.Scan() { - domain := scanner.Text() + for scanner.Scan() { + domain := scanner.Text() - if domain != "" { - domains <- domain - } + if domain != "" { + domains = append(domains, domain) } + } - if err := scanner.Err(); err != nil { - hqgolog.Error().Msg(err.Error()) - } + if err = scanner.Err(); err != nil { + hqgolog.Error().Msg(err.Error()) } + } - // input domains: stdin - if hasStdin() { - scanner := bufio.NewScanner(os.Stdin) + // input domains: stdin + if hasStdin() { + scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - domain := scanner.Text() + for scanner.Scan() { + domain := scanner.Text() - if domain != "" { - domains <- domain - } + if domain != "" { + domains = append(domains, domain) } + } - if err := scanner.Err(); err != nil { - hqgolog.Error().Msg(err.Error()) - } + if err = scanner.Err(); err != nil { + hqgolog.Error().Msg(err.Error()) } - }() + } // Find and output subdomains. var consolidatedWriter *bufio.Writer @@ -207,7 +196,9 @@ func main() { mkdir(directory) - consolidatedFile, err := os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + var consolidatedFile *os.File + + consolidatedFile, err = os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { hqgolog.Fatal().Msg(err.Error()) } @@ -221,45 +212,35 @@ func main() { mkdir(outputDirectory) } - wg := &sync.WaitGroup{} - - for i := 0; i < threads; i++ { - wg.Add(1) - - go func() { - defer wg.Done() + options := &xsubfind3r.Options{ + SourcesToExclude: sourcesToExclude, + SourcesToUSe: sourcesToUse, + Keys: config.Keys, + } - options := &xsubfind3r.Options{ - SourcesToExclude: sourcesToExclude, - SourcesToUSe: sourcesToUse, - Keys: config.Keys, - } + finder := xsubfind3r.New(options) - finder := xsubfind3r.New(options) + for _, domain := range domains { + subdomains := finder.Find(domain) - for domain := range domains { - subdomains := finder.Find(domain) + switch { + case output != "": + processSubdomains(consolidatedWriter, subdomains, verbosity) + case outputDirectory != "": + var domainFile *os.File - switch { - case output != "": - processSubdomains(consolidatedWriter, subdomains, verbosity) - case outputDirectory != "": - domainFile, err := os.OpenFile(filepath.Join(outputDirectory, domain+".txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - hqgolog.Fatal().Msg(err.Error()) - } + domainFile, err = os.OpenFile(filepath.Join(outputDirectory, domain+".txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + hqgolog.Fatal().Msg(err.Error()) + } - domainWriter := bufio.NewWriter(domainFile) + domainWriter := bufio.NewWriter(domainFile) - processSubdomains(domainWriter, subdomains, verbosity) - default: - processSubdomains(nil, subdomains, verbosity) - } - } - }() + processSubdomains(domainWriter, subdomains, verbosity) + default: + processSubdomains(nil, subdomains, verbosity) + } } - - wg.Wait() } func hasStdin() bool { From 462ec694e618264aa7b91c6d401d1a3b8b32da5f Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:18:48 +0300 Subject: [PATCH 02/50] chore: - --- cmd/xsubfind3r/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 6245432..8822539 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -189,6 +189,14 @@ func main() { } // Find and output subdomains. + options := &xsubfind3r.Options{ + SourcesToExclude: sourcesToExclude, + SourcesToUSe: sourcesToUse, + Keys: config.Keys, + } + + finder := xsubfind3r.New(options) + var consolidatedWriter *bufio.Writer if output != "" { @@ -212,14 +220,6 @@ func main() { mkdir(outputDirectory) } - options := &xsubfind3r.Options{ - SourcesToExclude: sourcesToExclude, - SourcesToUSe: sourcesToUse, - Keys: config.Keys, - } - - finder := xsubfind3r.New(options) - for _, domain := range domains { subdomains := finder.Find(domain) From 0425de4227bea0f3077618ccd5923be2dfac0727 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:29:42 +0300 Subject: [PATCH 03/50] docs: - --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7ee3a04..0b6482d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ## Features -* Fetches domains from curated passive sources to maximize results. +* [x] Fetches domains from curated passive sources to maximize results.
Sources: Click to expand! @@ -31,8 +31,8 @@ | WHOIS | AlienVault |
-* Supports `stdin` and `stdout` for easy integration into workflows. -* Cross-Platform (Windows, Linux & macOS). +* [x] Supports `stdin` and `stdout` for easy integration into workflows. +* [x] Cross-Platform (Windows, Linux & macOS). ## Installation @@ -116,6 +116,8 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest Example `config.yaml`: +> **NOTE:** The keys below are invalid, use your own keys! + ```yaml version: 0.3.0 sources: From 87ed838cda7f4c9d9f969fa8a092c83d0cf50045 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:32:04 +0300 Subject: [PATCH 04/50] docs: - --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0b6482d..6307e62 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ * [`go build ...` the development Version](#go-build--the-development-version) * [Post Installation](#post-installation) * [Usage](#usage) -* [Contribution](#contribution) +* [Contributing](#contributing) * [Licensing](#licensing) ## Features @@ -22,7 +22,7 @@ * [x] Fetches domains from curated passive sources to maximize results.
Sources: Click to expand! - + | Technique | Source | | :-------- | :----- | | APIs | AnubisDB, BeVigil, Chaos, FullHunt, GitHub, HackerTarget, IntelX, Shodan, URLScan | @@ -192,7 +192,7 @@ CONFIGURATION: -c, --configuration string configuration file path (default: ~/.hueristiq/xsubfind3r/config.yaml) ``` -## Contribution +## Contributing [Issues](https://github.com/hueristiq/xsubfind3r/issues) and [Pull Requests](https://github.com/hueristiq/xsubfind3r/pulls) are welcome! **Check out the [contribution guidelines](https://github.com/hueristiq/xsubfind3r/blob/master/CONTRIBUTING.md).** From b5d6f315df86f165b8fe3e2f6e31cbb5a56a839a Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:50:28 +0300 Subject: [PATCH 05/50] docs: Add issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 28 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 11 +++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..045fa93 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Create a report to help us improve +labels: 'Type: Bug' +--- + + + + +### `xsubfind3r` version: + + + +### Current Behavior: + + +### Expected Behavior: + + +### Steps To Reproduce: + + +### Anything else: + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..2623502 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,11 @@ +--- +name: Feature request +about: Request feature to implement in this project +labels: 'Type: Enhancement' +--- + +### Please describe your feature request: + + +### Describe the use case of this feature: + From 181a378298807519741c4e6f77773ba73c3745ae Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 3 Aug 2023 19:04:48 +0300 Subject: [PATCH 06/50] chore: - --- .gitignore | 2 +- Makefile | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index fa6c88d..c81e17d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # Executable -cmd/xsubfind3r/xsubfind3r +bin # Notes diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f87e716 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +# Go(Golang) Options +GOCMD=go +GOBUILD=$(GOCMD) build +GOMOD=$(GOCMD) mod +GOTEST=$(GOCMD) test +GOFLAGS := -v +LDFLAGS := -s -w + +# Golangci Options +GOLANGCILINTCMD=golangci-lint +GOLANGCILINTRUN=$(GOLANGCILINTCMD) run + +ifneq ($(shell go env GOOS),darwin) +LDFLAGS := -extldflags "-static" +endif + +tidy: + $(GOMOD) tidy +lint: + $(GOLANGCILINTRUN) ./... +test: + $(GOTEST) $(GOFLAGS) ./... +build: + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o bin/xsubfind3r cmd/xsubfind3r/main.go From 834bc1895ca03ecf550cb5e55a585a1b5978dd2e Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:22:32 +0300 Subject: [PATCH 07/50] refactor: Refine extraction from crtsh --- pkg/xsubfind3r/sources/crtsh/crtsh.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/xsubfind3r/sources/crtsh/crtsh.go b/pkg/xsubfind3r/sources/crtsh/crtsh.go index a789b34..f064181 100644 --- a/pkg/xsubfind3r/sources/crtsh/crtsh.go +++ b/pkg/xsubfind3r/sources/crtsh/crtsh.go @@ -3,8 +3,10 @@ package crtsh import ( "encoding/json" "fmt" + "regexp" "strings" + "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" "github.com/valyala/fasthttp" @@ -40,8 +42,21 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh return } + var regex *regexp.Regexp + + regex, err = extractor.New(domain) + if err != nil { + return + } + for _, record := range getNameValuesResData { - for _, subdomain := range strings.Split(record.NameValue, "\n") { + for _, value := range strings.Split(record.NameValue, "\n") { + subdomain := regex.FindString(value) + + if subdomain == "" { + continue + } + subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} } } From 2dfcf5c7b0d1223393477d4bc872c403ffe7137c Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:27:29 +0300 Subject: [PATCH 08/50] docs: - --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6307e62..5fb531b 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest Example `config.yaml`: -> **NOTE:** The keys below are invalid, use your own keys! +> **NOTE:** The keys/tokens below are invalid, use your own keys/tokens! ```yaml version: 0.3.0 From 4c84d043133d67fb5fa47c967858718441d033a4 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:32:26 +0300 Subject: [PATCH 09/50] refactor: Output source on debug only --- cmd/xsubfind3r/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 8822539..74777ee 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -265,10 +265,10 @@ func mkdir(path string) { func processSubdomains(writer *bufio.Writer, subdomains chan sources.Subdomain, verbosity string) { for subdomain := range subdomains { - if verbosity == string(levels.LevelSilent) { - hqgolog.Print().Msg(subdomain.Value) - } else { + if verbosity == string(levels.LevelDebug) { hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomain.Source), subdomain.Value) + } else { + hqgolog.Print().Msg(subdomain.Value) } if writer != nil { From 1540b2fd95fbb59052cd006e35bceab36eea2c68 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:56:31 +0300 Subject: [PATCH 10/50] chore: Track errors from sources --- cmd/xsubfind3r/main.go | 25 +++--- .../sources/alienvault/alienvault.go | 35 ++++++-- pkg/xsubfind3r/sources/anubis/anubis.go | 35 ++++++-- pkg/xsubfind3r/sources/bevigil/bevigil.go | 43 ++++++++-- pkg/xsubfind3r/sources/chaos/chaos.go | 32 ++++++-- .../sources/commoncrawl/commoncrawl.go | 82 +++++++++++++++++-- pkg/xsubfind3r/sources/crtsh/crtsh.go | 43 ++++++++-- pkg/xsubfind3r/sources/fullhunt/fullhunt.go | 43 ++++++++-- pkg/xsubfind3r/sources/github/github.go | 32 +++++--- .../sources/hackertarget/hackertarget.go | 40 +++++++-- pkg/xsubfind3r/sources/intelx/intelx.go | 70 ++++++++++++++-- pkg/xsubfind3r/sources/shodan/shodan.go | 43 ++++++++-- pkg/xsubfind3r/sources/source.go | 6 -- pkg/xsubfind3r/sources/sources.go | 9 ++ pkg/xsubfind3r/sources/sources_results.go | 18 ++++ pkg/xsubfind3r/sources/subdomain.go | 6 -- pkg/xsubfind3r/sources/urlscan/urlscan.go | 43 ++++++++-- pkg/xsubfind3r/sources/wayback/wayback.go | 59 +++++++++++-- pkg/xsubfind3r/xsubfind3r.go | 26 +++--- 19 files changed, 572 insertions(+), 118 deletions(-) delete mode 100644 pkg/xsubfind3r/sources/source.go create mode 100644 pkg/xsubfind3r/sources/sources_results.go delete mode 100644 pkg/xsubfind3r/sources/subdomain.go diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 74777ee..0a1eb9f 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -263,19 +263,24 @@ func mkdir(path string) { } } -func processSubdomains(writer *bufio.Writer, subdomains chan sources.Subdomain, verbosity string) { +func processSubdomains(writer *bufio.Writer, subdomains chan sources.Result, verbosity string) { for subdomain := range subdomains { - if verbosity == string(levels.LevelDebug) { - hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomain.Source), subdomain.Value) - } else { - hqgolog.Print().Msg(subdomain.Value) - } + switch subdomain.Type { + case sources.Error: + hqgolog.Warn().Msgf("Could not run source %s: %s\n", subdomain.Source, subdomain.Error) + case sources.Subdomain: + if verbosity == string(levels.LevelDebug) { + hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomain.Source), subdomain.Value) + } else { + hqgolog.Print().Msg(subdomain.Value) + } - if writer != nil { - fmt.Fprintln(writer, subdomain.Value) + if writer != nil { + fmt.Fprintln(writer, subdomain.Value) - if err := writer.Flush(); err != nil { - hqgolog.Fatal().Msg(err.Error()) + if err := writer.Flush(); err != nil { + hqgolog.Fatal().Msg(err.Error()) + } } } } diff --git a/pkg/xsubfind3r/sources/alienvault/alienvault.go b/pkg/xsubfind3r/sources/alienvault/alienvault.go index 643d565..8143995 100644 --- a/pkg/xsubfind3r/sources/alienvault/alienvault.go +++ b/pkg/xsubfind3r/sources/alienvault/alienvault.go @@ -19,11 +19,11 @@ type getPassiveDNSResponse struct { type Source struct{} -func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -33,12 +33,29 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh getPassiveDNSRes, err = httpclient.SimpleGet(getPassiveDNSReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getPassiveDNSResData getPassiveDNSResponse - if err = json.Unmarshal(getPassiveDNSRes.Body(), &getPassiveDNSResData); err != nil { + err = json.Unmarshal(getPassiveDNSRes.Body(), &getPassiveDNSResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -47,11 +64,17 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh } for _, record := range getPassiveDNSResData.PassiveDNS { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: record.Hostname} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: record.Hostname, + } + + results <- result } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/anubis/anubis.go b/pkg/xsubfind3r/sources/anubis/anubis.go index 52f105f..4a5009a 100644 --- a/pkg/xsubfind3r/sources/anubis/anubis.go +++ b/pkg/xsubfind3r/sources/anubis/anubis.go @@ -11,11 +11,11 @@ import ( type Source struct{} -func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -25,21 +25,44 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh getSubdomainsRes, err = httpclient.SimpleGet(getSubdomainsReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getSubdomainsResData []string - if err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData); err != nil { + err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } for _, subdomain := range getSubdomainsResData { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/bevigil/bevigil.go b/pkg/xsubfind3r/sources/bevigil/bevigil.go index e28452d..600bc85 100644 --- a/pkg/xsubfind3r/sources/bevigil/bevigil.go +++ b/pkg/xsubfind3r/sources/bevigil/bevigil.go @@ -16,11 +16,11 @@ type getSubdomainsResponse struct { type Source struct{} -func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -28,6 +28,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma key, err = sources.PickRandom(config.Keys.Bevigil) if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -43,21 +51,44 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getSubdomainsResData getSubdomainsResponse - if err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData); err != nil { + err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } for _, subdomain := range getSubdomainsResData.Subdomains { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/chaos/chaos.go b/pkg/xsubfind3r/sources/chaos/chaos.go index 41e94ce..0cb822f 100644 --- a/pkg/xsubfind3r/sources/chaos/chaos.go +++ b/pkg/xsubfind3r/sources/chaos/chaos.go @@ -17,11 +17,11 @@ type getSubdomainsResponse struct { type Source struct{} -func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -29,6 +29,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma key, err = sources.PickRandom(config.Keys.Chaos) if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -40,6 +48,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -50,13 +66,17 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma } for _, record := range getSubdomainsResData.Subdomains { - subdomain := fmt.Sprintf("%s.%s", record, getSubdomainsResData.Domain) + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: fmt.Sprintf("%s.%s", record, getSubdomainsResData.Domain), + } - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + results <- result } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go index 90ea089..1a5ece8 100644 --- a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go +++ b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go @@ -25,11 +25,11 @@ type getURLsResponse struct { type Source struct{} -func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) getIndexesReqURL := "https://index.commoncrawl.org/collinfo.json" @@ -39,12 +39,29 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh getIndexesRes, err = httpclient.SimpleGet(getIndexesReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getIndexesResData getIndexesResponse - if err = json.Unmarshal(getIndexesRes.Body(), &getIndexesResData); err != nil { + err = json.Unmarshal(getIndexesRes.Body(), &getIndexesResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -68,6 +85,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh getURLsRes, err = httpclient.Get(getURLsReqURL, "", getURLsReqHeaders) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -76,23 +101,62 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh for scanner.Scan() { var getURLsResData getURLsResponse - if err = json.Unmarshal(scanner.Bytes(), &getURLsResData); err != nil { - return + err = json.Unmarshal(scanner.Bytes(), &getURLsResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + continue } if getURLsResData.Error != "" { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: fmt.Errorf(getURLsResData.Error), + } + + results <- result + return } parsedURL, err := hqgourl.Parse(getURLsResData.URL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + continue } - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: parsedURL.Domain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: parsedURL.Domain, + } + + results <- result } - if scanner.Err() != nil { + if err = scanner.Err(); err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } }(indexData.API) @@ -101,7 +165,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh wg.Wait() }() - return subdomainsChannel + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/crtsh/crtsh.go b/pkg/xsubfind3r/sources/crtsh/crtsh.go index f064181..f818f47 100644 --- a/pkg/xsubfind3r/sources/crtsh/crtsh.go +++ b/pkg/xsubfind3r/sources/crtsh/crtsh.go @@ -19,11 +19,11 @@ type getNameValuesResponse []struct { type Source struct{} -func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -33,12 +33,29 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh getNameValuesRes, err = httpclient.SimpleGet(getNameValuesReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getNameValuesResData getNameValuesResponse - if err := json.Unmarshal(getNameValuesRes.Body(), &getNameValuesResData); err != nil { + err = json.Unmarshal(getNameValuesRes.Body(), &getNameValuesResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -46,6 +63,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh regex, err = extractor.New(domain) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -57,12 +82,18 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh continue } - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/fullhunt/fullhunt.go b/pkg/xsubfind3r/sources/fullhunt/fullhunt.go index bd831f0..099e1fe 100644 --- a/pkg/xsubfind3r/sources/fullhunt/fullhunt.go +++ b/pkg/xsubfind3r/sources/fullhunt/fullhunt.go @@ -17,11 +17,11 @@ type getSubdomainsResponse struct { type Source struct{} -func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -29,6 +29,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma key, err = sources.PickRandom(config.Keys.Fullhunt) if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -44,21 +52,44 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getSubdomainsResData getSubdomainsResponse - if err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData); err != nil { + err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } for _, subdomain := range getSubdomainsResData.Hosts { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/github/github.go b/pkg/xsubfind3r/sources/github/github.go index 367a6ec..65703c3 100644 --- a/pkg/xsubfind3r/sources/github/github.go +++ b/pkg/xsubfind3r/sources/github/github.go @@ -28,11 +28,11 @@ type searchResponse struct { type Source struct{} -func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) if len(config.Keys.GitHub) == 0 { return @@ -42,13 +42,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma searchReqURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=%q&sort=created&order=asc", domain) - source.Enumerate(searchReqURL, domainRegexp(domain), tokens, subdomainsChannel, config) + source.Enumerate(searchReqURL, domainRegexp(domain), tokens, results, config) }() - return subdomainsChannel + return results } -func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp, tokens *Tokens, subdomainsChannel chan sources.Subdomain, config *sources.Configuration) { +func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp, tokens *Tokens, results chan sources.Result, config *sources.Configuration) { token := tokens.Get() if token.RetryAfter > 0 { @@ -82,7 +82,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp tokens.setCurrentTokenExceeded(retryAfterSeconds) - source.Enumerate(searchReqURL, domainRegexp, tokens, subdomainsChannel, config) + source.Enumerate(searchReqURL, domainRegexp, tokens, results, config) } var searchResData searchResponse @@ -108,14 +108,26 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp subdomains := domainRegexp.FindAllString(string(getRawContentRes.Body()), -1) for _, subdomain := range subdomains { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } for _, textMatch := range item.TextMatches { subdomains := domainRegexp.FindAllString(textMatch.Fragment, -1) for _, subdomain := range subdomains { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } } } @@ -129,7 +141,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp return } - source.Enumerate(nextURL, domainRegexp, tokens, subdomainsChannel, config) + source.Enumerate(nextURL, domainRegexp, tokens, results, config) } } } diff --git a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go b/pkg/xsubfind3r/sources/hackertarget/hackertarget.go index 0781912..8ff34ce 100644 --- a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go +++ b/pkg/xsubfind3r/sources/hackertarget/hackertarget.go @@ -15,11 +15,11 @@ import ( type Source struct{} -func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -29,6 +29,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh hostSearchRes, err = httpclient.SimpleGet(hostSearchReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -36,6 +44,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh regex, err = extractor.New(domain) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -52,16 +68,30 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh match := regex.FindAllString(line, -1) for _, subdomain := range match { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } } if err = scanner.Err(); err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/intelx/intelx.go b/pkg/xsubfind3r/sources/intelx/intelx.go index d9e4e83..4efc38d 100644 --- a/pkg/xsubfind3r/sources/intelx/intelx.go +++ b/pkg/xsubfind3r/sources/intelx/intelx.go @@ -32,11 +32,11 @@ type getResultsResponse struct { type Source struct{} -func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -44,6 +44,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma key, err = sources.PickRandom(config.Keys.Intelx) if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -72,6 +80,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma searchReqBodyBytes, err = json.Marshal(searchReqBody) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -79,12 +95,29 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma searchRes, err = httpclient.SimplePost(searchReqURL, "application/json", searchReqBodyBytes) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var searchResData searchResponse - if err = json.Unmarshal(searchRes.Body(), &searchResData); err != nil { + err = json.Unmarshal(searchRes.Body(), &searchResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -96,24 +129,47 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma getResultsRes, err = httpclient.Get(getResultsReqURL, "", nil) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getResultsResData getResultsResponse - if err = json.Unmarshal(getResultsRes.Body(), &getResultsResData); err != nil { + err = json.Unmarshal(getResultsRes.Body(), &getResultsResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } status = getResultsResData.Status for _, record := range getResultsResData.Selectors { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: record.Selectvalue} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: record.Selectvalue, + } + + results <- result } } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/shodan/shodan.go b/pkg/xsubfind3r/sources/shodan/shodan.go index 1d5d83a..f5867a5 100644 --- a/pkg/xsubfind3r/sources/shodan/shodan.go +++ b/pkg/xsubfind3r/sources/shodan/shodan.go @@ -18,11 +18,11 @@ type getDNSResponse struct { type Source struct{} -func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -30,6 +30,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma key, err = sources.PickRandom(config.Keys.Shodan) if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -39,21 +47,44 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma getDNSRes, err = httpclient.SimpleGet(getDNSReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var getDNSResData getDNSResponse - if err := json.Unmarshal(getDNSRes.Body(), &getDNSResData); err != nil { + err = json.Unmarshal(getDNSRes.Body(), &getDNSResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } for _, subdomain := range getDNSResData.Subdomains { - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: fmt.Sprintf("%s.%s", subdomain, domain)} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: fmt.Sprintf("%s.%s", subdomain, domain), + } + + results <- result } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/source.go b/pkg/xsubfind3r/sources/source.go deleted file mode 100644 index 23509d6..0000000 --- a/pkg/xsubfind3r/sources/source.go +++ /dev/null @@ -1,6 +0,0 @@ -package sources - -type Source interface { - Run(config *Configuration, domain string) (subdomains chan Subdomain) - Name() string -} diff --git a/pkg/xsubfind3r/sources/sources.go b/pkg/xsubfind3r/sources/sources.go index 0887854..58bbe8b 100644 --- a/pkg/xsubfind3r/sources/sources.go +++ b/pkg/xsubfind3r/sources/sources.go @@ -1,5 +1,14 @@ package sources +// Source is an interface inherited by each source. +type Source interface { + // Run takes in configuration which includes keys/tokens and other stuff, + // and domain as arguments. + Run(config *Configuration, domain string) (results <-chan Result) + // Name returns the name of the source. + Name() string +} + var List = []string{ "alienvault", "anubis", diff --git a/pkg/xsubfind3r/sources/sources_results.go b/pkg/xsubfind3r/sources/sources_results.go new file mode 100644 index 0000000..a3ff380 --- /dev/null +++ b/pkg/xsubfind3r/sources/sources_results.go @@ -0,0 +1,18 @@ +package sources + +// Result is a result structure returned by a source. +type Result struct { + Type ResultType + Source string + Value string + Error error +} + +// ResultType is the type of result returned by the source. +type ResultType int + +// Types of results returned by the source. +const ( + Subdomain ResultType = iota + Error +) diff --git a/pkg/xsubfind3r/sources/subdomain.go b/pkg/xsubfind3r/sources/subdomain.go deleted file mode 100644 index 470c680..0000000 --- a/pkg/xsubfind3r/sources/subdomain.go +++ /dev/null @@ -1,6 +0,0 @@ -package sources - -type Subdomain struct { - Source string - Value string -} diff --git a/pkg/xsubfind3r/sources/urlscan/urlscan.go b/pkg/xsubfind3r/sources/urlscan/urlscan.go index 9fb4dcb..f7b5c01 100644 --- a/pkg/xsubfind3r/sources/urlscan/urlscan.go +++ b/pkg/xsubfind3r/sources/urlscan/urlscan.go @@ -28,11 +28,11 @@ type searchResponse struct { type Source struct{} -func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -40,6 +40,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma key, err = sources.PickRandom(config.Keys.URLScan) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -67,12 +75,29 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma searchRes, err = httpclient.Get(searchReqURL, "", searchReqHeaders) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var searchResData searchResponse - if err = json.Unmarshal(searchRes.Body(), &searchResData); err != nil { + err = json.Unmarshal(searchRes.Body(), &searchResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -85,7 +110,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma continue } - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: result.Page.Domain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: result.Page.Domain, + } + + results <- result } if !searchResData.HasMore { @@ -97,7 +128,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) (subdoma } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/sources/wayback/wayback.go b/pkg/xsubfind3r/sources/wayback/wayback.go index c97305a..ca5eb1f 100644 --- a/pkg/xsubfind3r/sources/wayback/wayback.go +++ b/pkg/xsubfind3r/sources/wayback/wayback.go @@ -17,11 +17,11 @@ import ( type Source struct{} -func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) { - subdomainsChannel = make(chan sources.Subdomain) +func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) go func() { - defer close(subdomainsChannel) + defer close(results) var err error @@ -31,12 +31,29 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh getPagesRes, err = httpclient.SimpleGet(getPagesReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } var pages uint - if err = json.Unmarshal(getPagesRes.Body(), &pages); err != nil { + err = json.Unmarshal(getPagesRes.Body(), &pages) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -44,6 +61,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh regex, err = extractor.New(domain) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -54,6 +79,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh getURLsRes, err = httpclient.SimpleGet(getURLsReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -74,17 +107,31 @@ func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsCh subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") - subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain} + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result } } if err = scanner.Err(); err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } } }() - return + return results } func (source *Source) Name() string { diff --git a/pkg/xsubfind3r/xsubfind3r.go b/pkg/xsubfind3r/xsubfind3r.go index 0e5ff9d..4a360b3 100644 --- a/pkg/xsubfind3r/xsubfind3r.go +++ b/pkg/xsubfind3r/xsubfind3r.go @@ -81,32 +81,36 @@ func New(options *Options) (finder *Finder) { return } -func (finder *Finder) Find(domain string) (subdomains chan sources.Subdomain) { - subdomains = make(chan sources.Subdomain) +func (finder *Finder) Find(domain string) (results chan sources.Result) { + results = make(chan sources.Result) go func() { - defer close(subdomains) + defer close(results) - wg := &sync.WaitGroup{} seenSubdomains := &sync.Map{} + wg := &sync.WaitGroup{} + for _, source := range finder.Sources { wg.Add(1) go func(source sources.Source) { defer wg.Done() - results := source.Run(finder.SourcesConfiguration, domain) + sResults := source.Run(finder.SourcesConfiguration, domain) - for subdomain := range results { - subdomain.Value = strings.ReplaceAll(strings.ToLower(subdomain.Value), "*.", "") + for sResult := range sResults { + if sResult.Type == sources.Subdomain { + sResult.Value = strings.ToLower(sResult.Value) + sResult.Value = strings.ReplaceAll(sResult.Value, "*.", "") - _, loaded := seenSubdomains.LoadOrStore(subdomain.Value, struct{}{}) - if loaded { - continue + _, loaded := seenSubdomains.LoadOrStore(sResult.Value, struct{}{}) + if loaded { + continue + } } - subdomains <- subdomain + results <- sResult } }(source) } From 4f8896799b8930f4faec1c93668a81c628000586 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Fri, 4 Aug 2023 17:05:46 +0300 Subject: [PATCH 11/50] chore: Add install command to makefile --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f87e716..ed7d1aa 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ # Go(Golang) Options GOCMD=go -GOBUILD=$(GOCMD) build GOMOD=$(GOCMD) mod GOTEST=$(GOCMD) test +GOBUILD=$(GOCMD) build +GOINSTALL=$(GOCMD) install GOFLAGS := -v LDFLAGS := -s -w @@ -22,3 +23,5 @@ test: $(GOTEST) $(GOFLAGS) ./... build: $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o bin/xsubfind3r cmd/xsubfind3r/main.go +install: + $(GOINSTALL) $(GOFLAGS) ./... From dc4f1c0a4e121268fb532d55964488c8d70bffed Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Fri, 4 Aug 2023 23:17:33 +0300 Subject: [PATCH 12/50] chore: - --- pkg/xsubfind3r/sources/github/github.go | 35 ++++++++++++++++++++++++- pkg/xsubfind3r/sources/sources.go | 2 +- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pkg/xsubfind3r/sources/github/github.go b/pkg/xsubfind3r/sources/github/github.go index 65703c3..4cc8b6a 100644 --- a/pkg/xsubfind3r/sources/github/github.go +++ b/pkg/xsubfind3r/sources/github/github.go @@ -73,6 +73,14 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp isForbidden := searchRes != nil && searchRes.StatusCode() == fasthttp.StatusForbidden if err != nil && !isForbidden { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -87,7 +95,16 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp var searchResData searchResponse - if err = json.Unmarshal(searchRes.Body(), &searchResData); err != nil { + err = json.Unmarshal(searchRes.Body(), &searchResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } @@ -98,6 +115,14 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp getRawContentRes, err = httpclient.SimpleGet(getRawContentReqURL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + continue } @@ -138,6 +163,14 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + return } diff --git a/pkg/xsubfind3r/sources/sources.go b/pkg/xsubfind3r/sources/sources.go index 58bbe8b..c905ea4 100644 --- a/pkg/xsubfind3r/sources/sources.go +++ b/pkg/xsubfind3r/sources/sources.go @@ -4,7 +4,7 @@ package sources type Source interface { // Run takes in configuration which includes keys/tokens and other stuff, // and domain as arguments. - Run(config *Configuration, domain string) (results <-chan Result) + Run(config *Configuration, domain string) <-chan Result // Name returns the name of the source. Name() string } From ac841f7bf4d719ea8f0b86c1b66a4a324580f179 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 24 Aug 2023 23:18:48 +0300 Subject: [PATCH 13/50] chore: - --- .github/workflows/build-test.yml | 2 +- .github/workflows/lint-test.yml | 11 ++++++---- .github/workflows/release.yml | 5 ++--- .goreleaser.yaml | 6 ----- CONTRIBUTING.md | 4 ++-- Makefile | 22 ++++++++++++++++++- go.mod | 10 ++++----- go.sum | 10 +++++++++ .../sources/commoncrawl/commoncrawl.go | 6 +++-- 9 files changed, 52 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 5cee769..2715a84 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -29,7 +29,7 @@ jobs: with: go-version: '>=1.20' - - name: Checkout the code + name: Checkout the repository uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 51d5f90..dfe62fc 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -15,13 +15,14 @@ on: - '**.mod' workflow_dispatch: -permissions: - contents: read - jobs: lint: name: Lint Test runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write steps: - name: Set up Go @@ -29,7 +30,7 @@ jobs: with: go-version: '>=1.20' - - name: Checkout code + name: Checkout the repository uses: actions/checkout@v3 with: fetch-depth: 0 @@ -38,3 +39,5 @@ jobs: uses: golangci/golangci-lint-action@v3 with: version: v1.52.2 + args: --timeout 5m + working-directory: . \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e20215..adf85f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: 🎉 release +name: 🎉 Release on: create: @@ -18,7 +18,7 @@ jobs: with: go-version: '>=1.20' - - name: Checkout code + name: Checkout the repository uses: actions/checkout@v3 with: fetch-depth: 0 @@ -32,6 +32,5 @@ jobs: workdir: . env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - SLACK_WEBHOOK: "${{ secrets.SLACK_WEBHOOK }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" diff --git a/.goreleaser.yaml b/.goreleaser.yaml index af61664..e6118bc 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -49,12 +49,6 @@ checksum: algorithm: sha256 announce: - slack: - enabled: true - channel: '#release' - username: GoReleaser - message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}' - discord: enabled: true message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f5dc44..455e10f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,10 +41,10 @@ Pull requests should target the `dev` branch. Please also reference the issue fr When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible. Here are a few points to keep in mind: -* Please run `go fmt ./...` before committing to ensure code aligns with go standards. -* We use [`golangci-lint`](https://golangci-lint.run/) for linting Go code, run `golangci-lint run --fix` before submitting PR. Editors such as Visual Studio Code or JetBrains IntelliJ; with Go support plugin will offer `golangci-lint` automatically. * All dependencies must be defined in the `go.mod` file. * Advanced IDEs and code editors (like VSCode) will take care of that, but to be sure, run `go mod tidy` to validate dependencies. +* Please run `go fmt ./...` before committing to ensure code aligns with go standards. +* We use [`golangci-lint`](https://golangci-lint.run/) for linting Go code, run `golangci-lint run --fix` before submitting PR. Editors such as Visual Studio Code or JetBrains IntelliJ; with Go support plugin will offer `golangci-lint` automatically. * For details on the approved style, check out [Effective Go](https://golang.org/doc/effective_go.html). ### License diff --git a/Makefile b/Makefile index ed7d1aa..6eaaad6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # Go(Golang) Options GOCMD=go GOMOD=$(GOCMD) mod +GOGET=$(GOCMD) get +GOFMT=$(GOCMD) fmt GOTEST=$(GOCMD) test GOBUILD=$(GOCMD) build GOINSTALL=$(GOCMD) install @@ -15,13 +17,31 @@ ifneq ($(shell go env GOOS),darwin) LDFLAGS := -extldflags "-static" endif +.PHONY: tidy tidy: $(GOMOD) tidy + +.PHONY: update-deps +update-deps: + $(GOGET) -f -t -u ./... + $(GOGET) -f -u ./... + +.PHONY: format +format: + $(GOFMT) ./... + +.PHONY: lint lint: $(GOLANGCILINTRUN) ./... + +.PHONY: test test: $(GOTEST) $(GOFLAGS) ./... + +.PHONY: build build: $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o bin/xsubfind3r cmd/xsubfind3r/main.go + +.PHONY: install install: - $(GOINSTALL) $(GOFLAGS) ./... + $(GOINSTALL) $(GOFLAGS) ./... \ No newline at end of file diff --git a/go.mod b/go.mod index 974728b..1d32099 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( dario.cat/mergo v1.0.0 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f - github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47 + github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 github.com/logrusorgru/aurora/v3 v3.0.0 github.com/spf13/cast v1.5.1 github.com/spf13/pflag v1.0.5 @@ -16,9 +16,9 @@ require ( require ( github.com/andybalholm/brotli v1.0.5 // indirect - github.com/klauspost/compress v1.16.3 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect ) diff --git a/go.sum b/go.sum index 7e61d03..55fcefd 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,12 @@ github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkEN github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47 h1:LSeeeVmzUmykvyAS/liWZ+yQuMjM/1462m9s+WWV9YI= github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47/go.mod h1:8NAT2ECb69qzGf2d/ty0PVE3M3HK/+fXLtri2c47wQE= +github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 h1:6pwdpEJoB1woSToh0cxLh5QirNOAp2z7DzvMKiaqdro= +github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53/go.mod h1:Fc2vfWpIVFWUmCv1S0xVsz3mIPYwdgsa6f2vCgL4CrA= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= @@ -27,10 +31,16 @@ github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79 github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go index 1a5ece8..f11a103 100644 --- a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go +++ b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go @@ -118,7 +118,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source result := sources.Result{ Type: sources.Error, Source: source.Name(), - Error: fmt.Errorf(getURLsResData.Error), + Error: fmt.Errorf("%s", getURLsResData.Error), } results <- result @@ -126,7 +126,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - parsedURL, err := hqgourl.Parse(getURLsResData.URL) + var parsedURL *hqgourl.URL + + parsedURL, err = hqgourl.Parse(getURLsResData.URL) if err != nil { result := sources.Result{ Type: sources.Error, From 4465c660d03ee2e21dd32b3e25d556c5c79d008a Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Fri, 25 Aug 2023 03:13:55 +0300 Subject: [PATCH 14/50] refactor: fasthttp -> net/http --- go.mod | 5 +- go.sum | 20 +--- pkg/httpclient/client.go | 93 +++++++++++++++++++ pkg/xsubfind3r/httpclient/client.go | 71 -------------- pkg/xsubfind3r/httpclient/user-agent.go | 52 ----------- .../sources/alienvault/alienvault.go | 20 +++- pkg/xsubfind3r/sources/anubis/anubis.go | 12 ++- pkg/xsubfind3r/sources/bevigil/bevigil.go | 12 ++- pkg/xsubfind3r/sources/chaos/chaos.go | 21 ++++- .../sources/commoncrawl/commoncrawl.go | 26 ++++-- pkg/xsubfind3r/sources/crtsh/crtsh.go | 12 ++- pkg/xsubfind3r/sources/fullhunt/fullhunt.go | 12 ++- pkg/xsubfind3r/sources/github/github.go | 59 +++++++++--- .../sources/hackertarget/hackertarget.go | 13 ++- pkg/xsubfind3r/sources/intelx/intelx.go | 23 +++-- pkg/xsubfind3r/sources/shodan/shodan.go | 12 ++- pkg/xsubfind3r/sources/urlscan/urlscan.go | 12 ++- pkg/xsubfind3r/sources/wayback/wayback.go | 21 +++-- 18 files changed, 279 insertions(+), 217 deletions(-) create mode 100644 pkg/httpclient/client.go delete mode 100644 pkg/xsubfind3r/httpclient/client.go delete mode 100644 pkg/xsubfind3r/httpclient/user-agent.go diff --git a/go.mod b/go.mod index 1d32099..72d5ab1 100644 --- a/go.mod +++ b/go.mod @@ -4,20 +4,17 @@ go 1.20 require ( dario.cat/mergo v1.0.0 + github.com/hueristiq/hqgohttp v0.0.0-20230820163100-a4393cbdabf4 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 github.com/logrusorgru/aurora/v3 v3.0.0 github.com/spf13/cast v1.5.1 github.com/spf13/pflag v1.0.5 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 - github.com/valyala/fasthttp v1.48.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/andybalholm/brotli v1.0.5 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/term v0.11.0 // indirect diff --git a/go.sum b/go.sum index 55fcefd..a19308f 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,13 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hueristiq/hqgohttp v0.0.0-20230820163100-a4393cbdabf4 h1:3YNS1MsuKyaFIWTchLS5O4vL6JQ5IeH+DAdGuF1NNRQ= +github.com/hueristiq/hqgohttp v0.0.0-20230820163100-a4393cbdabf4/go.mod h1:7r7OQaM4qHFDclvhmHaU49Rmfg576ayZN//tS+JmCRM= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= -github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47 h1:LSeeeVmzUmykvyAS/liWZ+yQuMjM/1462m9s+WWV9YI= -github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47/go.mod h1:8NAT2ECb69qzGf2d/ty0PVE3M3HK/+fXLtri2c47wQE= github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 h1:6pwdpEJoB1woSToh0cxLh5QirNOAp2z7DzvMKiaqdro= github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53/go.mod h1:Fc2vfWpIVFWUmCv1S0xVsz3mIPYwdgsa6f2vCgL4CrA= -github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= -github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= @@ -25,20 +19,10 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= -github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go new file mode 100644 index 0000000..6f39454 --- /dev/null +++ b/pkg/httpclient/client.go @@ -0,0 +1,93 @@ +package httpclient + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" +) + +var ( + client = &http.Client{} +) + +func init() { + timeout := 30 + + Transport := &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // intentonal + }, + Dial: (&net.Dialer{ + Timeout: time.Duration(timeout) * time.Second, + }).Dial, + } + + client = &http.Client{ + Transport: Transport, + Timeout: time.Duration(timeout) * time.Second, + } +} + +func httpRequestWrapper(request *http.Request) (*http.Response, error) { + response, err := client.Do(request) + if err != nil { + return nil, err + } + + if response.StatusCode != http.StatusOK { + requestURL, _ := url.QueryUnescape(request.URL.String()) + + return response, fmt.Errorf("unexpected status code %d received from %s", response.StatusCode, requestURL) + } + + return response, nil +} + +// HTTPRequest makes any HTTP request to a URL with extended parameters +func HTTPRequest(method, requestURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(context.Background(), method, requestURL, body) + if err != nil { + return nil, err + } + + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Language", "en") + req.Header.Set("Connection", "close") + + if cookies != "" { + req.Header.Set("Cookie", cookies) + } + + for key, value := range headers { + req.Header.Set(key, value) + } + + return httpRequestWrapper(req) +} + +// Get makes a GET request to a URL with extended parameters +func Get(getURL, cookies string, headers map[string]string) (*http.Response, error) { + return HTTPRequest(http.MethodGet, getURL, cookies, headers, nil) +} + +// SimpleGet makes a simple GET request to a URL +func SimpleGet(getURL string) (*http.Response, error) { + return HTTPRequest(http.MethodGet, getURL, "", map[string]string{}, nil) +} + +// Post makes a POST request to a URL with extended parameters +func Post(postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { + return HTTPRequest(http.MethodPost, postURL, cookies, headers, body) +} + +// SimplePost makes a simple POST request to a URL +func SimplePost(postURL, contentType string, body io.Reader) (*http.Response, error) { + return HTTPRequest(http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body) +} diff --git a/pkg/xsubfind3r/httpclient/client.go b/pkg/xsubfind3r/httpclient/client.go deleted file mode 100644 index d648501..0000000 --- a/pkg/xsubfind3r/httpclient/client.go +++ /dev/null @@ -1,71 +0,0 @@ -package httpclient - -import ( - "fmt" - - "github.com/valyala/fasthttp" -) - -var ( - client = &fasthttp.Client{} -) - -func httpRequestWrapper(req *fasthttp.Request) (res *fasthttp.Response, err error) { - res = fasthttp.AcquireResponse() - - if err = client.Do(req, res); err != nil { - return - } - - if res.StatusCode() != fasthttp.StatusOK { - err = fmt.Errorf("unexpected status code") - } - - return -} - -func Request(method, URL, cookies string, headers map[string]string, body []byte) (res *fasthttp.Response, err error) { - req := fasthttp.AcquireRequest() - - req.SetRequestURI(URL) - req.SetBody(body) - req.Header.SetMethod(method) - - var agent string - - agent, err = UserAgent() - if err != nil { - return - } - - req.Header.Set("User-Agent", agent) - req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Language", "en") - req.Header.Set("Connection", "close") - - if cookies != "" { - req.Header.Set("Cookie", cookies) - } - - for key, value := range headers { - req.Header.Set(key, value) - } - - return httpRequestWrapper(req) -} - -func SimpleGet(URL string) (*fasthttp.Response, error) { - return Request(fasthttp.MethodGet, URL, "", map[string]string{}, nil) -} - -func Get(URL, cookies string, headers map[string]string) (*fasthttp.Response, error) { - return Request(fasthttp.MethodGet, URL, cookies, headers, nil) -} - -func SimplePost(URL, contentType string, body []byte) (*fasthttp.Response, error) { - return Request(fasthttp.MethodPost, URL, "", map[string]string{"Content-Type": contentType}, body) -} - -func Post(URL, cookies string, headers map[string]string, body []byte) (*fasthttp.Response, error) { - return Request(fasthttp.MethodPost, URL, cookies, headers, body) -} diff --git a/pkg/xsubfind3r/httpclient/user-agent.go b/pkg/xsubfind3r/httpclient/user-agent.go deleted file mode 100644 index 639c3b3..0000000 --- a/pkg/xsubfind3r/httpclient/user-agent.go +++ /dev/null @@ -1,52 +0,0 @@ -package httpclient - -import ( - "crypto/rand" - "fmt" - "math/big" -) - -var agents = []string{ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0", - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12H321 Safari/600.1.4", - "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", - "Mozilla/5.0 (iPad; CPU OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", -} - -func UserAgent() (agent string, err error) { - return pickRandom(agents) -} - -func pickRandom[T any](v []T) (picked T, err error) { - length := len(v) - - if length == 0 { - return - } - - // Generate a cryptographically secure random index - max := big.NewInt(int64(length)) - - var indexBig *big.Int - - indexBig, err = rand.Int(rand.Reader, max) - if err != nil { - err = fmt.Errorf("failed to generate random index: %v", err) - - return - } - - index := indexBig.Int64() - - // Return the element at the random index - picked = v[index] - - return -} diff --git a/pkg/xsubfind3r/sources/alienvault/alienvault.go b/pkg/xsubfind3r/sources/alienvault/alienvault.go index 8143995..7d5e183 100644 --- a/pkg/xsubfind3r/sources/alienvault/alienvault.go +++ b/pkg/xsubfind3r/sources/alienvault/alienvault.go @@ -3,10 +3,10 @@ package alienvault import ( "encoding/json" "fmt" + "net/http" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type getPassiveDNSResponse struct { @@ -29,7 +29,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getPassiveDNSReqURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain) - var getPassiveDNSRes *fasthttp.Response + var getPassiveDNSRes *http.Response getPassiveDNSRes, err = httpclient.SimpleGet(getPassiveDNSReqURL) if err != nil { @@ -46,7 +46,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var getPassiveDNSResData getPassiveDNSResponse - err = json.Unmarshal(getPassiveDNSRes.Body(), &getPassiveDNSResData) + err = json.NewDecoder(getPassiveDNSRes.Body).Decode(&getPassiveDNSResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -56,10 +56,22 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getPassiveDNSRes.Body.Close() + return } + getPassiveDNSRes.Body.Close() + if getPassiveDNSResData.Error != "" { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: fmt.Errorf("%s, %s", getPassiveDNSResData.Detail, getPassiveDNSResData.Error), + } + + results <- result + return } diff --git a/pkg/xsubfind3r/sources/anubis/anubis.go b/pkg/xsubfind3r/sources/anubis/anubis.go index 4a5009a..53ca41c 100644 --- a/pkg/xsubfind3r/sources/anubis/anubis.go +++ b/pkg/xsubfind3r/sources/anubis/anubis.go @@ -3,10 +3,10 @@ package anubis import ( "encoding/json" "fmt" + "net/http" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type Source struct{} @@ -21,7 +21,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getSubdomainsReqURL := fmt.Sprintf("https://jldc.me/anubis/subdomains/%s", domain) - var getSubdomainsRes *fasthttp.Response + var getSubdomainsRes *http.Response getSubdomainsRes, err = httpclient.SimpleGet(getSubdomainsReqURL) if err != nil { @@ -38,7 +38,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var getSubdomainsResData []string - err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData) + err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -48,9 +48,13 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getSubdomainsRes.Body.Close() + return } + getSubdomainsRes.Body.Close() + for _, subdomain := range getSubdomainsResData { result := sources.Result{ Type: sources.Subdomain, diff --git a/pkg/xsubfind3r/sources/bevigil/bevigil.go b/pkg/xsubfind3r/sources/bevigil/bevigil.go index 600bc85..4202035 100644 --- a/pkg/xsubfind3r/sources/bevigil/bevigil.go +++ b/pkg/xsubfind3r/sources/bevigil/bevigil.go @@ -3,10 +3,10 @@ package bevigil import ( "encoding/json" "fmt" + "net/http" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type getSubdomainsResponse struct { @@ -47,7 +47,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsReqURL := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) - var getSubdomainsRes *fasthttp.Response + var getSubdomainsRes *http.Response getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders) if err != nil { @@ -64,7 +64,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var getSubdomainsResData getSubdomainsResponse - err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData) + err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -74,9 +74,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getSubdomainsRes.Body.Close() + return } + getSubdomainsRes.Body.Close() + for _, subdomain := range getSubdomainsResData.Subdomains { result := sources.Result{ Type: sources.Subdomain, diff --git a/pkg/xsubfind3r/sources/chaos/chaos.go b/pkg/xsubfind3r/sources/chaos/chaos.go index 0cb822f..4392fc7 100644 --- a/pkg/xsubfind3r/sources/chaos/chaos.go +++ b/pkg/xsubfind3r/sources/chaos/chaos.go @@ -3,10 +3,10 @@ package chaos import ( "encoding/json" "fmt" + "net/http" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type getSubdomainsResponse struct { @@ -44,7 +44,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsReqURL := fmt.Sprintf("https://dns.projectdiscovery.io/dns/%s/subdomains", domain) - var getSubdomainsRes *fasthttp.Response + var getSubdomainsRes *http.Response getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders) if err != nil { @@ -61,10 +61,23 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var getSubdomainsResData getSubdomainsResponse - if err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData); err != nil { + err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + getSubdomainsRes.Body.Close() + return } + getSubdomainsRes.Body.Close() + for _, record := range getSubdomainsResData.Subdomains { result := sources.Result{ Type: sources.Subdomain, diff --git a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go index f11a103..2cebd08 100644 --- a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go +++ b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go @@ -2,15 +2,14 @@ package commoncrawl import ( "bufio" - "bytes" "encoding/json" "fmt" + "net/http" "sync" "github.com/hueristiq/hqgourl" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type getIndexesResponse []struct { @@ -35,7 +34,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var err error - var getIndexesRes *fasthttp.Response + var getIndexesRes *http.Response getIndexesRes, err = httpclient.SimpleGet(getIndexesReqURL) if err != nil { @@ -52,7 +51,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var getIndexesResData getIndexesResponse - err = json.Unmarshal(getIndexesRes.Body(), &getIndexesResData) + err = json.NewDecoder(getIndexesRes.Body).Decode(&getIndexesResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -62,9 +61,13 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getIndexesRes.Body.Close() + return } + getIndexesRes.Body.Close() + wg := new(sync.WaitGroup) for _, indexData := range getIndexesResData { @@ -81,7 +84,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var err error - var getURLsRes *fasthttp.Response + var getURLsRes *http.Response getURLsRes, err = httpclient.Get(getURLsReqURL, "", getURLsReqHeaders) if err != nil { @@ -96,9 +99,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - scanner := bufio.NewScanner(bytes.NewReader(getURLsRes.Body())) + scanner := bufio.NewScanner(getURLsRes.Body) for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + var getURLsResData getURLsResponse err = json.Unmarshal(scanner.Bytes(), &getURLsResData) @@ -159,8 +167,12 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getURLsRes.Body.Close() + return } + + getURLsRes.Body.Close() }(indexData.API) } diff --git a/pkg/xsubfind3r/sources/crtsh/crtsh.go b/pkg/xsubfind3r/sources/crtsh/crtsh.go index f818f47..b1db270 100644 --- a/pkg/xsubfind3r/sources/crtsh/crtsh.go +++ b/pkg/xsubfind3r/sources/crtsh/crtsh.go @@ -3,13 +3,13 @@ package crtsh import ( "encoding/json" "fmt" + "net/http" "regexp" "strings" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type getNameValuesResponse []struct { @@ -29,7 +29,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getNameValuesReqURL := fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain) - var getNameValuesRes *fasthttp.Response + var getNameValuesRes *http.Response getNameValuesRes, err = httpclient.SimpleGet(getNameValuesReqURL) if err != nil { @@ -46,7 +46,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var getNameValuesResData getNameValuesResponse - err = json.Unmarshal(getNameValuesRes.Body(), &getNameValuesResData) + err = json.NewDecoder(getNameValuesRes.Body).Decode(&getNameValuesResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -56,9 +56,13 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getNameValuesRes.Body.Close() + return } + getNameValuesRes.Body.Close() + var regex *regexp.Regexp regex, err = extractor.New(domain) diff --git a/pkg/xsubfind3r/sources/fullhunt/fullhunt.go b/pkg/xsubfind3r/sources/fullhunt/fullhunt.go index 099e1fe..35c890f 100644 --- a/pkg/xsubfind3r/sources/fullhunt/fullhunt.go +++ b/pkg/xsubfind3r/sources/fullhunt/fullhunt.go @@ -3,10 +3,10 @@ package fullhunt import ( "encoding/json" "fmt" + "net/http" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type getSubdomainsResponse struct { @@ -48,7 +48,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsReqURL := fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain) - var getSubdomainsRes *fasthttp.Response + var getSubdomainsRes *http.Response getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders) if err != nil { @@ -65,7 +65,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var getSubdomainsResData getSubdomainsResponse - err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData) + err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -75,9 +75,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getSubdomainsRes.Body.Close() + return } + getSubdomainsRes.Body.Close() + for _, subdomain := range getSubdomainsResData.Hosts { result := sources.Result{ Type: sources.Subdomain, diff --git a/pkg/xsubfind3r/sources/github/github.go b/pkg/xsubfind3r/sources/github/github.go index 4cc8b6a..5b39cdc 100644 --- a/pkg/xsubfind3r/sources/github/github.go +++ b/pkg/xsubfind3r/sources/github/github.go @@ -1,18 +1,20 @@ package github import ( + "bufio" "encoding/json" "fmt" + "net/http" "net/url" "regexp" "strings" "time" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + hqgohttpstatus "github.com/hueristiq/hqgohttp/status" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" "github.com/spf13/cast" "github.com/tomnomnom/linkheader" - "github.com/valyala/fasthttp" ) type searchResponse struct { @@ -66,11 +68,11 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp var err error - var searchRes *fasthttp.Response + var searchRes *http.Response searchRes, err = httpclient.Get(searchReqURL, "", searchReqHeaders) - isForbidden := searchRes != nil && searchRes.StatusCode() == fasthttp.StatusForbidden + isForbidden := searchRes != nil && searchRes.StatusCode == hqgohttpstatus.Forbidden if err != nil && !isForbidden { result := sources.Result{ @@ -84,9 +86,9 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp return } - ratelimitRemaining := cast.ToInt64(searchRes.Header.Peek("X-Ratelimit-Remaining")) + ratelimitRemaining := cast.ToInt64(searchRes.Header.Get("X-Ratelimit-Remaining")) if isForbidden && ratelimitRemaining == 0 { - retryAfterSeconds := cast.ToInt64(searchRes.Header.Peek("Retry-After")) + retryAfterSeconds := cast.ToInt64(searchRes.Header.Get("Retry-After")) tokens.setCurrentTokenExceeded(retryAfterSeconds) @@ -95,7 +97,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp var searchResData searchResponse - err = json.Unmarshal(searchRes.Body(), &searchResData) + err = json.NewDecoder(searchRes.Body).Decode(&searchResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -105,13 +107,17 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp results <- result + searchRes.Body.Close() + return } + searchRes.Body.Close() + for _, item := range searchResData.Items { getRawContentReqURL := getRawContentURL(item.HTMLURL) - var getRawContentRes *fasthttp.Response + var getRawContentRes *http.Response getRawContentRes, err = httpclient.SimpleGet(getRawContentReqURL) if err != nil { @@ -126,22 +132,47 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp continue } - if getRawContentRes.StatusCode() != fasthttp.StatusOK { + if getRawContentRes.StatusCode != hqgohttpstatus.OK { continue } - subdomains := domainRegexp.FindAllString(string(getRawContentRes.Body()), -1) + scanner := bufio.NewScanner(getRawContentRes.Body) + + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + + subdomains := domainRegexp.FindAllString(line, -1) + + for _, subdomain := range subdomains { + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result + } + } - for _, subdomain := range subdomains { + if err = scanner.Err(); err != nil { result := sources.Result{ - Type: sources.Subdomain, + Type: sources.Error, Source: source.Name(), - Value: subdomain, + Error: err, } results <- result + + getRawContentRes.Body.Close() + + return } + getRawContentRes.Body.Close() + for _, textMatch := range item.TextMatches { subdomains := domainRegexp.FindAllString(textMatch.Fragment, -1) @@ -157,7 +188,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp } } - linksHeader := linkheader.Parse(string(searchRes.Header.Peek("Link"))) + linksHeader := linkheader.Parse(searchRes.Header.Get("Link")) for _, link := range linksHeader { if link.Rel == "next" { diff --git a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go b/pkg/xsubfind3r/sources/hackertarget/hackertarget.go index 8ff34ce..327012b 100644 --- a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go +++ b/pkg/xsubfind3r/sources/hackertarget/hackertarget.go @@ -2,15 +2,14 @@ package hackertarget import ( "bufio" - "bytes" "fmt" + "net/http" "net/url" "regexp" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type Source struct{} @@ -25,7 +24,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source hostSearchReqURL := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) - var hostSearchRes *fasthttp.Response + var hostSearchRes *http.Response hostSearchRes, err = httpclient.SimpleGet(hostSearchReqURL) if err != nil { @@ -55,7 +54,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - scanner := bufio.NewScanner(bytes.NewReader(hostSearchRes.Body())) + scanner := bufio.NewScanner(hostSearchRes.Body) for scanner.Scan() { line := scanner.Text() @@ -87,8 +86,12 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + hostSearchRes.Body.Close() + return } + + hostSearchRes.Body.Close() }() return results diff --git a/pkg/xsubfind3r/sources/intelx/intelx.go b/pkg/xsubfind3r/sources/intelx/intelx.go index 4efc38d..5afe84c 100644 --- a/pkg/xsubfind3r/sources/intelx/intelx.go +++ b/pkg/xsubfind3r/sources/intelx/intelx.go @@ -1,14 +1,15 @@ package intelx import ( + "bytes" "encoding/json" "fmt" + "net/http" "strings" "time" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type searchRequest struct { @@ -91,9 +92,9 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s return } - var searchRes *fasthttp.Response + var searchRes *http.Response - searchRes, err = httpclient.SimplePost(searchReqURL, "application/json", searchReqBodyBytes) + searchRes, err = httpclient.SimplePost(searchReqURL, "application/json", bytes.NewBuffer(searchReqBodyBytes)) if err != nil { result := sources.Result{ Type: sources.Error, @@ -108,7 +109,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var searchResData searchResponse - err = json.Unmarshal(searchRes.Body(), &searchResData) + err = json.NewDecoder(searchRes.Body).Decode(&searchResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -118,14 +119,18 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + searchRes.Body.Close() + return } + searchRes.Body.Close() + getResultsReqURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", intelXHost, intelXKey, searchResData.ID) status := 0 for status == 0 || status == 3 { - var getResultsRes *fasthttp.Response + var getResultsRes *http.Response getResultsRes, err = httpclient.Get(getResultsReqURL, "", nil) if err != nil { @@ -142,7 +147,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var getResultsResData getResultsResponse - err = json.Unmarshal(getResultsRes.Body(), &getResultsResData) + err = json.NewDecoder(getResultsRes.Body).Decode(&getResultsResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -152,9 +157,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getResultsRes.Body.Close() + return } + getResultsRes.Body.Close() + status = getResultsResData.Status for _, record := range getResultsResData.Selectors { diff --git a/pkg/xsubfind3r/sources/shodan/shodan.go b/pkg/xsubfind3r/sources/shodan/shodan.go index f5867a5..0bbcafa 100644 --- a/pkg/xsubfind3r/sources/shodan/shodan.go +++ b/pkg/xsubfind3r/sources/shodan/shodan.go @@ -3,10 +3,10 @@ package shodan import ( "encoding/json" "fmt" + "net/http" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type getDNSResponse struct { @@ -43,7 +43,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getDNSReqURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s", domain, key) - var getDNSRes *fasthttp.Response + var getDNSRes *http.Response getDNSRes, err = httpclient.SimpleGet(getDNSReqURL) if err != nil { @@ -60,7 +60,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var getDNSResData getDNSResponse - err = json.Unmarshal(getDNSRes.Body(), &getDNSResData) + err = json.NewDecoder(getDNSRes.Body).Decode(&getDNSResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -70,9 +70,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getDNSRes.Body.Close() + return } + getDNSRes.Body.Close() + for _, subdomain := range getDNSResData.Subdomains { result := sources.Result{ Type: sources.Subdomain, diff --git a/pkg/xsubfind3r/sources/urlscan/urlscan.go b/pkg/xsubfind3r/sources/urlscan/urlscan.go index f7b5c01..f365752 100644 --- a/pkg/xsubfind3r/sources/urlscan/urlscan.go +++ b/pkg/xsubfind3r/sources/urlscan/urlscan.go @@ -3,11 +3,11 @@ package urlscan import ( "encoding/json" "fmt" + "net/http" "strings" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type searchResponse struct { @@ -71,7 +71,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s searchReqURL := fmt.Sprintf("https://urlscan.io/api/v1/search/?q=domain:%s&size=100", domain) + after - var searchRes *fasthttp.Response + var searchRes *http.Response searchRes, err = httpclient.Get(searchReqURL, "", searchReqHeaders) if err != nil { @@ -88,7 +88,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var searchResData searchResponse - err = json.Unmarshal(searchRes.Body(), &searchResData) + err = json.NewDecoder(searchRes.Body).Decode(&searchResData) if err != nil { result := sources.Result{ Type: sources.Error, @@ -98,9 +98,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + searchRes.Body.Close() + return } + searchRes.Body.Close() + if searchResData.Status == 429 { break } diff --git a/pkg/xsubfind3r/sources/wayback/wayback.go b/pkg/xsubfind3r/sources/wayback/wayback.go index ca5eb1f..9f2de3d 100644 --- a/pkg/xsubfind3r/sources/wayback/wayback.go +++ b/pkg/xsubfind3r/sources/wayback/wayback.go @@ -2,17 +2,16 @@ package wayback import ( "bufio" - "bytes" "encoding/json" "fmt" + "net/http" "net/url" "regexp" "strings" + "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/valyala/fasthttp" ) type Source struct{} @@ -27,7 +26,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getPagesReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey&showNumPages=true", domain) - var getPagesRes *fasthttp.Response + var getPagesRes *http.Response getPagesRes, err = httpclient.SimpleGet(getPagesReqURL) if err != nil { @@ -44,7 +43,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var pages uint - err = json.Unmarshal(getPagesRes.Body(), &pages) + err = json.NewDecoder(getPagesRes.Body).Decode(&pages) if err != nil { result := sources.Result{ Type: sources.Error, @@ -54,9 +53,13 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getPagesRes.Body.Close() + return } + getPagesRes.Body.Close() + var regex *regexp.Regexp regex, err = extractor.New(domain) @@ -75,7 +78,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source for page := uint(0); page < pages; page++ { getURLsReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey&page=%d", domain, page) - var getURLsRes *fasthttp.Response + var getURLsRes *http.Response getURLsRes, err = httpclient.SimpleGet(getURLsReqURL) if err != nil { @@ -90,7 +93,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - scanner := bufio.NewScanner(bytes.NewReader(getURLsRes.Body())) + scanner := bufio.NewScanner(getURLsRes.Body) for scanner.Scan() { line := scanner.Text() @@ -126,8 +129,12 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getURLsRes.Body.Close() + return } + + getURLsRes.Body.Close() } }() From 7956a58d9f5fc36e4283142c6e1cbb3093b3d752 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 02:45:32 +0300 Subject: [PATCH 15/50] refactor: net/http -> hqgohttp/client --- go.mod | 4 +++- go.sum | 8 ++++++-- pkg/httpclient/client.go | 32 +++++++++++--------------------- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 72d5ab1..0418f4e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( dario.cat/mergo v1.0.0 - github.com/hueristiq/hqgohttp v0.0.0-20230820163100-a4393cbdabf4 + github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 github.com/logrusorgru/aurora/v3 v3.0.0 @@ -15,7 +15,9 @@ require ( ) require ( + github.com/Mzack9999/go-http-digest-auth-client v0.6.0 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index a19308f..be293e4 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,11 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Mzack9999/go-http-digest-auth-client v0.6.0 h1:LXVNMsj7qiNVmlZByFbjJmXf6SOm/uoo04XmnNcWPms= +github.com/Mzack9999/go-http-digest-auth-client v0.6.0/go.mod h1:gbwaYYXwA15ZfIxMyY5QU1acATDyNKEuG5TylBCL7AM= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/hueristiq/hqgohttp v0.0.0-20230820163100-a4393cbdabf4 h1:3YNS1MsuKyaFIWTchLS5O4vL6JQ5IeH+DAdGuF1NNRQ= -github.com/hueristiq/hqgohttp v0.0.0-20230820163100-a4393cbdabf4/go.mod h1:7r7OQaM4qHFDclvhmHaU49Rmfg576ayZN//tS+JmCRM= +github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21 h1:liOyeZd487AUwCWbTOXKMgdvg82JOBms6xb3RYAdSO0= +github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21/go.mod h1:HewuEHc3F5aA5op40SzMuW/S0ZxW98pnrqXBeF+HsIs= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 h1:6pwdpEJoB1woSToh0cxLh5QirNOAp2z7DzvMKiaqdro= @@ -25,6 +27,8 @@ golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 6f39454..5611199 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -2,41 +2,31 @@ package httpclient import ( "context" - "crypto/tls" "fmt" "io" - "net" "net/http" "net/url" - "time" + + hqgohttpclient "github.com/hueristiq/hqgohttp/client" ) var ( - client = &http.Client{} + client *hqgohttpclient.Client ) func init() { - timeout := 30 - - Transport := &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // intentonal - }, - Dial: (&net.Dialer{ - Timeout: time.Duration(timeout) * time.Second, - }).Dial, - } + options := hqgohttpclient.DefaultOptionsSpraying - client = &http.Client{ - Transport: Transport, - Timeout: time.Duration(timeout) * time.Second, - } + client, _ = hqgohttpclient.New(options) } func httpRequestWrapper(request *http.Request) (*http.Response, error) { - response, err := client.Do(request) + r, err := hqgohttpclient.FromRequest(request) + if err != nil { + return nil, err + } + + response, err := client.Do(r) if err != nil { return nil, err } From 703604ed7c84f5480ca9a8276c38de935cc0084d Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 15:55:11 +0300 Subject: [PATCH 16/50] refactor(wayback): - --- pkg/xsubfind3r/sources/wayback/wayback.go | 56 ++++++++++------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/pkg/xsubfind3r/sources/wayback/wayback.go b/pkg/xsubfind3r/sources/wayback/wayback.go index 9f2de3d..a290469 100644 --- a/pkg/xsubfind3r/sources/wayback/wayback.go +++ b/pkg/xsubfind3r/sources/wayback/wayback.go @@ -1,13 +1,10 @@ package wayback import ( - "bufio" "encoding/json" "fmt" "net/http" - "net/url" "regexp" - "strings" "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" @@ -76,7 +73,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source } for page := uint(0); page < pages; page++ { - getURLsReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey&page=%d", domain, page) + getURLsReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&fl=original&page=%d", domain, page) var getURLsRes *http.Response @@ -93,34 +90,10 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - scanner := bufio.NewScanner(getURLsRes.Body) + var getURLsResData [][]string - for scanner.Scan() { - line := scanner.Text() - - if line == "" { - continue - } - - line, _ = url.QueryUnescape(line) - subdomain := regex.FindString(line) - - if subdomain != "" { - subdomain = strings.ToLower(subdomain) - subdomain = strings.TrimPrefix(subdomain, "25") - subdomain = strings.TrimPrefix(subdomain, "2f") - - result := sources.Result{ - Type: sources.Subdomain, - Source: source.Name(), - Value: subdomain, - } - - results <- result - } - } - - if err = scanner.Err(); err != nil { + err = json.NewDecoder(getURLsRes.Body).Decode(&getURLsResData) + if err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), @@ -135,6 +108,27 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source } getURLsRes.Body.Close() + + // check if there's results, wayback's pagination response + // is not always correct when using a filter + if len(getURLsResData) == 0 { + break + } + + // Slicing as [1:] to skip first result by default + for _, entry := range getURLsResData[1:] { + subdomain := regex.FindString(entry[0]) + + if subdomain != "" { + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result + } + } } }() From 9f11d6323a1d1ed819b1275e5c3470e282376d75 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 16:56:22 +0300 Subject: [PATCH 17/50] refactor(urlscan): - --- pkg/xsubfind3r/sources/urlscan/urlscan.go | 46 +++++++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/pkg/xsubfind3r/sources/urlscan/urlscan.go b/pkg/xsubfind3r/sources/urlscan/urlscan.go index f365752..596c7e0 100644 --- a/pkg/xsubfind3r/sources/urlscan/urlscan.go +++ b/pkg/xsubfind3r/sources/urlscan/urlscan.go @@ -4,9 +4,10 @@ import ( "encoding/json" "fmt" "net/http" - "strings" + "regexp" "github.com/hueristiq/xsubfind3r/pkg/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" ) @@ -65,7 +66,21 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s after := "" if searchAfter != nil { - searchAfterJSON, _ := json.Marshal(searchAfter) + var searchAfterJSON []byte + + searchAfterJSON, err = json.Marshal(searchAfter) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + return + } + after = "&search_after=" + string(searchAfterJSON) } @@ -109,18 +124,33 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s break } - for _, result := range searchResData.Results { - if !strings.HasSuffix(result.Page.Domain, "."+domain) { - continue - } + var regex *regexp.Regexp + regex, err = extractor.New(domain) + if err != nil { result := sources.Result{ - Type: sources.Subdomain, + Type: sources.Error, Source: source.Name(), - Value: result.Page.Domain, + Error: err, } results <- result + + return + } + + for _, result := range searchResData.Results { + subdomain := regex.FindString(result.Page.Domain) + + if subdomain != "" { + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } + + results <- result + } } if !searchResData.HasMore { From a8664de50ba8f8340ecd1cd17da83a8843aa9657 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:31:22 +0300 Subject: [PATCH 18/50] refactor(hackertarget): - --- pkg/xsubfind3r/sources/hackertarget/hackertarget.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go b/pkg/xsubfind3r/sources/hackertarget/hackertarget.go index 327012b..cc21d30 100644 --- a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go +++ b/pkg/xsubfind3r/sources/hackertarget/hackertarget.go @@ -4,7 +4,6 @@ import ( "bufio" "fmt" "net/http" - "net/url" "regexp" "github.com/hueristiq/xsubfind3r/pkg/httpclient" @@ -63,7 +62,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source continue } - line, _ = url.QueryUnescape(line) match := regex.FindAllString(line, -1) for _, subdomain := range match { From 269d24faf46e04ecbe67d1b3ec140674e2f62243 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:35:02 +0300 Subject: [PATCH 19/50] refactor(urlscan): - --- pkg/xsubfind3r/sources/urlscan/urlscan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/xsubfind3r/sources/urlscan/urlscan.go b/pkg/xsubfind3r/sources/urlscan/urlscan.go index 596c7e0..6eb8e6f 100644 --- a/pkg/xsubfind3r/sources/urlscan/urlscan.go +++ b/pkg/xsubfind3r/sources/urlscan/urlscan.go @@ -140,9 +140,9 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s } for _, result := range searchResData.Results { - subdomain := regex.FindString(result.Page.Domain) + match := regex.FindAllString(result.Page.Domain, -1) - if subdomain != "" { + for _, subdomain := range match { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), From 6a24664a1c7ce1d1b7f30d62fbd15f8e932bd23a Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:38:02 +0300 Subject: [PATCH 20/50] refactor(wayback): - --- pkg/xsubfind3r/sources/wayback/wayback.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/xsubfind3r/sources/wayback/wayback.go b/pkg/xsubfind3r/sources/wayback/wayback.go index a290469..2b0e295 100644 --- a/pkg/xsubfind3r/sources/wayback/wayback.go +++ b/pkg/xsubfind3r/sources/wayback/wayback.go @@ -117,9 +117,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source // Slicing as [1:] to skip first result by default for _, entry := range getURLsResData[1:] { - subdomain := regex.FindString(entry[0]) + match := regex.FindAllString(entry[0], -1) - if subdomain != "" { + for _, subdomain := range match { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), From a43f596de001aa736bdb2e01de51f61a1907ee79 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 19:11:52 +0300 Subject: [PATCH 21/50] refactor(github): - --- go.mod | 2 +- go.sum | 2 ++ pkg/xsubfind3r/sources/github/github.go | 28 +++++++++++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 0418f4e..c69bc55 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( dario.cat/mergo v1.0.0 - github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21 + github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 github.com/logrusorgru/aurora/v3 v3.0.0 diff --git a/go.sum b/go.sum index be293e4..00a3ac7 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21 h1:liOyeZd487AUwCWbTOXKMgdvg82JOBms6xb3RYAdSO0= github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21/go.mod h1:HewuEHc3F5aA5op40SzMuW/S0ZxW98pnrqXBeF+HsIs= +github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf h1:qnTJknI2w4zIAKtY/5K8iAW91oj/wZgrHxzWvmes18Q= +github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf/go.mod h1:HewuEHc3F5aA5op40SzMuW/S0ZxW98pnrqXBeF+HsIs= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 h1:6pwdpEJoB1woSToh0cxLh5QirNOAp2z7DzvMKiaqdro= diff --git a/pkg/xsubfind3r/sources/github/github.go b/pkg/xsubfind3r/sources/github/github.go index 5b39cdc..21246d4 100644 --- a/pkg/xsubfind3r/sources/github/github.go +++ b/pkg/xsubfind3r/sources/github/github.go @@ -10,8 +10,10 @@ import ( "strings" "time" + hqgohttpheaders "github.com/hueristiq/hqgohttp/headers" hqgohttpstatus "github.com/hueristiq/hqgohttp/status" "github.com/hueristiq/xsubfind3r/pkg/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" "github.com/spf13/cast" "github.com/tomnomnom/linkheader" @@ -44,7 +46,20 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s searchReqURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=%q&sort=created&order=asc", domain) - source.Enumerate(searchReqURL, domainRegexp(domain), tokens, results, config) + regex, err := extractor.New(domain) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + return + } + + source.Enumerate(searchReqURL, regex, tokens, results, config) }() return results @@ -86,9 +101,9 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp return } - ratelimitRemaining := cast.ToInt64(searchRes.Header.Get("X-Ratelimit-Remaining")) + ratelimitRemaining := cast.ToInt64(searchRes.Header.Get(hqgohttpheaders.XRatelimitRemaining)) if isForbidden && ratelimitRemaining == 0 { - retryAfterSeconds := cast.ToInt64(searchRes.Header.Get("Retry-After")) + retryAfterSeconds := cast.ToInt64(searchRes.Header.Get(hqgohttpheaders.RetryAfter)) tokens.setCurrentTokenExceeded(retryAfterSeconds) @@ -188,7 +203,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp } } - linksHeader := linkheader.Parse(searchRes.Header.Get("Link")) + linksHeader := linkheader.Parse(searchRes.Header.Get(hqgohttpheaders.Link)) for _, link := range linksHeader { if link.Rel == "next" { @@ -215,11 +230,6 @@ func getRawContentURL(htmlURL string) string { return strings.ReplaceAll(domain, "/blob/", "/") } -func domainRegexp(domain string) *regexp.Regexp { - rdomain := strings.ReplaceAll(domain, ".", "\\.") - return regexp.MustCompile("(\\w+[.])*" + rdomain) -} - func (source *Source) Name() string { return "github" } From 6cf22e432211e0b1e36219c10e71e9fd48c28fe7 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 19:15:32 +0300 Subject: [PATCH 22/50] refactor(crtsh): - --- pkg/xsubfind3r/sources/crtsh/crtsh.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/xsubfind3r/sources/crtsh/crtsh.go b/pkg/xsubfind3r/sources/crtsh/crtsh.go index b1db270..177e082 100644 --- a/pkg/xsubfind3r/sources/crtsh/crtsh.go +++ b/pkg/xsubfind3r/sources/crtsh/crtsh.go @@ -80,19 +80,17 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source for _, record := range getNameValuesResData { for _, value := range strings.Split(record.NameValue, "\n") { - subdomain := regex.FindString(value) + match := regex.FindAllString(value, -1) - if subdomain == "" { - continue - } + for _, subdomain := range match { + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: subdomain, + } - result := sources.Result{ - Type: sources.Subdomain, - Source: source.Name(), - Value: subdomain, + results <- result } - - results <- result } } }() From b543e4180e3b6cffdd7acbc747d46b5341c6e5d2 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 19:20:40 +0300 Subject: [PATCH 23/50] refactor(alienvault): alienvault -> otx --- .../sources/{alienvault/alienvault.go => otx/otx.go} | 4 ++-- pkg/xsubfind3r/sources/sources.go | 2 +- pkg/xsubfind3r/xsubfind3r.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) rename pkg/xsubfind3r/sources/{alienvault/alienvault.go => otx/otx.go} (97%) diff --git a/pkg/xsubfind3r/sources/alienvault/alienvault.go b/pkg/xsubfind3r/sources/otx/otx.go similarity index 97% rename from pkg/xsubfind3r/sources/alienvault/alienvault.go rename to pkg/xsubfind3r/sources/otx/otx.go index 7d5e183..803416a 100644 --- a/pkg/xsubfind3r/sources/alienvault/alienvault.go +++ b/pkg/xsubfind3r/sources/otx/otx.go @@ -1,4 +1,4 @@ -package alienvault +package otx import ( "encoding/json" @@ -90,5 +90,5 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source } func (source *Source) Name() string { - return "alienvault" + return "otx" } diff --git a/pkg/xsubfind3r/sources/sources.go b/pkg/xsubfind3r/sources/sources.go index c905ea4..ac2b440 100644 --- a/pkg/xsubfind3r/sources/sources.go +++ b/pkg/xsubfind3r/sources/sources.go @@ -10,7 +10,6 @@ type Source interface { } var List = []string{ - "alienvault", "anubis", "bevigil", "chaos", @@ -20,6 +19,7 @@ var List = []string{ "github", "hackertarget", "intelx", + "otx", "shodan", "urlscan", "wayback", diff --git a/pkg/xsubfind3r/xsubfind3r.go b/pkg/xsubfind3r/xsubfind3r.go index 4a360b3..7c579fa 100644 --- a/pkg/xsubfind3r/xsubfind3r.go +++ b/pkg/xsubfind3r/xsubfind3r.go @@ -5,7 +5,6 @@ import ( "sync" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/alienvault" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/anubis" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/bevigil" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/chaos" @@ -15,6 +14,7 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/github" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/hackertarget" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/intelx" + "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/otx" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/shodan" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/urlscan" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/wayback" @@ -45,8 +45,8 @@ func New(options *Options) (finder *Finder) { for _, source := range options.SourcesToUSe { switch source { - case "alienvault": - finder.Sources[source] = &alienvault.Source{} + case "otx": + finder.Sources[source] = &otx.Source{} case "anubis": finder.Sources[source] = &anubis.Source{} case "bevigil": From 0be1a9f62a661c49a79647354f4d1bdd64a0fb62 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 28 Aug 2023 19:27:52 +0300 Subject: [PATCH 24/50] chore(*): - --- pkg/xsubfind3r/{xsubfind3r.go => finder.go} | 90 ++++++++++----------- pkg/xsubfind3r/options.go | 9 +++ 2 files changed, 51 insertions(+), 48 deletions(-) rename pkg/xsubfind3r/{xsubfind3r.go => finder.go} (96%) create mode 100644 pkg/xsubfind3r/options.go diff --git a/pkg/xsubfind3r/xsubfind3r.go b/pkg/xsubfind3r/finder.go similarity index 96% rename from pkg/xsubfind3r/xsubfind3r.go rename to pkg/xsubfind3r/finder.go index 7c579fa..4e3e1ce 100644 --- a/pkg/xsubfind3r/xsubfind3r.go +++ b/pkg/xsubfind3r/finder.go @@ -20,17 +20,51 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/wayback" ) -type Options struct { - SourcesToExclude []string - SourcesToUSe []string - Keys sources.Keys -} - type Finder struct { Sources map[string]sources.Source SourcesConfiguration *sources.Configuration } +func (finder *Finder) Find(domain string) (results chan sources.Result) { + results = make(chan sources.Result) + + go func() { + defer close(results) + + seenSubdomains := &sync.Map{} + + wg := &sync.WaitGroup{} + + for _, source := range finder.Sources { + wg.Add(1) + + go func(source sources.Source) { + defer wg.Done() + + sResults := source.Run(finder.SourcesConfiguration, domain) + + for sResult := range sResults { + if sResult.Type == sources.Subdomain { + sResult.Value = strings.ToLower(sResult.Value) + sResult.Value = strings.ReplaceAll(sResult.Value, "*.", "") + + _, loaded := seenSubdomains.LoadOrStore(sResult.Value, struct{}{}) + if loaded { + continue + } + } + + results <- sResult + } + }(source) + } + + wg.Wait() + }() + + return +} + func New(options *Options) (finder *Finder) { finder = &Finder{ Sources: map[string]sources.Source{}, @@ -45,8 +79,6 @@ func New(options *Options) (finder *Finder) { for _, source := range options.SourcesToUSe { switch source { - case "otx": - finder.Sources[source] = &otx.Source{} case "anubis": finder.Sources[source] = &anubis.Source{} case "bevigil": @@ -65,6 +97,8 @@ func New(options *Options) (finder *Finder) { finder.Sources[source] = &hackertarget.Source{} case "intelx": finder.Sources[source] = &intelx.Source{} + case "otx": + finder.Sources[source] = &otx.Source{} case "shodan": finder.Sources[source] = &shodan.Source{} case "urlscan": @@ -80,43 +114,3 @@ func New(options *Options) (finder *Finder) { return } - -func (finder *Finder) Find(domain string) (results chan sources.Result) { - results = make(chan sources.Result) - - go func() { - defer close(results) - - seenSubdomains := &sync.Map{} - - wg := &sync.WaitGroup{} - - for _, source := range finder.Sources { - wg.Add(1) - - go func(source sources.Source) { - defer wg.Done() - - sResults := source.Run(finder.SourcesConfiguration, domain) - - for sResult := range sResults { - if sResult.Type == sources.Subdomain { - sResult.Value = strings.ToLower(sResult.Value) - sResult.Value = strings.ReplaceAll(sResult.Value, "*.", "") - - _, loaded := seenSubdomains.LoadOrStore(sResult.Value, struct{}{}) - if loaded { - continue - } - } - - results <- sResult - } - }(source) - } - - wg.Wait() - }() - - return -} diff --git a/pkg/xsubfind3r/options.go b/pkg/xsubfind3r/options.go new file mode 100644 index 0000000..dabec4a --- /dev/null +++ b/pkg/xsubfind3r/options.go @@ -0,0 +1,9 @@ +package xsubfind3r + +import "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + +type Options struct { + SourcesToExclude []string + SourcesToUSe []string + Keys sources.Keys +} From 1333bdbcddb22d24f53bc3a77e3fb5dd96dc2edf Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Tue, 29 Aug 2023 03:18:42 +0300 Subject: [PATCH 25/50] refactor(commoncrawl): - --- .../sources/commoncrawl/commoncrawl.go | 108 ++++++++++++------ 1 file changed, 74 insertions(+), 34 deletions(-) diff --git a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go index 2cebd08..3c2e589 100644 --- a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go +++ b/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go @@ -5,10 +5,10 @@ import ( "encoding/json" "fmt" "net/http" - "sync" + "regexp" - "github.com/hueristiq/hqgourl" "github.com/hueristiq/xsubfind3r/pkg/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" ) @@ -17,6 +17,12 @@ type getIndexesResponse []struct { API string `json:"cdx-API"` } +type getPaginationResponse struct { + Blocks uint `json:"blocks"` + PageSize uint `json:"pageSize"` + Pages uint `json:"pages"` +} + type getURLsResponse struct { URL string `json:"url"` Error string `json:"error"` @@ -30,10 +36,25 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source go func() { defer close(results) - getIndexesReqURL := "https://index.commoncrawl.org/collinfo.json" - var err error + var regex *regexp.Regexp + + regex, err = extractor.New(domain) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + return + } + + getIndexesReqURL := "https://index.commoncrawl.org/collinfo.json" + var getIndexesRes *http.Response getIndexesRes, err = httpclient.SimpleGet(getIndexesReqURL) @@ -68,21 +89,53 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getIndexesRes.Body.Close() - wg := new(sync.WaitGroup) - for _, indexData := range getIndexesResData { - wg.Add(1) + getURLsReqHeaders := map[string]string{ + "Host": "index.commoncrawl.org", + } + + getPaginationReqURL := fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url&showNumPages=true", indexData.API, domain) + + var getPaginationRes *http.Response + + getPaginationRes, err = httpclient.SimpleGet(getPaginationReqURL) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + continue + } - go func(API string) { - defer wg.Done() + var getPaginationData getPaginationResponse - getURLsReqHeaders := map[string]string{ - "Host": "index.commoncrawl.org", + err = json.NewDecoder(getPaginationRes.Body).Decode(&getPaginationData) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, } - getURLsReqURL := fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url", API, domain) + results <- result + + getPaginationRes.Body.Close() + + continue + } + + getPaginationRes.Body.Close() + + if getPaginationData.Pages < 1 { + continue + } - var err error + for page := uint(0); page < getPaginationData.Pages; page++ { + getURLsReqURL := fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url&page=%d", indexData.API, domain, page) var getURLsRes *http.Response @@ -96,7 +149,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - return + continue } scanner := bufio.NewScanner(getURLsRes.Body) @@ -131,31 +184,20 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - return + continue } - var parsedURL *hqgourl.URL + match := regex.FindAllString(getURLsResData.URL, -1) - parsedURL, err = hqgourl.Parse(getURLsResData.URL) - if err != nil { + for _, subdomain := range match { result := sources.Result{ - Type: sources.Error, + Type: sources.Subdomain, Source: source.Name(), - Error: err, + Value: subdomain, } results <- result - - continue - } - - result := sources.Result{ - Type: sources.Subdomain, - Source: source.Name(), - Value: parsedURL.Domain, } - - results <- result } if err = scanner.Err(); err != nil { @@ -169,14 +211,12 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getURLsRes.Body.Close() - return + continue } getURLsRes.Body.Close() - }(indexData.API) + } } - - wg.Wait() }() return results From a9359db98cc25fe81b7a2763bb536cdd1f068c97 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Tue, 29 Aug 2023 03:24:15 +0300 Subject: [PATCH 26/50] refactor(intelx): - --- pkg/xsubfind3r/sources/intelx/intelx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/xsubfind3r/sources/intelx/intelx.go b/pkg/xsubfind3r/sources/intelx/intelx.go index 5afe84c..ced0577 100644 --- a/pkg/xsubfind3r/sources/intelx/intelx.go +++ b/pkg/xsubfind3r/sources/intelx/intelx.go @@ -70,7 +70,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s searchReqURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", intelXHost, intelXKey) searchReqBody := searchRequest{ - Term: "*" + domain, + Term: domain, MaxResults: 100000, Media: 0, Target: 1, // 1 = Domains | 2 = Emails | 3 = URLs From 9851ddaaa747321bf1f25fdf14c63c0c044b74d4 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Tue, 29 Aug 2023 03:30:41 +0300 Subject: [PATCH 27/50] chore(*): - --- go.mod | 1 - go.sum | 4 ---- 2 files changed, 5 deletions(-) diff --git a/go.mod b/go.mod index c69bc55..c32fe29 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( dario.cat/mergo v1.0.0 github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f - github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 github.com/logrusorgru/aurora/v3 v3.0.0 github.com/spf13/cast v1.5.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 00a3ac7..fbdcd60 100644 --- a/go.sum +++ b/go.sum @@ -4,14 +4,10 @@ github.com/Mzack9999/go-http-digest-auth-client v0.6.0 h1:LXVNMsj7qiNVmlZByFbjJm github.com/Mzack9999/go-http-digest-auth-client v0.6.0/go.mod h1:gbwaYYXwA15ZfIxMyY5QU1acATDyNKEuG5TylBCL7AM= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21 h1:liOyeZd487AUwCWbTOXKMgdvg82JOBms6xb3RYAdSO0= -github.com/hueristiq/hqgohttp v0.0.0-20230827233921-93961bc4da21/go.mod h1:HewuEHc3F5aA5op40SzMuW/S0ZxW98pnrqXBeF+HsIs= github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf h1:qnTJknI2w4zIAKtY/5K8iAW91oj/wZgrHxzWvmes18Q= github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf/go.mod h1:HewuEHc3F5aA5op40SzMuW/S0ZxW98pnrqXBeF+HsIs= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= -github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53 h1:6pwdpEJoB1woSToh0cxLh5QirNOAp2z7DzvMKiaqdro= -github.com/hueristiq/hqgourl v0.0.0-20230821112831-e12f907b5a53/go.mod h1:Fc2vfWpIVFWUmCv1S0xVsz3mIPYwdgsa6f2vCgL4CrA= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= From 62d44297c7c4dc41115701a59fbd67f39094d4cc Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Wed, 30 Aug 2023 01:56:51 +0300 Subject: [PATCH 28/50] refactor(*): Update extraction regex --- pkg/xsubfind3r/extractor/extractor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/xsubfind3r/extractor/extractor.go b/pkg/xsubfind3r/extractor/extractor.go index 8024f85..4a4424c 100644 --- a/pkg/xsubfind3r/extractor/extractor.go +++ b/pkg/xsubfind3r/extractor/extractor.go @@ -11,7 +11,7 @@ func New(domain string) (extractor *regexp.Regexp, err error) { mutex.Lock() defer mutex.Unlock() - pattern := `(?i)[a-zA-Z0-9\*_.-]+\.` + domain + pattern := `(\w+[.])*` + regexp.QuoteMeta(domain) extractor, err = regexp.Compile(pattern) if err != nil { From dcd8a889d6838580163deb9a5f5c8a32f4a75a2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 05:15:36 +0000 Subject: [PATCH 29/50] chore(deps): bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2715a84..b17f594 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -30,7 +30,7 @@ jobs: go-version: '>=1.20' - name: Checkout the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index dfe62fc..28669fd 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -31,7 +31,7 @@ jobs: go-version: '>=1.20' - name: Checkout the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index adf85f5..26fe801 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: go-version: '>=1.20' - name: Checkout the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - From 2a40808ac38974eab984313e656579be74b07239 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 17 Sep 2023 22:10:12 +0300 Subject: [PATCH 30/50] chore: - --- .golangci.yaml | 183 ++++++++++++++++++------ Makefile | 2 +- README.md | 44 +++--- cmd/xsubfind3r/main.go | 77 +++++----- go.mod | 10 +- go.sum | 20 +-- internal/configuration/configuration.go | 37 +++-- pkg/httpclient/client.go | 6 +- pkg/xsubfind3r/sources/github/github.go | 13 +- pkg/xsubfind3r/sources/utils.go | 2 +- 10 files changed, 251 insertions(+), 143 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 5de582c..870c8eb 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,61 +1,160 @@ run: - issues-exit-code: 1 + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 5m linters: + # Disable all linters. + # Default: false disable-all: true + # Enable specific linter enable: - - bodyclose - - depguard - - dogsled - - dupl - - errcheck - - exportloopref - - exhaustive - - goconst - - gocritic - - gofmt - - goimports - - gocyclo - - gosec - - gosimple - - govet - - ineffassign - - misspell - - nolintlint - - prealloc - - predeclared - - revive - - staticcheck - - stylecheck - - thelper - - tparallel - - typecheck - - unconvert - - unparam - - unused - - whitespace - - wsl + # Enabled by Default + - errcheck # errcheck is a program for checking for unchecked errors in Go code. These unchecked errors can be critical bugs in some cases [fast: false, auto-fix: false] + - gosimple # (megacheck) # Linter for Go source code that specializes in simplifying code [fast: false, auto-fix: false] + - govet # (vet, vetshadow) # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string [fast: false, auto-fix: false] + - ineffassign # Detects when assignments to existing variables are not used [fast: true, auto-fix: false] + - staticcheck # (megacheck) # It's a set of rules from staticcheck. It's not the same thing as the staticcheck binary. The author of staticcheck doesn't support or approve the use of staticcheck as a library inside golangci-lint. [fast: false, auto-fix: false] + - unused # (megacheck) # Checks Go code for unused constants, variables, functions and types [fast: false, auto-fix: false] + # Disabled by Default + - asasalint # check for pass []any as any in variadic func(...any) [fast: false, auto-fix: false] + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false] + - bidichk # Checks for dangerous unicode character sequences [fast: true, auto-fix: false] + - bodyclose # checks whether HTTP response body is closed successfully [fast: false, auto-fix: false] + - containedctx # containedctx is a linter that detects struct contained context.Context field [fast: false, auto-fix: false] + - contextcheck # check whether the function uses a non-inherited context [fast: false, auto-fix: false] + - cyclop # checks function and package cyclomatic complexity [fast: false, auto-fix: false] + # - deadcode # [deprecated] # Finds unused code [fast: false, auto-fix: false] + - decorder # check declaration order and count of types, constants, variables and functions [fast: true, auto-fix: false] + # - depguard # Go linter that checks if package imports are in a list of acceptable packages [fast: true, auto-fix: false] + - dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) [fast: true, auto-fix: false] + # - dupl # Tool for code clone detection [fast: true, auto-fix: false] + - dupword # checks for duplicate words in the source code [fast: true, auto-fix: true] + - durationcheck # check for two durations multiplied together [fast: false, auto-fix: false] + - errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occasions, where the check for the returned error can be omitted. [fast: false, auto-fix: false] + - errname # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`. [fast: false, auto-fix: false] + - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13. [fast: false, auto-fix: false] + - execinquery # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds [fast: false, auto-fix: false] + - exhaustive # check exhaustiveness of enum switch statements [fast: false, auto-fix: false] + # - exhaustivestruct # [deprecated] # Checks if all struct's fields are initialized [fast: false, auto-fix: false] + # - exhaustruct # Checks if all structure fields are initialized [fast: false, auto-fix: false] + - exportloopref # checks for pointers to enclosing loop variables [fast: false, auto-fix: false] + # - forbidigo # Forbids identifiers [fast: false, auto-fix: false] + - forcetypeassert # finds forced type assertions [fast: true, auto-fix: false] + - funlen # Tool for detection of long functions [fast: true, auto-fix: false] + - gci # Gci controls Go package import order and makes it always deterministic. [fast: true, auto-fix: false] + - ginkgolinter # enforces standards of using ginkgo and gomega [fast: false, auto-fix: false] + - gocheckcompilerdirectives # Checks that go compiler directive comments (//go:) are valid. [fast: true, auto-fix: false] + # - gochecknoglobals # check that no global variables exist [fast: false, auto-fix: false] + # - gochecknoinits # Checks that no init functions are present in Go code [fast: true, auto-fix: false] + - gocognit # Computes and checks the cognitive complexity of functions [fast: true, auto-fix: false] + - goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false] + - gocritic # Provides diagnostics that check for bugs, performance and style issues. [fast: false, auto-fix: false] + - gocyclo # Computes and checks the cyclomatic complexity of functions [fast: true, auto-fix: false] + # - godot # Check if comments end in a period [fast: true, auto-fix: true] + # - godox # Tool for detection of FIXME, TODO and other comment keywords [fast: true, auto-fix: false] + - goerr113 # Go linter to check the errors handling expressions [fast: false, auto-fix: false] + - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification [fast: true, auto-fix: true] + - gofumpt # Gofumpt checks whether code was gofumpt-ed. [fast: true, auto-fix: true] + - goheader # Checks is file header matches to pattern [fast: true, auto-fix: false] + - goimports # Check import statements are formatted according to the 'goimport' command. Reformat imports in autofix mode. [fast: true, auto-fix: true] + # - golint # [deprecated] # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes [fast: false, auto-fix: false] + # - gomnd # An analyzer to detect magic numbers. [fast: true, auto-fix: false] + - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod. [fast: true, auto-fix: false] + - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. [fast: true, auto-fix: false] + - goprintffuncname # Checks that printf-like functions are named with `f` at the end [fast: true, auto-fix: false] + # - gosec # (gas) # Inspects source code for security problems [fast: false, auto-fix: false] + - gosmopolitan # Report certain i18n/l10n anti-patterns in your Go codebase [fast: false, auto-fix: false] + - grouper # An analyzer to analyze expression groups. [fast: true, auto-fix: false] + # - ifshort # [deprecated] # Checks that your code uses short syntax for if-statements whenever possible [fast: true, auto-fix: false] + - importas # Enforces consistent import aliases [fast: false, auto-fix: false] + - interfacebloat # A linter that checks the number of methods inside an interface. [fast: true, auto-fix: false] + # - interfacer # [deprecated] # Linter that suggests narrower interface types [fast: false, auto-fix: false] + - ireturn # Accept Interfaces, Return Concrete Types [fast: false, auto-fix: false] + # - lll # Reports long lines [fast: true, auto-fix: false] + - loggercheck # (logrlint) # Checks key value pairs for common logger libraries (kitlog,klog,logr,zap). [fast: false, auto-fix: false] + - maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false] + - makezero # Finds slice declarations with non-zero initial length [fast: false, auto-fix: false] + # - maligned # [deprecated] # Tool to detect Go structs that would take less memory if their fields were sorted [fast: false, auto-fix: false] + - mirror # reports wrong mirror patterns of bytes/strings usage [fast: false, auto-fix: false] + - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true] + - musttag # enforce field tags in (un)marshaled structs [fast: false, auto-fix: false] + - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] + - nestif # Reports deeply nested if statements [fast: true, auto-fix: false] + - nilerr # Finds the code that returns nil even if it checks that the error is not nil. [fast: false, auto-fix: false] + - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. [fast: false, auto-fix: false] + - nlreturn # nlreturn checks for a new line before return and branch statements to increase code clarity [fast: true, auto-fix: false] + - noctx # noctx finds sending http request without context.Context [fast: false, auto-fix: false] + - nolintlint # Reports ill-formed or insufficient nolint directives [fast: true, auto-fix: false] + # - nonamedreturns # Reports all named returns [fast: false, auto-fix: false] + # - nosnakecase # [deprecated] # nosnakecase is a linter that detects snake case of variable naming and function name. [fast: true, auto-fix: false] + - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. [fast: true, auto-fix: false] + - paralleltest # paralleltest detects missing usage of t.Parallel() method in your Go test [fast: false, auto-fix: false] + - prealloc # Finds slice declarations that could potentially be pre-allocated [fast: true, auto-fix: false] + - predeclared # find code that shadows one of Go's predeclared identifiers [fast: true, auto-fix: false] + # - promlinter # Check Prometheus metrics naming via promlint [fast: true, auto-fix: false] + - reassign # Checks that package variables are not reassigned [fast: false, auto-fix: false] + - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. [fast: false, auto-fix: false] + - rowserrcheck # checks whether Err of rows is checked successfully [fast: false, auto-fix: false] + # - scopelint # [deprecated] # Scopelint checks for unpinned variables in go programs [fast: true, auto-fix: false] + - sqlclosecheck # Checks that sql.Rows and sql.Stmt are closed. [fast: false, auto-fix: false] + # - structcheck # [deprecated] # Finds unused struct fields [fast: false, auto-fix: false] + - stylecheck # Stylecheck is a replacement for golint [fast: false, auto-fix: false] + - tagalign # check that struct tags are well aligned [fast: true, auto-fix: true] + # - tagliatelle # Checks the struct tags. [fast: true, auto-fix: false] + - tenv # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17 [fast: false, auto-fix: false] + - testableexamples # linter checks if examples are testable (have an expected output) [fast: true, auto-fix: false] + - testpackage # linter that makes you use a separate _test package [fast: true, auto-fix: false] + - thelper # thelper detects Go test helpers without t.Helper() call and checks the consistency of test helpers [fast: false, auto-fix: false] + - tparallel # tparallel detects inappropriate usage of t.Parallel() method in your Go test codes [fast: false, auto-fix: false] + - unconvert # Remove unnecessary type conversions [fast: false, auto-fix: false] + - unparam # Reports unused function parameters [fast: false, auto-fix: false] + - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library. [fast: true, auto-fix: false] + # - varcheck # [deprecated] # Finds unused global variables and constants [fast: false, auto-fix: false] + # - varnamelen # checks that the length of a variable's name matches its scope [fast: false, auto-fix: false] + - wastedassign # wastedassign finds wasted assignment statements. [fast: false, auto-fix: false] + - whitespace # Tool for detection of leading and trailing whitespace [fast: true, auto-fix: true] + - wrapcheck # Checks that errors returned from external packages are wrapped [fast: false, auto-fix: false] + - wsl # Whitespace Linter - Forces you to use empty lines! [fast: true, auto-fix: false] + - zerologlint # Detects the wrong usage of `zerolog` that a user forgets to dispatch with `Send` or `Msg`. [fast: false, auto-fix: false] linters-settings: - errcheck: - check-type-assertions: true goconst: min-len: 2 min-occurrences: 3 gocritic: enabled-tags: - - style - - diagnostic - performance - experimental + - style - opinionated disabled-checks: - captLocal - - octalLiteral + - whyNoLint + gocyclo: + # Minimal code complexity to report. + # Default: 30 (but we recommend 10-20) + min-complexity: 10 govet: check-shadowing: true - disabled-checks: - - fieldalignment - nolintlint: - require-explanation: true - require-specific: true \ No newline at end of file + varnamelen: + # The minimum length of a variable's name that is considered "long". + # Variable names that are at least this long will be ignored. + # Default: 3 + min-name-length: 2 + # Check method receivers. + # Default: false + check-receiver: true + # Check named return values. + # Default: false + check-return: true + # Check type parameters. + # Default: false + check-type-param: true + whitespace: + # Enforces newlines (or comments) after every multi-line if statement. + # Default: false + multi-if: true + # Enforces newlines (or comments) after every multi-line function signature. + # Default: false + multi-func: true \ No newline at end of file diff --git a/Makefile b/Makefile index 6eaaad6..fe7a072 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ format: .PHONY: lint lint: - $(GOLANGCILINTRUN) ./... + $(GOLANGCILINTRUN) ./... --fix .PHONY: test test: diff --git a/README.md b/README.md index 5fb531b..b92496a 100644 --- a/README.md +++ b/README.md @@ -20,18 +20,7 @@ ## Features * [x] Fetches domains from curated passive sources to maximize results. -
- Sources: Click to expand! - - | Technique | Source | - | :-------- | :----- | - | APIs | AnubisDB, BeVigil, Chaos, FullHunt, GitHub, HackerTarget, IntelX, Shodan, URLScan | - | Certificates | Crtsh | - | Web Archives | CommonCrawl, Wayback | - | WHOIS | AlienVault | - -
-* [x] Supports `stdin` and `stdout` for easy integration into workflows. +* [x] `stdin` and `stdout` for easy integration into workflows. * [x] Cross-Platform (Windows, Linux & macOS). ## Installation @@ -112,11 +101,11 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ## Post Installation -`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.hueristiq/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. +`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. Example `config.yaml`: -> **NOTE:** The keys/tokens below are invalid, use your own keys/tokens! +> **NOTE:** The keys/tokens below are invalid and used as examples, use your own keys/tokens! ```yaml version: 0.3.0 @@ -164,18 +153,23 @@ help message: ```text - _ __ _ _ _____ -__ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ + _ __ _ _ _____ +__ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _` | |_ \| '__| - > <\__ \ |_| | |_) | _| | | | | (_| |___) | | -/_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| v0.3.0 + > <\__ \ |_| | |_) | _| | | | | (_| |___) | | +/_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| + v0.3.0 + with <3 by Hueristiq Open Source USAGE: xsubfind3r [OPTIONS] +CONFIGURATION: + -c, --configuration string configuration file path (default: $HOME/.config/xsubfind3r/config.yaml) + INPUT: -d, --domain string[] target domains - -l, --list string target domains' list file path + -l, --list string target domains list file path SOURCES: --sources bool list supported sources @@ -183,13 +177,13 @@ SOURCES: -e, --sources-to-exclude string[] comma(,) separeted sources to exclude OUTPUT: - --no-color bool disable colored output - -o, --output string output subdomains' file path - -O, --output-directory string output subdomains' directory path - -v, --verbosity string debug, info, warning, error, fatal or silent (default: info) + --monochrome bool display no color output + -o, --output string output subdomains file path + -O, --output-directory string output subdomains directory path + --silent bool display output subdomains only + --verbose bool display verbose output -CONFIGURATION: - -c, --configuration string configuration file path (default: ~/.hueristiq/xsubfind3r/config.yaml) +pflag: help requested ``` ## Contributing diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 0a1eb9f..46578ce 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -22,40 +22,42 @@ import ( var ( au aurora.Aurora - domains []string - domainsListFilePath string - listSources bool - sourcesToUse []string - sourcesToExclude []string - monochrome bool - output string - outputDirectory string - verbosity string - YAMLConfigFile string + configurationFilePath string + domains []string + domainsListFilePath string + listSources bool + sourcesToUse []string + sourcesToExclude []string + monochrome bool + output string + outputDirectory string + silent bool + verbose bool ) func init() { - // defaults - defaultYAMLConfigFile := fmt.Sprintf("~/.hueristiq/%s/config.yaml", configuration.NAME) - // Handle CLI arguments, flags & help message (pflag) + pflag.StringVarP(&configurationFilePath, "configuration", "c", configuration.ConfigurationFilePath, "") pflag.StringSliceVarP(&domains, "domain", "d", []string{}, "") pflag.StringVarP(&domainsListFilePath, "list", "l", "", "") pflag.BoolVar(&listSources, "sources", false, "") pflag.StringSliceVarP(&sourcesToUse, "use-sources", "u", []string{}, "") pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "") - pflag.BoolVar(&monochrome, "no-color", false, "") + pflag.BoolVar(&monochrome, "monochromer", false, "") pflag.StringVarP(&output, "output", "o", "", "") pflag.StringVarP(&outputDirectory, "outputDirectory", "O", "", "") - pflag.StringVarP(&verbosity, "verbosity", "v", string(levels.LevelInfo), "") - pflag.StringVarP(&YAMLConfigFile, "configuration", "c", defaultYAMLConfigFile, "") + pflag.BoolVar(&silent, "silent", false, "") + pflag.BoolVar(&verbose, "verbose", false, "") pflag.CommandLine.SortFlags = false pflag.Usage = func() { fmt.Fprintln(os.Stderr, configuration.BANNER) h := "USAGE:\n" - h += " xsubfind3r [OPTIONS]\n" + h += fmt.Sprintf(" %s [OPTIONS]\n", configuration.NAME) + + h += "\nCONFIGURATION:\n" + h += fmt.Sprintf(" -c, --configuration string configuration file path (default: %s)\n", configuration.ConfigurationFilePath) h += "\nINPUT:\n" h += " -d, --domain string[] target domains\n" @@ -67,13 +69,11 @@ func init() { h += " -e, --sources-to-exclude string[] comma(,) separeted sources to exclude\n" h += "\nOUTPUT:\n" - h += " --no-color bool disable colored output\n" + h += " --monochrome bool display no color output\n" h += " -o, --output string output subdomains file path\n" h += " -O, --output-directory string output subdomains directory path\n" - h += fmt.Sprintf(" -v, --verbosity string debug, info, warning, error, fatal or silent (default: %s)\n", string(levels.LevelInfo)) - - h += "\nCONFIGURATION:\n" - h += fmt.Sprintf(" -c, --configuration string configuration file path (default: %s)\n", defaultYAMLConfigFile) + h += " --silent bool display output subdomains only\n" + h += " --verbose bool display verbose output\n" fmt.Fprintln(os.Stderr, h) } @@ -81,22 +81,27 @@ func init() { pflag.Parse() // Initialize logger (hqgolog) - hqgolog.DefaultLogger.SetMaxLevel(levels.LevelStr(verbosity)) + hqgolog.DefaultLogger.SetMaxLevel(levels.LevelInfo) + + if verbose { + hqgolog.DefaultLogger.SetMaxLevel(levels.LevelDebug) + } + hqgolog.DefaultLogger.SetFormatter(formatter.NewCLI(&formatter.CLIOptions{ Colorize: !monochrome, })) // Create | Update configuration - if strings.HasPrefix(YAMLConfigFile, "~") { + if strings.HasPrefix(configurationFilePath, "$HOME") { home, err := os.UserHomeDir() if err != nil { hqgolog.Fatal().Msg(err.Error()) } - YAMLConfigFile = strings.Replace(YAMLConfigFile, "~", home, 1) + configurationFilePath = strings.Replace(configurationFilePath, "$HOME", home, 1) } - if err := configuration.CreateUpdate(YAMLConfigFile); err != nil { + if err := configuration.CreateUpdate(configurationFilePath); err != nil { hqgolog.Fatal().Msg(err.Error()) } @@ -105,7 +110,7 @@ func init() { func main() { // Print banner. - if verbosity != string(levels.LevelSilent) { + if !silent { fmt.Fprintln(os.Stderr, configuration.BANNER) } @@ -114,12 +119,12 @@ func main() { var config configuration.Configuration // Read in configuration. - config, err = configuration.Read(YAMLConfigFile) + config, err = configuration.Read(configurationFilePath) if err != nil { hqgolog.Fatal().Msg(err.Error()) } - // If --sources: List suported sources & exit. + // If `--sources`: List suported sources & exit. if listSources { hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(config.Sources))).Bold()) hqgolog.Info().Msgf("sources marked with %v take in key(s) or token(s).", au.Underline("*").Bold()) @@ -206,7 +211,7 @@ func main() { var consolidatedFile *os.File - consolidatedFile, err = os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + consolidatedFile, err = os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { hqgolog.Fatal().Msg(err.Error()) } @@ -225,20 +230,20 @@ func main() { switch { case output != "": - processSubdomains(consolidatedWriter, subdomains, verbosity) + processSubdomains(consolidatedWriter, subdomains) case outputDirectory != "": var domainFile *os.File - domainFile, err = os.OpenFile(filepath.Join(outputDirectory, domain+".txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + domainFile, err = os.OpenFile(filepath.Join(outputDirectory, domain+".txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { hqgolog.Fatal().Msg(err.Error()) } domainWriter := bufio.NewWriter(domainFile) - processSubdomains(domainWriter, subdomains, verbosity) + processSubdomains(domainWriter, subdomains) default: - processSubdomains(nil, subdomains, verbosity) + processSubdomains(nil, subdomains) } } } @@ -263,13 +268,13 @@ func mkdir(path string) { } } -func processSubdomains(writer *bufio.Writer, subdomains chan sources.Result, verbosity string) { +func processSubdomains(writer *bufio.Writer, subdomains chan sources.Result) { for subdomain := range subdomains { switch subdomain.Type { case sources.Error: hqgolog.Warn().Msgf("Could not run source %s: %s\n", subdomain.Source, subdomain.Error) case sources.Subdomain: - if verbosity == string(levels.LevelDebug) { + if verbose { hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomain.Source), subdomain.Value) } else { hqgolog.Print().Msg(subdomain.Value) diff --git a/go.mod b/go.mod index c32fe29..dbb0131 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( dario.cat/mergo v1.0.0 - github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf + github.com/hueristiq/hqgohttp v0.0.0-20230917162130-697d8e95e15d github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/logrusorgru/aurora/v3 v3.0.0 github.com/spf13/cast v1.5.1 @@ -15,8 +15,8 @@ require ( require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect ) diff --git a/go.sum b/go.sum index fbdcd60..a6c1f52 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/Mzack9999/go-http-digest-auth-client v0.6.0 h1:LXVNMsj7qiNVmlZByFbjJm github.com/Mzack9999/go-http-digest-auth-client v0.6.0/go.mod h1:gbwaYYXwA15ZfIxMyY5QU1acATDyNKEuG5TylBCL7AM= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf h1:qnTJknI2w4zIAKtY/5K8iAW91oj/wZgrHxzWvmes18Q= -github.com/hueristiq/hqgohttp v0.0.0-20230828153804-6cb564391caf/go.mod h1:HewuEHc3F5aA5op40SzMuW/S0ZxW98pnrqXBeF+HsIs= +github.com/hueristiq/hqgohttp v0.0.0-20230917162130-697d8e95e15d h1:HZY/au1fr6CV/s3iZsTImOXaFgiGW8RW5+lTAUqfODE= +github.com/hueristiq/hqgohttp v0.0.0-20230917162130-697d8e95e15d/go.mod h1:To9Bfohm6oIXZge4tRiWGusF1/Xb7KdTT4YPgsB/1YI= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -19,14 +19,14 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 4192929..377850e 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -5,6 +5,7 @@ import ( "path/filepath" "dario.cat/mergo" + "github.com/hueristiq/hqgolog" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" "github.com/logrusorgru/aurora/v3" "gopkg.in/yaml.v3" @@ -17,9 +18,7 @@ type Configuration struct { } func (configuration *Configuration) Write(path string) (err error) { - var ( - file *os.File - ) + var file *os.File directory := filepath.Dir(path) identation := 4 @@ -32,7 +31,7 @@ func (configuration *Configuration) Write(path string) (err error) { } } - file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) if err != nil { return } @@ -58,16 +57,31 @@ var ( __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _`+"`"+` | |_ \| '__| > <\__ \ |_| | |_) | _| | | | | (_| |___) | | -/_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| %s +/_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| + %s + %s `).Bold(), - aurora.BrightYellow("v"+VERSION).Bold(), + aurora.BrightRed("v"+VERSION).Bold(), + aurora.BrightYellow("with <3 by Hueristiq Open Source").Italic(), ) + userDotConfigDirectoryPath = func() (userDotConfig string) { + var err error + + userDotConfig, err = os.UserConfigDir() + if err != nil { + hqgolog.Fatal().Msg(err.Error()) + } + + return + }() + projectRootDirectoryName = NAME + ProjectRootDirectoryPath = filepath.Join(userDotConfigDirectoryPath, projectRootDirectoryName) + configurationFileName = "config.yaml" + ConfigurationFilePath = filepath.Join(ProjectRootDirectoryPath, configurationFileName) ) func CreateUpdate(path string) (err error) { - var ( - config Configuration - ) + var config Configuration defaultConfig := Configuration{ Version: VERSION, @@ -102,6 +116,7 @@ func CreateUpdate(path string) (err error) { if config.Version != VERSION || len(config.Sources) != len(sources.List) { + if err = mergo.Merge(&config, defaultConfig); err != nil { return } @@ -119,9 +134,7 @@ func CreateUpdate(path string) (err error) { } func Read(path string) (configuration Configuration, err error) { - var ( - file *os.File - ) + var file *os.File file, err = os.Open(path) if err != nil { diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 5611199..11bd022 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -7,12 +7,10 @@ import ( "net/http" "net/url" - hqgohttpclient "github.com/hueristiq/hqgohttp/client" + hqgohttpclient "github.com/hueristiq/hqgohttp" ) -var ( - client *hqgohttpclient.Client -) +var client *hqgohttpclient.Client func init() { options := hqgohttpclient.DefaultOptionsSpraying diff --git a/pkg/xsubfind3r/sources/github/github.go b/pkg/xsubfind3r/sources/github/github.go index 21246d4..2b65f9c 100644 --- a/pkg/xsubfind3r/sources/github/github.go +++ b/pkg/xsubfind3r/sources/github/github.go @@ -10,8 +10,7 @@ import ( "strings" "time" - hqgohttpheaders "github.com/hueristiq/hqgohttp/headers" - hqgohttpstatus "github.com/hueristiq/hqgohttp/status" + "github.com/hueristiq/hqgohttp" "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" @@ -87,7 +86,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp searchRes, err = httpclient.Get(searchReqURL, "", searchReqHeaders) - isForbidden := searchRes != nil && searchRes.StatusCode == hqgohttpstatus.Forbidden + isForbidden := searchRes != nil && searchRes.StatusCode == hqgohttp.StatusForbidden if err != nil && !isForbidden { result := sources.Result{ @@ -101,9 +100,9 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp return } - ratelimitRemaining := cast.ToInt64(searchRes.Header.Get(hqgohttpheaders.XRatelimitRemaining)) + ratelimitRemaining := cast.ToInt64(searchRes.Header.Get(hqgohttp.HeaderXRatelimitRemaining)) if isForbidden && ratelimitRemaining == 0 { - retryAfterSeconds := cast.ToInt64(searchRes.Header.Get(hqgohttpheaders.RetryAfter)) + retryAfterSeconds := cast.ToInt64(searchRes.Header.Get(hqgohttp.HeaderRetryAfter)) tokens.setCurrentTokenExceeded(retryAfterSeconds) @@ -147,7 +146,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp continue } - if getRawContentRes.StatusCode != hqgohttpstatus.OK { + if getRawContentRes.StatusCode != hqgohttp.StatusOK { continue } @@ -203,7 +202,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp } } - linksHeader := linkheader.Parse(searchRes.Header.Get(hqgohttpheaders.Link)) + linksHeader := linkheader.Parse(searchRes.Header.Get(hqgohttp.HeaderLink)) for _, link := range linksHeader { if link.Rel == "next" { diff --git a/pkg/xsubfind3r/sources/utils.go b/pkg/xsubfind3r/sources/utils.go index e540af5..272082c 100644 --- a/pkg/xsubfind3r/sources/utils.go +++ b/pkg/xsubfind3r/sources/utils.go @@ -20,7 +20,7 @@ func PickRandom[T any](v []T) (picked T, err error) { indexBig, err = rand.Int(rand.Reader, max) if err != nil { - err = fmt.Errorf("failed to generate random index: %v", err) + err = fmt.Errorf("failed to generate random index: %w", err) return } From a80131a3446a7a6a370283d5e92c0630b843595b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 05:47:20 +0000 Subject: [PATCH 31/50] chore(deps): bump goreleaser/goreleaser-action from 4 to 5 Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 4 to 5. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/v4...v5) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26fe801..c4add6b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 + uses: goreleaser/goreleaser-action@v5 with: distribution: goreleaser version: latest From 7cf332d674bf0daf974ef0bf9b76c81195f84a35 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 12 Oct 2023 04:24:53 +0300 Subject: [PATCH 32/50] chore: - --- .gitignore | 6 +- .golangci.yaml | 16 +-- Makefile | 2 +- README.md | 23 ++-- cmd/xsubfind3r/main.go | 68 +++++----- go.mod | 6 +- go.sum | 12 +- internal/configuration/configuration.go | 50 ++++---- pkg/{xsubfind3r => }/extractor/extractor.go | 0 pkg/httpclient/client.go | 1 + pkg/{xsubfind3r => scraper}/options.go | 4 +- pkg/scraper/scraper.go | 116 ++++++++++++++++++ .../sources/anubis/anubis.go | 6 +- .../sources/bevigil/bevigil.go | 6 +- .../sources/chaos/chaos.go | 8 +- .../sources/commoncrawl/commoncrawl.go | 43 ++++++- .../sources/configuration.go | 0 .../sources/crtsh/crtsh.go | 8 +- .../sources/fullhunt/fullhunt.go | 6 +- .../sources/github/github.go | 5 +- .../sources/github/tokenmanager.go | 0 .../sources/hackertarget/hackertarget.go | 8 +- .../sources/intelx/intelx.go | 6 +- .../sources/otx/otx.go | 6 +- .../sources/shodan/shodan.go | 6 +- .../sources/sources.go | 0 .../sources/sources_results.go | 0 .../sources/urlscan/urlscan.go | 11 +- pkg/{xsubfind3r => scraper}/sources/utils.go | 0 .../sources/wayback/wayback.go | 11 +- pkg/xsubfind3r/finder.go | 116 ------------------ 31 files changed, 302 insertions(+), 248 deletions(-) rename pkg/{xsubfind3r => }/extractor/extractor.go (100%) rename pkg/{xsubfind3r => scraper}/options.go (57%) create mode 100644 pkg/scraper/scraper.go rename pkg/{xsubfind3r => scraper}/sources/anubis/anubis.go (89%) rename pkg/{xsubfind3r => scraper}/sources/bevigil/bevigil.go (91%) rename pkg/{xsubfind3r => scraper}/sources/chaos/chaos.go (87%) rename pkg/{xsubfind3r => scraper}/sources/commoncrawl/commoncrawl.go (82%) rename pkg/{xsubfind3r => scraper}/sources/configuration.go (100%) rename pkg/{xsubfind3r => scraper}/sources/crtsh/crtsh.go (90%) rename pkg/{xsubfind3r => scraper}/sources/fullhunt/fullhunt.go (92%) rename pkg/{xsubfind3r => scraper}/sources/github/github.go (97%) rename pkg/{xsubfind3r => scraper}/sources/github/tokenmanager.go (100%) rename pkg/{xsubfind3r => scraper}/sources/hackertarget/hackertarget.go (90%) rename pkg/{xsubfind3r => scraper}/sources/intelx/intelx.go (95%) rename pkg/{xsubfind3r => scraper}/sources/otx/otx.go (91%) rename pkg/{xsubfind3r => scraper}/sources/shodan/shodan.go (91%) rename pkg/{xsubfind3r => scraper}/sources/sources.go (100%) rename pkg/{xsubfind3r => scraper}/sources/sources_results.go (100%) rename pkg/{xsubfind3r => scraper}/sources/urlscan/urlscan.go (92%) rename pkg/{xsubfind3r => scraper}/sources/utils.go (100%) rename pkg/{xsubfind3r => scraper}/sources/wayback/wayback.go (91%) delete mode 100644 pkg/xsubfind3r/finder.go diff --git a/.gitignore b/.gitignore index c81e17d..77a98d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,3 @@ # Executable -bin - -# Notes - -notes.txt \ No newline at end of file +bin \ No newline at end of file diff --git a/.golangci.yaml b/.golangci.yaml index 870c8eb..2f183b5 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -23,7 +23,7 @@ linters: - bodyclose # checks whether HTTP response body is closed successfully [fast: false, auto-fix: false] - containedctx # containedctx is a linter that detects struct contained context.Context field [fast: false, auto-fix: false] - contextcheck # check whether the function uses a non-inherited context [fast: false, auto-fix: false] - - cyclop # checks function and package cyclomatic complexity [fast: false, auto-fix: false] + # - cyclop # checks function and package cyclomatic complexity [fast: false, auto-fix: false] # - deadcode # [deprecated] # Finds unused code [fast: false, auto-fix: false] - decorder # check declaration order and count of types, constants, variables and functions [fast: true, auto-fix: false] # - depguard # Go linter that checks if package imports are in a list of acceptable packages [fast: true, auto-fix: false] @@ -41,19 +41,19 @@ linters: - exportloopref # checks for pointers to enclosing loop variables [fast: false, auto-fix: false] # - forbidigo # Forbids identifiers [fast: false, auto-fix: false] - forcetypeassert # finds forced type assertions [fast: true, auto-fix: false] - - funlen # Tool for detection of long functions [fast: true, auto-fix: false] + # - funlen # Tool for detection of long functions [fast: true, auto-fix: false] - gci # Gci controls Go package import order and makes it always deterministic. [fast: true, auto-fix: false] - ginkgolinter # enforces standards of using ginkgo and gomega [fast: false, auto-fix: false] - gocheckcompilerdirectives # Checks that go compiler directive comments (//go:) are valid. [fast: true, auto-fix: false] # - gochecknoglobals # check that no global variables exist [fast: false, auto-fix: false] # - gochecknoinits # Checks that no init functions are present in Go code [fast: true, auto-fix: false] - - gocognit # Computes and checks the cognitive complexity of functions [fast: true, auto-fix: false] + # - gocognit # Computes and checks the cognitive complexity of functions [fast: true, auto-fix: false] - goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false] - gocritic # Provides diagnostics that check for bugs, performance and style issues. [fast: false, auto-fix: false] - - gocyclo # Computes and checks the cyclomatic complexity of functions [fast: true, auto-fix: false] + # - gocyclo # Computes and checks the cyclomatic complexity of functions [fast: true, auto-fix: false] # - godot # Check if comments end in a period [fast: true, auto-fix: true] # - godox # Tool for detection of FIXME, TODO and other comment keywords [fast: true, auto-fix: false] - - goerr113 # Go linter to check the errors handling expressions [fast: false, auto-fix: false] + # - goerr113 # Go linter to check the errors handling expressions [fast: false, auto-fix: false] - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification [fast: true, auto-fix: true] - gofumpt # Gofumpt checks whether code was gofumpt-ed. [fast: true, auto-fix: true] - goheader # Checks is file header matches to pattern [fast: true, auto-fix: false] @@ -70,7 +70,7 @@ linters: - importas # Enforces consistent import aliases [fast: false, auto-fix: false] - interfacebloat # A linter that checks the number of methods inside an interface. [fast: true, auto-fix: false] # - interfacer # [deprecated] # Linter that suggests narrower interface types [fast: false, auto-fix: false] - - ireturn # Accept Interfaces, Return Concrete Types [fast: false, auto-fix: false] + # - ireturn # Accept Interfaces, Return Concrete Types [fast: false, auto-fix: false] # - lll # Reports long lines [fast: true, auto-fix: false] - loggercheck # (logrlint) # Checks key value pairs for common logger libraries (kitlog,klog,logr,zap). [fast: false, auto-fix: false] - maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false] @@ -79,7 +79,7 @@ linters: - mirror # reports wrong mirror patterns of bytes/strings usage [fast: false, auto-fix: false] - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true] - musttag # enforce field tags in (un)marshaled structs [fast: false, auto-fix: false] - - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] + # - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] - nestif # Reports deeply nested if statements [fast: true, auto-fix: false] - nilerr # Finds the code that returns nil even if it checks that the error is not nil. [fast: false, auto-fix: false] - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. [fast: false, auto-fix: false] @@ -114,7 +114,7 @@ linters: # - varnamelen # checks that the length of a variable's name matches its scope [fast: false, auto-fix: false] - wastedassign # wastedassign finds wasted assignment statements. [fast: false, auto-fix: false] - whitespace # Tool for detection of leading and trailing whitespace [fast: true, auto-fix: true] - - wrapcheck # Checks that errors returned from external packages are wrapped [fast: false, auto-fix: false] + # - wrapcheck # Checks that errors returned from external packages are wrapped [fast: false, auto-fix: false] - wsl # Whitespace Linter - Forces you to use empty lines! [fast: true, auto-fix: false] - zerologlint # Detects the wrong usage of `zerolog` that a user forgets to dispatch with `Send` or `Msg`. [fast: false, auto-fix: false] diff --git a/Makefile b/Makefile index fe7a072..6eaaad6 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ format: .PHONY: lint lint: - $(GOLANGCILINTRUN) ./... --fix + $(GOLANGCILINTRUN) ./... .PHONY: test test: diff --git a/README.md b/README.md index b92496a..c2dc75c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![made with go](https://img.shields.io/badge/made%20with-Go-0000FF.svg) [![release](https://img.shields.io/github/release/hueristiq/xsubfind3r?style=flat&color=0000FF)](https://github.com/hueristiq/xsubfind3r/releases) [![license](https://img.shields.io/badge/license-MIT-gray.svg?color=0000FF)](https://github.com/hueristiq/xsubfind3r/blob/master/LICENSE) ![maintenance](https://img.shields.io/badge/maintained%3F-yes-0000FF.svg) [![open issues](https://img.shields.io/github/issues-raw/hueristiq/xsubfind3r.svg?style=flat&color=0000FF)](https://github.com/hueristiq/xsubfind3r/issues?q=is:issue+is:open) [![closed issues](https://img.shields.io/github/issues-closed-raw/hueristiq/xsubfind3r.svg?style=flat&color=0000FF)](https://github.com/hueristiq/xsubfind3r/issues?q=is:issue+is:closed) [![contribution](https://img.shields.io/badge/contributions-welcome-0000FF.svg)](https://github.com/hueristiq/xsubfind3r/blob/master/CONTRIBUTING.md) -`xsubfind3r` is a command-line interface (CLI) utility to find domain's known subdomains from curated passive online sources. +`xsubfind3r` is a command-line interface (CLI) utility to find domain's known subdomains from curated, passive online sources. ## Resource @@ -19,9 +19,9 @@ ## Features -* [x] Fetches domains from curated passive sources to maximize results. -* [x] `stdin` and `stdout` for easy integration into workflows. -* [x] Cross-Platform (Windows, Linux & macOS). +* Fetches domains from curated passive sources to maximize results. +* `stdin` and `stdout` for easy integration into workflows. +* Cross-Platform (Windows, Linux & macOS). ## Installation @@ -152,25 +152,28 @@ xsubfind3r -h help message: ```text - _ __ _ _ _____ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _` | |_ \| '__| > <\__ \ |_| | |_) | _| | | | | (_| |___) | | /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| v0.3.0 + with <3 by Hueristiq Open Source USAGE: xsubfind3r [OPTIONS] CONFIGURATION: - -c, --configuration string configuration file path (default: $HOME/.config/xsubfind3r/config.yaml) + -c, --configuration string configuration file path (default: $HOME/.config/xsubfind3r/config.yaml) INPUT: - -d, --domain string[] target domains + -d, --domain string[] target domain -l, --list string target domains list file path +TIP: For multiple input domains use comma(,) separated value with `-d`, + specify multiple `-d`, load from file with `-l` or load from stdin. + SOURCES: --sources bool list supported sources -u, --sources-to-use string[] comma(,) separeted sources to use @@ -180,10 +183,8 @@ OUTPUT: --monochrome bool display no color output -o, --output string output subdomains file path -O, --output-directory string output subdomains directory path - --silent bool display output subdomains only - --verbose bool display verbose output - -pflag: help requested + -s, --silent bool display output subdomains only + -v, --verbose bool display verbose output ``` ## Contributing diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 46578ce..acc2fad 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -13,8 +13,8 @@ import ( "github.com/hueristiq/hqgolog/formatter" "github.com/hueristiq/hqgolog/levels" "github.com/hueristiq/xsubfind3r/internal/configuration" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" "github.com/logrusorgru/aurora/v3" "github.com/spf13/pflag" ) @@ -43,26 +43,30 @@ func init() { pflag.BoolVar(&listSources, "sources", false, "") pflag.StringSliceVarP(&sourcesToUse, "use-sources", "u", []string{}, "") pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "") - pflag.BoolVar(&monochrome, "monochromer", false, "") + pflag.BoolVar(&monochrome, "monochrome", false, "") pflag.StringVarP(&output, "output", "o", "", "") pflag.StringVarP(&outputDirectory, "outputDirectory", "O", "", "") - pflag.BoolVar(&silent, "silent", false, "") - pflag.BoolVar(&verbose, "verbose", false, "") + pflag.BoolVarP(&silent, "silent", "s", false, "") + pflag.BoolVarP(&verbose, "verbose", "v", false, "") pflag.CommandLine.SortFlags = false pflag.Usage = func() { fmt.Fprintln(os.Stderr, configuration.BANNER) - h := "USAGE:\n" + h := "\nUSAGE:\n" h += fmt.Sprintf(" %s [OPTIONS]\n", configuration.NAME) h += "\nCONFIGURATION:\n" - h += fmt.Sprintf(" -c, --configuration string configuration file path (default: %s)\n", configuration.ConfigurationFilePath) + defaultConfigurationFilePath := strings.ReplaceAll(configuration.ConfigurationFilePath, configuration.UserDotConfigDirectoryPath, "$HOME/.config") + h += fmt.Sprintf(" -c, --configuration string configuration file path (default: %s)\n", defaultConfigurationFilePath) h += "\nINPUT:\n" - h += " -d, --domain string[] target domains\n" + h += " -d, --domain string[] target domain\n" h += " -l, --list string target domains list file path\n" + h += "\nTIP: For multiple input domains use comma(,) separated value with `-d`,\n" + h += " specify multiple `-d`, load from file with `-l` or load from stdin.\n" + h += "\nSOURCES:\n" h += " --sources bool list supported sources\n" h += " -u, --sources-to-use string[] comma(,) separeted sources to use\n" @@ -72,8 +76,8 @@ func init() { h += " --monochrome bool display no color output\n" h += " -o, --output string output subdomains file path\n" h += " -O, --output-directory string output subdomains directory path\n" - h += " --silent bool display output subdomains only\n" - h += " --verbose bool display verbose output\n" + h += " -s, --silent bool display output subdomains only\n" + h += " -v, --verbose bool display verbose output\n" fmt.Fprintln(os.Stderr, h) } @@ -91,16 +95,7 @@ func init() { Colorize: !monochrome, })) - // Create | Update configuration - if strings.HasPrefix(configurationFilePath, "$HOME") { - home, err := os.UserHomeDir() - if err != nil { - hqgolog.Fatal().Msg(err.Error()) - } - - configurationFilePath = strings.Replace(configurationFilePath, "$HOME", home, 1) - } - + // Create or Update configuration if err := configuration.CreateUpdate(configurationFilePath); err != nil { hqgolog.Fatal().Msg(err.Error()) } @@ -109,7 +104,7 @@ func init() { } func main() { - // Print banner. + // print banner. if !silent { fmt.Fprintln(os.Stderr, configuration.BANNER) } @@ -118,14 +113,15 @@ func main() { var config configuration.Configuration - // Read in configuration. + // read in configuration. config, err = configuration.Read(configurationFilePath) if err != nil { hqgolog.Fatal().Msg(err.Error()) } - // If `--sources`: List suported sources & exit. + // if `--sources`: List suported sources & exit. if listSources { + hqgolog.Print().Msg("") hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(config.Sources))).Bold()) hqgolog.Info().Msgf("sources marked with %v take in key(s) or token(s).", au.Underline("*").Bold()) hqgolog.Print().Msg("") @@ -150,9 +146,7 @@ func main() { os.Exit(0) } - // Load input domains. - - // input domains: file + // load input domains from file if domainsListFilePath != "" { var file *os.File @@ -176,7 +170,7 @@ func main() { } } - // input domains: stdin + // load input domains from stdin if hasStdin() { scanner := bufio.NewScanner(os.Stdin) @@ -193,14 +187,14 @@ func main() { } } - // Find and output subdomains. - options := &xsubfind3r.Options{ + // scrape and output subdomains. + options := &scraper.Options{ SourcesToExclude: sourcesToExclude, SourcesToUSe: sourcesToUse, Keys: config.Keys, } - finder := xsubfind3r.New(options) + spr := scraper.New(options) var consolidatedWriter *bufio.Writer @@ -225,8 +219,16 @@ func main() { mkdir(outputDirectory) } - for _, domain := range domains { - subdomains := finder.Find(domain) + for index := range domains { + domain := domains[index] + + if !silent { + hqgolog.Print().Msg("") + hqgolog.Info().Msgf("Finding subdomains for %v...", au.Underline(domain).Bold()) + hqgolog.Print().Msg("") + } + + subdomains := spr.Scrape(domain) switch { case output != "": @@ -272,7 +274,7 @@ func processSubdomains(writer *bufio.Writer, subdomains chan sources.Result) { for subdomain := range subdomains { switch subdomain.Type { case sources.Error: - hqgolog.Warn().Msgf("Could not run source %s: %s\n", subdomain.Source, subdomain.Error) + hqgolog.Error().Msgf("%s: %s\n", subdomain.Source, subdomain.Error) case sources.Subdomain: if verbose { hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomain.Source), subdomain.Value) diff --git a/go.mod b/go.mod index dbb0131..034264f 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect ) diff --git a/go.sum b/go.sum index a6c1f52..d23332e 100644 --- a/go.sum +++ b/go.sum @@ -19,12 +19,12 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 377850e..4c2002e 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -6,7 +6,7 @@ import ( "dario.cat/mergo" "github.com/hueristiq/hqgolog" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" "github.com/logrusorgru/aurora/v3" "gopkg.in/yaml.v3" ) @@ -17,7 +17,7 @@ type Configuration struct { Keys sources.Keys `yaml:"keys"` } -func (configuration *Configuration) Write(path string) (err error) { +func (cfg *Configuration) Write(path string) (err error) { var file *os.File directory := filepath.Dir(path) @@ -40,7 +40,7 @@ func (configuration *Configuration) Write(path string) (err error) { enc := yaml.NewEncoder(file) enc.SetIndent(identation) - err = enc.Encode(&configuration) + err = enc.Encode(&cfg) return } @@ -59,12 +59,12 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ > <\__ \ |_| | |_) | _| | | | | (_| |___) | | /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| %s - %s -`).Bold(), + + %s`).Bold(), aurora.BrightRed("v"+VERSION).Bold(), aurora.BrightYellow("with <3 by Hueristiq Open Source").Italic(), ) - userDotConfigDirectoryPath = func() (userDotConfig string) { + UserDotConfigDirectoryPath = func() (userDotConfig string) { var err error userDotConfig, err = os.UserConfigDir() @@ -75,13 +75,13 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ return }() projectRootDirectoryName = NAME - ProjectRootDirectoryPath = filepath.Join(userDotConfigDirectoryPath, projectRootDirectoryName) + ProjectRootDirectoryPath = filepath.Join(UserDotConfigDirectoryPath, projectRootDirectoryName) configurationFileName = "config.yaml" ConfigurationFilePath = filepath.Join(ProjectRootDirectoryPath, configurationFileName) ) func CreateUpdate(path string) (err error) { - var config Configuration + var cfg Configuration defaultConfig := Configuration{ Version: VERSION, @@ -98,33 +98,31 @@ func CreateUpdate(path string) (err error) { } _, err = os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - config = defaultConfig - if err = config.Write(path); err != nil { - return - } - } else { + switch { + case err != nil && os.IsNotExist(err): + cfg = defaultConfig + + if err = cfg.Write(path); err != nil { return } - } else { - config, err = Read(path) + case err != nil: + return + default: + cfg, err = Read(path) if err != nil { return } - if config.Version != VERSION || - len(config.Sources) != len(sources.List) { - - if err = mergo.Merge(&config, defaultConfig); err != nil { + if cfg.Version != VERSION || len(cfg.Sources) != len(sources.List) { + if err = mergo.Merge(&cfg, defaultConfig); err != nil { return } - config.Version = VERSION - config.Sources = sources.List + cfg.Version = VERSION + cfg.Sources = sources.List - if err = config.Write(path); err != nil { + if err = cfg.Write(path); err != nil { return } } @@ -133,7 +131,7 @@ func CreateUpdate(path string) (err error) { return } -func Read(path string) (configuration Configuration, err error) { +func Read(path string) (cfg Configuration, err error) { var file *os.File file, err = os.Open(path) @@ -143,7 +141,7 @@ func Read(path string) (configuration Configuration, err error) { defer file.Close() - if err = yaml.NewDecoder(file).Decode(&configuration); err != nil { + if err = yaml.NewDecoder(file).Decode(&cfg); err != nil { return } diff --git a/pkg/xsubfind3r/extractor/extractor.go b/pkg/extractor/extractor.go similarity index 100% rename from pkg/xsubfind3r/extractor/extractor.go rename to pkg/extractor/extractor.go diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 11bd022..f40e147 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "net/url" hqgohttpclient "github.com/hueristiq/hqgohttp" diff --git a/pkg/xsubfind3r/options.go b/pkg/scraper/options.go similarity index 57% rename from pkg/xsubfind3r/options.go rename to pkg/scraper/options.go index dabec4a..042f721 100644 --- a/pkg/xsubfind3r/options.go +++ b/pkg/scraper/options.go @@ -1,6 +1,6 @@ -package xsubfind3r +package scraper -import "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" +import "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" type Options struct { SourcesToExclude []string diff --git a/pkg/scraper/scraper.go b/pkg/scraper/scraper.go new file mode 100644 index 0000000..ac4f748 --- /dev/null +++ b/pkg/scraper/scraper.go @@ -0,0 +1,116 @@ +package scraper + +import ( + "strings" + "sync" + + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/anubis" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/bevigil" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/chaos" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/commoncrawl" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/crtsh" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/fullhunt" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/github" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/hackertarget" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/intelx" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/otx" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/shodan" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/urlscan" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/wayback" +) + +type Scraper struct { + Sources map[string]sources.Source + SourcesConfiguration *sources.Configuration +} + +func (scraper *Scraper) Scrape(domain string) (results chan sources.Result) { + results = make(chan sources.Result) + + go func() { + defer close(results) + + seenSubdomains := &sync.Map{} + + wg := &sync.WaitGroup{} + + for _, source := range scraper.Sources { + wg.Add(1) + + go func(source sources.Source) { + defer wg.Done() + + sResults := source.Run(scraper.SourcesConfiguration, domain) + + for sResult := range sResults { + if sResult.Type == sources.Subdomain { + sResult.Value = strings.ToLower(sResult.Value) + sResult.Value = strings.ReplaceAll(sResult.Value, "*.", "") + + _, loaded := seenSubdomains.LoadOrStore(sResult.Value, struct{}{}) + if loaded { + continue + } + } + + results <- sResult + } + }(source) + } + + wg.Wait() + }() + + return +} + +func New(options *Options) (scraper *Scraper) { + scraper = &Scraper{ + Sources: map[string]sources.Source{}, + SourcesConfiguration: &sources.Configuration{ + Keys: options.Keys, + }, + } + + if len(options.SourcesToUSe) < 1 { + options.SourcesToUSe = sources.List + } + + for _, source := range options.SourcesToUSe { + switch source { + case "anubis": + scraper.Sources[source] = &anubis.Source{} + case "bevigil": + scraper.Sources[source] = &bevigil.Source{} + case "chaos": + scraper.Sources[source] = &chaos.Source{} + case "commoncrawl": + scraper.Sources[source] = &commoncrawl.Source{} + case "crtsh": + scraper.Sources[source] = &crtsh.Source{} + case "fullhunt": + scraper.Sources[source] = &fullhunt.Source{} + case "github": + scraper.Sources[source] = &github.Source{} + case "hackertarget": + scraper.Sources[source] = &hackertarget.Source{} + case "intelx": + scraper.Sources[source] = &intelx.Source{} + case "otx": + scraper.Sources[source] = &otx.Source{} + case "shodan": + scraper.Sources[source] = &shodan.Source{} + case "urlscan": + scraper.Sources[source] = &urlscan.Source{} + case "wayback": + scraper.Sources[source] = &wayback.Source{} + } + } + + for _, source := range options.SourcesToExclude { + delete(scraper.Sources, source) + } + + return +} diff --git a/pkg/xsubfind3r/sources/anubis/anubis.go b/pkg/scraper/sources/anubis/anubis.go similarity index 89% rename from pkg/xsubfind3r/sources/anubis/anubis.go rename to pkg/scraper/sources/anubis/anubis.go index 53ca41c..de69fe9 100644 --- a/pkg/xsubfind3r/sources/anubis/anubis.go +++ b/pkg/scraper/sources/anubis/anubis.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type Source struct{} @@ -55,7 +55,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getSubdomainsRes.Body.Close() - for _, subdomain := range getSubdomainsResData { + for index := range getSubdomainsResData { + subdomain := getSubdomainsResData[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/bevigil/bevigil.go b/pkg/scraper/sources/bevigil/bevigil.go similarity index 91% rename from pkg/xsubfind3r/sources/bevigil/bevigil.go rename to pkg/scraper/sources/bevigil/bevigil.go index 4202035..f8c52d9 100644 --- a/pkg/xsubfind3r/sources/bevigil/bevigil.go +++ b/pkg/scraper/sources/bevigil/bevigil.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type getSubdomainsResponse struct { @@ -81,7 +81,9 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsRes.Body.Close() - for _, subdomain := range getSubdomainsResData.Subdomains { + for index := range getSubdomainsResData.Subdomains { + subdomain := getSubdomainsResData.Subdomains[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/chaos/chaos.go b/pkg/scraper/sources/chaos/chaos.go similarity index 87% rename from pkg/xsubfind3r/sources/chaos/chaos.go rename to pkg/scraper/sources/chaos/chaos.go index 4392fc7..455556f 100644 --- a/pkg/xsubfind3r/sources/chaos/chaos.go +++ b/pkg/scraper/sources/chaos/chaos.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type getSubdomainsResponse struct { @@ -78,11 +78,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsRes.Body.Close() - for _, record := range getSubdomainsResData.Subdomains { + for index := range getSubdomainsResData.Subdomains { + subdomain := getSubdomainsResData.Subdomains[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), - Value: fmt.Sprintf("%s.%s", record, getSubdomainsResData.Domain), + Value: fmt.Sprintf("%s.%s", subdomain, getSubdomainsResData.Domain), } results <- result diff --git a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go b/pkg/scraper/sources/commoncrawl/commoncrawl.go similarity index 82% rename from pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go rename to pkg/scraper/sources/commoncrawl/commoncrawl.go index 3c2e589..dca9c3f 100644 --- a/pkg/xsubfind3r/sources/commoncrawl/commoncrawl.go +++ b/pkg/scraper/sources/commoncrawl/commoncrawl.go @@ -6,10 +6,13 @@ import ( "fmt" "net/http" "regexp" + "strconv" + "strings" + "time" + "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type getIndexesResponse []struct { @@ -89,12 +92,38 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getIndexesRes.Body.Close() - for _, indexData := range getIndexesResData { + year := time.Now().Year() + years := make([]string, 0) + maxYearsBack := 5 + + for i := 0; i < maxYearsBack; i++ { + years = append(years, strconv.Itoa(year-i)) + } + + searchIndexes := make(map[string]string) + + for index := range years { + year := years[index] + + for index := range getIndexesResData { + CCIndex := getIndexesResData[index] + + if strings.Contains(CCIndex.ID, year) { + if _, ok := searchIndexes[year]; !ok { + searchIndexes[year] = CCIndex.API + + break + } + } + } + } + + for _, CCIndexAPI := range searchIndexes { getURLsReqHeaders := map[string]string{ "Host": "index.commoncrawl.org", } - getPaginationReqURL := fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url&showNumPages=true", indexData.API, domain) + getPaginationReqURL := fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url&showNumPages=true", CCIndexAPI, domain) var getPaginationRes *http.Response @@ -135,7 +164,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source } for page := uint(0); page < getPaginationData.Pages; page++ { - getURLsReqURL := fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url&page=%d", indexData.API, domain, page) + getURLsReqURL := fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url&page=%d", CCIndexAPI, domain, page) var getURLsRes *http.Response @@ -189,7 +218,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source match := regex.FindAllString(getURLsResData.URL, -1) - for _, subdomain := range match { + for index := range match { + subdomain := match[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/configuration.go b/pkg/scraper/sources/configuration.go similarity index 100% rename from pkg/xsubfind3r/sources/configuration.go rename to pkg/scraper/sources/configuration.go diff --git a/pkg/xsubfind3r/sources/crtsh/crtsh.go b/pkg/scraper/sources/crtsh/crtsh.go similarity index 90% rename from pkg/xsubfind3r/sources/crtsh/crtsh.go rename to pkg/scraper/sources/crtsh/crtsh.go index 177e082..5d6868d 100644 --- a/pkg/xsubfind3r/sources/crtsh/crtsh.go +++ b/pkg/scraper/sources/crtsh/crtsh.go @@ -7,9 +7,9 @@ import ( "regexp" "strings" + "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type getNameValuesResponse []struct { @@ -78,7 +78,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - for _, record := range getNameValuesResData { + for index := range getNameValuesResData { + record := getNameValuesResData[index] + for _, value := range strings.Split(record.NameValue, "\n") { match := regex.FindAllString(value, -1) diff --git a/pkg/xsubfind3r/sources/fullhunt/fullhunt.go b/pkg/scraper/sources/fullhunt/fullhunt.go similarity index 92% rename from pkg/xsubfind3r/sources/fullhunt/fullhunt.go rename to pkg/scraper/sources/fullhunt/fullhunt.go index 35c890f..b179746 100644 --- a/pkg/xsubfind3r/sources/fullhunt/fullhunt.go +++ b/pkg/scraper/sources/fullhunt/fullhunt.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type getSubdomainsResponse struct { @@ -82,7 +82,9 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsRes.Body.Close() - for _, subdomain := range getSubdomainsResData.Hosts { + for index := range getSubdomainsResData.Hosts { + subdomain := getSubdomainsResData.Hosts[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/github/github.go b/pkg/scraper/sources/github/github.go similarity index 97% rename from pkg/xsubfind3r/sources/github/github.go rename to pkg/scraper/sources/github/github.go index 2b65f9c..8116142 100644 --- a/pkg/xsubfind3r/sources/github/github.go +++ b/pkg/scraper/sources/github/github.go @@ -11,9 +11,9 @@ import ( "time" "github.com/hueristiq/hqgohttp" + "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" "github.com/spf13/cast" "github.com/tomnomnom/linkheader" ) @@ -226,6 +226,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp func getRawContentURL(htmlURL string) string { domain := strings.ReplaceAll(htmlURL, "https://github.com/", "https://raw.githubusercontent.com/") + return strings.ReplaceAll(domain, "/blob/", "/") } diff --git a/pkg/xsubfind3r/sources/github/tokenmanager.go b/pkg/scraper/sources/github/tokenmanager.go similarity index 100% rename from pkg/xsubfind3r/sources/github/tokenmanager.go rename to pkg/scraper/sources/github/tokenmanager.go diff --git a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go b/pkg/scraper/sources/hackertarget/hackertarget.go similarity index 90% rename from pkg/xsubfind3r/sources/hackertarget/hackertarget.go rename to pkg/scraper/sources/hackertarget/hackertarget.go index cc21d30..86ae9e0 100644 --- a/pkg/xsubfind3r/sources/hackertarget/hackertarget.go +++ b/pkg/scraper/sources/hackertarget/hackertarget.go @@ -6,9 +6,9 @@ import ( "net/http" "regexp" + "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type Source struct{} @@ -64,7 +64,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source match := regex.FindAllString(line, -1) - for _, subdomain := range match { + for index := range match { + subdomain := match[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/intelx/intelx.go b/pkg/scraper/sources/intelx/intelx.go similarity index 95% rename from pkg/xsubfind3r/sources/intelx/intelx.go rename to pkg/scraper/sources/intelx/intelx.go index ced0577..8e9e019 100644 --- a/pkg/xsubfind3r/sources/intelx/intelx.go +++ b/pkg/scraper/sources/intelx/intelx.go @@ -9,7 +9,7 @@ import ( "time" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type searchRequest struct { @@ -166,7 +166,9 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s status = getResultsResData.Status - for _, record := range getResultsResData.Selectors { + for index := range getResultsResData.Selectors { + record := getResultsResData.Selectors[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/otx/otx.go b/pkg/scraper/sources/otx/otx.go similarity index 91% rename from pkg/xsubfind3r/sources/otx/otx.go rename to pkg/scraper/sources/otx/otx.go index 803416a..50435c2 100644 --- a/pkg/xsubfind3r/sources/otx/otx.go +++ b/pkg/scraper/sources/otx/otx.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type getPassiveDNSResponse struct { @@ -75,7 +75,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - for _, record := range getPassiveDNSResData.PassiveDNS { + for index := range getPassiveDNSResData.PassiveDNS { + record := getPassiveDNSResData.PassiveDNS[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/shodan/shodan.go b/pkg/scraper/sources/shodan/shodan.go similarity index 91% rename from pkg/xsubfind3r/sources/shodan/shodan.go rename to pkg/scraper/sources/shodan/shodan.go index 0bbcafa..b609acd 100644 --- a/pkg/xsubfind3r/sources/shodan/shodan.go +++ b/pkg/scraper/sources/shodan/shodan.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type getDNSResponse struct { @@ -77,7 +77,9 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getDNSRes.Body.Close() - for _, subdomain := range getDNSResData.Subdomains { + for index := range getDNSResData.Subdomains { + subdomain := getDNSResData.Subdomains[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/sources.go b/pkg/scraper/sources/sources.go similarity index 100% rename from pkg/xsubfind3r/sources/sources.go rename to pkg/scraper/sources/sources.go diff --git a/pkg/xsubfind3r/sources/sources_results.go b/pkg/scraper/sources/sources_results.go similarity index 100% rename from pkg/xsubfind3r/sources/sources_results.go rename to pkg/scraper/sources/sources_results.go diff --git a/pkg/xsubfind3r/sources/urlscan/urlscan.go b/pkg/scraper/sources/urlscan/urlscan.go similarity index 92% rename from pkg/xsubfind3r/sources/urlscan/urlscan.go rename to pkg/scraper/sources/urlscan/urlscan.go index 6eb8e6f..ad5f0b2 100644 --- a/pkg/xsubfind3r/sources/urlscan/urlscan.go +++ b/pkg/scraper/sources/urlscan/urlscan.go @@ -6,9 +6,9 @@ import ( "net/http" "regexp" + "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type searchResponse struct { @@ -139,10 +139,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s return } - for _, result := range searchResData.Results { + for index := range searchResData.Results { + result := searchResData.Results[index] match := regex.FindAllString(result.Page.Domain, -1) - for _, subdomain := range match { + for index := range match { + subdomain := match[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/sources/utils.go b/pkg/scraper/sources/utils.go similarity index 100% rename from pkg/xsubfind3r/sources/utils.go rename to pkg/scraper/sources/utils.go diff --git a/pkg/xsubfind3r/sources/wayback/wayback.go b/pkg/scraper/sources/wayback/wayback.go similarity index 91% rename from pkg/xsubfind3r/sources/wayback/wayback.go rename to pkg/scraper/sources/wayback/wayback.go index 2b0e295..5d8ae41 100644 --- a/pkg/xsubfind3r/sources/wayback/wayback.go +++ b/pkg/scraper/sources/wayback/wayback.go @@ -6,9 +6,9 @@ import ( "net/http" "regexp" + "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/extractor" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) type Source struct{} @@ -116,10 +116,13 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source } // Slicing as [1:] to skip first result by default - for _, entry := range getURLsResData[1:] { + for index := range getURLsResData[1:] { + entry := getURLsResData[1:][index] match := regex.FindAllString(entry[0], -1) - for _, subdomain := range match { + for index := range match { + subdomain := match[index] + result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/xsubfind3r/finder.go b/pkg/xsubfind3r/finder.go deleted file mode 100644 index 4e3e1ce..0000000 --- a/pkg/xsubfind3r/finder.go +++ /dev/null @@ -1,116 +0,0 @@ -package xsubfind3r - -import ( - "strings" - "sync" - - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/anubis" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/bevigil" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/chaos" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/commoncrawl" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/crtsh" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/fullhunt" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/github" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/hackertarget" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/intelx" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/otx" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/shodan" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/urlscan" - "github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources/wayback" -) - -type Finder struct { - Sources map[string]sources.Source - SourcesConfiguration *sources.Configuration -} - -func (finder *Finder) Find(domain string) (results chan sources.Result) { - results = make(chan sources.Result) - - go func() { - defer close(results) - - seenSubdomains := &sync.Map{} - - wg := &sync.WaitGroup{} - - for _, source := range finder.Sources { - wg.Add(1) - - go func(source sources.Source) { - defer wg.Done() - - sResults := source.Run(finder.SourcesConfiguration, domain) - - for sResult := range sResults { - if sResult.Type == sources.Subdomain { - sResult.Value = strings.ToLower(sResult.Value) - sResult.Value = strings.ReplaceAll(sResult.Value, "*.", "") - - _, loaded := seenSubdomains.LoadOrStore(sResult.Value, struct{}{}) - if loaded { - continue - } - } - - results <- sResult - } - }(source) - } - - wg.Wait() - }() - - return -} - -func New(options *Options) (finder *Finder) { - finder = &Finder{ - Sources: map[string]sources.Source{}, - SourcesConfiguration: &sources.Configuration{ - Keys: options.Keys, - }, - } - - if len(options.SourcesToUSe) < 1 { - options.SourcesToUSe = sources.List - } - - for _, source := range options.SourcesToUSe { - switch source { - case "anubis": - finder.Sources[source] = &anubis.Source{} - case "bevigil": - finder.Sources[source] = &bevigil.Source{} - case "chaos": - finder.Sources[source] = &chaos.Source{} - case "commoncrawl": - finder.Sources[source] = &commoncrawl.Source{} - case "crtsh": - finder.Sources[source] = &crtsh.Source{} - case "fullhunt": - finder.Sources[source] = &fullhunt.Source{} - case "github": - finder.Sources[source] = &github.Source{} - case "hackertarget": - finder.Sources[source] = &hackertarget.Source{} - case "intelx": - finder.Sources[source] = &intelx.Source{} - case "otx": - finder.Sources[source] = &otx.Source{} - case "shodan": - finder.Sources[source] = &shodan.Source{} - case "urlscan": - finder.Sources[source] = &urlscan.Source{} - case "wayback": - finder.Sources[source] = &wayback.Source{} - } - } - - for _, source := range options.SourcesToExclude { - delete(finder.Sources, source) - } - - return -} From 21e68a61decc107e75e21802426a721de8697e9a Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 12 Oct 2023 04:34:56 +0300 Subject: [PATCH 33/50] ci: Add codeql analysis workflow --- .github/workflows/codeql-analysis.yaml | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yaml diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 0000000..eac5074 --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,40 @@ +name: 🚨 CodeQL Analysis + +on: + pull_request: + paths: + - '**.go' + - '**.mod' + workflow_dispatch: + +jobs: + analyze: + name: CodeQL Analysis + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + + steps: + - + name: Checkout repository + uses: actions/checkout@v3 + # Initializes the CodeQL tools for scanning. + - + name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + - + name: Autobuild + uses: github/codeql-action/autobuild@v2 + - + name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 \ No newline at end of file From 9b7ee8fea8eea9aa567ce869d51f14479a4ad05c Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 12 Oct 2023 04:38:40 +0300 Subject: [PATCH 34/50] ci: Update codeql analysis workflow events --- .github/workflows/codeql-analysis.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index eac5074..e935927 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -1,8 +1,16 @@ name: 🚨 CodeQL Analysis on: + push: + branches: + - "main" + paths: + - '**.go' + - '**.mod' pull_request: - paths: + branches: + - "main" + paths: - '**.go' - '**.mod' workflow_dispatch: From a4f38dfefabaa9de1d73847bba8e6db2a58efad9 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 12 Oct 2023 04:44:47 +0300 Subject: [PATCH 35/50] ci: Update release workflow events --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c4add6b..b75aaf6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,12 @@ name: 🎉 Release on: - create: + push: branches: - main tags: - v*.*.* + workflow_dispatch: jobs: release: From 032d4f17c3363b15ce1d1c4c004fd9c22b6f797a Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 12 Oct 2023 04:50:08 +0300 Subject: [PATCH 36/50] refactor: Don't show errors if not in verbose mode --- cmd/xsubfind3r/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index acc2fad..b35baa7 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -274,7 +274,9 @@ func processSubdomains(writer *bufio.Writer, subdomains chan sources.Result) { for subdomain := range subdomains { switch subdomain.Type { case sources.Error: - hqgolog.Error().Msgf("%s: %s\n", subdomain.Source, subdomain.Error) + if verbose { + hqgolog.Error().Msgf("%s: %s\n", subdomain.Source, subdomain.Error) + } case sources.Subdomain: if verbose { hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomain.Source), subdomain.Value) From 11943cb11203af7464da5a1e88ab843060d66c1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 05:28:20 +0000 Subject: [PATCH 37/50] chore(deps): bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index e935927..4dda1b1 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -33,7 +33,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL From bde2b5b44cc453277ca3c96e4e39b0d5955b4025 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:56:58 +0300 Subject: [PATCH 38/50] chore: - --- README.md | 10 +++--- cmd/xsubfind3r/main.go | 16 +++++----- pkg/httpclient/client.go | 1 - pkg/scraper/options.go | 2 +- pkg/scraper/scraper.go | 16 ++++++---- pkg/scraper/sources/anubis/anubis.go | 5 +-- pkg/scraper/sources/bevigil/bevigil.go | 5 +-- pkg/scraper/sources/chaos/chaos.go | 5 +-- .../sources/commoncrawl/commoncrawl.go | 15 +++++---- pkg/scraper/sources/configuration.go | 15 --------- pkg/scraper/sources/crtsh/crtsh.go | 5 +-- pkg/scraper/sources/fullhunt/fullhunt.go | 2 ++ pkg/scraper/sources/github/github.go | 7 +++-- .../sources/hackertarget/hackertarget.go | 2 ++ pkg/scraper/sources/intelx/intelx.go | 11 ++++--- pkg/scraper/sources/otx/otx.go | 5 +-- pkg/scraper/sources/shodan/shodan.go | 5 +-- pkg/scraper/sources/sources.go | 31 +++++++++++++++++++ pkg/scraper/sources/sources_results.go | 18 ----------- pkg/scraper/sources/urlscan/urlscan.go | 5 +-- pkg/scraper/sources/wayback/wayback.go | 5 +-- 21 files changed, 105 insertions(+), 81 deletions(-) delete mode 100644 pkg/scraper/sources/configuration.go delete mode 100644 pkg/scraper/sources/sources_results.go diff --git a/README.md b/README.md index c2dc75c..75a95ec 100644 --- a/README.md +++ b/README.md @@ -162,10 +162,10 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ with <3 by Hueristiq Open Source USAGE: - xsubfind3r [OPTIONS] + xsubfind3r [OPTIONS] CONFIGURATION: - -c, --configuration string configuration file path (default: $HOME/.config/xsubfind3r/config.yaml) + -c, --configuration string configuration file (default: $HOME/.config/xsubfind3r/config.yaml) INPUT: -d, --domain string[] target domain @@ -175,9 +175,9 @@ TIP: For multiple input domains use comma(,) separated value with `-d`, specify multiple `-d`, load from file with `-l` or load from stdin. SOURCES: - --sources bool list supported sources - -u, --sources-to-use string[] comma(,) separeted sources to use - -e, --sources-to-exclude string[] comma(,) separeted sources to exclude + --sources bool list supported sources + -u, --sources-to-use string[] comma(,) separeted sources to use + -e, --sources-to-exclude string[] comma(,) separeted sources to exclude OUTPUT: --monochrome bool display no color output diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index b35baa7..35fe56b 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -54,11 +54,11 @@ func init() { fmt.Fprintln(os.Stderr, configuration.BANNER) h := "\nUSAGE:\n" - h += fmt.Sprintf(" %s [OPTIONS]\n", configuration.NAME) + h += fmt.Sprintf(" %s [OPTIONS]\n", configuration.NAME) h += "\nCONFIGURATION:\n" defaultConfigurationFilePath := strings.ReplaceAll(configuration.ConfigurationFilePath, configuration.UserDotConfigDirectoryPath, "$HOME/.config") - h += fmt.Sprintf(" -c, --configuration string configuration file path (default: %s)\n", defaultConfigurationFilePath) + h += fmt.Sprintf(" -c, --configuration string configuration file (default: %s)\n", defaultConfigurationFilePath) h += "\nINPUT:\n" h += " -d, --domain string[] target domain\n" @@ -68,9 +68,9 @@ func init() { h += " specify multiple `-d`, load from file with `-l` or load from stdin.\n" h += "\nSOURCES:\n" - h += " --sources bool list supported sources\n" - h += " -u, --sources-to-use string[] comma(,) separeted sources to use\n" - h += " -e, --sources-to-exclude string[] comma(,) separeted sources to exclude\n" + h += " --sources bool list supported sources\n" + h += " -u, --sources-to-use string[] comma(,) separeted sources to use\n" + h += " -e, --sources-to-exclude string[] comma(,) separeted sources to exclude\n" h += "\nOUTPUT:\n" h += " --monochrome bool display no color output\n" @@ -133,7 +133,9 @@ func main() { needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface() } - for _, source := range config.Sources { + for index := range config.Sources { + source := config.Sources[index] + if _, ok := needsKey[source]; ok { hqgolog.Print().Msgf("> %s *", source) } else { @@ -189,8 +191,8 @@ func main() { // scrape and output subdomains. options := &scraper.Options{ - SourcesToExclude: sourcesToExclude, SourcesToUSe: sourcesToUse, + SourcesToExclude: sourcesToExclude, Keys: config.Keys, } diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index f40e147..11bd022 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "net/http" - "net/url" hqgohttpclient "github.com/hueristiq/hqgohttp" diff --git a/pkg/scraper/options.go b/pkg/scraper/options.go index 042f721..14e9b43 100644 --- a/pkg/scraper/options.go +++ b/pkg/scraper/options.go @@ -3,7 +3,7 @@ package scraper import "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" type Options struct { - SourcesToExclude []string SourcesToUSe []string + SourcesToExclude []string Keys sources.Keys } diff --git a/pkg/scraper/scraper.go b/pkg/scraper/scraper.go index ac4f748..341c022 100644 --- a/pkg/scraper/scraper.go +++ b/pkg/scraper/scraper.go @@ -21,8 +21,8 @@ import ( ) type Scraper struct { - Sources map[string]sources.Source - SourcesConfiguration *sources.Configuration + Sources map[string]sources.Source + Configuration *sources.Configuration } func (scraper *Scraper) Scrape(domain string) (results chan sources.Result) { @@ -41,7 +41,7 @@ func (scraper *Scraper) Scrape(domain string) (results chan sources.Result) { go func(source sources.Source) { defer wg.Done() - sResults := source.Run(scraper.SourcesConfiguration, domain) + sResults := source.Run(scraper.Configuration, domain) for sResult := range sResults { if sResult.Type == sources.Subdomain { @@ -68,7 +68,7 @@ func (scraper *Scraper) Scrape(domain string) (results chan sources.Result) { func New(options *Options) (scraper *Scraper) { scraper = &Scraper{ Sources: map[string]sources.Source{}, - SourcesConfiguration: &sources.Configuration{ + Configuration: &sources.Configuration{ Keys: options.Keys, }, } @@ -77,7 +77,9 @@ func New(options *Options) (scraper *Scraper) { options.SourcesToUSe = sources.List } - for _, source := range options.SourcesToUSe { + for index := range options.SourcesToUSe { + source := options.SourcesToUSe[index] + switch source { case "anubis": scraper.Sources[source] = &anubis.Source{} @@ -108,7 +110,9 @@ func New(options *Options) (scraper *Scraper) { } } - for _, source := range options.SourcesToExclude { + for index := range options.SourcesToExclude { + source := options.SourcesToExclude[index] + delete(scraper.Sources, source) } diff --git a/pkg/scraper/sources/anubis/anubis.go b/pkg/scraper/sources/anubis/anubis.go index de69fe9..fc79cdf 100644 --- a/pkg/scraper/sources/anubis/anubis.go +++ b/pkg/scraper/sources/anubis/anubis.go @@ -33,13 +33,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getSubdomainsRes.Body.Close() + return } var getSubdomainsResData []string - err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData) - if err != nil { + if err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/bevigil/bevigil.go b/pkg/scraper/sources/bevigil/bevigil.go index f8c52d9..bf32884 100644 --- a/pkg/scraper/sources/bevigil/bevigil.go +++ b/pkg/scraper/sources/bevigil/bevigil.go @@ -59,13 +59,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getSubdomainsRes.Body.Close() + return } var getSubdomainsResData getSubdomainsResponse - err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData) - if err != nil { + if err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/chaos/chaos.go b/pkg/scraper/sources/chaos/chaos.go index 455556f..0a049b1 100644 --- a/pkg/scraper/sources/chaos/chaos.go +++ b/pkg/scraper/sources/chaos/chaos.go @@ -56,13 +56,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getSubdomainsRes.Body.Close() + return } var getSubdomainsResData getSubdomainsResponse - err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData) - if err != nil { + if err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/commoncrawl/commoncrawl.go b/pkg/scraper/sources/commoncrawl/commoncrawl.go index dca9c3f..19a9b43 100644 --- a/pkg/scraper/sources/commoncrawl/commoncrawl.go +++ b/pkg/scraper/sources/commoncrawl/commoncrawl.go @@ -70,13 +70,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getIndexesRes.Body.Close() + return } var getIndexesResData getIndexesResponse - err = json.NewDecoder(getIndexesRes.Body).Decode(&getIndexesResData) - if err != nil { + if err = json.NewDecoder(getIndexesRes.Body).Decode(&getIndexesResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), @@ -137,13 +138,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getPaginationRes.Body.Close() + continue } var getPaginationData getPaginationResponse - err = json.NewDecoder(getPaginationRes.Body).Decode(&getPaginationData) - if err != nil { + if err = json.NewDecoder(getPaginationRes.Body).Decode(&getPaginationData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), @@ -178,6 +180,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getURLsRes.Body.Close() + continue } @@ -191,8 +195,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var getURLsResData getURLsResponse - err = json.Unmarshal(scanner.Bytes(), &getURLsResData) - if err != nil { + if err = json.Unmarshal(scanner.Bytes(), &getURLsResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/configuration.go b/pkg/scraper/sources/configuration.go deleted file mode 100644 index 2710769..0000000 --- a/pkg/scraper/sources/configuration.go +++ /dev/null @@ -1,15 +0,0 @@ -package sources - -type Configuration struct { - Keys Keys -} - -type Keys struct { - Bevigil []string `yaml:"bevigil"` - Chaos []string `yaml:"chaos"` - Fullhunt []string `yaml:"fullhunt"` - GitHub []string `yaml:"github"` - Intelx []string `yaml:"intelx"` - Shodan []string `yaml:"shodan"` - URLScan []string `yaml:"urlscan"` -} diff --git a/pkg/scraper/sources/crtsh/crtsh.go b/pkg/scraper/sources/crtsh/crtsh.go index 5d6868d..fa8df28 100644 --- a/pkg/scraper/sources/crtsh/crtsh.go +++ b/pkg/scraper/sources/crtsh/crtsh.go @@ -41,13 +41,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getNameValuesRes.Body.Close() + return } var getNameValuesResData getNameValuesResponse - err = json.NewDecoder(getNameValuesRes.Body).Decode(&getNameValuesResData) - if err != nil { + if err = json.NewDecoder(getNameValuesRes.Body).Decode(&getNameValuesResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/fullhunt/fullhunt.go b/pkg/scraper/sources/fullhunt/fullhunt.go index b179746..fe9368f 100644 --- a/pkg/scraper/sources/fullhunt/fullhunt.go +++ b/pkg/scraper/sources/fullhunt/fullhunt.go @@ -60,6 +60,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getSubdomainsRes.Body.Close() + return } diff --git a/pkg/scraper/sources/github/github.go b/pkg/scraper/sources/github/github.go index 8116142..74159e9 100644 --- a/pkg/scraper/sources/github/github.go +++ b/pkg/scraper/sources/github/github.go @@ -97,6 +97,8 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp results <- result + searchRes.Body.Close() + return } @@ -111,8 +113,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp var searchResData searchResponse - err = json.NewDecoder(searchRes.Body).Decode(&searchResData) - if err != nil { + if err = json.NewDecoder(searchRes.Body).Decode(&searchResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), @@ -143,6 +144,8 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp results <- result + getRawContentRes.Body.Close() + continue } diff --git a/pkg/scraper/sources/hackertarget/hackertarget.go b/pkg/scraper/sources/hackertarget/hackertarget.go index 86ae9e0..0564cf7 100644 --- a/pkg/scraper/sources/hackertarget/hackertarget.go +++ b/pkg/scraper/sources/hackertarget/hackertarget.go @@ -35,6 +35,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + hostSearchRes.Body.Close() + return } diff --git a/pkg/scraper/sources/intelx/intelx.go b/pkg/scraper/sources/intelx/intelx.go index 8e9e019..2012f4e 100644 --- a/pkg/scraper/sources/intelx/intelx.go +++ b/pkg/scraper/sources/intelx/intelx.go @@ -19,6 +19,7 @@ type searchRequest struct { MaxResults int `json:"maxresults"` Media int `json:"media"` } + type searchResponse struct { ID string `json:"id"` Status int `json:"status"` @@ -104,13 +105,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + searchRes.Body.Close() + return } var searchResData searchResponse - err = json.NewDecoder(searchRes.Body).Decode(&searchResData) - if err != nil { + if err = json.NewDecoder(searchRes.Body).Decode(&searchResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), @@ -142,13 +144,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getResultsRes.Body.Close() + return } var getResultsResData getResultsResponse - err = json.NewDecoder(getResultsRes.Body).Decode(&getResultsResData) - if err != nil { + if err = json.NewDecoder(getResultsRes.Body).Decode(&getResultsResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/otx/otx.go b/pkg/scraper/sources/otx/otx.go index 50435c2..5811a52 100644 --- a/pkg/scraper/sources/otx/otx.go +++ b/pkg/scraper/sources/otx/otx.go @@ -41,13 +41,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getPassiveDNSRes.Body.Close() + return } var getPassiveDNSResData getPassiveDNSResponse - err = json.NewDecoder(getPassiveDNSRes.Body).Decode(&getPassiveDNSResData) - if err != nil { + if err = json.NewDecoder(getPassiveDNSRes.Body).Decode(&getPassiveDNSResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/shodan/shodan.go b/pkg/scraper/sources/shodan/shodan.go index b609acd..61bad1e 100644 --- a/pkg/scraper/sources/shodan/shodan.go +++ b/pkg/scraper/sources/shodan/shodan.go @@ -55,13 +55,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + getDNSRes.Body.Close() + return } var getDNSResData getDNSResponse - err = json.NewDecoder(getDNSRes.Body).Decode(&getDNSResData) - if err != nil { + if err = json.NewDecoder(getDNSRes.Body).Decode(&getDNSResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/sources.go b/pkg/scraper/sources/sources.go index ac2b440..c1ed2c2 100644 --- a/pkg/scraper/sources/sources.go +++ b/pkg/scraper/sources/sources.go @@ -9,6 +9,37 @@ type Source interface { Name() string } +type Configuration struct { + Keys Keys +} + +type Keys struct { + Bevigil []string `yaml:"bevigil"` + Chaos []string `yaml:"chaos"` + Fullhunt []string `yaml:"fullhunt"` + GitHub []string `yaml:"github"` + Intelx []string `yaml:"intelx"` + Shodan []string `yaml:"shodan"` + URLScan []string `yaml:"urlscan"` +} + +// Result is a result structure returned by a source. +type Result struct { + Type ResultType + Source string + Value string + Error error +} + +// ResultType is the type of result returned by the source. +type ResultType int + +// Types of results returned by the source. +const ( + Subdomain ResultType = iota + Error +) + var List = []string{ "anubis", "bevigil", diff --git a/pkg/scraper/sources/sources_results.go b/pkg/scraper/sources/sources_results.go deleted file mode 100644 index a3ff380..0000000 --- a/pkg/scraper/sources/sources_results.go +++ /dev/null @@ -1,18 +0,0 @@ -package sources - -// Result is a result structure returned by a source. -type Result struct { - Type ResultType - Source string - Value string - Error error -} - -// ResultType is the type of result returned by the source. -type ResultType int - -// Types of results returned by the source. -const ( - Subdomain ResultType = iota - Error -) diff --git a/pkg/scraper/sources/urlscan/urlscan.go b/pkg/scraper/sources/urlscan/urlscan.go index ad5f0b2..c3c09d8 100644 --- a/pkg/scraper/sources/urlscan/urlscan.go +++ b/pkg/scraper/sources/urlscan/urlscan.go @@ -98,13 +98,14 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result + searchRes.Body.Close() + return } var searchResData searchResponse - err = json.NewDecoder(searchRes.Body).Decode(&searchResData) - if err != nil { + if err = json.NewDecoder(searchRes.Body).Decode(&searchResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), diff --git a/pkg/scraper/sources/wayback/wayback.go b/pkg/scraper/sources/wayback/wayback.go index 5d8ae41..d65960f 100644 --- a/pkg/scraper/sources/wayback/wayback.go +++ b/pkg/scraper/sources/wayback/wayback.go @@ -87,13 +87,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result + getURLsRes.Body.Close() + return } var getURLsResData [][]string - err = json.NewDecoder(getURLsRes.Body).Decode(&getURLsResData) - if err != nil { + if err = json.NewDecoder(getURLsRes.Body).Decode(&getURLsResData); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), From 60674a71d7088364283ebd3697857e57593b1536 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:08:44 +0300 Subject: [PATCH 39/50] fix: Typo separeted -> separated --- README.md | 4 ++-- cmd/xsubfind3r/main.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 75a95ec..6d268cb 100644 --- a/README.md +++ b/README.md @@ -176,8 +176,8 @@ TIP: For multiple input domains use comma(,) separated value with `-d`, SOURCES: --sources bool list supported sources - -u, --sources-to-use string[] comma(,) separeted sources to use - -e, --sources-to-exclude string[] comma(,) separeted sources to exclude + -u, --sources-to-use string[] comma(,) separated sources to use + -e, --sources-to-exclude string[] comma(,) separated sources to exclude OUTPUT: --monochrome bool display no color output diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 35fe56b..16ac260 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -69,8 +69,8 @@ func init() { h += "\nSOURCES:\n" h += " --sources bool list supported sources\n" - h += " -u, --sources-to-use string[] comma(,) separeted sources to use\n" - h += " -e, --sources-to-exclude string[] comma(,) separeted sources to exclude\n" + h += " -u, --sources-to-use string[] comma(,) separated sources to use\n" + h += " -e, --sources-to-exclude string[] comma(,) separated sources to exclude\n" h += "\nOUTPUT:\n" h += " --monochrome bool display no color output\n" From 055a0fb5fcc291b6f97310e51001a2b2130c7541 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:56:28 +0300 Subject: [PATCH 40/50] feat: Add BuiltWith as a source --- README.md | 5 +- internal/configuration/configuration.go | 15 +-- pkg/scraper/scraper.go | 3 + pkg/scraper/sources/builtwith/builtwith.go | 114 +++++++++++++++++++++ pkg/scraper/sources/sources.go | 16 +-- 5 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 pkg/scraper/sources/builtwith/builtwith.go diff --git a/README.md b/README.md index 6d268cb..2adff0d 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ## Post Installation -`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. +`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. Example `config.yaml`: @@ -113,6 +113,7 @@ sources: - alienvault - anubis - bevigil + - builtwith - chaos - commoncrawl - crtsh @@ -126,6 +127,8 @@ sources: keys: bevigil: - awA5nvpKU3N8ygkZ + builtwith: + - 7fcbaec4-dc49-472c-b837-3896cb255823 chaos: - d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39asdsd54bbc1aabb208c9acfb fullhunt: diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 4c2002e..9aea3c6 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -87,13 +87,14 @@ func CreateUpdate(path string) (err error) { Version: VERSION, Sources: sources.List, Keys: sources.Keys{ - Bevigil: []string{}, - Chaos: []string{}, - Fullhunt: []string{}, - GitHub: []string{}, - Intelx: []string{}, - Shodan: []string{}, - URLScan: []string{}, + Bevigil: []string{}, + BuiltWith: []string{}, + Chaos: []string{}, + Fullhunt: []string{}, + GitHub: []string{}, + Intelx: []string{}, + Shodan: []string{}, + URLScan: []string{}, }, } diff --git a/pkg/scraper/scraper.go b/pkg/scraper/scraper.go index 341c022..5d5aab0 100644 --- a/pkg/scraper/scraper.go +++ b/pkg/scraper/scraper.go @@ -7,6 +7,7 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/anubis" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/bevigil" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/builtwith" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/chaos" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/commoncrawl" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/crtsh" @@ -85,6 +86,8 @@ func New(options *Options) (scraper *Scraper) { scraper.Sources[source] = &anubis.Source{} case "bevigil": scraper.Sources[source] = &bevigil.Source{} + case "builtwith": + scraper.Sources[source] = &builtwith.Source{} case "chaos": scraper.Sources[source] = &chaos.Source{} case "commoncrawl": diff --git a/pkg/scraper/sources/builtwith/builtwith.go b/pkg/scraper/sources/builtwith/builtwith.go new file mode 100644 index 0000000..c75aa34 --- /dev/null +++ b/pkg/scraper/sources/builtwith/builtwith.go @@ -0,0 +1,114 @@ +package builtwith + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/hueristiq/xsubfind3r/pkg/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" +) + +type getDomainInfoResponse struct { + Results []struct { + Result struct { + Paths []struct { + Domain string `json:"Domain"` + URL string `json:"Url"` + SubDomain string `json:"SubDomain"` + } `json:"Paths"` + } `json:"Result"` + } `json:"Results"` +} + +type Source struct{} + +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) + + go func() { + defer close(results) + + var err error + + var key string + + key, err = sources.PickRandom(config.Keys.BuiltWith) + if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + return + } + + getDomainInfoReqURL := fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", key, domain) + + var getDomainInfoRes *http.Response + + getDomainInfoRes, err = httpclient.SimpleGet(getDomainInfoReqURL) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + getDomainInfoRes.Body.Close() + + return + } + + var getDomainInfoResData getDomainInfoResponse + + if err = json.NewDecoder(getDomainInfoRes.Body).Decode(&getDomainInfoResData); err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + getDomainInfoRes.Body.Close() + + return + } + + getDomainInfoRes.Body.Close() + + for index := range getDomainInfoResData.Results { + item := getDomainInfoResData.Results[index] + + for index := range item.Result.Paths { + path := item.Result.Paths[index] + + value := path.Domain + + if path.SubDomain != "" { + value = path.SubDomain + "." + value + } + + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: value, + } + + results <- result + } + } + }() + + return results +} + +func (source *Source) Name() string { + return "builtwith" +} diff --git a/pkg/scraper/sources/sources.go b/pkg/scraper/sources/sources.go index c1ed2c2..f511b02 100644 --- a/pkg/scraper/sources/sources.go +++ b/pkg/scraper/sources/sources.go @@ -14,13 +14,14 @@ type Configuration struct { } type Keys struct { - Bevigil []string `yaml:"bevigil"` - Chaos []string `yaml:"chaos"` - Fullhunt []string `yaml:"fullhunt"` - GitHub []string `yaml:"github"` - Intelx []string `yaml:"intelx"` - Shodan []string `yaml:"shodan"` - URLScan []string `yaml:"urlscan"` + Bevigil []string `yaml:"bevigil"` + BuiltWith []string `yaml:"builtwith"` + Chaos []string `yaml:"chaos"` + Fullhunt []string `yaml:"fullhunt"` + GitHub []string `yaml:"github"` + Intelx []string `yaml:"intelx"` + Shodan []string `yaml:"shodan"` + URLScan []string `yaml:"urlscan"` } // Result is a result structure returned by a source. @@ -43,6 +44,7 @@ const ( var List = []string{ "anubis", "bevigil", + "builtwith", "chaos", "commoncrawl", "crtsh", From ad7b7546c78891ef24bef5d1271ee614821e8fa5 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Tue, 24 Oct 2023 04:20:00 +0300 Subject: [PATCH 41/50] chore: - --- go.mod | 10 ++++-- go.sum | 23 +++++++++--- pkg/httpclient/client.go | 52 ++++++++++++---------------- pkg/scraper/sources/github/github.go | 13 +++---- pkg/scraper/sources/intelx/intelx.go | 20 +++++++---- 5 files changed, 68 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index 034264f..d4256e6 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/hueristiq/xsubfind3r -go 1.20 +go 1.21 + +toolchain go1.21.0 require ( dario.cat/mergo v1.0.0 - github.com/hueristiq/hqgohttp v0.0.0-20230917162130-697d8e95e15d + github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/logrusorgru/aurora/v3 v3.0.0 github.com/spf13/cast v1.5.1 @@ -14,9 +16,11 @@ require ( ) require ( - github.com/Mzack9999/go-http-digest-auth-client v0.6.0 // indirect + github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index d23332e..f0c8eb2 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,30 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/Mzack9999/go-http-digest-auth-client v0.6.0 h1:LXVNMsj7qiNVmlZByFbjJmXf6SOm/uoo04XmnNcWPms= -github.com/Mzack9999/go-http-digest-auth-client v0.6.0/go.mod h1:gbwaYYXwA15ZfIxMyY5QU1acATDyNKEuG5TylBCL7AM= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/hueristiq/hqgohttp v0.0.0-20230917162130-697d8e95e15d h1:HZY/au1fr6CV/s3iZsTImOXaFgiGW8RW5+lTAUqfODE= -github.com/hueristiq/hqgohttp v0.0.0-20230917162130-697d8e95e15d/go.mod h1:To9Bfohm6oIXZge4tRiWGusF1/Xb7KdTT4YPgsB/1YI= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hueristiq/hqgohttp v0.0.0-20231021223900-c1e2cdd9b08d h1:7MfmMbTb+Zgn0kzhemDb1qir+5KFeaJ1/iMMb3StIgQ= +github.com/hueristiq/hqgohttp v0.0.0-20231021223900-c1e2cdd9b08d/go.mod h1:+ZLulIeXY2OQS296uZcaQcrS2iqvr3SDJqINQv0uDWQ= +github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead h1:Iep2G2h3hSwc7w0qr1iVVAptgXqjn7DRXVQ33luPmhk= +github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead/go.mod h1:Faf/mOhyfNnLIfhoYj2vfPrjt0nKBr4WaU+OQ0C7r6U= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= +github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 h1:dpHAa9c74HgAXkZ2WPd84q2cCiF76eluuSGRw7bk7To= +github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440/go.mod h1:NlZ117o///yWDbRAbgYD7/Y44qce8z1Dj4caUsjunSY= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -27,7 +39,8 @@ golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 11bd022..2c1642a 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -1,53 +1,52 @@ package httpclient import ( - "context" "fmt" "io" "net/http" "net/url" - hqgohttpclient "github.com/hueristiq/hqgohttp" + "github.com/hueristiq/hqgohttp" + "github.com/hueristiq/hqgohttp/methods" + "github.com/hueristiq/hqgohttp/status" + "github.com/hueristiq/xsubfind3r/internal/configuration" ) -var client *hqgohttpclient.Client +var client *hqgohttp.Client func init() { - options := hqgohttpclient.DefaultOptionsSpraying + options := hqgohttp.DefaultOptionsSpraying - client, _ = hqgohttpclient.New(options) + client, _ = hqgohttp.New(options) } -func httpRequestWrapper(request *http.Request) (*http.Response, error) { - r, err := hqgohttpclient.FromRequest(request) +func httpRequestWrapper(req *hqgohttp.Request) (res *http.Response, err error) { + res, err = client.Do(req) if err != nil { - return nil, err + return } - response, err := client.Do(r) - if err != nil { - return nil, err - } + if res.StatusCode != status.OK { + requestURL, _ := url.QueryUnescape(req.URL.String()) - if response.StatusCode != http.StatusOK { - requestURL, _ := url.QueryUnescape(request.URL.String()) + err = fmt.Errorf("unexpected status code %d received from %s", res.StatusCode, requestURL) - return response, fmt.Errorf("unexpected status code %d received from %s", response.StatusCode, requestURL) + return } - return response, nil + return } // HTTPRequest makes any HTTP request to a URL with extended parameters func HTTPRequest(method, requestURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequestWithContext(context.Background(), method, requestURL, body) + req, err := hqgohttp.NewRequest(method, requestURL, body) if err != nil { return nil, err } req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") - req.Header.Set("Connection", "close") + req.Header.Set("User-Agent", fmt.Sprintf("%s v%s (https://github.com/hueristiq/%s)", configuration.NAME, configuration.VERSION, configuration.NAME)) if cookies != "" { req.Header.Set("Cookie", cookies) @@ -61,21 +60,16 @@ func HTTPRequest(method, requestURL, cookies string, headers map[string]string, } // Get makes a GET request to a URL with extended parameters -func Get(getURL, cookies string, headers map[string]string) (*http.Response, error) { - return HTTPRequest(http.MethodGet, getURL, cookies, headers, nil) +func Get(URL, cookies string, headers map[string]string) (*http.Response, error) { + return HTTPRequest(methods.Get, URL, cookies, headers, nil) } // SimpleGet makes a simple GET request to a URL -func SimpleGet(getURL string) (*http.Response, error) { - return HTTPRequest(http.MethodGet, getURL, "", map[string]string{}, nil) +func SimpleGet(URL string) (*http.Response, error) { + return HTTPRequest(methods.Get, URL, "", map[string]string{}, nil) } // Post makes a POST request to a URL with extended parameters -func Post(postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { - return HTTPRequest(http.MethodPost, postURL, cookies, headers, body) -} - -// SimplePost makes a simple POST request to a URL -func SimplePost(postURL, contentType string, body io.Reader) (*http.Response, error) { - return HTTPRequest(http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body) +func Post(URL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { + return HTTPRequest(methods.Post, URL, cookies, headers, body) } diff --git a/pkg/scraper/sources/github/github.go b/pkg/scraper/sources/github/github.go index 74159e9..5bd0193 100644 --- a/pkg/scraper/sources/github/github.go +++ b/pkg/scraper/sources/github/github.go @@ -10,7 +10,8 @@ import ( "strings" "time" - "github.com/hueristiq/hqgohttp" + "github.com/hueristiq/hqgohttp/headers" + "github.com/hueristiq/hqgohttp/status" "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" @@ -86,7 +87,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp searchRes, err = httpclient.Get(searchReqURL, "", searchReqHeaders) - isForbidden := searchRes != nil && searchRes.StatusCode == hqgohttp.StatusForbidden + isForbidden := searchRes != nil && searchRes.StatusCode == status.Forbidden if err != nil && !isForbidden { result := sources.Result{ @@ -102,9 +103,9 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp return } - ratelimitRemaining := cast.ToInt64(searchRes.Header.Get(hqgohttp.HeaderXRatelimitRemaining)) + ratelimitRemaining := cast.ToInt64(searchRes.Header.Get(headers.XRatelimitRemaining)) if isForbidden && ratelimitRemaining == 0 { - retryAfterSeconds := cast.ToInt64(searchRes.Header.Get(hqgohttp.HeaderRetryAfter)) + retryAfterSeconds := cast.ToInt64(searchRes.Header.Get(headers.RetryAfter)) tokens.setCurrentTokenExceeded(retryAfterSeconds) @@ -149,7 +150,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp continue } - if getRawContentRes.StatusCode != hqgohttp.StatusOK { + if getRawContentRes.StatusCode != status.OK { continue } @@ -205,7 +206,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp } } - linksHeader := linkheader.Parse(searchRes.Header.Get(hqgohttp.HeaderLink)) + linksHeader := linkheader.Parse(searchRes.Header.Get(headers.Link)) for _, link := range linksHeader { if link.Rel == "next" { diff --git a/pkg/scraper/sources/intelx/intelx.go b/pkg/scraper/sources/intelx/intelx.go index 2012f4e..d7e6a2e 100644 --- a/pkg/scraper/sources/intelx/intelx.go +++ b/pkg/scraper/sources/intelx/intelx.go @@ -12,17 +12,20 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" ) -type searchRequest struct { +type searchRequestBody struct { Term string `json:"term"` - Timeout time.Duration `json:"timeout"` - Target int `json:"target"` MaxResults int `json:"maxresults"` Media int `json:"media"` + Target int `json:"target"` + Timeout time.Duration `json:"timeout"` } type searchResponse struct { - ID string `json:"id"` - Status int `json:"status"` + ID string `json:"id"` + SelfSelectWarning bool `json:"selfselectwarning"` + Status int `json:"status"` + AltTerm string `json:"altterm"` + AltTermH string `json:"alttermh"` } type getResultsResponse struct { @@ -70,7 +73,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s } searchReqURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", intelXHost, intelXKey) - searchReqBody := searchRequest{ + searchReqHeaders := map[string]string{ + "Content-Type": "application/json", + } + searchReqBody := searchRequestBody{ Term: domain, MaxResults: 100000, Media: 0, @@ -95,7 +101,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s var searchRes *http.Response - searchRes, err = httpclient.SimplePost(searchReqURL, "application/json", bytes.NewBuffer(searchReqBodyBytes)) + searchRes, err = httpclient.Post(searchReqURL, "", searchReqHeaders, bytes.NewBuffer(searchReqBodyBytes)) if err != nil { result := sources.Result{ Type: sources.Error, From 1d9f7dbbfb5b0c069f8bce6805f769d33a2683b2 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sat, 28 Oct 2023 18:45:28 +0300 Subject: [PATCH 42/50] feat: Add LeakIX as a source --- README.md | 5 +- go.sum | 8 +- pkg/scraper/scraper.go | 3 + pkg/scraper/sources/leakix/leakix.go | 107 +++++++++++++++++++++++++++ pkg/scraper/sources/sources.go | 2 + 5 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 pkg/scraper/sources/leakix/leakix.go diff --git a/README.md b/README.md index 2adff0d..b70e7f5 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ## Post Installation -`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. +`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)**, **[LeakIX](https://leakix.net)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. Example `config.yaml`: @@ -121,6 +121,7 @@ sources: - github - hackertarget - intelx + - leakix - shodan - urlscan - wayback @@ -138,6 +139,8 @@ keys: - asdsd54bbc1aabb208c9acfbd2dd41ce7fc9db39 intelx: - 2.intelx.io:00000000-0000-0000-0000-000000000000 + leakix: + - xhDsgKejYTUWVNLn9R6f8afhsG6h6KM69lqEBoMJbfcvDk1v shodan: - AAAAClP1bJJSRMEYJazgwhJKrggRwKA urlscan: diff --git a/go.sum b/go.sum index f0c8eb2..c1aba5f 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,12 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hueristiq/hqgohttp v0.0.0-20231021223900-c1e2cdd9b08d h1:7MfmMbTb+Zgn0kzhemDb1qir+5KFeaJ1/iMMb3StIgQ= -github.com/hueristiq/hqgohttp v0.0.0-20231021223900-c1e2cdd9b08d/go.mod h1:+ZLulIeXY2OQS296uZcaQcrS2iqvr3SDJqINQv0uDWQ= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead h1:Iep2G2h3hSwc7w0qr1iVVAptgXqjn7DRXVQ33luPmhk= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead/go.mod h1:Faf/mOhyfNnLIfhoYj2vfPrjt0nKBr4WaU+OQ0C7r6U= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= @@ -23,12 +23,16 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= diff --git a/pkg/scraper/scraper.go b/pkg/scraper/scraper.go index 5d5aab0..a6938c0 100644 --- a/pkg/scraper/scraper.go +++ b/pkg/scraper/scraper.go @@ -15,6 +15,7 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/github" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/hackertarget" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/intelx" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/leakix" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/otx" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/shodan" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/urlscan" @@ -102,6 +103,8 @@ func New(options *Options) (scraper *Scraper) { scraper.Sources[source] = &hackertarget.Source{} case "intelx": scraper.Sources[source] = &intelx.Source{} + case "leakix": + scraper.Sources[source] = &leakix.Source{} case "otx": scraper.Sources[source] = &otx.Source{} case "shodan": diff --git a/pkg/scraper/sources/leakix/leakix.go b/pkg/scraper/sources/leakix/leakix.go new file mode 100644 index 0000000..aecd246 --- /dev/null +++ b/pkg/scraper/sources/leakix/leakix.go @@ -0,0 +1,107 @@ +package leakix + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/hueristiq/xsubfind3r/pkg/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" +) + +type getSubdomainsResponse struct { + Subdomain string `json:"subdomain"` + DistinctIps int `json:"distinct_ips"` + LastSeen time.Time `json:"last_seen"` +} + +type Source struct{} + +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) + + go func() { + defer close(results) + + var err error + + var key string + + key, err = sources.PickRandom(config.Keys.LeakIX) + if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + return + } + + getSubdomainsReqHeaders := map[string]string{ + "accept": "application/json", + } + + if len(config.Keys.Bevigil) > 0 { + getSubdomainsReqHeaders["api-key"] = key + } + + getSubdomainsReqURL := fmt.Sprintf("https://leakix.net/api/subdomains/%s", domain) + + var getSubdomainsRes *http.Response + + getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + getSubdomainsRes.Body.Close() + + return + } + + var getSubdomainsResData []getSubdomainsResponse + + if err = json.NewDecoder(getSubdomainsRes.Body).Decode(&getSubdomainsResData); err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + getSubdomainsRes.Body.Close() + + return + } + + getSubdomainsRes.Body.Close() + + for index := range getSubdomainsResData { + record := getSubdomainsResData[index] + + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: record.Subdomain, + } + + results <- result + } + }() + + return results +} + +func (source *Source) Name() string { + return "leakix" +} diff --git a/pkg/scraper/sources/sources.go b/pkg/scraper/sources/sources.go index f511b02..403311c 100644 --- a/pkg/scraper/sources/sources.go +++ b/pkg/scraper/sources/sources.go @@ -20,6 +20,7 @@ type Keys struct { Fullhunt []string `yaml:"fullhunt"` GitHub []string `yaml:"github"` Intelx []string `yaml:"intelx"` + LeakIX []string `yaml:"leakix"` Shodan []string `yaml:"shodan"` URLScan []string `yaml:"urlscan"` } @@ -52,6 +53,7 @@ var List = []string{ "github", "hackertarget", "intelx", + "leakix", "otx", "shodan", "urlscan", From 6412bb8e299a7377a9cc8b620b7f3d56a7018fb7 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sat, 28 Oct 2023 18:50:33 +0300 Subject: [PATCH 43/50] chore: - --- pkg/httpclient/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 2c1642a..6f69d51 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -46,6 +46,7 @@ func HTTPRequest(method, requestURL, cookies string, headers map[string]string, req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") + req.Header.Set("Connection", "close") req.Header.Set("User-Agent", fmt.Sprintf("%s v%s (https://github.com/hueristiq/%s)", configuration.NAME, configuration.VERSION, configuration.NAME)) if cookies != "" { From ce6096fa8cd86cf6dd4733f5b41c6a4518d963a6 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sat, 28 Oct 2023 18:54:23 +0300 Subject: [PATCH 44/50] chore: Bump up version to 0.4.0 --- README.md | 4 ++-- internal/configuration/configuration.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b70e7f5..060586a 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Example `config.yaml`: > **NOTE:** The keys/tokens below are invalid and used as examples, use your own keys/tokens! ```yaml -version: 0.3.0 +version: 0.4.0 sources: - alienvault - anubis @@ -163,7 +163,7 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _` | |_ \| '__| > <\__ \ |_| | |_) | _| | | | | (_| |___) | | /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| - v0.3.0 + v0.4.0 with <3 by Hueristiq Open Source diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 9aea3c6..92129ee 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -47,7 +47,7 @@ func (cfg *Configuration) Write(path string) (err error) { const ( NAME string = "xsubfind3r" - VERSION string = "0.3.0" + VERSION string = "0.4.0" ) var ( From d45ef9d7b7af76f716fa3c0226438f703b334172 Mon Sep 17 00:00:00 2001 From: Hue'Q <129622697+hue0x2751@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:10:34 +0300 Subject: [PATCH 45/50] ci: Update linting workflow --- .github/workflows/lint-test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 28669fd..26346b7 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -14,21 +14,21 @@ on: - '**.go' - '**.mod' workflow_dispatch: - + +permissions: + contents: read + jobs: lint: name: Lint Test runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write steps: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '>=1.20' + go-version: '>=1.21' + cache: false - name: Checkout the repository uses: actions/checkout@v4 @@ -38,6 +38,6 @@ jobs: name: Run golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.52.2 + version: v1.54.2 args: --timeout 5m - working-directory: . \ No newline at end of file + working-directory: . From f7e0bf11126e821c83511456c2d5e571ac091262 Mon Sep 17 00:00:00 2001 From: Hue'Q <129622697+hue0x2751@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:12:12 +0300 Subject: [PATCH 46/50] ci: Update build workflow --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b17f594..7db574d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -27,7 +27,7 @@ jobs: name: Set up Go uses: actions/setup-go@v4 with: - go-version: '>=1.20' + go-version: '>=1.21' - name: Checkout the repository uses: actions/checkout@v4 @@ -42,4 +42,4 @@ jobs: - name: Go build run: go build -v . - working-directory: ./cmd/xsubfind3r \ No newline at end of file + working-directory: ./cmd/xsubfind3r From 19b4aaaefe3c2c9ac0897cd76f45ca6a58fdbace Mon Sep 17 00:00:00 2001 From: Hue'Q <129622697+hue0x2751@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:13:42 +0300 Subject: [PATCH 47/50] ci: Update release workflow --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b75aaf6..97b6f87 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: name: Set up Go uses: actions/setup-go@v4 with: - go-version: '>=1.20' + go-version: '>=1.21' - name: Checkout the repository uses: actions/checkout@v4 From e746ac1a482c84a661ef2138a1f9a68136722829 Mon Sep 17 00:00:00 2001 From: Hue'Q <129622697+hue0x2751@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:22:15 +0300 Subject: [PATCH 48/50] ci: Update codeql analysis workflow --- .github/workflows/codeql-analysis.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index 4dda1b1..5e0fab1 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -28,13 +28,11 @@ jobs: fail-fast: false matrix: language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] steps: - name: Checkout repository uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 @@ -45,4 +43,4 @@ jobs: uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + uses: github/codeql-action/analyze@v2 From dbfd40f7c03f640a201efe4453e300ed946059b1 Mon Sep 17 00:00:00 2001 From: Hue'Q <129622697+hue0x2751@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:24:18 +0300 Subject: [PATCH 49/50] build: - --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6eaaad6..452c63d 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,8 @@ ifneq ($(shell go env GOOS),darwin) LDFLAGS := -extldflags "-static" endif +all: build + .PHONY: tidy tidy: $(GOMOD) tidy @@ -44,4 +46,4 @@ build: .PHONY: install install: - $(GOINSTALL) $(GOFLAGS) ./... \ No newline at end of file + $(GOINSTALL) $(GOFLAGS) ./... From c9f60d7e0a7ec507586c9a6164f7b308ccf6335b Mon Sep 17 00:00:00 2001 From: Hue'Q <129622697+hue0x2751@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:30:29 +0300 Subject: [PATCH 50/50] build: - --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index d4256e6..689f346 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/hueristiq/xsubfind3r go 1.21 -toolchain go1.21.0 - require ( dario.cat/mergo v1.0.0 github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead