Merge pull request #8 from hueristiq/dev

Development v0.3.0

README.md

@@ -20,6 +20,17 @@
 ## Features
 
 * Fetches domains from curated passive sources to maximize results.
+    <details>
+    <summary>Sources: Click to expand!</summary>
+
+    | Technique | Source |
+    | :-------- | :----- |
+    | APIs | AnubisDB, BeVigil, Chaos, FullHunt, GitHub, HackerTarget, IntelX, Shodan, URLScan |
+    | Certificates | Crtsh |
+    | Web Archives | CommonCrawl, Wayback |
+    | WHOIS | AlienVault |
+
+    </details>
 * Supports `stdin` and `stdout` for easy integration into workflows.
 * Cross-Platform (Windows, Linux & macOS).
 
@@ -101,12 +112,12 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest
 
 ## Post Installation
 
-`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)** and **[Intelligence X](https://intelx.io)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.hueristiq/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used.
+`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.hueristiq/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used.
 
 Example `config.yaml`:
 
 ```yaml
-version: 0.2.0
+version: 0.3.0
 sources:
     - alienvault
     - anubis
@@ -118,6 +129,7 @@ sources:
     - github
     - hackertarget
     - intelx
+    - shodan
     - urlscan
     - wayback
 keys:
@@ -132,6 +144,8 @@ keys:
         - asdsd54bbc1aabb208c9acfbd2dd41ce7fc9db39
     intelx:
         - 2.intelx.io:00000000-0000-0000-0000-000000000000
+    shodan:
+        - AAAAClP1bJJSRMEYJazgwhJKrggRwKA
     urlscan:
         - d4c85d34-e425-446e-d4ab-f5a3412acbe8
 ```
@@ -152,7 +166,7 @@ help message:
 __  _____ _   _| |__  / _(_)_ __   __| |___ / _ __ 
 \ \/ / __| | | | '_ \| |_| | '_ \ / _` | |_ \| '__|
  >  <\__ \ |_| | |_) |  _| | | | | (_| |___) | |   
-/_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| v0.2.0
+/_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| v0.3.0
 
 USAGE:
   xsubfind3r [OPTIONS]

cmd/xsubfind3r/main.go

@@ -25,41 +25,33 @@ var (
 
 	domainsSlice        []string
 	domainsListFilePath string
-
-	listSources      bool
-	sourcesToUse     []string
-	sourcesToExclude []string
-
-	threads int
-
-	monochrome      bool
-	output          string
-	outputDirectory string
-	verbosity       string
-
-	YAMLConfigFile string
+	listSources         bool
+	sourcesToUse        []string
+	sourcesToExclude    []string
+	threads             int
+	monochrome          bool
+	output              string
+	outputDirectory     string
+	verbosity           string
+	YAMLConfigFile      string
 )
 
 func init() {
 	// defaults
 	defaultThreads := 50
-	defaultYAMLConfigFile := "~/.hueristiq/xsubfind3r/config.yaml"
+	defaultYAMLConfigFile := fmt.Sprintf("~/.hueristiq/%s/config.yaml", configuration.NAME)
 
 	// Handle CLI arguments, flags & help message (pflag)
 	pflag.StringSliceVarP(&domainsSlice, "domain", "d", []string{}, "")
 	pflag.StringVarP(&domainsListFilePath, "list", "l", "", "")
-
 	pflag.BoolVar(&listSources, "sources", false, "")
 	pflag.StringSliceVarP(&sourcesToUse, "use-sources", "u", []string{}, "")
 	pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "")
-
 	pflag.IntVarP(&threads, "threads", "t", defaultThreads, "")
-
 	pflag.BoolVar(&monochrome, "no-color", false, "")
 	pflag.StringVarP(&output, "output", "o", "", "")
 	pflag.StringVarP(&outputDirectory, "outputDirectory", "O", "", "")
 	pflag.StringVarP(&verbosity, "verbosity", "v", string(levels.LevelInfo), "")
-
 	pflag.StringVarP(&YAMLConfigFile, "configuration", "c", defaultYAMLConfigFile, "")
 
 	pflag.CommandLine.SortFlags = false
@@ -71,7 +63,7 @@ func init() {
 
 		h += "\nINPUT:\n"
 		h += " -d, --domain string[]                 target domains\n"
-		h += " -l, --list string                     target domains' list file path\n"
+		h += " -l, --list string                     target domains list file path\n"
 
 		h += "\nSOURCES:\n"
 		h += "      --sources bool                   list supported sources\n"
@@ -83,8 +75,8 @@ func init() {
 
 		h += "\nOUTPUT:\n"
 		h += "     --no-color bool                   disable colored output\n"
-		h += " -o, --output string                   output subdomains' file path\n"
-		h += " -O, --output-directory string         output subdomains' directory path\n"
+		h += " -o, --output string                   output subdomains file path\n"
+		h += " -O, --output-directory string         output subdomains directory path\n"
 		h += fmt.Sprintf(" -v, --verbosity string                debug, info, warning, error, fatal or silent (default: %s)\n", string(levels.LevelInfo))
 
 		h += "\nCONFIGURATION:\n"

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f
 	github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47
 	github.com/logrusorgru/aurora/v3 v3.0.0
+	github.com/spf13/cast v1.5.1
 	github.com/spf13/pflag v1.0.5
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
 	github.com/valyala/fasthttp v1.48.0

go.sum

@@ -2,14 +2,21 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg=
 github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47 h1:LSeeeVmzUmykvyAS/liWZ+yQuMjM/1462m9s+WWV9YI=
 github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47/go.mod h1:8NAT2ECb69qzGf2d/ty0PVE3M3HK/+fXLtri2c47wQE=
 github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
 github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4=
 github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
+github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=

internal/configuration/configuration.go

@@ -48,7 +48,7 @@ func (configuration *Configuration) Write(path string) (err error) {
 
 const (
 	NAME    string = "xsubfind3r"
-	VERSION string = "0.2.0"
+	VERSION string = "0.3.0"
 )
 
 var (
@@ -78,6 +78,7 @@ func CreateUpdate(path string) (err error) {
 			Fullhunt: []string{},
 			GitHub:   []string{},
 			Intelx:   []string{},
+			Shodan:   []string{},
 			URLScan:  []string{},
 		},
 	}

pkg/xsubfind3r/extractor/extractor.go

@@ -11,7 +11,9 @@ func New(domain string) (extractor *regexp.Regexp, err error) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
-	extractor, err = regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain)
+	pattern := `(?i)[a-zA-Z0-9\*_.-]+\.` + domain
+
+	extractor, err = regexp.Compile(pattern)
 	if err != nil {
 		return
 	}

pkg/xsubfind3r/sources/alienvault/alienvault.go

@@ -9,7 +9,7 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-type response struct {
+type getPassiveDNSResponse struct {
 	Detail     string `json:"detail"`
 	Error      string `json:"error"`
 	PassiveDNS []struct {
@@ -19,36 +19,35 @@ type response struct {
 
 type Source struct{}
 
-func (source *Source) Run(_ *sources.Configuration, domain string) (subdomains chan sources.Subdomain) {
-	subdomains = make(chan sources.Subdomain)
+func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) {
+	subdomainsChannel = make(chan sources.Subdomain)
 
 	go func() {
-		defer close(subdomains)
+		defer close(subdomainsChannel)
 
-		var (
-			err error
-			res *fasthttp.Response
-		)
+		var err error
 
-		reqURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)
+		getPassiveDNSReqURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)
 
-		res, err = httpclient.SimpleGet(reqURL)
+		var getPassiveDNSRes *fasthttp.Response
+
+		getPassiveDNSRes, err = httpclient.SimpleGet(getPassiveDNSReqURL)
 		if err != nil {
 			return
 		}
 
-		var results response
+		var getPassiveDNSResData getPassiveDNSResponse
 
-		if err = json.Unmarshal(res.Body(), &results); err != nil {
+		if err = json.Unmarshal(getPassiveDNSRes.Body(), &getPassiveDNSResData); err != nil {
 			return
 		}
 
-		if results.Error != "" {
+		if getPassiveDNSResData.Error != "" {
 			return
 		}
 
-		for _, record := range results.PassiveDNS {
-			subdomains <- sources.Subdomain{Source: source.Name(), Value: record.Hostname}
+		for _, record := range getPassiveDNSResData.PassiveDNS {
+			subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: record.Hostname}
 		}
 	}()
 

pkg/xsubfind3r/sources/anubis/anubis.go

@@ -11,32 +11,31 @@ import (
 
 type Source struct{}
 
-func (source *Source) Run(_ *sources.Configuration, domain string) (subdomains chan sources.Subdomain) {
-	subdomains = make(chan sources.Subdomain)
+func (source *Source) Run(_ *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) {
+	subdomainsChannel = make(chan sources.Subdomain)
 
 	go func() {
-		defer close(subdomains)
+		defer close(subdomainsChannel)
 
-		var (
-			err error
-			res *fasthttp.Response
-		)
+		var err error
 
-		reqURL := fmt.Sprintf("https://jldc.me/anubis/subdomains/%s", domain)
+		getSubdomainsReqURL := fmt.Sprintf("https://jldc.me/anubis/subdomains/%s", domain)
 
-		res, err = httpclient.SimpleGet(reqURL)
+		var getSubdomainsRes *fasthttp.Response
+
+		getSubdomainsRes, err = httpclient.SimpleGet(getSubdomainsReqURL)
 		if err != nil {
 			return
 		}
 
-		var results []string
+		var getSubdomainsResData []string
 
-		if err = json.Unmarshal(res.Body(), &results); err != nil {
+		if err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData); err != nil {
 			return
 		}
 
-		for _, subdomain := range results {
-			subdomains <- sources.Subdomain{Source: source.Name(), Value: subdomain}
+		for _, subdomain := range getSubdomainsResData {
+			subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain}
 		}
 	}()
 

pkg/xsubfind3r/sources/bevigil/bevigil.go

@@ -9,52 +9,51 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-type response struct {
+type getSubdomainsResponse struct {
 	Domain     string   `json:"domain"`
 	Subdomains []string `json:"subdomains"`
 }
 
 type Source struct{}
 
-func (source *Source) Run(config *sources.Configuration, domain string) (subdomains chan sources.Subdomain) {
-	subdomains = make(chan sources.Subdomain)
+func (source *Source) Run(config *sources.Configuration, domain string) (subdomainsChannel chan sources.Subdomain) {
+	subdomainsChannel = make(chan sources.Subdomain)
 
 	go func() {
-		defer close(subdomains)
+		defer close(subdomainsChannel)
 
-		var (
-			key     string
-			err     error
-			res     *fasthttp.Response
-			headers = map[string]string{}
-		)
+		var err error
+
+		var key string
 
 		key, err = sources.PickRandom(config.Keys.Bevigil)
 		if key == "" || err != nil {
 			return
 		}
 
+		getSubdomainsReqHeaders := map[string]string{}
+
 		if len(config.Keys.Bevigil) > 0 {
-			headers["X-Access-Token"] = key
+			getSubdomainsReqHeaders["X-Access-Token"] = key
 		}
 
-		reqURL := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain)
+		getSubdomainsReqURL := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain)
 
-		res, err = httpclient.Request(fasthttp.MethodGet, reqURL, "", headers, nil)
+		var getSubdomainsRes *fasthttp.Response
+
+		getSubdomainsRes, err = httpclient.Get(getSubdomainsReqURL, "", getSubdomainsReqHeaders)
 		if err != nil {
 			return
 		}
 
-		body := res.Body()
-
-		var results response
+		var getSubdomainsResData getSubdomainsResponse
 
-		if err = json.Unmarshal(body, &results); err != nil {
+		if err = json.Unmarshal(getSubdomainsRes.Body(), &getSubdomainsResData); err != nil {
 			return
 		}
 
-		for _, subdomain := range results.Subdomains {
-			subdomains <- sources.Subdomain{Source: source.Name(), Value: subdomain}
+		for _, subdomain := range getSubdomainsResData.Subdomains {
+			subdomainsChannel <- sources.Subdomain{Source: source.Name(), Value: subdomain}
 		}
 	}()