pax_global_header00006660000000000000000000000064136455620630014524gustar00rootroot0000000000000052 comment=4e95d8701aae8cff1c27af2626eb22ba110ad583 assetfinder-0.1.1/000077500000000000000000000000001364556206300140325ustar00rootroot00000000000000assetfinder-0.1.1/.gitignore000066400000000000000000000000441364556206300160200ustar00rootroot00000000000000assetfinder *.sw* *.tgz *.zip *.exe assetfinder-0.1.1/LICENSE000066400000000000000000000020531364556206300150370ustar00rootroot00000000000000MIT License Copyright (c) 2019 Tom Hudson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. assetfinder-0.1.1/README.md000066400000000000000000000026521364556206300153160ustar00rootroot00000000000000# assetfinder Find domains and subdomains potentially related to a given domain. ## Install If you have Go installed and configured (i.e. with `$GOPATH/bin` in your `$PATH`): ``` go get -u github.com/tomnomnom/assetfinder ``` Otherwise [download a release for your platform](https://github.com/tomnomnom/assetfinder/releases). To make it easier to execute you can put the binary in your `$PATH`. ## Usage ``` assetfinder [--subs-only] ``` ## Sources Please feel free to issue pull requests with new sources! :) ### Implemented * crt.sh * certspotter * hackertarget * threatcrowd * wayback machine * dns.bufferover.run * facebook * Needs `FB_APP_ID` and `FB_APP_SECRET` environment variables set (https://developers.facebook.com/) * You need to be careful with your app's rate limits * virustotal * Needs `VT_API_KEY` environment variable set (https://developers.virustotal.com/reference) * findsubdomains * Needs `SPYSE_API_TOKEN` environment variable set (the free version always gives the first response page, and you also get "25 unlimited requests") — (https://spyse.com/apidocs) ### Sources to be implemented * http://api.passivetotal.org/api/docs/ * https://community.riskiq.com/ (?) * https://riddler.io/ * http://www.dnsdb.org/ * https://certdb.com/api-documentation ## TODO * Flags to control which sources are used * Likely to be all on by default and a flag to disable * Read domains from stdin assetfinder-0.1.1/bufferoverrun.go000066400000000000000000000007741364556206300172630ustar00rootroot00000000000000package main import ( "fmt" "strings" ) func fetchBufferOverrun(domain string) ([]string, error) { out := make([]string, 0) fetchURL := fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain) wrapper := struct { Records []string `json:"FDNS_A"` }{} err := fetchJSON(fetchURL, &wrapper) if err != nil { return out, err } for _, r := range wrapper.Records { parts := strings.SplitN(r, ",", 2) if len(parts) != 2 { continue } out = append(out, parts[1]) } return out, nil } assetfinder-0.1.1/certspotter.go000066400000000000000000000006601364556206300167410ustar00rootroot00000000000000package main import ( "fmt" ) func fetchCertSpotter(domain string) ([]string, error) { out := make([]string, 0) fetchURL := fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain) wrapper := []struct { DNSNames []string `json:"dns_names"` }{} err := fetchJSON(fetchURL, &wrapper) if err != nil { return out, err } for _, w := range wrapper { out = append(out, w.DNSNames...) } return out, nil } assetfinder-0.1.1/crtsh.go000066400000000000000000000011601364556206300155020ustar00rootroot00000000000000package main import ( "encoding/json" "fmt" "io/ioutil" "net/http" ) type CrtShResult struct { Name string `json:"name_value"` } func fetchCrtSh(domain string) ([]string, error) { var results []CrtShResult resp, err := http.Get( fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain), ) if err != nil { return []string{}, err } defer resp.Body.Close() output := make([]string, 0) body, _ := ioutil.ReadAll(resp.Body) if err := json.Unmarshal(body, &results); err != nil { return []string{}, err } for _, res := range results { output = append(output, res.Name) } return output, nil } assetfinder-0.1.1/facebook.go000066400000000000000000000035451364556206300161410ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "net/http" "os" ) func fetchFacebook(domain string) ([]string, error) { appId := os.Getenv("FB_APP_ID") appSecret := os.Getenv("FB_APP_SECRET") if appId == "" || appSecret == "" { // fail silently because it's reasonable not to have // the Facebook API creds return []string{}, nil } accessToken, err := facebookAuth(appId, appSecret) if err != nil { return []string{}, err } domains, err := getFacebookCerts(accessToken, domain) if err != nil { return []string{}, err } return domains, nil } func getFacebookCerts(accessToken, query string) ([]string, error) { out := make([]string, 0) fetchURL := fmt.Sprintf( "https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=*.%s", accessToken, query, ) for { wrapper := struct { Data []struct { Domains []string `json:"domains"` } `json:"data"` Paging struct { Next string `json:"next"` } `json:"paging"` }{} err := fetchJSON(fetchURL, &wrapper) if err != nil { return out, err } for _, data := range wrapper.Data { for _, d := range data.Domains { out = append(out, d) } } fetchURL = wrapper.Paging.Next if fetchURL == "" { break } } return out, nil } func facebookAuth(appId, appSecret string) (string, error) { authUrl := fmt.Sprintf( "https://graph.facebook.com/oauth/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials", appId, appSecret, ) resp, err := http.Get(authUrl) if err != nil { return "", err } defer resp.Body.Close() dec := json.NewDecoder(resp.Body) auth := struct { AccessToken string `json:"access_token"` }{} err = dec.Decode(&auth) if err != nil { return "", err } if auth.AccessToken == "" { return "", errors.New("no access token in Facebook API response") } return auth.AccessToken, nil } assetfinder-0.1.1/findsubdomains.go000066400000000000000000000046271364556206300173770ustar00rootroot00000000000000package main import ( "fmt" "os" ) var apiToken = os.Getenv("SPYSE_API_TOKEN") func callSubdomainsAggregateEndpoint(domain string) []string { out := make([]string, 0) fetchURL := fmt.Sprintf( "https://api.spyse.com/v1/subdomains-aggregate?api_token=%s&domain=%s", apiToken, domain, ) type Cidr struct { Results []struct { Data struct { Domains []string `json:"domains"` } `json:"data"` } `json:"results"` } type Cidrs struct { Cidr16, Cidr24 Cidr } wrapper := struct { Cidrs Cidrs `json:"cidr"` }{} err := fetchJSON(fetchURL, &wrapper) if err != nil { // Fail silently return []string{} } for _, result := range wrapper.Cidrs.Cidr16.Results { for _, domain := range result.Data.Domains { out = append(out, domain) } } for _, result := range wrapper.Cidrs.Cidr24.Results { for _, domain := range result.Data.Domains { out = append(out, domain) } } return out } /** */ func callSubdomainsEndpoint(domain string) []string { out := make([]string, 0) // Start querying the Spyse API from page 1 page := 1 for { wrapper := struct { Records []struct { Domain string `json:"domain"` } `json:"records"` }{} fetchURL := fmt.Sprintf( "https://api.spyse.com/v1/subdomains?api_token=%s&domain=%s&page=%d", apiToken, domain, page, ) err := fetchJSON(fetchURL, &wrapper) if err != nil { // Fail silently, by returning what we got so far return out } // The API does not respond with any paging, nor does it give us any idea of // the total amount of domains, so we just have to keep asking for a new page until // the returned `records` array is empty // NOTE: The free tier always gives you the first page for free, and you get "25 unlimited search requests" if len(wrapper.Records) == 0 { break } for _, record := range wrapper.Records { out = append(out, record.Domain) } page++ } return out } func fetchFindSubDomains(domain string) ([]string, error) { out := make([]string, 0) apiToken := os.Getenv("SPYSE_API_TOKEN") if apiToken == "" { // Must have an API token return []string{}, nil } // The Subdomains-Aggregate endpoint returns some, but not all available domains out = append(out, callSubdomainsAggregateEndpoint(domain)...) // The Subdomains endpoint only guarantees the first 30 domains, the rest needs credit at Spyze out = append(out, callSubdomainsEndpoint(domain)...) return out, nil } assetfinder-0.1.1/hackertarget.go000066400000000000000000000007511364556206300170300ustar00rootroot00000000000000package main import ( "bufio" "bytes" "fmt" "strings" ) func fetchHackerTarget(domain string) ([]string, error) { out := make([]string, 0) raw, err := httpGet( fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain), ) if err != nil { return out, err } sc := bufio.NewScanner(bytes.NewReader(raw)) for sc.Scan() { parts := strings.SplitN(sc.Text(), ",", 2) if len(parts) != 2 { continue } out = append(out, parts[0]) } return out, sc.Err() } assetfinder-0.1.1/main.go000066400000000000000000000044331364556206300153110ustar00rootroot00000000000000package main import ( "bufio" "encoding/json" "flag" "fmt" "io" "io/ioutil" "net/http" "os" "strings" "sync" "time" ) func main() { var subsOnly bool flag.BoolVar(&subsOnly, "subs-only", false, "Only include subdomains of search domain") flag.Parse() var domains io.Reader domains = os.Stdin domain := flag.Arg(0) if domain != "" { domains = strings.NewReader(domain) } sources := []fetchFn{ fetchCertSpotter, fetchHackerTarget, fetchThreatCrowd, fetchCrtSh, fetchFacebook, //fetchWayback, // A little too slow :( fetchVirusTotal, fetchFindSubDomains, fetchUrlscan, fetchBufferOverrun, } out := make(chan string) var wg sync.WaitGroup sc := bufio.NewScanner(domains) rl := newRateLimiter(time.Second) for sc.Scan() { domain := strings.ToLower(sc.Text()) // call each of the source workers in a goroutine for _, source := range sources { wg.Add(1) fn := source go func() { defer wg.Done() rl.Block(fmt.Sprintf("%#v", fn)) names, err := fn(domain) if err != nil { //fmt.Fprintf(os.Stderr, "err: %s\n", err) return } for _, n := range names { n = cleanDomain(n) if subsOnly && !strings.HasSuffix(n, domain) { continue } out <- n } }() } } // close the output channel when all the workers are done go func() { wg.Wait() close(out) }() // track what we've already printed to avoid duplicates printed := make(map[string]bool) for n := range out { if _, ok := printed[n]; ok { continue } printed[n] = true fmt.Println(n) } } type fetchFn func(string) ([]string, error) func httpGet(url string) ([]byte, error) { res, err := http.Get(url) if err != nil { return []byte{}, err } raw, err := ioutil.ReadAll(res.Body) res.Body.Close() if err != nil { return []byte{}, err } return raw, nil } func cleanDomain(d string) string { d = strings.ToLower(d) // no idea what this is, but we can't clean it ¯\_(ツ)_/¯ if len(d) < 2 { return d } if d[0] == '*' || d[0] == '%' { d = d[1:] } if d[0] == '.' { d = d[1:] } return d } func fetchJSON(url string, wrapper interface{}) error { resp, err := http.Get(url) if err != nil { return err } defer resp.Body.Close() dec := json.NewDecoder(resp.Body) return dec.Decode(wrapper) } assetfinder-0.1.1/ratelimit.go000066400000000000000000000021221364556206300163500ustar00rootroot00000000000000package main import ( "sync" "time" ) // a rateLimiter allows you to delay operations // on a per-key basis. I.e. only one operation for // a given key can be done within the delay time type rateLimiter struct { sync.Mutex delay time.Duration ops map[string]time.Time } // newRateLimiter returns a new *rateLimiter for the // provided delay func newRateLimiter(delay time.Duration) *rateLimiter { return &rateLimiter{ delay: delay, ops: make(map[string]time.Time), } } // Block blocks until an operation for key is // allowed to proceed func (r *rateLimiter) Block(key string) { now := time.Now() r.Lock() // if there's nothing in the map we can // return straight away if _, ok := r.ops[key]; !ok { r.ops[key] = now r.Unlock() return } // if time is up we can return straight away t := r.ops[key] deadline := t.Add(r.delay) if now.After(deadline) { r.ops[key] = now r.Unlock() return } remaining := deadline.Sub(now) // Set the time of the operation r.ops[key] = now.Add(remaining) r.Unlock() // Block for the remaining time <-time.After(remaining) } assetfinder-0.1.1/script/000077500000000000000000000000001364556206300153365ustar00rootroot00000000000000assetfinder-0.1.1/script/release000077500000000000000000000030671364556206300167120ustar00rootroot00000000000000#!/bin/bash PROJDIR=$(cd `dirname $0`/.. && pwd) VERSION="${1}" TAG="v${VERSION}" USER="tomnomnom" REPO="assetfinder" BINARY="${REPO}" if [[ -z "${VERSION}" ]]; then echo "Usage: ${0} " exit 1 fi if [[ -z "${GITHUB_TOKEN}" ]]; then echo "You forgot to set your GITHUB_TOKEN" exit 2 fi cd ${PROJDIR} # Run the tests go test if [ $? -ne 0 ]; then echo "Tests failed. Aborting." exit 3 fi # Check if tag exists git fetch --tags git tag | grep "^${TAG}$" if [ $? -ne 0 ]; then github-release release \ --user ${USER} \ --repo ${REPO} \ --tag ${TAG} \ --name "${REPO} ${TAG}" \ --description "${TAG}" \ --pre-release fi for ARCH in "amd64" "386"; do for OS in "darwin" "linux" "windows" "freebsd"; do BINFILE="${BINARY}" if [[ "${OS}" == "windows" ]]; then BINFILE="${BINFILE}.exe" fi rm -f ${BINFILE} GOOS=${OS} GOARCH=${ARCH} go build -ldflags "-X main.gronVersion=${VERSION}" github.com/${USER}/${REPO} if [[ "${OS}" == "windows" ]]; then ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.zip" zip ${ARCHIVE} ${BINFILE} else ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.tgz" tar --create --gzip --file=${ARCHIVE} ${BINFILE} fi echo "Uploading ${ARCHIVE}..." github-release upload \ --user ${USER} \ --repo ${REPO} \ --tag ${TAG} \ --name "${ARCHIVE}" \ --file ${PROJDIR}/${ARCHIVE} done done assetfinder-0.1.1/threatcrowd.go000066400000000000000000000006531364556206300167130ustar00rootroot00000000000000package main import ( "fmt" ) func fetchThreatCrowd(domain string) ([]string, error) { out := make([]string, 0) fetchURL := fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain) wrapper := struct { Subdomains []string `json:"subdomains"` }{} err := fetchJSON(fetchURL, &wrapper) if err != nil { return out, err } out = append(out, wrapper.Subdomains...) return out, nil } assetfinder-0.1.1/urlscan.go000066400000000000000000000016741364556206300160400ustar00rootroot00000000000000package main import ( "encoding/json" "fmt" "net/http" "net/url" ) func fetchUrlscan(domain string) ([]string, error) { resp, err := http.Get( fmt.Sprintf("https://urlscan.io/api/v1/search/?q=domain:%s", domain), ) if err != nil { return []string{}, err } defer resp.Body.Close() output := make([]string, 0) dec := json.NewDecoder(resp.Body) wrapper := struct { Results []struct { Task struct { URL string `json:"url"` } `json:"task"` Page struct { URL string `json:"url"` } `json:"page"` } `json:"results"` }{} err = dec.Decode(&wrapper) if err != nil { return []string{}, err } for _, r := range wrapper.Results { u, err := url.Parse(r.Task.URL) if err != nil { continue } output = append(output, u.Hostname()) } for _, r := range wrapper.Results { u, err := url.Parse(r.Page.URL) if err != nil { continue } output = append(output, u.Hostname()) } return output, nil } assetfinder-0.1.1/virustotal.go000066400000000000000000000007651364556206300166050ustar00rootroot00000000000000package main import ( "fmt" "os" ) func fetchVirusTotal(domain string) ([]string, error) { apiKey := os.Getenv("VT_API_KEY") if apiKey == "" { // swallow not having an API key, just // don't fetch return []string{}, nil } fetchURL := fmt.Sprintf( "https://www.virustotal.com/vtapi/v2/domain/report?domain=%s&apikey=%s", domain, apiKey, ) wrapper := struct { Subdomains []string `json:"subdomains"` }{} err := fetchJSON(fetchURL, &wrapper) return wrapper.Subdomains, err } assetfinder-0.1.1/wayback.go000066400000000000000000000012671364556206300160100ustar00rootroot00000000000000package main import ( "fmt" "net/url" ) func fetchWayback(domain string) ([]string, error) { fetchURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey", domain) var wrapper [][]string err := fetchJSON(fetchURL, &wrapper) if err != nil { return []string{}, err } out := make([]string, 0) skip := true for _, item := range wrapper { // The first item is always just the string "original", // so we should skip the first item if skip { skip = false continue } if len(item) < 3 { continue } u, err := url.Parse(item[2]) if err != nil { continue } out = append(out, u.Hostname()) } return out, nil }