pax_global_header00006660000000000000000000000064133204212460014507gustar00rootroot0000000000000052 comment=bf3c39034624b45eb3e9729acaf95e443b608b50 hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/000077500000000000000000000000001332042124600202245ustar00rootroot00000000000000hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/.gitignore000066400000000000000000000000001332042124600222020ustar00rootroot00000000000000hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/.travis.yml000066400000000000000000000001371332042124600223360ustar00rootroot00000000000000language: go go_import_path: pathspider.net/hellfire/ go: - 1.x - 1.6 - 1.7.x - master hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/README.md000066400000000000000000000056571332042124600215200ustar00rootroot00000000000000Hellfire ======== _PAT**H**spider **E**ffects **Li**st **Re**solver_ [![Travis](https://img.shields.io/travis/irl/hellfire.svg)](https://travis-ci.org/irl/hellfire) [![GoDoc](https://godoc.org/github.com/irl/hellfire?status.svg)](https://godoc.org/github.com/irl/hellfire) Hellfire is a parallelised DNS resolver. It is written in Go and for the purpose of generating input lists to [PATHspider](https://pathspider.net/), though may be useful for other applications. Install ------- Debian 10 or above: ```$ apt install hellfire``` Go: ```$ go get pathspider.net/hellfire/cmd/hellfire``` Input and Output Formats ------------------------ The following input formats are supported: * [Alexa Topsites](http://www.alexa.com/topsites) * [Cisco Umbrella Popularity List](http://s3-us-west-1.amazonaws.com/umbrella-static/index.html) * [Citizen Lab Test Lists](https://github.com/citizenlab/test-lists/tree/master/lists) (by country) * [OpenDNS Public Domain Lists](https://github.com/opendns/public-domain-lists) * Plain text (not implemented yet) * JSON (not implemented yet) Extra metadata can be declared to Hellfire that will be present in the jobs when output. The output format is [NDJSON](http://specs.okfnlabs.org/ndjson/) (not yet implemented) using the native input schema for PATHspider. Services -------- As well as looking up the A and AAAA records for the domain directly, it is also possible to select indirect lookups for the addresses of the mail exchanger (not yet implemented) and the name server (not yet implemented). Copyright --------- Copyright (c) 2016 Iain R. Learmonth All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/alexa.go000066400000000000000000000021411332042124600216430ustar00rootroot00000000000000package hellfire // import "pathspider.net/hellfire" import ( "archive/zip" "log" ) type AlexaTopsitesList struct { TestList filename string } // URL to download the latest Alexa Topsites list from const AlexaTopsitesURL string = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip" func (l *AlexaTopsitesList) SetFilename(filename string) { l.filename = filename } func (l *AlexaTopsitesList) FeedJobs(jobs chan map[string]interface{}) { var topsites *CSVList if l.filename == "" { urlReader, err := getReaderFromUrl(AlexaTopsitesURL) if err != nil { log.Fatalf("Unable to get <%s>: %s", AlexaTopsitesURL, err) } zr, err := zip.NewReader(urlReader, int64(urlReader.Len())) if err != nil { log.Fatalf("Unable to read zip: %s", err) } for _, zf := range zr.File { if zf.Name == "top-1m.csv" { f, _ := zf.Open() topsites = CSVListFromReader(f) break } } if topsites == nil { panic("Did not find top-1m.csv in the zip archive") } } else { topsites = CSVListFromFile(l.filename) } topsites.SetHeader([]string{"rank", "domain"}) topsites.FeedJobs(jobs) } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/canid.go000066400000000000000000000007571332042124600216420ustar00rootroot00000000000000package hellfire // import "pathspider.net/hellfire" import ( "encoding/json" "fmt" "io/ioutil" "net" "net/http" ) func GetAdditionalInfo(ip net.IP, canidAddress string) map[string]interface{} { url := fmt.Sprintf("http://%s/prefix.json?addr=%s", canidAddress, ip.String()) res, err := http.Get(url) if err != nil { panic(err) } defer res.Body.Close() body, err := ioutil.ReadAll(res.Body) var info map[string]interface{} json.Unmarshal([]byte(body), &info) return info } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/cisco.go000066400000000000000000000021561332042124600216570ustar00rootroot00000000000000package hellfire // import "pathspider.net/hellfire" import ( "archive/zip" "log" ) type CiscoUmbrellaList struct { TestList filename string } // URL to download the latest Cisco Umbrella list from const CiscoUmbrellaURL string = "http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" func (l *CiscoUmbrellaList) SetFilename(filename string) { l.filename = filename } func (l *CiscoUmbrellaList) FeedJobs(jobs chan map[string]interface{}) { var topsites *CSVList if l.filename == "" { urlReader, err := getReaderFromUrl(CiscoUmbrellaURL) if err != nil { log.Fatalf("Unable to get <%s>: %s", CiscoUmbrellaURL, err) } zr, err := zip.NewReader(urlReader, int64(urlReader.Len())) if err != nil { log.Fatalf("Unable to read zip: %s", err) } for _, zf := range zr.File { if zf.Name == "top-1m.csv" { f, _ := zf.Open() topsites = CSVListFromReader(f) break } } if topsites == nil { panic("Did not find top-1m.csv in the zip archive") } } else { topsites = CSVListFromFile(l.filename) } topsites.SetHeader([]string{"rank", "domain"}) topsites.FeedJobs(jobs) } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/citizenlab.go000066400000000000000000000037371332042124600227110ustar00rootroot00000000000000package hellfire // import "pathspider.net/hellfire" import ( "fmt" "log" "strings" ) type CitizenLabCountryList struct { TestList country string filename string } // URL format string to download the latest Citizen Lab test list from. The %s // will be replaced with either the two letter country code, or with "global" // as specified in the call to SetCountry before the job feeder is activated. const CitizenLabCountryListURL string = "https://raw.githubusercontent.com/citizenlab/test-lists/master/lists/%s.csv" func (l *CitizenLabCountryList) SetFilename(filename string) { l.filename = filename } // The SetCountry method allows selection of the Citizen Lab test list to use. // The full list of countries available can be found at // https://github.com/citizenlab/test-lists/tree/master/lists. This method will // also accept "global" as a country name, selecting the global test list. // // This function must be called before FeedJobs() or the application will panic. func (l *CitizenLabCountryList) SetCountry(country string) { country = strings.ToLower(country) if country == "global" || len(country) == 2 { l.country = country } else { panic("Country code must be two characters, or 'global'.") } } func (l *CitizenLabCountryList) FeedJobs(jobs chan map[string]interface{}) { var citizenLabList *CSVList if l.filename == "" { if l.country == "" { panic("The country to use for the Citizen Lab test was not specified") } listUrl := fmt.Sprintf(CitizenLabCountryListURL, l.country) urlReader, err := getReaderFromUrl(listUrl) if err != nil { log.Fatalf("Unable to get <%s>: %s", listUrl, err) } citizenLabList = CSVListFromReader(urlReader) } else { // Note that country codes starting with X are "private use" // and XF in this case is to indicate that a file was used. // BUG(irl): Maybe a hint could be provided on the command line // later. l.SetCountry("xf") citizenLabList = CSVListFromFile(l.filename) } citizenLabList.FeedJobs(jobs) } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/cmd/000077500000000000000000000000001332042124600207675ustar00rootroot00000000000000hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/cmd/hellfire/000077500000000000000000000000001332042124600225615ustar00rootroot00000000000000hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/cmd/hellfire/main.go000066400000000000000000000123711332042124600240400ustar00rootroot00000000000000// Hellfire is a parallelised DNS resolver. It builds effects lists for input // to PATHspider measurements. For sources where the filename is optional, the // latest source will be downloaded from the Internet when the filename is // omitted. // // BASIC USAGE // // Usage: // hellfire --topsites [--file=] [--output=] [--type=] [--canid=] [--rate=] // hellfire --cisco [--file=] [--output=] [--type=] [--canid=] [--rate=] // hellfire --citizenlab [--country=|--file=] [--output=] [--type=] [--canid=] [--rate=] // hellfire --opendns [--list=|--file=] [--output=] [--type=] [--canid=] [--rate=] // hellfire --csv --file= [--output=] [--type=] [--canid=] [--rate=] // hellfire --txt --file= [--output=] [--type=] [--canid=] [--rate=] // // Options: // -h --help Show this screen. // --version Show version. // // OUTPUT TYPES // // * "individual" - One record output per IP address looked up, discarding no // addresses. // * "array" - One record output per domain name, with an array of all // addresses resolved. // * "oneeach" - One record output per IP address, only printing one IPv4 and // one IPv6 at most for each domain. // // SEE ALSO // // The PATHspider website can be found at https://pathspider.net/. package main import ( "fmt" "os" "strings" "strconv" docopt "github.com/docopt/docopt-go" "pathspider.net/hellfire" ) func main() { usage := `Hellfire: PATHspider Effects List Resolver Hellfire is a parallelised DNS resolver. It builds effects lists for input to PATHspider measurements. For sources where the filename is optional, the latest source will be downloaded from the Internet when the filename is omitted. Usage: hellfire --topsites [--file=] [--output=] [--type=] [--canid=] [--rate=] hellfire --cisco [--file=] [--output=] [--type=] [--canid=] [--rate=] hellfire --citizenlab [--country=|--file=] [--output=] [--type=] [--canid=] [--rate=] hellfire --opendns [--list=|--file=] [--output=] [--type=] [--canid=] [--rate=] hellfire --csv --file= [--output=] [--type=] [--canid=] [--rate=] hellfire --txt --file= [--output=] [--type=] [--canid=] [--rate=] Options: -h --help Show this screen. --version Show version.` arguments, _ := docopt.Parse(usage, nil, true, "Hellfire dev", false) var listName string var listVariant string var listFilename string if arguments["--topsites"].(bool) { listName = "topsites" } else if arguments["--cisco"].(bool) { listName = "cisco" } else if arguments["--citizenlab"].(bool) { listName = "citizenlab" if arguments["--country"] != nil { listVariant = arguments["--country"].(string) } else { listVariant = "global" } } else if arguments["--opendns"].(bool) { listName = "opendns" if arguments["--list"] != nil { listVariant = arguments["--list"].(string) } else { listVariant = "top" } } else if arguments["--csv"].(bool) { listName = "csv" } else if arguments["--txt"].(bool) { listName = "txt" } if arguments["--file"] != nil { listFilename = arguments["--file"].(string) } var queriesPerSecond = 10 if arguments["--rate"] != nil { var err error queriesPerSecond, err = strconv.Atoi(arguments["--rate"].(string)) if err != nil { fmt.Println(err) os.Exit(2) } } var lookupType string supportedLookupTypes := []string{"host", "mx", "ns"} if arguments["--type"] != nil { for _, supportedType := range supportedLookupTypes { if arguments["--type"].(string) == supportedType { lookupType = arguments["--type"].(string) } } if lookupType == "" { panic("Unsupported lookup type requested.") //BUG(irl): Should list the supported types. } } else { lookupType = "host" } var outputType string supportedOutputTypes := []string{"individual", "array", "oneeach"} if arguments["--output"] != nil { for _, supportedType := range supportedOutputTypes { if arguments["--output"].(string) == supportedType { outputType = arguments["--output"].(string) } } if outputType == "" { panic("Unsupported lookup type requested.") //BUG(irl): Should list the supported types. } } else { outputType = "individual" } var canidAddress string if arguments["--canid"] == nil { canidAddress = "" } else { canidAddress = arguments["--canid"].(string) } testListOptions := strings.Join([]string{listName, listVariant, listFilename}, ";") hellfire.PerformLookups(testListOptions, lookupType, outputType, canidAddress, queriesPerSecond) } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/common.go000066400000000000000000000025261332042124600220500ustar00rootroot00000000000000// Package inputs provides means of importing reconnaissance missions into // hellfire. A number of input formats can be interpreted, including missions // formatted using CSV and JSON schemas. // // Any method of importing missions must implement the TestList interface. package hellfire // import "pathspider.net/hellfire" import ( "bytes" "io" "net/http" ) // The TestList interface describes the methods that are used by hellfire // to import missions. type TestList interface { // The FeedJobs method should submit jobs, one at a time, into the // chan that has been passed to it. The map must contain one of the // keys "domain" or "url" that is either a fully-qualified domain // name or a URL with a fully-qualified domain name in the host // portion. // // If there is no value set for the "domain" key, the host portion of // the URL will be used for the lookup. If there is both a value for // "domain" and "url", the "url" value will be ignored and the "domain" // value used directly. FeedJobs(chan map[string]interface{}) SetFilename(string) } func getReaderFromUrl(url string) (*bytes.Reader, error) { res, err := http.Get(url) if err != nil { return nil, err } defer res.Body.Close() buf := &bytes.Buffer{} _, err = io.Copy(buf, res.Body) if err != nil { return nil, err } return bytes.NewReader(buf.Bytes()), nil } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/csv.go000066400000000000000000000025461332042124600213550ustar00rootroot00000000000000package hellfire // import "pathspider.net/hellfire" import ( "bufio" "encoding/csv" "io" "net/url" "os" ) // A CSVList handles input in CSV format. There may be a more specific type // available and that should be used if that is the case (e.g. for the // Alexa or Citizen Lab test lists). type CSVList struct { TestList reader io.Reader header []string } func CSVListFromFile(filename string) *CSVList { f, err := os.Open(filename) if err != nil { panic("Error opening file") } return CSVListFromReader(f) } func CSVListFromReader(reader io.Reader) *CSVList { l := new(CSVList) l.reader = reader return l } func (l *CSVList) SetHeader(header []string) { l.header = header } func (l *CSVList) FeedJobs(jobs chan map[string]interface{}) { reader := csv.NewReader(bufio.NewReader(l.reader)) if reader == nil { panic("CSVList not initialised with a reader") } var header []string if l.header == nil { var err error header, err = reader.Read() if err != nil { panic("Error reading the header from the CSV") } } else { header = l.header } for { record, err := reader.Read() if err == io.EOF { break } r := make(map[string]interface{}) for idx, name := range header { r[name] = record[idx] } if r["domain"] == nil && r["url"] != nil { u, _ := url.Parse(r["url"].(string)) r["domain"] = u.Host } jobs <- r } } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/hellfire.go000066400000000000000000000134511332042124600223510ustar00rootroot00000000000000package hellfire // import "pathspider.net/hellfire" import ( "encoding/json" "fmt" "net" "strings" "sync" "time" ) type LookupQueryResult struct { attempts int result []net.IP } func prepareTestList(testListOptions string) TestList { var testList TestList options := strings.Split(testListOptions, ";") if len(options) < 3 { return nil } if options[0] == "topsites" { testList = new(AlexaTopsitesList) } else if options[0] == "cisco" { testList = new(CiscoUmbrellaList) } else if options[0] == "citizenlab" { testList = new(CitizenLabCountryList) if options[1] != "" { testList.(*CitizenLabCountryList).SetCountry(options[1]) } } else if options[0] == "opendns" { testList = new(OpenDNSList) if options[1] != "" { testList.(*OpenDNSList).SetListName(options[1]) } } if options[2] != "" { if options[0] == "csv" { testList = CSVListFromFile(options[2]) } else if options[0] == "txt" { testList = CSVListFromFile(options[2]) testList.(*CSVList).SetHeader([]string{"domain"}) } else if testList != nil { testList.SetFilename(options[2]) } } if testList == nil { panic("Could not initialise a test list!") } return testList } func makeQuery(domain string, lookupType string) LookupQueryResult { result := []net.IP{} domains := []string{} lookupAttempt := 1 //BUG(irl): Need to add support for MX lookups //BUG(irl): Need to add support for SRV lookups if lookupType == "host" { domains = append(domains, domain) } else if lookupType == "ns" { var nss []*net.NS for { nss, _ = net.LookupNS(domain) if len(nss) == 0 { time.Sleep(1) } else { break } lookupAttempt++ if lookupAttempt == 4 { lookupAttempt = 3 break } } for _, ns := range nss { domains = append(domains, ns.Host) } } else if lookupType == "mx" { var nss []*net.MX for { nss, _ = net.LookupMX(domain) if len(nss) == 0 { time.Sleep(1) } else { break } lookupAttempt++ if lookupAttempt == 4 { lookupAttempt = 3 break } } for _, ns := range nss { domains = append(domains, ns.Host) } } for _, d := range domains { var ips []net.IP for { ips, _ = net.LookupIP(d) if len(ips) == 0 { time.Sleep(1) } else { break } lookupAttempt++ if lookupAttempt == 4 { lookupAttempt = 3 break } } result = append(result, ips...) } return LookupQueryResult{lookupAttempt, result} } func lookupWorker(id int, lookupWaitGroup *sync.WaitGroup, jobs chan map[string]interface{}, results chan map[string]interface{}, lookupType string, canidAddress string, rateLimiter <-chan time.Time) { lookupWaitGroup.Add(1) go func(id int, lookupWaitGroup *sync.WaitGroup, jobs chan map[string]interface{}, results chan map[string]interface{}, lookupType string, canidAddress string) { defer lookupWaitGroup.Done() for job := range jobs { if job["domain"] == nil { jobs <- make(map[string]interface{}) break } // wait for a tick <-rateLimiter lookupResult := makeQuery(job["domain"].(string), lookupType) job["hellfire_lookup_attempts"] = lookupResult.attempts job["hellfire_lookup_type"] = lookupType for _, ip := range lookupResult.result { thisResult := make(map[string]interface{}) for key, value := range job { thisResult[key] = value } thisResult["ips"] = []net.IP{ip} if canidAddress != "" { thisResult["canid_info"] = GetAdditionalInfo(ip, canidAddress) } results <- thisResult } } }(id, lookupWaitGroup, jobs, results, lookupType, canidAddress) } func outputPrinter(outputWaitGroup *sync.WaitGroup, results chan map[string]interface{}, outputType string) { outputWaitGroup.Add(1) go func(results chan map[string]interface{}) { defer outputWaitGroup.Done() for { result := <-results if result["domain"] == nil { break } if outputType == "all" { b, _ := json.Marshal(result) fmt.Println(string(b)) } else if outputType == "individual" { ips := result["ips"] if ips == nil { continue } delete(result, "ips") for _, ipo := range ips.([]net.IP) { ip := ipo.String() result["dip"] = ip b, _ := json.Marshal(result) fmt.Println(string(b)) delete(result, "dip") } } else if outputType == "oneeach" { found4 := false found6 := false ips := result["ips"].([]net.IP) delete(result, "ips") for _, ipo := range ips { ip := ipo.String() if strings.Contains(ip, ".") { if found4 { continue } else { found4 = true } } else { if found6 { continue } else { found6 = true } } result["dip"] = ip b, _ := json.Marshal(result) fmt.Println(string(b)) delete(result, "dip") } } } }(results) } func PerformLookups(testListOptions string, lookupType string, outputType string, canidAddress string, queriesPerSecond int) { var lookupWaitGroup sync.WaitGroup var outputWaitGroup sync.WaitGroup jobs := make(chan map[string]interface{}, 1) results := make(chan map[string]interface{}) testList := prepareTestList(testListOptions) // Create a rate limiting ticker rateLimiter := time.Tick(time.Second / time.Duration(queriesPerSecond)) // Spawn lookup workers for i := 0; i < 300; i++ { lookupWorker(i, &lookupWaitGroup, jobs, results, lookupType, canidAddress, rateLimiter) } // Spawn output printer outputPrinter(&outputWaitGroup, results, outputType) // Submit jobs testList.FeedJobs(jobs) jobs <- make(map[string]interface{}) lookupWaitGroup.Wait() <-jobs // Read last shutdown sentinel from the queue left by the // final worker to exit // https://blog.golang.org/pipelines - This is a better way close(jobs) // Shutdown the output printer results <- make(map[string]interface{}) outputWaitGroup.Wait() close(results) } hellfire-bf3c39034624b45eb3e9729acaf95e443b608b50/opendns.go000066400000000000000000000030531332042124600222220ustar00rootroot00000000000000package hellfire // import "pathspider.net/hellfire" import ( "fmt" "log" "strings" ) type OpenDNSList struct { TestList listname string filename string } // URL format string to download the OpenDNS public domain list from. The %s // will be replaced with the name of the list as specified in the call to // SetListName before the job feeder is activated. const OpenDNSListURL string = "https://raw.githubusercontent.com/opendns/public-domain-lists/master/opendns-%s-domains.txt" func (l *OpenDNSList) SetFilename(filename string) { l.filename = filename } // The SetListName method allows selection of the OpenDNS list to use. This // function accepts either "top" or "random" as the list name. // // This function must be called before FeedJobs() or the application will panic. func (l *OpenDNSList) SetListName(listname string) { listname = strings.ToLower(listname) if listname == "top" || listname == "random" { l.listname = listname } else { panic("List name must be either \"top\" or \"random\".") } } func (l *OpenDNSList) FeedJobs(jobs chan map[string]interface{}) { var openDNSList *CSVList if l.filename == "" { if l.listname == "" { panic("The list name to use was not specified.") } listUrl := fmt.Sprintf(OpenDNSListURL, l.listname) urlReader, err := getReaderFromUrl(listUrl) if err != nil { log.Fatalf("Unable to get <%s>: %s", listUrl, err) } openDNSList = CSVListFromReader(urlReader) } else { openDNSList = CSVListFromFile(l.filename) } openDNSList.SetHeader([]string{"domain"}) openDNSList.FeedJobs(jobs) }