refactor gobusterdir and show correct number of requests, fixes #211

gobusterdir/gobusterdir.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"path"
 	"strings"
 	"text/tabwriter"
 
@@ -14,6 +13,11 @@ import (
 	"github.com/google/uuid"
 )
 
+var (
+	backupExtensions    = []string{"~", ".bak", ".bak2", ".old", ".1"}
+	backupDotExtensions = []string{".swp"}
+)
+
 // ErrWildcard is returned if a wildcard response is found
 type ErrWildcard struct {
 	url        string
@@ -27,9 +31,10 @@ func (e *ErrWildcard) Error() string {
 
 // GobusterDir is the main type to implement the interface
 type GobusterDir struct {
-	options    *OptionsDir
-	globalopts *libgobuster.Options
-	http       *libgobuster.HTTPClient
+	options        *OptionsDir
+	globalopts     *libgobuster.Options
+	http           *libgobuster.HTTPClient
+	requestsPerRun *int // helper variable so we do not recalculate this over and over
 }
 
 // NewGobusterDir creates a new initialized GobusterDir
@@ -77,6 +82,25 @@ func (d *GobusterDir) Name() string {
 	return "directory enumeration"
 }
 
+// RequestsPerRun returns the number of requests this plugin makes per single wordlist item
+func (d *GobusterDir) RequestsPerRun() int {
+	if d.requestsPerRun != nil {
+		return *d.requestsPerRun
+	}
+
+	num := 1 + len(d.options.ExtensionsParsed.Set)
+	if d.options.DiscoverBackup {
+		// default word
+		num += len(backupExtensions)
+		num += len(backupDotExtensions)
+		// backups of filenames
+		num += len(d.options.ExtensionsParsed.Set) * len(backupExtensions)
+		num += len(d.options.ExtensionsParsed.Set) * len(backupDotExtensions)
+	}
+	d.requestsPerRun = &num
+	return *d.requestsPerRun
+}
+
 // PreRun is the pre run implementation of gobusterdir
 func (d *GobusterDir) PreRun() error {
 	// add trailing slash
@@ -115,136 +139,82 @@ func (d *GobusterDir) PreRun() error {
 	return nil
 }
 
+func getBackupFilenames(word string) []string {
+	var ret []string
+	for _, b := range backupExtensions {
+		ret = append(ret, fmt.Sprintf("%s%s", word, b))
+	}
+	for _, b := range backupDotExtensions {
+		ret = append(ret, fmt.Sprintf(".%s%s", word, b))
+	}
+	return ret
+}
+
 // Run is the process implementation of gobusterdir
 func (d *GobusterDir) Run(word string) ([]libgobuster.Result, error) {
 	suffix := ""
 	if d.options.UseSlash {
 		suffix = "/"
 	}
 
-	// Try the DIR first
-	url := fmt.Sprintf("%s%s%s", d.options.URL, word, suffix)
-	dirResp, dirSize, _, err := d.http.Request(url, libgobuster.RequestOptions{})
-	if err != nil {
-		return nil, err
+	// build list of urls to check
+	//   1: No extension
+	//   2: With extension
+	//   3: backupextension
+	urlsToCheck := make(map[string]string)
+	entity := fmt.Sprintf("%s%s", word, suffix)
+	dirURL := fmt.Sprintf("%s%s", d.options.URL, entity)
+	urlsToCheck[entity] = dirURL
+	if d.options.DiscoverBackup {
+		for _, u := range getBackupFilenames(word) {
+			url := fmt.Sprintf("%s%s", d.options.URL, u)
+			urlsToCheck[u] = url
+		}
 	}
-	var ret []libgobuster.Result
-	if dirResp != nil {
-		resultStatus := libgobuster.StatusMissed
-
-		if d.options.StatusCodesBlacklistParsed.Length() > 0 {
-			if !d.options.StatusCodesBlacklistParsed.Contains(*dirResp) {
-				resultStatus = libgobuster.StatusFound
-			}
-		} else if d.options.StatusCodesParsed.Length() > 0 {
-			if d.options.StatusCodesParsed.Contains(*dirResp) {
-				resultStatus = libgobuster.StatusFound
+	for ext := range d.options.ExtensionsParsed.Set {
+		filename := fmt.Sprintf("%s.%s", word, ext)
+		url := fmt.Sprintf("%s%s", d.options.URL, filename)
+		urlsToCheck[filename] = url
+		if d.options.DiscoverBackup {
+			for _, u := range getBackupFilenames(filename) {
+				url2 := fmt.Sprintf("%s%s", d.options.URL, u)
+				urlsToCheck[u] = url2
 			}
-		} else {
-			return nil, fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen")
-		}
-
-		if (resultStatus == libgobuster.StatusFound && !helper.SliceContains(d.options.ExcludeLength, int(dirSize))) || d.globalopts.Verbose {
-			ret = append(ret, libgobuster.Result{
-				Entity:     fmt.Sprintf("%s%s", word, suffix),
-				StatusCode: *dirResp,
-				Size:       &dirSize,
-				Status:     resultStatus,
-			})
 		}
 	}
 
-	// Follow up with files using each ext.
-	for ext := range d.options.ExtensionsParsed.Set {
-		file := fmt.Sprintf("%s.%s", word, ext)
-		url = fmt.Sprintf("%s%s", d.options.URL, file)
-		fileResp, fileSize, _, err := d.http.Request(url, libgobuster.RequestOptions{})
+	var ret []libgobuster.Result
+	for entity, url := range urlsToCheck {
+		resp, size, _, err := d.http.Request(url, libgobuster.RequestOptions{})
 		if err != nil {
 			return nil, err
 		}
-
-		if fileResp != nil {
+		if resp != nil {
 			resultStatus := libgobuster.StatusMissed
 
 			if d.options.StatusCodesBlacklistParsed.Length() > 0 {
-				if !d.options.StatusCodesBlacklistParsed.Contains(*fileResp) {
+				if !d.options.StatusCodesBlacklistParsed.Contains(*resp) {
 					resultStatus = libgobuster.StatusFound
 				}
 			} else if d.options.StatusCodesParsed.Length() > 0 {
-				if d.options.StatusCodesParsed.Contains(*fileResp) {
+				if d.options.StatusCodesParsed.Contains(*resp) {
 					resultStatus = libgobuster.StatusFound
 				}
 			} else {
 				return nil, fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen")
 			}
 
-			if (resultStatus == libgobuster.StatusFound && !helper.SliceContains(d.options.ExcludeLength, int(fileSize))) || d.globalopts.Verbose {
+			if (resultStatus == libgobuster.StatusFound && !helper.SliceContains(d.options.ExcludeLength, int(size))) || d.globalopts.Verbose {
 				ret = append(ret, libgobuster.Result{
-					Entity:     file,
-					StatusCode: *fileResp,
-					Size:       &fileSize,
+					Entity:     entity,
+					StatusCode: *resp,
+					Size:       &size,
 					Status:     resultStatus,
 				})
 			}
 		}
 	}
 
-	// Discover Backup: Pull all 200's create common backup names.
-	if d.options.DiscoverBackup {
-		for _, r := range ret {
-			fileNames := make([]string, 0)
-			// Common Backup Extensions
-			backupExtensions := strings.Fields("~ .bak .bak2 .old .1")
-			for _, backupExtension := range backupExtensions {
-				// Append Backup Extension to File Name
-				filename := fmt.Sprintf("%s%s", r.Entity, backupExtension)
-				fileNames = append(fileNames, filename)
-				// Strip extension, then append backup extension
-				noExtension := strings.TrimSuffix(r.Entity, path.Ext(r.Entity))
-				if noExtension != r.Entity {
-					filename2 := fmt.Sprintf("%s%s", noExtension, backupExtension)
-					fileNames = append(fileNames, filename2)
-				}
-			}
-
-			// Vim Swap File
-			vimFilename := fmt.Sprintf(".%s.swp", r.Entity)
-			fileNames = append(fileNames, vimFilename)
-
-			for _, file := range fileNames {
-				url = fmt.Sprintf("%s%s", d.options.URL, file)
-				fileResp, fileSize, _, err := d.http.Request(url, libgobuster.RequestOptions{})
-				if err != nil {
-					return nil, err
-				}
-
-				if fileResp != nil {
-					resultStatus := libgobuster.StatusMissed
-
-					if d.options.StatusCodesBlacklistParsed.Length() > 0 {
-						if !d.options.StatusCodesBlacklistParsed.Contains(*fileResp) {
-							resultStatus = libgobuster.StatusFound
-						}
-					} else if d.options.StatusCodesParsed.Length() > 0 {
-						if d.options.StatusCodesParsed.Contains(*fileResp) {
-							resultStatus = libgobuster.StatusFound
-						}
-					} else {
-						return nil, fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen")
-					}
-
-					if (resultStatus == libgobuster.StatusFound && !helper.SliceContains(d.options.ExcludeLength, int(fileSize))) || d.globalopts.Verbose {
-						ret = append(ret, libgobuster.Result{
-							Entity:     file,
-							StatusCode: *fileResp,
-							Size:       &fileSize,
-							Status:     resultStatus,
-						})
-					}
-				}
-			}
-		}
-	}
 	return ret, nil
 }
 

gobusterdns/gobusterdns.go

@@ -76,6 +76,11 @@ func (d *GobusterDNS) Name() string {
 	return "DNS enumeration"
 }
 
+// RequestsPerRun returns the number of requests this plugin makes per single wordlist item
+func (d *GobusterDNS) RequestsPerRun() int {
+	return 1
+}
+
 // PreRun is the pre run implementation of gobusterdns
 func (d *GobusterDNS) PreRun() error {
 	// Resolve a subdomain that probably shouldn't exist

gobusterfuzz/gobusterfuzz.go

@@ -75,6 +75,11 @@ func (d *GobusterFuzz) Name() string {
 	return "fuzzing"
 }
 
+// RequestsPerRun returns the number of requests this plugin makes per single wordlist item
+func (d *GobusterFuzz) RequestsPerRun() int {
+	return 1
+}
+
 // PreRun is the pre run implementation of gobusterfuzz
 func (d *GobusterFuzz) PreRun() error {
 	return nil

gobusters3/gobusters3.go

@@ -64,6 +64,11 @@ func (s *GobusterS3) Name() string {
 	return "S3 bucket enumeration"
 }
 
+// RequestsPerRun returns the number of requests this plugin makes per single wordlist item
+func (s *GobusterS3) RequestsPerRun() int {
+	return 1
+}
+
 // PreRun is the pre run implementation of GobusterS3
 func (s *GobusterS3) PreRun() error {
 	return nil

gobustervhost/gobustervhost.go

@@ -68,6 +68,11 @@ func (v *GobusterVhost) Name() string {
 	return "VHOST enumeration"
 }
 
+// RequestsPerRun returns the number of requests this plugin makes per single wordlist item
+func (v *GobusterVhost) RequestsPerRun() int {
+	return 1
+}
+
 // PreRun is the pre run implementation of gobusterdir
 func (v *GobusterVhost) PreRun() error {
 	// add trailing slash

libgobuster/interfaces.go

@@ -3,6 +3,7 @@ package libgobuster
 // GobusterPlugin is an interface which plugins must implement
 type GobusterPlugin interface {
 	Name() string
+	RequestsPerRun() int
 	PreRun() error
 	Run(string) ([]Result, error)
 	ResultToString(*Result) (*string, error)

libgobuster/libgobuster.go

@@ -64,7 +64,7 @@ func (g *Gobuster) Errors() <-chan error {
 
 func (g *Gobuster) incrementRequests() {
 	g.RequestsCountMutex.Lock()
-	g.RequestsIssued++
+	g.RequestsIssued += g.plugin.RequestsPerRun()
 	g.RequestsCountMutex.Unlock()
 }
 
@@ -131,6 +131,8 @@ func (g *Gobuster) getWordlist() (*bufio.Scanner, error) {
 		g.RequestsExpected += (lines * len(g.Opts.Patterns))
 	}
 
+	g.RequestsExpected = g.RequestsExpected * g.plugin.RequestsPerRun()
+
 	// rewind wordlist
 	_, err = wordlist.Seek(0, 0)
 	if err != nil {