Add support for filtering results with a unique size for each status code

gobusterdir/gobusterdir.go

@@ -117,7 +117,7 @@ func (d GobusterDir) ResultToString(g *libgobuster.Gobuster, r *libgobuster.Resu
 			}
 		}
 
-		if r.Size != nil {
+		if g.Opts.IncludeLength && r.Size != nil {
 			if _, err := fmt.Fprintf(buf, " [Size: %d]", *r.Size); err != nil {
 				return nil, err
 			}

libgobuster/http.go

@@ -60,7 +60,7 @@ func newHTTPClient(c context.Context, opt *Options) (*httpClient, error) {
 	client.context = c
 	client.username = opt.Username
 	client.password = opt.Password
-	client.includeLength = opt.IncludeLength
+	client.includeLength = opt.IncludeLength || opt.UniqueResponseLength
 	client.userAgent = opt.UserAgent
 	return &client, nil
 }

libgobuster/libgobuster.go

@@ -25,11 +25,18 @@ type ProcessFunc func(*Gobuster, string) ([]Result, error)
 // ResultToStringFunc is the "to string" function prototype for implementations
 type ResultToStringFunc func(*Gobuster, *Result) (*string, error)
 
+// ResponseSizeSet is a set of unique response sizes
+type ResponseSizeSet map[int64]bool
+
+// StatusCodeToResponseSizeSet maps each status code to a set of unique response sizes
+type StatusCodeToResponseSizeSet map[int]ResponseSizeSet
+
 // Gobuster is the main object when creating a new run
 type Gobuster struct {
 	Opts             *Options
 	http             *httpClient
 	WildcardIps      stringSet
+	statusCodeToResponseSizeSet StatusCodeToResponseSizeSet
 	context          context.Context
 	requestsExpected int
 	requestsIssued   int
@@ -57,6 +64,7 @@ func NewGobuster(c context.Context, opts *Options, plugin GobusterPlugin) (*Gobu
 
 	var g Gobuster
 	g.WildcardIps = newStringSet()
+	g.statusCodeToResponseSizeSet = make(StatusCodeToResponseSizeSet)
 	g.context = c
 	g.Opts = opts
 	h, err := newHTTPClient(c, opts)
@@ -145,13 +153,41 @@ func (g *Gobuster) worker(wordChan <-chan string, wg *sync.WaitGroup) {
 				continue
 			} else {
 				for _, r := range res {
-					g.resultChan <- r
+					g.addResult(&r)
 				}
 			}
 		}
 	}
 }
 
+func (g *Gobuster) addResult(r *Result) {
+	if !g.Opts.UniqueResponseLength {
+		g.resultChan <- *r
+		return
+	}
+
+	responseSizeSet, ok := g.statusCodeToResponseSizeSet[r.Status]
+
+	// Create a bucket to collect all unique response sizes for
+	// the given status code
+	if !ok {
+		g.statusCodeToResponseSizeSet[r.Status] = make(ResponseSizeSet)
+		g.statusCodeToResponseSizeSet[r.Status][*r.Size] = true
+		g.resultChan <- *r
+		return
+	}
+
+	// Skip this result if we already have one with a known response size
+	if responseSizeSet[*r.Size] {
+		return
+	}
+
+	// Mark this result size as one we know and return the response since
+	// we count it as unique
+	responseSizeSet[*r.Size] = true
+	g.resultChan <- *r
+}
+
 func (g *Gobuster) getWordlist() (*bufio.Scanner, error) {
 	if g.Opts.Wordlist == "-" {
 		// Read directly from stdin

libgobuster/options.go

@@ -36,6 +36,7 @@ type Options struct {
 	Timeout           time.Duration
 	FollowRedirect    bool
 	IncludeLength     bool
+	UniqueResponseLength     bool
 	NoStatus          bool
 	NoProgress        bool
 	Expanded          bool

main.go

@@ -126,6 +126,7 @@ func main() {
 	flag.BoolVar(&o.Expanded, "e", false, "Expanded mode, print full URLs")
 	flag.BoolVar(&o.NoStatus, "n", false, "Don't print status codes")
 	flag.BoolVar(&o.IncludeLength, "l", false, "Include the length of the body in the output (dir mode only)")
+	flag.BoolVar(&o.UniqueResponseLength, "lu", false, "Only output results with a unique size for each status code")
 	flag.BoolVar(&o.UseSlash, "f", false, "Append a forward-slash to each directory request (dir mode only)")
 	flag.BoolVar(&o.WildcardForced, "fw", false, "Force continued operation when wildcard found")
 	flag.BoolVar(&o.InsecureSSL, "k", false, "Skip SSL certificate verification")