feat(minipipeline): add ControlFinalResponseExpectations

internal/cmd/minipipeline/testdata/observations.json

@@ -381,5 +381,8 @@
       },
       "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/cmd/minipipeline/testdata/observations_classic.json

@@ -140,5 +140,8 @@
       },
       "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/classic.go

@@ -52,5 +52,8 @@ func ClassicFilter(input *WebObservationsContainer) (output *WebObservationsCont
 		output.KnownTCPEndpoints[txid] = entry
 	}
 
+	// ControlFinalResponseExpectations
+	output.ControlFinalResponseExpectations = input.ControlFinalResponseExpectations
+
 	return
 }

internal/minipipeline/observation.go

@@ -253,6 +253,14 @@ type WebObservation struct {
 	ControlHTTPResponseTitle optional.Value[string]
 }
 
+// WebObservationsControlFinalResponseExpectation summarizes the expectations
+// on the final response based on what the control has observed.
+type WebObservationsControlFinalResponseExpectation struct {
+	// Failure is the failure observed by the control when attempting
+	// to fetch the final webpage associated with a URL.
+	Failure optional.Value[string]
+}
+
 // WebObservationsContainer contains [*WebObservations].
 //
 // The zero value of this struct is not ready to use, please use [NewWebObservationsContainer].
@@ -273,6 +281,10 @@ type WebObservationsContainer struct {
 	// KnownTCPEndpoints maps transaction IDs to TCP observations.
 	KnownTCPEndpoints map[int64]*WebObservation
 
+	// ControlFinalResponseExpectations summarizes the expectations we have
+	// for the control based on the final response.
+	ControlFinalResponseExpectations optional.Value[*WebObservationsControlFinalResponseExpectation]
+
 	// knownIPAddresses is an internal field that maps an IP address to the
 	// corresponding DNS observation that discovered it.
 	knownIPAddresses map[string]*WebObservation
@@ -584,7 +596,6 @@ func (c *WebObservationsContainer) controlXrefTLSFailures(resp *model.THResponse
 }
 
 func (c *WebObservationsContainer) controlSetHTTPFinalResponseExpectation(resp *model.THResponse) {
-
 	// We need to set expectations for each type of observation. For example, to detect
 	// NXDOMAIN blocking with redirects when there's the expectation of success, we need
 	// to have the expectation inside the DNS-lookup-failure observation.
@@ -595,6 +606,13 @@ func (c *WebObservationsContainer) controlSetHTTPFinalResponseExpectation(resp *
 		observations = append(observations, obs)
 	}
 
+	// make sure we have a final expectation based on what the control observed, which
+	// is in turn necessary to figure out whether unexplained probe failures during redirects
+	// are expected or unexpected.
+	c.ControlFinalResponseExpectations = optional.Some(&WebObservationsControlFinalResponseExpectation{
+		Failure: optional.Some(utilsStringPointerToString(resp.HTTPRequest.Failure)),
+	})
+
 	for _, obs := range observations {
 		obs.ControlHTTPFailure = optional.Some(utilsStringPointerToString(resp.HTTPRequest.Failure))
 

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json

@@ -186,5 +186,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json

@@ -95,5 +95,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json

@@ -276,5 +276,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations_classic.json

@@ -95,5 +95,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json

@@ -260,5 +260,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations.json

@@ -186,5 +186,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations_classic.json

@@ -95,5 +95,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations.json

@@ -183,5 +183,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations_classic.json

@@ -96,5 +96,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/observations.json

@@ -227,5 +227,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/observations_classic.json

@@ -96,5 +96,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/observations.json

@@ -209,5 +209,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/observations_classic.json

@@ -51,5 +51,8 @@
     }
   ],
   "DNSLookupSuccesses": [],
-  "KnownTCPEndpoints": {}
+  "KnownTCPEndpoints": {},
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
+  }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json

@@ -262,5 +262,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/observations.json

@@ -209,5 +209,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/observations_classic.json

@@ -51,5 +51,8 @@
     }
   ],
   "DNSLookupSuccesses": [],
-  "KnownTCPEndpoints": {}
+  "KnownTCPEndpoints": {},
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
+  }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/observations.json

@@ -260,5 +260,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/observations_classic.json

@@ -110,5 +110,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/observations.json

@@ -359,5 +359,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/observations_classic.json

@@ -109,5 +109,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/observations.json

@@ -257,5 +257,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/observations.json

@@ -257,5 +257,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations.json

@@ -355,5 +355,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/observations.json

@@ -453,5 +453,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/observations_classic.json

@@ -207,5 +207,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/observations.json

@@ -404,5 +404,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/observations_classic.json

@@ -207,5 +207,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }