feat(minipipeline): add ControlFinalResponseExpectations (ooni#1451)

This PR adds the ControlFinalResponseExpectations field to
WebObservationsContainer. We need this field to determine whether
unexplained failures observed by the probe (i.e., failures occurring
during redirects) are to be expected (because for some reason also the
TH failed) or unexpected (because the TH succeeded).

Part of ooni/probe#2640.

internal/cmd/minipipeline/testdata/observations.json

@@ -381,5 +381,8 @@
       },
       "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/cmd/minipipeline/testdata/observations_classic.json

@@ -140,5 +140,8 @@
       },
       "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/classic.go

@@ -52,5 +52,8 @@ func ClassicFilter(input *WebObservationsContainer) (output *WebObservationsCont
 		output.KnownTCPEndpoints[txid] = entry
 	}
 
+	// ControlFinalResponseExpectations
+	output.ControlFinalResponseExpectations = input.ControlFinalResponseExpectations
+
 	return
 }

internal/minipipeline/observation.go

@@ -253,6 +253,14 @@ type WebObservation struct {
 	ControlHTTPResponseTitle optional.Value[string]
 }
 
+// WebObservationsControlFinalResponseExpectation summarizes the expectations
+// on the final response based on what the control has observed.
+type WebObservationsControlFinalResponseExpectation struct {
+	// Failure is the failure observed by the control when attempting
+	// to fetch the final webpage associated with a URL.
+	Failure optional.Value[string]
+}
+
 // WebObservationsContainer contains [*WebObservations].
 //
 // The zero value of this struct is not ready to use, please use [NewWebObservationsContainer].
@@ -273,6 +281,10 @@ type WebObservationsContainer struct {
 	// KnownTCPEndpoints maps transaction IDs to TCP observations.
 	KnownTCPEndpoints map[int64]*WebObservation
 
+	// ControlFinalResponseExpectations summarizes the expectations we have
+	// for the control based on the final response.
+	ControlFinalResponseExpectations optional.Value[*WebObservationsControlFinalResponseExpectation]
+
 	// knownIPAddresses is an internal field that maps an IP address to the
 	// corresponding DNS observation that discovered it.
 	knownIPAddresses map[string]*WebObservation
@@ -584,7 +596,6 @@ func (c *WebObservationsContainer) controlXrefTLSFailures(resp *model.THResponse
 }
 
 func (c *WebObservationsContainer) controlSetHTTPFinalResponseExpectation(resp *model.THResponse) {
-
 	// We need to set expectations for each type of observation. For example, to detect
 	// NXDOMAIN blocking with redirects when there's the expectation of success, we need
 	// to have the expectation inside the DNS-lookup-failure observation.
@@ -595,6 +606,13 @@ func (c *WebObservationsContainer) controlSetHTTPFinalResponseExpectation(resp *
 		observations = append(observations, obs)
 	}
 
+	// make sure we have a final expectation based on what the control observed, which
+	// is in turn necessary to figure out whether unexplained probe failures during redirects
+	// are expected or unexpected.
+	c.ControlFinalResponseExpectations = optional.Some(&WebObservationsControlFinalResponseExpectation{
+		Failure: optional.Some(utilsStringPointerToString(resp.HTTPRequest.Failure)),
+	})
+
 	for _, obs := range observations {
 		obs.ControlHTTPFailure = optional.Some(utilsStringPointerToString(resp.HTTPRequest.Failure))
 

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json

@@ -186,5 +186,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json

@@ -95,5 +95,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json

@@ -276,5 +276,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations_classic.json

@@ -95,5 +95,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json

@@ -260,5 +260,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations.json

@@ -186,5 +186,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations_classic.json

@@ -95,5 +95,8 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": "unknown_error"
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations.json

@@ -183,5 +183,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations_classic.json

@@ -96,5 +96,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/observations.json

@@ -227,5 +227,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/observations_classic.json

@@ -96,5 +96,6 @@
       "ControlHTTPResponseHeadersKeys": null,
       "ControlHTTPResponseTitle": null
     }
-  }
+  },
+  "ControlFinalResponseExpectations": null
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/observations.json

@@ -209,5 +209,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/observations_classic.json

@@ -51,5 +51,8 @@
     }
   ],
   "DNSLookupSuccesses": [],
-  "KnownTCPEndpoints": {}
+  "KnownTCPEndpoints": {},
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
+  }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json

@@ -262,5 +262,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/observations.json

@@ -209,5 +209,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/observations_classic.json

@@ -51,5 +51,8 @@
     }
   ],
   "DNSLookupSuccesses": [],
-  "KnownTCPEndpoints": {}
+  "KnownTCPEndpoints": {},
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
+  }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/observations.json

@@ -260,5 +260,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/observations_classic.json

@@ -110,5 +110,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/observations.json

@@ -359,5 +359,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/observations_classic.json

@@ -109,5 +109,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/observations.json

@@ -257,5 +257,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/observations.json

@@ -257,5 +257,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations.json

@@ -355,5 +355,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations_classic.json

@@ -105,5 +105,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/observations.json

@@ -453,5 +453,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/observations_classic.json

@@ -207,5 +207,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/observations.json

@@ -404,5 +404,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }

internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/observations_classic.json

@@ -207,5 +207,8 @@
       },
       "ControlHTTPResponseTitle": "Default Web Page"
     }
+  },
+  "ControlFinalResponseExpectations": {
+    "Failure": ""
   }
 }