From c369bb3ebb509756312e3be0f1d834e6ff1fc4e1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 15:28:34 +0200 Subject: [PATCH 1/8] refactor(httpclientx): use better naming Prodded by @ainghazal to avoid confusion when discussing. --- internal/enginelocate/cloudflare.go | 2 +- internal/enginelocate/ubuntu.go | 2 +- internal/httpclientx/DESIGN.md | 32 ++++++------ internal/httpclientx/baseurl.go | 49 +++++++++++++++++++ .../{endpoint_test.go => baseurl_test.go} | 24 ++++----- internal/httpclientx/endpoint.go | 49 ------------------- internal/httpclientx/getjson.go | 6 +-- internal/httpclientx/getjson_test.go | 16 +++--- internal/httpclientx/getraw.go | 8 +-- internal/httpclientx/getraw_test.go | 8 +-- internal/httpclientx/getxml.go | 6 +-- internal/httpclientx/getxml_test.go | 10 ++-- internal/httpclientx/httpclientx.go | 4 +- internal/httpclientx/httpclientx_test.go | 14 +++--- internal/httpclientx/overlapped.go | 38 +++++++------- internal/httpclientx/overlapped_test.go | 38 +++++++------- internal/httpclientx/postjson.go | 10 ++-- internal/httpclientx/postjson_test.go | 26 +++++----- internal/oonirun/v2.go | 2 +- internal/probeservices/bouncer.go | 2 +- internal/probeservices/checkin.go | 2 +- internal/probeservices/collector.go | 4 +- internal/probeservices/login.go | 2 +- internal/probeservices/measurementmeta.go | 2 +- internal/probeservices/openvpn.go | 2 +- internal/probeservices/psiphon.go | 2 +- internal/probeservices/register.go | 2 +- internal/probeservices/tor.go | 2 +- .../webconnectivityalgo/calltesthelpers.go | 2 +- 29 files changed, 183 insertions(+), 183 deletions(-) create mode 100644 internal/httpclientx/baseurl.go rename internal/httpclientx/{endpoint_test.go => baseurl_test.go} (63%) delete mode 100644 internal/httpclientx/endpoint.go diff --git a/internal/enginelocate/cloudflare.go b/internal/enginelocate/cloudflare.go index 66884eb32a..13f02cd807 100644 --- a/internal/enginelocate/cloudflare.go +++ b/internal/enginelocate/cloudflare.go @@ -20,7 +20,7 @@ func cloudflareIPLookup( // get the raw response body data, err := httpclientx.GetRaw( ctx, - httpclientx.NewEndpoint("https://www.cloudflare.com/cdn-cgi/trace"), + httpclientx.NewBaseURL("https://www.cloudflare.com/cdn-cgi/trace"), &httpclientx.Config{ Authorization: "", // not needed Client: httpClient, diff --git a/internal/enginelocate/ubuntu.go b/internal/enginelocate/ubuntu.go index 61393a90a1..84257800c0 100644 --- a/internal/enginelocate/ubuntu.go +++ b/internal/enginelocate/ubuntu.go @@ -24,7 +24,7 @@ func ubuntuIPLookup( // read the HTTP response and parse as XML v, err := httpclientx.GetXML[*ubuntuResponse]( ctx, - httpclientx.NewEndpoint("https://geoip.ubuntu.com/lookup"), + httpclientx.NewBaseURL("https://geoip.ubuntu.com/lookup"), &httpclientx.Config{ Authorization: "", // not needed Client: httpClient, diff --git a/internal/httpclientx/DESIGN.md b/internal/httpclientx/DESIGN.md index dd3c396f43..96efc78555 100644 --- a/internal/httpclientx/DESIGN.md +++ b/internal/httpclientx/DESIGN.md @@ -76,18 +76,18 @@ type Config struct { UserAgent string } -type Endpoint struct { - URL string - Host string // optional for cloudfronting +type BaseURL struct { + Value string // mandatory base-URL value + HostOverride string // optional for cloudfronting } -func GetJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) +func GetJSON[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) -func GetRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) +func GetRaw(ctx context.Context, base *BaseURL, config *Config) ([]byte, error) -func GetXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) +func GetXML[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) -func PostJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) +func PostJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input, config *Config) (Output, error) ``` (The `*Config` is the last argument because it is handy to create it inline when calling @@ -124,9 +124,9 @@ To avoid logging bodies, one just needs to pass `model.DiscardLogger` as the The code at `./internal/httpapi` performs sequential function calls. This design does not interact well with the `enginenetx` package and its dial tactics. A better strategy is to allow calls to be overlapped. This means that, if the `enginenetx` -is busy trying tactics for a given API endpoint, we eventually try to use the -subsequent (semantically-equivalent) endpoint after a given time, without waiting -for the first endpoint to complete. +is busy trying tactics for a given API base URL, we eventually try to use the +subsequent (semantically-equivalent) base URL after a given time, without waiting +for the first base URL to complete. We allow for overlapped operations by defining these constructors: @@ -144,7 +144,7 @@ They all construct the same `*Overlapped` struct, which looks like this: ```Go type Overlapped[Output any] struct { - RunFunc func(ctx context.Context, epnt *Endpoint) (Output, error) + RunFunc func(ctx context.Context, base *BaseURL) (Output, error) ScheduleInterval time.Duration } @@ -156,15 +156,15 @@ name (i.e., `NewOverlappedGetXML` configures `RunFunc` to run `GetXML`). Then, we define the following method: ```Go -func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*Endpoint) (Output, error) +func (ovx *Overlapped[Output]) Run(ctx context.Context, bases ...*BaseURL) (Output, error) ``` -This method starts N goroutines to issue the API calls with each endpoint URL. (A classic example +This method starts N goroutines to issue the API calls with each base URL. (A classic example is for the URLs to be `https://0.th.ooni.org/`, `https://1.th.ooni.org/` and so on.) -By default, `ScheduleInterval` is 15 seconds. If the first endpoint URL does not provide a result +By default, `ScheduleInterval` is 15 seconds. If the first base URL does not provide a result within 15 seconds, we try the second one. That is, every 15 seconds, we will attempt using -another endpoint URL, until there's a successful response or we run out of URLs. +another base URL, until there's a successful response or we run out of URLs. As soon as we have a successful response, we cancel all the other pending operations that may exist. Once all operations have terminated, we return to the caller. @@ -329,7 +329,7 @@ that is part of the same package, while in this package we marshal in `PostJSON` Consider the following code snippet: ```Go -resp, err := httpclientx.GetJSON[*APIResponse](ctx, epnt, config) +resp, err := httpclientx.GetJSON[*APIResponse](ctx, base, config) runtimex.Assert((resp == nil && err != nil) || (resp != nil && err == nil), "ouch") ``` diff --git a/internal/httpclientx/baseurl.go b/internal/httpclientx/baseurl.go new file mode 100644 index 0000000000..c977bdc185 --- /dev/null +++ b/internal/httpclientx/baseurl.go @@ -0,0 +1,49 @@ +package httpclientx + +import "github.com/ooni/probe-cli/v3/internal/model" + +// BaseURL is an HTTP-endpoint base URL. +// +// The zero value is invalid; construct using [NewBaseURL]. +type BaseURL struct { + // Value is the MANDATORY base-URL Value. + Value string + + // HostOverride is the OPTIONAL host header to use for cloudfronting. + HostOverride string +} + +// NewBaseURL constructs a new [*BaseURL] instance using the given URL. +func NewBaseURL(URL string) *BaseURL { + return &BaseURL{ + Value: URL, + HostOverride: "", + } +} + +// WithHostOverride returns a copy of the [*BaseURL] using the given host header override. +func (e *BaseURL) WithHostOverride(host string) *BaseURL { + return &BaseURL{ + Value: e.Value, + HostOverride: host, + } +} + +// NewBaseURLsFromModelOOAPIServices constructs new [*BaseURL] instances from the +// given [model.OOAPIService] instances, assigning the host header if "cloudfront", and +// skipping all the entries that are neither "https" not "cloudfront". +func NewBaseURLsFromModelOOAPIServices(svcs ...model.OOAPIService) (epnts []*BaseURL) { + for _, svc := range svcs { + epnt := NewBaseURL(svc.Address) + switch svc.Type { + case "cloudfront": + epnt = epnt.WithHostOverride(svc.Front) + fallthrough + case "https": + epnts = append(epnts, epnt) + default: + // skip entry + } + } + return +} diff --git a/internal/httpclientx/endpoint_test.go b/internal/httpclientx/baseurl_test.go similarity index 63% rename from internal/httpclientx/endpoint_test.go rename to internal/httpclientx/baseurl_test.go index e3ee6ed406..0643680fb7 100644 --- a/internal/httpclientx/endpoint_test.go +++ b/internal/httpclientx/baseurl_test.go @@ -7,23 +7,23 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -func TestEndpoint(t *testing.T) { +func TestBaseURL(t *testing.T) { t.Run("the constructor only assigns the URL", func(t *testing.T) { - epnt := NewEndpoint("https://www.example.com/") - if epnt.URL != "https://www.example.com/" { + epnt := NewBaseURL("https://www.example.com/") + if epnt.Value != "https://www.example.com/" { t.Fatal("unexpected URL") } - if epnt.Host != "" { + if epnt.HostOverride != "" { t.Fatal("unexpected host") } }) t.Run("we can optionally get a copy with an assigned host header", func(t *testing.T) { - epnt := NewEndpoint("https://www.example.com/").WithHostOverride("www.cloudfront.com") - if epnt.URL != "https://www.example.com/" { + epnt := NewBaseURL("https://www.example.com/").WithHostOverride("www.cloudfront.com") + if epnt.Value != "https://www.example.com/" { t.Fatal("unexpected URL") } - if epnt.Host != "www.cloudfront.com" { + if epnt.HostOverride != "www.cloudfront.com" { t.Fatal("unexpected host") } }) @@ -50,14 +50,14 @@ func TestEndpoint(t *testing.T) { Front: "", }} - expect := []*Endpoint{{ - URL: "https://www.example.com/", + expect := []*BaseURL{{ + Value: "https://www.example.com/", }, { - URL: "https://www.example.com/", - Host: "www.cloudfront.com", + Value: "https://www.example.com/", + HostOverride: "www.cloudfront.com", }} - got := NewEndpointFromModelOOAPIServices(services...) + got := NewBaseURLsFromModelOOAPIServices(services...) if diff := cmp.Diff(expect, got); diff != "" { t.Fatal(diff) } diff --git a/internal/httpclientx/endpoint.go b/internal/httpclientx/endpoint.go deleted file mode 100644 index abe5d86895..0000000000 --- a/internal/httpclientx/endpoint.go +++ /dev/null @@ -1,49 +0,0 @@ -package httpclientx - -import "github.com/ooni/probe-cli/v3/internal/model" - -// Endpoint is an HTTP endpoint. -// -// The zero value is invalid; construct using [NewEndpoint]. -type Endpoint struct { - // URL is the MANDATORY endpoint URL. - URL string - - // Host is the OPTIONAL host header to use for cloudfronting. - Host string -} - -// NewEndpoint constructs a new [*Endpoint] instance using the given URL. -func NewEndpoint(URL string) *Endpoint { - return &Endpoint{ - URL: URL, - Host: "", - } -} - -// WithHostOverride returns a copy of the [*Endpoint] using the given host header override. -func (e *Endpoint) WithHostOverride(host string) *Endpoint { - return &Endpoint{ - URL: e.URL, - Host: host, - } -} - -// NewEndpointFromModelOOAPIServices constructs new [*Endpoint] instances from the -// given [model.OOAPIService] instances, assigning the host header if "cloudfront", and -// skipping all the entries that are neither "https" not "cloudfront". -func NewEndpointFromModelOOAPIServices(svcs ...model.OOAPIService) (epnts []*Endpoint) { - for _, svc := range svcs { - epnt := NewEndpoint(svc.Address) - switch svc.Type { - case "cloudfront": - epnt = epnt.WithHostOverride(svc.Front) - fallthrough - case "https": - epnts = append(epnts, epnt) - default: - // skip entry - } - } - return -} diff --git a/internal/httpclientx/getjson.go b/internal/httpclientx/getjson.go index 346c524238..587582e128 100644 --- a/internal/httpclientx/getjson.go +++ b/internal/httpclientx/getjson.go @@ -15,16 +15,16 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*Endpoint] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - config contains the config. // // This function either returns an error or a valid Output. -func GetJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { +func GetJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { return OverlappedIgnoreIndex(NewOverlappedGetJSON[Output](config).Run(ctx, epnt)) } -func getJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { +func getJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { // read the raw body rawrespbody, err := GetRaw(ctx, epnt, config) diff --git a/internal/httpclientx/getjson_test.go b/internal/httpclientx/getjson_test.go index 9cc2dc3a79..ac6fd31a8a 100644 --- a/internal/httpclientx/getjson_test.go +++ b/internal/httpclientx/getjson_test.go @@ -28,7 +28,7 @@ func TestGetJSON(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -59,7 +59,7 @@ func TestGetJSON(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -90,7 +90,7 @@ func TestGetJSON(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -137,7 +137,7 @@ func TestGetJSONHeadersOkay(t *testing.T) { // send the request and receive the response apiresp, err := GetJSON[*apiResponse]( context.Background(), - NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), + NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), &Config{ Authorization: "scribai", Client: http.DefaultClient, @@ -194,7 +194,7 @@ func TestGetJSONLoggingOkay(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: logger, @@ -238,7 +238,7 @@ func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := GetJSON[map[string]string]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -268,7 +268,7 @@ func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -298,7 +298,7 @@ func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := GetJSON[[]string]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, diff --git a/internal/httpclientx/getraw.go b/internal/httpclientx/getraw.go index da155215aa..d3bb46d316 100644 --- a/internal/httpclientx/getraw.go +++ b/internal/httpclientx/getraw.go @@ -15,18 +15,18 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*Endpoint] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) { +func GetRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { return OverlappedIgnoreIndex(NewOverlappedGetRaw(config).Run(ctx, epnt)) } -func getRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) { +func getRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { // construct the request to use - req, err := http.NewRequestWithContext(ctx, "GET", epnt.URL, nil) + req, err := http.NewRequestWithContext(ctx, "GET", epnt.Value, nil) if err != nil { return nil, err } diff --git a/internal/httpclientx/getraw_test.go b/internal/httpclientx/getraw_test.go index e9616ea7a7..0965805428 100644 --- a/internal/httpclientx/getraw_test.go +++ b/internal/httpclientx/getraw_test.go @@ -18,7 +18,7 @@ func TestGetRaw(t *testing.T) { rawrespbody, err := GetRaw( context.Background(), - NewEndpoint("\t"), // <- invalid URL that we cannot parse + NewBaseURL("\t"), // <- invalid URL that we cannot parse &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -49,7 +49,7 @@ func TestGetRaw(t *testing.T) { rawrespbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -93,7 +93,7 @@ func TestGetRawHeadersOkay(t *testing.T) { // send the request and receive the response rawresp, err := GetRaw( context.Background(), - NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), + NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), &Config{ Authorization: "scribai", Client: http.DefaultClient, @@ -151,7 +151,7 @@ func TestGetRawLoggingOkay(t *testing.T) { rawrespbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: logger, diff --git a/internal/httpclientx/getxml.go b/internal/httpclientx/getxml.go index 8163f3df36..fe5e6dc857 100644 --- a/internal/httpclientx/getxml.go +++ b/internal/httpclientx/getxml.go @@ -15,16 +15,16 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*Endpoint] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { +func GetXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { return OverlappedIgnoreIndex(NewOverlappedGetXML[Output](config).Run(ctx, epnt)) } -func getXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { +func getXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { // read the raw body rawrespbody, err := GetRaw(ctx, epnt, config) diff --git a/internal/httpclientx/getxml_test.go b/internal/httpclientx/getxml_test.go index 519bda2056..91012f7cdd 100644 --- a/internal/httpclientx/getxml_test.go +++ b/internal/httpclientx/getxml_test.go @@ -24,7 +24,7 @@ func TestGetXML(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -55,7 +55,7 @@ func TestGetXML(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -86,7 +86,7 @@ func TestGetXML(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -133,7 +133,7 @@ func TestGetXMLHeadersOkay(t *testing.T) { // send the request and receive the response apiresp, err := GetXML[*apiResponse]( context.Background(), - NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), + NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), &Config{ Authorization: "scribai", Client: http.DefaultClient, @@ -190,7 +190,7 @@ func TestGetXMLLoggingOkay(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: logger, diff --git a/internal/httpclientx/httpclientx.go b/internal/httpclientx/httpclientx.go index 97514b3536..b9c145fd70 100644 --- a/internal/httpclientx/httpclientx.go +++ b/internal/httpclientx/httpclientx.go @@ -42,7 +42,7 @@ func zeroValue[T any]() T { var ErrTruncated = errors.New("httpapi: truncated response body") // do is the internal function to finish preparing the request and getting a raw response. -func do(ctx context.Context, req *http.Request, epnt *Endpoint, config *Config) ([]byte, error) { +func do(ctx context.Context, req *http.Request, epnt *BaseURL, config *Config) ([]byte, error) { // optionally assign authorization if value := config.Authorization; value != "" { req.Header.Set("Authorization", value) @@ -56,7 +56,7 @@ func do(ctx context.Context, req *http.Request, epnt *Endpoint, config *Config) // OPTIONALLY allow for cloudfronting (the default in net/http is for // the req.Host to be empty and to use req.URL.Host) - req.Host = epnt.Host + req.Host = epnt.HostOverride // get the response resp, err := config.Client.Do(req) diff --git a/internal/httpclientx/httpclientx_test.go b/internal/httpclientx/httpclientx_test.go index 41c5faca21..a7a19385c3 100644 --- a/internal/httpclientx/httpclientx_test.go +++ b/internal/httpclientx/httpclientx_test.go @@ -47,7 +47,7 @@ func TestGzipDecompression(t *testing.T) { // make sure we can read it respbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -79,7 +79,7 @@ func TestGzipDecompression(t *testing.T) { // attempt to get a response body respbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -109,7 +109,7 @@ func TestGzipDecompression(t *testing.T) { // make sure we can read it respbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -135,7 +135,7 @@ func TestHTTPStatusCodeHandling(t *testing.T) { respbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -168,7 +168,7 @@ func TestHTTPReadBodyErrorsHandling(t *testing.T) { respbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -200,7 +200,7 @@ func TestLimitMaximumBodySize(t *testing.T) { // note: here we're using a small max body size, definitely smaller than what we send respbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -233,7 +233,7 @@ func TestLimitMaximumBodySize(t *testing.T) { // note: here we're using a small max body size, definitely smaller than the gzip bomb respbody, err := GetRaw( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, diff --git a/internal/httpclientx/overlapped.go b/internal/httpclientx/overlapped.go index 8577e119f8..242df1a9e8 100644 --- a/internal/httpclientx/overlapped.go +++ b/internal/httpclientx/overlapped.go @@ -28,14 +28,14 @@ const OverlappedDefaultWatchdogTimeout = 5 * time.Minute // call to start while the previous one is still in progress and very slowly downloading // a response. A future implementation MIGHT want to account for this possibility. type Overlapped[Output any] struct { - // RunFunc is the MANDATORY function that fetches the given [*Endpoint]. + // RunFunc is the MANDATORY function that fetches the given [*BaseURL]. // // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw], // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be the proper function that // makes sense for the operation that you requested with the constructor. // // If you set it manually, you MUST modify it before calling [*Overlapped.Run]. - RunFunc func(ctx context.Context, epnt *Endpoint) (Output, error) + RunFunc func(ctx context.Context, epnt *BaseURL) (Output, error) // ScheduleInterval is the MANDATORY scheduling interval. // @@ -55,7 +55,7 @@ type Overlapped[Output any] struct { WatchdogTimeout time.Duration } -func newOverlappedWithFunc[Output any](fx func(context.Context, *Endpoint) (Output, error)) *Overlapped[Output] { +func newOverlappedWithFunc[Output any](fx func(context.Context, *BaseURL) (Output, error)) *Overlapped[Output] { return &Overlapped[Output]{ RunFunc: fx, ScheduleInterval: OverlappedDefaultScheduleInterval, @@ -65,28 +65,28 @@ func newOverlappedWithFunc[Output any](fx func(context.Context, *Endpoint) (Outp // NewOverlappedGetJSON constructs a [*Overlapped] for calling [GetJSON] with multiple URLs. func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { return getJSON[Output](ctx, epnt, config) }) } // NewOverlappedGetRaw constructs a [*Overlapped] for calling [GetRaw] with multiple URLs. func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) ([]byte, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) ([]byte, error) { return getRaw(ctx, epnt, config) }) } // NewOverlappedGetXML constructs a [*Overlapped] for calling [GetXML] with multiple URLs. func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { return getXML[Output](ctx, epnt, config) }) } // NewOverlappedPostJSON constructs a [*Overlapped] for calling [PostJSON] with multiple URLs. func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { return postJSON[Input, Output](ctx, epnt, input, config) }) } @@ -95,17 +95,17 @@ func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Over var ErrGenericOverlappedFailure = errors.New("overlapped: generic failure") // Run runs the overlapped operations, returning the result of the first operation -// that succeeds and its endpoint index, or the error that occurred. -func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*Endpoint) (Output, int, error) { +// that succeeds and its base URL index, or the error that occurred. +func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*BaseURL) (Output, int, error) { return OverlappedReduce[Output](ovx.Map(ctx, epnts...)) } -// OverlappedErrorOr combines error information, result information and the endpoint index. +// OverlappedErrorOr combines error information, result information and the base URL index. type OverlappedErrorOr[Output any] struct { // Err is the error or nil. Err error - // Index is the endpoint index. + // Index is the base URL index. Index int // Value is the result. @@ -120,7 +120,7 @@ type OverlappedErrorOr[Output any] struct { // of each operation, which is mostly useful when running unit tests. // // Note that this function will return a zero length slice if epnts lenth is also zero. -func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*Endpoint) []*OverlappedErrorOr[Output] { +func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*OverlappedErrorOr[Output] { // create cancellable context for early cancellation and also apply the // watchdog timeout so that eventually this code returns. // @@ -142,7 +142,7 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*Endpoint) []*O ticker := time.NewTicker(ovx.ScheduleInterval) defer ticker.Stop() - // create index for the next endpoint to try + // create index for the next base URL to try idx := 0 // create vector for collecting results @@ -152,10 +152,10 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*Endpoint) []*O // then we're going to filter the results and produce a final result results := []*OverlappedErrorOr[Output]{} - // keep looping until we have results for each endpoints + // keep looping until we have results for each base URLs for len(results) < len(epnts) { - // if there are more endpoints to try, spawn a goroutine to try, + // if there are more base URLs to try, spawn a goroutine to try, // and, otherwise, we can safely stop ticking if idx < len(epnts) { go ovx.transact(ctx, idx, epnts[idx], output) @@ -180,7 +180,7 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*Endpoint) []*O } // this means the ticker ticked, so we should loop again and - // attempt another endpoint because it's time to do that + // attempt another base URL because it's time to do that case <-ticker.C: } } @@ -208,7 +208,7 @@ func OverlappedReduce[Output any](results []*OverlappedErrorOr[Output]) (Output, // handle the case where there's no error // - // this happens if the user provided no endpoints to measure + // this happens if the user provided no base URLs to measure if len(errorv) <= 0 { errorv = append(errorv, ErrGenericOverlappedFailure) } @@ -222,14 +222,14 @@ func OverlappedReduce[Output any](results []*OverlappedErrorOr[Output]) (Output, // transact performs an HTTP transaction with the given URL and writes results to the output channel. func (ovx *Overlapped[Output]) transact( - ctx context.Context, idx int, epnt *Endpoint, output chan<- *OverlappedErrorOr[Output]) { + ctx context.Context, idx int, epnt *BaseURL, output chan<- *OverlappedErrorOr[Output]) { // obtain the results value, err := ovx.RunFunc(ctx, epnt) // emit the results // // note that this unconditional channel write REQUIRES that we keep reading from - // the results channel in Run until we have a result per input endpoint + // the results channel in Run until we have a result per input base URL output <- &OverlappedErrorOr[Output]{ Err: err, Index: idx, diff --git a/internal/httpclientx/overlapped_test.go b/internal/httpclientx/overlapped_test.go index e09d4c95da..a0a2912300 100644 --- a/internal/httpclientx/overlapped_test.go +++ b/internal/httpclientx/overlapped_test.go @@ -34,7 +34,7 @@ func TestNewOverlappedPostJSONFastRecoverFromEarlyErrors(t *testing.T) { // // Because the first three THs fail fast but the schedule interval is the default (i.e., // 15 seconds), we're testing whether the algorithm allows us to recover quickly from - // failure and check the other endpoints without waiting for too much time. + // failure and check the other base URLs without waiting for too much time. // // Note: before changing the algorithm, this test ran for 45 seconds. Now it runs // for 1s because a previous goroutine terminating with error causes the next @@ -81,10 +81,10 @@ func TestNewOverlappedPostJSONFastRecoverFromEarlyErrors(t *testing.T) { results := overlapped.Map( context.Background(), - NewEndpoint(zeroTh.URL), - NewEndpoint(oneTh.URL), - NewEndpoint(twoTh.URL), - NewEndpoint(threeTh.URL), + NewBaseURL(zeroTh.URL), + NewBaseURL(oneTh.URL), + NewBaseURL(twoTh.URL), + NewBaseURL(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -195,10 +195,10 @@ func TestNewOverlappedPostJSONFirstCallSucceeds(t *testing.T) { results := overlapped.Map( context.Background(), - NewEndpoint(zeroTh.URL), - NewEndpoint(oneTh.URL), - NewEndpoint(twoTh.URL), - NewEndpoint(threeTh.URL), + NewBaseURL(zeroTh.URL), + NewBaseURL(oneTh.URL), + NewBaseURL(twoTh.URL), + NewBaseURL(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -254,7 +254,7 @@ func TestNewOverlappedPostJSONHandlesAllTimeouts(t *testing.T) { // - 2.th.ooni.org causes timeout // - 3.th.ooni.org causes timeout // - // We expect to loop for all endpoints and then discover that all of them + // We expect to loop for all base URLs and then discover that all of them // failed. To make the test ~quick, we reduce the scheduling interval, and // the watchdog timeout. // @@ -312,10 +312,10 @@ func TestNewOverlappedPostJSONHandlesAllTimeouts(t *testing.T) { results := overlapped.Map( context.Background(), - NewEndpoint(zeroTh.URL), - NewEndpoint(oneTh.URL), - NewEndpoint(twoTh.URL), - NewEndpoint(threeTh.URL), + NewBaseURL(zeroTh.URL), + NewBaseURL(oneTh.URL), + NewBaseURL(twoTh.URL), + NewBaseURL(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -422,10 +422,10 @@ func TestNewOverlappedPostJSONResetTimeoutSuccessCanceled(t *testing.T) { results := overlapped.Map( context.Background(), - NewEndpoint(zeroTh.URL), - NewEndpoint(oneTh.URL), - NewEndpoint(twoTh.URL), - NewEndpoint(threeTh.URL), + NewBaseURL(zeroTh.URL), + NewBaseURL(oneTh.URL), + NewBaseURL(twoTh.URL), + NewBaseURL(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -517,7 +517,7 @@ func TestNewOverlappedPostJSONWithNoURLs(t *testing.T) { } func TestNewOverlappedWithFuncDefaultsAreCorrect(t *testing.T) { - overlapped := newOverlappedWithFunc(func(ctx context.Context, e *Endpoint) (int, error) { + overlapped := newOverlappedWithFunc(func(ctx context.Context, e *BaseURL) (int, error) { return 1, nil }) if overlapped.ScheduleInterval != 15*time.Second { diff --git a/internal/httpclientx/postjson.go b/internal/httpclientx/postjson.go index 1dd2efe67d..a6f044e4dc 100644 --- a/internal/httpclientx/postjson.go +++ b/internal/httpclientx/postjson.go @@ -17,18 +17,18 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*Endpoint] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - input is the input structure to JSON serialize as the request body; // // - config is the config to use. // // This function either returns an error or a valid Output. -func PostJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) { +func PostJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { return OverlappedIgnoreIndex(NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, epnt)) } -func postJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) { +func postJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { // ensure we're not sending a nil map, pointer, or slice if _, err := NilSafetyErrorIfNil(input); err != nil { return zeroValue[Output](), err @@ -41,10 +41,10 @@ func postJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Inpu } // log the raw request body - config.Logger.Debugf("POST %s: raw request body: %s", epnt.URL, string(rawreqbody)) + config.Logger.Debugf("POST %s: raw request body: %s", epnt.Value, string(rawreqbody)) // construct the request to use - req, err := http.NewRequestWithContext(ctx, "POST", epnt.URL, bytes.NewReader(rawreqbody)) + req, err := http.NewRequestWithContext(ctx, "POST", epnt.Value, bytes.NewReader(rawreqbody)) if err != nil { return zeroValue[Output](), err } diff --git a/internal/httpclientx/postjson_test.go b/internal/httpclientx/postjson_test.go index 7d4a4957d0..b4e80a55de 100644 --- a/internal/httpclientx/postjson_test.go +++ b/internal/httpclientx/postjson_test.go @@ -28,7 +28,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[chan int, *apiResponse]( context.Background(), - NewEndpoint(""), + NewBaseURL(""), req, &Config{ Client: http.DefaultClient, @@ -53,7 +53,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint("\t"), // <- invalid URL that we cannot parse + NewBaseURL("\t"), // <- invalid URL that we cannot parse req, &Config{ Client: http.DefaultClient, @@ -82,7 +82,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), req, &Config{ Client: http.DefaultClient, @@ -112,7 +112,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), req, &Config{ Client: http.DefaultClient, @@ -153,7 +153,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), req, &Config{ Client: http.DefaultClient, @@ -211,7 +211,7 @@ func TestPostJSONCommunicationOkay(t *testing.T) { // send the request and receive the response apiresp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), + NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), apireq, &Config{ Authorization: "scribai", @@ -288,7 +288,7 @@ func TestPostJSONLoggingOkay(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), req, &Config{ Client: http.DefaultClient, @@ -335,7 +335,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[map[string]string, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), nil, &Config{ Client: http.DefaultClient, @@ -366,7 +366,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), nil, &Config{ Client: http.DefaultClient, @@ -397,7 +397,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[[]string, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), nil, &Config{ Client: http.DefaultClient, @@ -431,7 +431,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, map[string]string]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), apireq, &Config{ Client: http.DefaultClient, @@ -465,7 +465,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), apireq, &Config{ Client: http.DefaultClient, @@ -499,7 +499,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, []string]( context.Background(), - NewEndpoint(server.URL), + NewBaseURL(server.URL), apireq, &Config{ Client: http.DefaultClient, diff --git a/internal/oonirun/v2.go b/internal/oonirun/v2.go index 4330b07b76..fe93f663dd 100644 --- a/internal/oonirun/v2.go +++ b/internal/oonirun/v2.go @@ -66,7 +66,7 @@ func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient, logger model.Logger, URL string) (*V2Descriptor, error) { return httpclientx.GetJSON[*V2Descriptor]( ctx, - httpclientx.NewEndpoint(URL), + httpclientx.NewBaseURL(URL), &httpclientx.Config{ Authorization: "", // not needed Client: client, diff --git a/internal/probeservices/bouncer.go b/internal/probeservices/bouncer.go index 392c78c839..4fe41abf93 100644 --- a/internal/probeservices/bouncer.go +++ b/internal/probeservices/bouncer.go @@ -23,7 +23,7 @@ func (c *Client) GetTestHelpers(ctx context.Context) (map[string][]model.OOAPISe // get the response return httpclientx.GetJSON[map[string][]model.OOAPIService]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), &httpclientx.Config{ Client: c.HTTPClient, Logger: c.Logger, diff --git a/internal/probeservices/checkin.go b/internal/probeservices/checkin.go index 3b766c5a81..625e17f042 100644 --- a/internal/probeservices/checkin.go +++ b/internal/probeservices/checkin.go @@ -26,7 +26,7 @@ func (c Client) CheckIn( // issue the API call resp, err := httpclientx.PostJSON[*model.OOAPICheckInConfig, *model.OOAPICheckInResult]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), &config, &httpclientx.Config{ Authorization: "", // not needed diff --git a/internal/probeservices/collector.go b/internal/probeservices/collector.go index 907c749b01..5d28c83bba 100644 --- a/internal/probeservices/collector.go +++ b/internal/probeservices/collector.go @@ -68,7 +68,7 @@ func (c Client) OpenReport(ctx context.Context, rt model.OOAPIReportTemplate) (R cor, err := httpclientx.PostJSON[model.OOAPIReportTemplate, *model.OOAPICollectorOpenResponse]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), rt, &httpclientx.Config{ Client: c.HTTPClient, @@ -120,7 +120,7 @@ func (r reportChan) SubmitMeasurement(ctx context.Context, m *model.Measurement) updateResponse, err := httpclientx.PostJSON[ model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(r.client.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(r.client.Host), apiReq, &httpclientx.Config{ Client: r.client.HTTPClient, diff --git a/internal/probeservices/login.go b/internal/probeservices/login.go index 2b42339547..cc08e8ec47 100644 --- a/internal/probeservices/login.go +++ b/internal/probeservices/login.go @@ -27,7 +27,7 @@ func (c Client) MaybeLogin(ctx context.Context) error { auth, err := httpclientx.PostJSON[*model.OOAPILoginCredentials, *model.OOAPILoginAuth]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), creds, &httpclientx.Config{ Client: c.HTTPClient, diff --git a/internal/probeservices/measurementmeta.go b/internal/probeservices/measurementmeta.go index 4f9bea5416..121a4b5e1d 100644 --- a/internal/probeservices/measurementmeta.go +++ b/internal/probeservices/measurementmeta.go @@ -35,7 +35,7 @@ func (c Client) GetMeasurementMeta( // get the response return httpclientx.GetJSON[*model.OOAPIMeasurementMeta]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), &httpclientx.Config{ Client: c.HTTPClient, Logger: c.Logger, diff --git a/internal/probeservices/openvpn.go b/internal/probeservices/openvpn.go index 326cb70cb7..f2bab0556b 100644 --- a/internal/probeservices/openvpn.go +++ b/internal/probeservices/openvpn.go @@ -36,7 +36,7 @@ func (c Client) FetchOpenVPNConfig(ctx context.Context, provider, cc string) (re // use a model.DiscardLogger to avoid logging config return httpclientx.GetJSON[model.OOAPIVPNProviderConfig]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), &httpclientx.Config{ Client: c.HTTPClient, Logger: model.DiscardLogger, diff --git a/internal/probeservices/psiphon.go b/internal/probeservices/psiphon.go index 7c5faac3d5..2c519b2b5e 100644 --- a/internal/probeservices/psiphon.go +++ b/internal/probeservices/psiphon.go @@ -31,7 +31,7 @@ func (c Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { // use a model.DiscardLogger to avoid logging config return httpclientx.GetRaw( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), &httpclientx.Config{ Authorization: s, Client: c.HTTPClient, diff --git a/internal/probeservices/register.go b/internal/probeservices/register.go index de2ad472fb..c358fc45f4 100644 --- a/internal/probeservices/register.go +++ b/internal/probeservices/register.go @@ -39,7 +39,7 @@ func (c Client) MaybeRegister(ctx context.Context, metadata model.OOAPIProbeMeta resp, err := httpclientx.PostJSON[*model.OOAPIRegisterRequest, *model.OOAPIRegisterResponse]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), req, &httpclientx.Config{ Client: c.HTTPClient, diff --git a/internal/probeservices/tor.go b/internal/probeservices/tor.go index 262cb05d1f..b46a6f25f4 100644 --- a/internal/probeservices/tor.go +++ b/internal/probeservices/tor.go @@ -36,7 +36,7 @@ func (c Client) FetchTorTargets(ctx context.Context, cc string) (map[string]mode // use a model.DiscardLogger to avoid logging bridges return httpclientx.GetJSON[map[string]model.OOAPITorTarget]( ctx, - httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), &httpclientx.Config{ Authorization: s, Client: c.HTTPClient, diff --git a/internal/webconnectivityalgo/calltesthelpers.go b/internal/webconnectivityalgo/calltesthelpers.go index 49a8379771..d28bc6f165 100644 --- a/internal/webconnectivityalgo/calltesthelpers.go +++ b/internal/webconnectivityalgo/calltesthelpers.go @@ -37,7 +37,7 @@ func CallWebConnectivityTestHelper(ctx context.Context, creq *model.THRequest, ) // perform the overlapped HTTP API calls - cresp, idx, err := overlapped.Run(ctx, httpclientx.NewEndpointFromModelOOAPIServices(testhelpers...)...) + cresp, idx, err := overlapped.Run(ctx, httpclientx.NewBaseURLsFromModelOOAPIServices(testhelpers...)...) // handle the case where all test helpers failed if err != nil { From 2da0ed8642e4a14346d73cb294c0fdfab6b76328 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 15:45:33 +0200 Subject: [PATCH 2/8] x --- internal/engine/session.go | 4 +- internal/engine/session_integration_test.go | 2 +- internal/probeservices/benchselect.go | 23 ++-- internal/probeservices/probeservices.go | 20 ++-- internal/probeservices/probeservices_test.go | 116 +++++++++---------- 5 files changed, 83 insertions(+), 82 deletions(-) diff --git a/internal/engine/session.go b/internal/engine/session.go index 1a6bc5443b..fbd17d22bf 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -673,8 +673,8 @@ func (s *Session) MaybeLookupBackendsContext(ctx context.Context) error { if selected == nil { return ErrAllProbeServicesFailed } - s.logger.Infof("session: using probe services: %+v", selected.Endpoint) - s.selectedProbeService = &selected.Endpoint + s.logger.Infof("session: using probe services: %+v", selected.Service) + s.selectedProbeService = &selected.Service s.availableTestHelpers = selected.TestHelpers return nil } diff --git a/internal/engine/session_integration_test.go b/internal/engine/session_integration_test.go index 77a1e22fc4..6175891446 100644 --- a/internal/engine/session_integration_test.go +++ b/internal/engine/session_integration_test.go @@ -483,7 +483,7 @@ func TestNewOrchestraClientProbeServicesNewClientFailure(t *testing.T) { svc.Type = "antani" // should really not be supported for a long time } client, err := sess.newOrchestraClient(context.Background()) - if !errors.Is(err, probeservices.ErrUnsupportedEndpoint) { + if !errors.Is(err, probeservices.ErrUnsupportedService) { t.Fatal("not the error we expected") } if client != nil { diff --git a/internal/probeservices/benchselect.go b/internal/probeservices/benchselect.go index 154cc4a2af..894100d256 100644 --- a/internal/probeservices/benchselect.go +++ b/internal/probeservices/benchselect.go @@ -19,8 +19,8 @@ func Default() []model.OOAPIService { }} } -// SortEndpoints gives priority to https, then cloudfronted, then onion. -func SortEndpoints(in []model.OOAPIService) (out []model.OOAPIService) { +// SortServices gives priority to https, then cloudfronted, then onion. +func SortServices(in []model.OOAPIService) (out []model.OOAPIService) { for _, entry := range in { if entry.Type == "https" { out = append(out, entry) @@ -39,7 +39,7 @@ func SortEndpoints(in []model.OOAPIService) (out []model.OOAPIService) { return } -// OnlyHTTPS returns the HTTPS endpoints only. +// OnlyHTTPS returns the HTTPS services only. func OnlyHTTPS(in []model.OOAPIService) (out []model.OOAPIService) { for _, entry := range in { if entry.Type == "https" { @@ -49,9 +49,9 @@ func OnlyHTTPS(in []model.OOAPIService) (out []model.OOAPIService) { return } -// OnlyFallbacks returns the fallback endpoints only. +// OnlyFallbacks returns the fallback services only. func OnlyFallbacks(in []model.OOAPIService) (out []model.OOAPIService) { - for _, entry := range SortEndpoints(in) { + for _, entry := range SortServices(in) { if entry.Type != "https" { out = append(out, entry) } @@ -67,15 +67,16 @@ type Candidate struct { // Err indicates whether the service works. Err error - // Endpoint is the service endpoint. - Endpoint model.OOAPIService + // Service is the service to use. + Service model.OOAPIService - // TestHelpers contains the data returned by the endpoint. + // TestHelpers contains the data returned by the service when + // querying the /api/v1/test-helpers API endpoint. TestHelpers map[string][]model.OOAPIService } func (c *Candidate) try(ctx context.Context, sess Session) { - client, err := NewClient(sess, c.Endpoint) + client, err := NewClient(sess, c.Service) if err != nil { c.Err = err return @@ -85,11 +86,11 @@ func (c *Candidate) try(ctx context.Context, sess Session) { c.Duration = time.Since(start) c.Err = err c.TestHelpers = testhelpers - sess.Logger().Debugf("probe services: %+v: %+v %s", c.Endpoint, err, c.Duration) + sess.Logger().Debugf("probe services: %+v: %+v %s", c.Service, err, c.Duration) } func try(ctx context.Context, sess Session, svc model.OOAPIService) *Candidate { - candidate := &Candidate{Endpoint: svc} + candidate := &Candidate{Service: svc} candidate.try(ctx, sess) return candidate } diff --git a/internal/probeservices/probeservices.go b/internal/probeservices/probeservices.go index 16c8a195cf..2e85bcdf04 100644 --- a/internal/probeservices/probeservices.go +++ b/internal/probeservices/probeservices.go @@ -1,6 +1,6 @@ // Package probeservices contains code to contact OONI probe services. // -// The probe services are HTTPS endpoints distributed across a bunch of data +// The probe services are HTTPS services distributed across a bunch of data // centres implementing a bunch of OONI APIs. When started, OONI will benchmark // the available probe services and select the fastest one. Eventually all the // possible OONI APIs will run as probe services. @@ -32,8 +32,8 @@ import ( ) var ( - // ErrUnsupportedEndpoint indicates that we don't support this endpoint type. - ErrUnsupportedEndpoint = errors.New("probe services: unsupported endpoint type") + // ErrUnsupportedService indicates that we don't support this service type. + ErrUnsupportedService = errors.New("probe services: unsupported service type") // ErrUnsupportedCloudFrontAddress indicates that we don't support this // cloudfront address (e.g. wrong scheme, presence of port). @@ -90,11 +90,11 @@ func (c Client) GetCredsAndAuth() (*model.OOAPILoginCredentials, *model.OOAPILog return creds, auth, nil } -// NewClient creates a new client for the specified probe services endpoint. This -// function fails, e.g., we don't support the specified endpoint. -func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) { +// NewClient creates a new client for the specified probe services service. This +// function fails, e.g., we don't support the specified service. +func NewClient(sess Session, service model.OOAPIService) (*Client, error) { client := &Client{ - BaseURL: endpoint.Address, + BaseURL: service.Address, HTTPClient: sess.DefaultHTTPClient(), Host: "", KVStore: sess.KeyValueStore(), @@ -104,7 +104,7 @@ func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) { StateFile: NewStateFile(sess.KeyValueStore()), UserAgent: sess.UserAgent(), } - switch endpoint.Type { + switch service.Type { case "https": return client, nil case "cloudfront": @@ -119,13 +119,13 @@ func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) { return nil, ErrUnsupportedCloudFrontAddress } client.Host = URL.Hostname() - URL.Host = endpoint.Front + URL.Host = service.Front client.BaseURL = URL.String() if _, err := url.Parse(client.BaseURL); err != nil { return nil, err } return client, nil default: - return nil, ErrUnsupportedEndpoint + return nil, ErrUnsupportedService } } diff --git a/internal/probeservices/probeservices_test.go b/internal/probeservices/probeservices_test.go index dcb32667bf..40063a485a 100644 --- a/internal/probeservices/probeservices_test.go +++ b/internal/probeservices/probeservices_test.go @@ -47,13 +47,13 @@ func TestNewClientHTTPS(t *testing.T) { } } -func TestNewClientUnsupportedEndpoint(t *testing.T) { +func TestNewClientUnsupportedService(t *testing.T) { client, err := NewClient( &mockable.Session{}, model.OOAPIService{ Address: "https://x.org", Type: "onion", }) - if !errors.Is(err, ErrUnsupportedEndpoint) { + if !errors.Is(err, ErrUnsupportedService) { t.Fatal("not the error we expected") } if client != nil { @@ -193,7 +193,7 @@ func TestDefaultProbeServicesWorkAsIntended(t *testing.T) { } } -func TestSortEndpoints(t *testing.T) { +func TestSortServices(t *testing.T) { in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -216,7 +216,7 @@ func TestSortEndpoints(t *testing.T) { Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", }} - out := SortEndpoints(in) + out := SortServices(in) diff := cmp.Diff(out, expect) if diff != "" { t.Fatal(diff) @@ -259,7 +259,7 @@ func TestOnlyHTTPS(t *testing.T) { } func TestOnlyFallbacks(t *testing.T) { - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -293,7 +293,7 @@ func TestOnlyFallbacks(t *testing.T) { } func TestTryAllCanceledContext(t *testing.T) { - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -328,11 +328,11 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[0].Err, context.Canceled) { t.Fatal("invalid error") } - if out[0].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[0].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[0].Endpoint.Address != "https://ams-ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[0].Service.Address != "https://ams-ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[1].Duration <= 0 { @@ -341,11 +341,11 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[1].Err, context.Canceled) { t.Fatal("invalid error") } - if out[1].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[1].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[1].Service.Address != "https://hkg-ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[2].Duration <= 0 { @@ -354,11 +354,11 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[2].Err, context.Canceled) { t.Fatal("invalid error") } - if out[2].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[2].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[2].Endpoint.Address != "https://mia-ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[2].Service.Address != "https://mia-ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[3].Duration <= 0 { @@ -367,28 +367,28 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[3].Err, context.Canceled) { t.Fatal("invalid error") } - if out[3].Endpoint.Type != "cloudfront" { - t.Fatal("invalid endpoint type") + if out[3].Service.Type != "cloudfront" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" { - t.Fatal("invalid endpoint type") + if out[3].Service.Front != "dkyhjv0wpi2dk.cloudfront.net" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { - t.Fatal("invalid endpoint type") + if out[3].Service.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { + t.Fatal("invalid service type") } // - // Note: here duration may be zero because the endpoint is not supported + // Note: here duration may be zero because the service is not supported // and so we don't basically do anything. But it also may be nonzero since // we also run tests in the cloud, which is slower than my desktop. So, I // have not written a specific test concerning out[4].Duration. - if !errors.Is(out[4].Err, ErrUnsupportedEndpoint) { + if !errors.Is(out[4].Err, ErrUnsupportedService) { t.Fatal("invalid error") } - if out[4].Endpoint.Type != "onion" { - t.Fatal("invalid endpoint type") + if out[4].Service.Type != "onion" { + t.Fatal("invalid service type") } - if out[4].Endpoint.Address != "httpo://jehhrikjjqrlpufu.onion" { - t.Fatal("invalid endpoint type") + if out[4].Service.Address != "httpo://jehhrikjjqrlpufu.onion" { + t.Fatal("invalid service type") } } @@ -396,7 +396,7 @@ func TestTryAllIntegrationWeRaceForFastestHTTPS(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -423,11 +423,11 @@ func TestTryAllIntegrationWeRaceForFastestHTTPS(t *testing.T) { if out[0].Err != nil { t.Fatal("invalid error") } - if out[0].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[0].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[0].Endpoint.Address != "https://api.ooni.io" { - t.Fatal("invalid endpoint address") + if out[0].Service.Address != "https://api.ooni.io" { + t.Fatal("invalid service address") } } @@ -435,7 +435,7 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -468,11 +468,11 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if !strings.HasSuffix(out[0].Err.Error(), "no such host") { t.Fatal("invalid error") } - if out[0].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[0].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[0].Endpoint.Address != "https://ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[0].Service.Address != "https://ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[1].Duration <= 0 { @@ -481,11 +481,11 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if !strings.HasSuffix(out[1].Err.Error(), "no such host") { t.Fatal("invalid error") } - if out[1].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[1].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.nu" { - t.Fatal("invalid endpoint type") + if out[1].Service.Address != "https://hkg-ps-nonexistent.ooni.nu" { + t.Fatal("invalid service type") } // if out[2].Duration <= 0 { @@ -494,11 +494,11 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if !strings.HasSuffix(out[2].Err.Error(), "no such host") { t.Fatal("invalid error") } - if out[2].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[2].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[2].Endpoint.Address != "https://mia-ps2-nonexistent.ooni.nu" { - t.Fatal("invalid endpoint type") + if out[2].Service.Address != "https://mia-ps2-nonexistent.ooni.nu" { + t.Fatal("invalid service type") } // if out[3].Duration <= 0 { @@ -507,13 +507,13 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if out[3].Err != nil { t.Fatal("invalid error") } - if out[3].Endpoint.Type != "cloudfront" { - t.Fatal("invalid endpoint type") + if out[3].Service.Type != "cloudfront" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { - t.Fatal("invalid endpoint type") + if out[3].Service.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" { + if out[3].Service.Front != "dkyhjv0wpi2dk.cloudfront.net" { t.Fatal("invalid front") } } @@ -537,32 +537,32 @@ func TestSelectBestOnlyFailures(t *testing.T) { func TestSelectBestSelectsTheFastest(t *testing.T) { in := []*Candidate{{ Duration: 10 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps1.ooni.nonexistent", Type: "https", }, }, { Duration: 4 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps2.ooni.nonexistent", Type: "https", }, }, { Duration: 7 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps3.ooni.nonexistent", Type: "https", }, }, { Duration: 11 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps4.ooni.nonexistent", Type: "https", }, }} expected := &Candidate{ Duration: 4 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps2.ooni.nonexistent", Type: "https", }, From a8de19cef465e73485d588728635ab55453daa6f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 15:48:59 +0200 Subject: [PATCH 3/8] x --- internal/httpclientx/baseurl.go | 8 +++--- internal/httpclientx/baseurl_test.go | 12 ++++----- internal/httpclientx/getjson.go | 10 +++---- internal/httpclientx/getraw.go | 12 ++++----- internal/httpclientx/getxml.go | 10 +++---- internal/httpclientx/httpclientx.go | 4 +-- internal/httpclientx/overlapped.go | 40 ++++++++++++++-------------- internal/httpclientx/postjson.go | 14 +++++----- 8 files changed, 55 insertions(+), 55 deletions(-) diff --git a/internal/httpclientx/baseurl.go b/internal/httpclientx/baseurl.go index c977bdc185..6836bdcafe 100644 --- a/internal/httpclientx/baseurl.go +++ b/internal/httpclientx/baseurl.go @@ -32,15 +32,15 @@ func (e *BaseURL) WithHostOverride(host string) *BaseURL { // NewBaseURLsFromModelOOAPIServices constructs new [*BaseURL] instances from the // given [model.OOAPIService] instances, assigning the host header if "cloudfront", and // skipping all the entries that are neither "https" not "cloudfront". -func NewBaseURLsFromModelOOAPIServices(svcs ...model.OOAPIService) (epnts []*BaseURL) { +func NewBaseURLsFromModelOOAPIServices(svcs ...model.OOAPIService) (bases []*BaseURL) { for _, svc := range svcs { - epnt := NewBaseURL(svc.Address) + base := NewBaseURL(svc.Address) switch svc.Type { case "cloudfront": - epnt = epnt.WithHostOverride(svc.Front) + base = base.WithHostOverride(svc.Front) fallthrough case "https": - epnts = append(epnts, epnt) + bases = append(bases, base) default: // skip entry } diff --git a/internal/httpclientx/baseurl_test.go b/internal/httpclientx/baseurl_test.go index 0643680fb7..ab0c4ef416 100644 --- a/internal/httpclientx/baseurl_test.go +++ b/internal/httpclientx/baseurl_test.go @@ -9,21 +9,21 @@ import ( func TestBaseURL(t *testing.T) { t.Run("the constructor only assigns the URL", func(t *testing.T) { - epnt := NewBaseURL("https://www.example.com/") - if epnt.Value != "https://www.example.com/" { + base := NewBaseURL("https://www.example.com/") + if base.Value != "https://www.example.com/" { t.Fatal("unexpected URL") } - if epnt.HostOverride != "" { + if base.HostOverride != "" { t.Fatal("unexpected host") } }) t.Run("we can optionally get a copy with an assigned host header", func(t *testing.T) { - epnt := NewBaseURL("https://www.example.com/").WithHostOverride("www.cloudfront.com") - if epnt.Value != "https://www.example.com/" { + base := NewBaseURL("https://www.example.com/").WithHostOverride("www.cloudfront.com") + if base.Value != "https://www.example.com/" { t.Fatal("unexpected URL") } - if epnt.HostOverride != "www.cloudfront.com" { + if base.HostOverride != "www.cloudfront.com" { t.Fatal("unexpected host") } }) diff --git a/internal/httpclientx/getjson.go b/internal/httpclientx/getjson.go index 587582e128..ffdc6405c7 100644 --- a/internal/httpclientx/getjson.go +++ b/internal/httpclientx/getjson.go @@ -15,18 +15,18 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - base is the HTTP [*BaseURL] to use; // // - config contains the config. // // This function either returns an error or a valid Output. -func GetJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { - return OverlappedIgnoreIndex(NewOverlappedGetJSON[Output](config).Run(ctx, epnt)) +func GetJSON[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { + return OverlappedIgnoreIndex(NewOverlappedGetJSON[Output](config).Run(ctx, base)) } -func getJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { +func getJSON[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { // read the raw body - rawrespbody, err := GetRaw(ctx, epnt, config) + rawrespbody, err := GetRaw(ctx, base, config) // handle the case of error if err != nil { diff --git a/internal/httpclientx/getraw.go b/internal/httpclientx/getraw.go index d3bb46d316..f7c0788f62 100644 --- a/internal/httpclientx/getraw.go +++ b/internal/httpclientx/getraw.go @@ -15,22 +15,22 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - base is the HTTP [*BaseURL] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { - return OverlappedIgnoreIndex(NewOverlappedGetRaw(config).Run(ctx, epnt)) +func GetRaw(ctx context.Context, base *BaseURL, config *Config) ([]byte, error) { + return OverlappedIgnoreIndex(NewOverlappedGetRaw(config).Run(ctx, base)) } -func getRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { +func getRaw(ctx context.Context, base *BaseURL, config *Config) ([]byte, error) { // construct the request to use - req, err := http.NewRequestWithContext(ctx, "GET", epnt.Value, nil) + req, err := http.NewRequestWithContext(ctx, "GET", base.Value, nil) if err != nil { return nil, err } // get raw response body - return do(ctx, req, epnt, config) + return do(ctx, req, base, config) } diff --git a/internal/httpclientx/getxml.go b/internal/httpclientx/getxml.go index fe5e6dc857..f3f1cedf0d 100644 --- a/internal/httpclientx/getxml.go +++ b/internal/httpclientx/getxml.go @@ -15,18 +15,18 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - base is the HTTP [*BaseURL] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { - return OverlappedIgnoreIndex(NewOverlappedGetXML[Output](config).Run(ctx, epnt)) +func GetXML[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { + return OverlappedIgnoreIndex(NewOverlappedGetXML[Output](config).Run(ctx, base)) } -func getXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { +func getXML[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { // read the raw body - rawrespbody, err := GetRaw(ctx, epnt, config) + rawrespbody, err := GetRaw(ctx, base, config) // handle the case of error if err != nil { diff --git a/internal/httpclientx/httpclientx.go b/internal/httpclientx/httpclientx.go index b9c145fd70..45d2b5b143 100644 --- a/internal/httpclientx/httpclientx.go +++ b/internal/httpclientx/httpclientx.go @@ -42,7 +42,7 @@ func zeroValue[T any]() T { var ErrTruncated = errors.New("httpapi: truncated response body") // do is the internal function to finish preparing the request and getting a raw response. -func do(ctx context.Context, req *http.Request, epnt *BaseURL, config *Config) ([]byte, error) { +func do(ctx context.Context, req *http.Request, base *BaseURL, config *Config) ([]byte, error) { // optionally assign authorization if value := config.Authorization; value != "" { req.Header.Set("Authorization", value) @@ -56,7 +56,7 @@ func do(ctx context.Context, req *http.Request, epnt *BaseURL, config *Config) ( // OPTIONALLY allow for cloudfronting (the default in net/http is for // the req.Host to be empty and to use req.URL.Host) - req.Host = epnt.HostOverride + req.Host = base.HostOverride // get the response resp, err := config.Client.Do(req) diff --git a/internal/httpclientx/overlapped.go b/internal/httpclientx/overlapped.go index 242df1a9e8..0abee96458 100644 --- a/internal/httpclientx/overlapped.go +++ b/internal/httpclientx/overlapped.go @@ -35,7 +35,7 @@ type Overlapped[Output any] struct { // makes sense for the operation that you requested with the constructor. // // If you set it manually, you MUST modify it before calling [*Overlapped.Run]. - RunFunc func(ctx context.Context, epnt *BaseURL) (Output, error) + RunFunc func(ctx context.Context, base *BaseURL) (Output, error) // ScheduleInterval is the MANDATORY scheduling interval. // @@ -65,29 +65,29 @@ func newOverlappedWithFunc[Output any](fx func(context.Context, *BaseURL) (Outpu // NewOverlappedGetJSON constructs a [*Overlapped] for calling [GetJSON] with multiple URLs. func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { - return getJSON[Output](ctx, epnt, config) + return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) (Output, error) { + return getJSON[Output](ctx, base, config) }) } // NewOverlappedGetRaw constructs a [*Overlapped] for calling [GetRaw] with multiple URLs. func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) ([]byte, error) { - return getRaw(ctx, epnt, config) + return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) ([]byte, error) { + return getRaw(ctx, base, config) }) } // NewOverlappedGetXML constructs a [*Overlapped] for calling [GetXML] with multiple URLs. func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { - return getXML[Output](ctx, epnt, config) + return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) (Output, error) { + return getXML[Output](ctx, base, config) }) } // NewOverlappedPostJSON constructs a [*Overlapped] for calling [PostJSON] with multiple URLs. func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { - return postJSON[Input, Output](ctx, epnt, input, config) + return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) (Output, error) { + return postJSON[Input, Output](ctx, base, input, config) }) } @@ -96,8 +96,8 @@ var ErrGenericOverlappedFailure = errors.New("overlapped: generic failure") // Run runs the overlapped operations, returning the result of the first operation // that succeeds and its base URL index, or the error that occurred. -func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*BaseURL) (Output, int, error) { - return OverlappedReduce[Output](ovx.Map(ctx, epnts...)) +func (ovx *Overlapped[Output]) Run(ctx context.Context, bases ...*BaseURL) (Output, int, error) { + return OverlappedReduce[Output](ovx.Map(ctx, bases...)) } // OverlappedErrorOr combines error information, result information and the base URL index. @@ -112,15 +112,15 @@ type OverlappedErrorOr[Output any] struct { Value Output } -// Map applies the [*Overlapped.RunFunc] function to each epnts entry, thus producing +// Map applies the [*Overlapped.RunFunc] function to each bases entry, thus producing // a result for each entry. This function will cancel subsequent operations until there // is a success: subsequent results will be [context.Canceled] errors. // // Note that you SHOULD use [*Overlapped.Run] unless you want to observe the result // of each operation, which is mostly useful when running unit tests. // -// Note that this function will return a zero length slice if epnts lenth is also zero. -func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*OverlappedErrorOr[Output] { +// Note that this function will return a zero length slice if bases lenth is also zero. +func (ovx *Overlapped[Output]) Map(ctx context.Context, bases ...*BaseURL) []*OverlappedErrorOr[Output] { // create cancellable context for early cancellation and also apply the // watchdog timeout so that eventually this code returns. // @@ -153,12 +153,12 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*Ov results := []*OverlappedErrorOr[Output]{} // keep looping until we have results for each base URLs - for len(results) < len(epnts) { + for len(results) < len(bases) { // if there are more base URLs to try, spawn a goroutine to try, // and, otherwise, we can safely stop ticking - if idx < len(epnts) { - go ovx.transact(ctx, idx, epnts[idx], output) + if idx < len(bases) { + go ovx.transact(ctx, idx, bases[idx], output) idx++ } else { ticker.Stop() @@ -170,7 +170,7 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*Ov // background goroutines and stop ticking // // note that we MUST continue reading until we have - // exactly `len(epnts)` results because the inner + // exactly `len(bases)` results because the inner // goroutine performs blocking writes on the channel case res := <-output: results = append(results, res) @@ -222,9 +222,9 @@ func OverlappedReduce[Output any](results []*OverlappedErrorOr[Output]) (Output, // transact performs an HTTP transaction with the given URL and writes results to the output channel. func (ovx *Overlapped[Output]) transact( - ctx context.Context, idx int, epnt *BaseURL, output chan<- *OverlappedErrorOr[Output]) { + ctx context.Context, idx int, base *BaseURL, output chan<- *OverlappedErrorOr[Output]) { // obtain the results - value, err := ovx.RunFunc(ctx, epnt) + value, err := ovx.RunFunc(ctx, base) // emit the results // diff --git a/internal/httpclientx/postjson.go b/internal/httpclientx/postjson.go index a6f044e4dc..d6a8bcb677 100644 --- a/internal/httpclientx/postjson.go +++ b/internal/httpclientx/postjson.go @@ -17,18 +17,18 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - base is the HTTP [*BaseURL] to use; // // - input is the input structure to JSON serialize as the request body; // // - config is the config to use. // // This function either returns an error or a valid Output. -func PostJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { - return OverlappedIgnoreIndex(NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, epnt)) +func PostJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input, config *Config) (Output, error) { + return OverlappedIgnoreIndex(NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, base)) } -func postJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { +func postJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input, config *Config) (Output, error) { // ensure we're not sending a nil map, pointer, or slice if _, err := NilSafetyErrorIfNil(input); err != nil { return zeroValue[Output](), err @@ -41,10 +41,10 @@ func postJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input } // log the raw request body - config.Logger.Debugf("POST %s: raw request body: %s", epnt.Value, string(rawreqbody)) + config.Logger.Debugf("POST %s: raw request body: %s", base.Value, string(rawreqbody)) // construct the request to use - req, err := http.NewRequestWithContext(ctx, "POST", epnt.Value, bytes.NewReader(rawreqbody)) + req, err := http.NewRequestWithContext(ctx, "POST", base.Value, bytes.NewReader(rawreqbody)) if err != nil { return zeroValue[Output](), err } @@ -53,7 +53,7 @@ func postJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input req.Header.Set("Content-Type", "application/json") // get the raw response body - rawrespbody, err := do(ctx, req, epnt, config) + rawrespbody, err := do(ctx, req, base, config) // handle the case of error if err != nil { From dbccd53cc496f3d384a7b9d53361a6d490525f05 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 15:49:54 +0200 Subject: [PATCH 4/8] x --- internal/engine/experiment_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine/experiment_integration_test.go b/internal/engine/experiment_integration_test.go index 1d9a7f91a3..08bd8a191e 100644 --- a/internal/engine/experiment_integration_test.go +++ b/internal/engine/experiment_integration_test.go @@ -384,7 +384,7 @@ func TestOpenReportNewClientFailure(t *testing.T) { Type: "antani", } err = exp.OpenReportContext(context.Background()) - if err.Error() != "probe services: unsupported endpoint type" { + if err.Error() != "probe services: unsupported service type" { t.Fatal(err) } } From 2dfa5f58418a1b06792da1408107c04fe1070303 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 15:51:20 +0200 Subject: [PATCH 5/8] x --- internal/engine/session_integration_test.go | 2 +- internal/probeservices/probeservices.go | 6 +++--- internal/probeservices/probeservices_test.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/engine/session_integration_test.go b/internal/engine/session_integration_test.go index 6175891446..9f211e930e 100644 --- a/internal/engine/session_integration_test.go +++ b/internal/engine/session_integration_test.go @@ -483,7 +483,7 @@ func TestNewOrchestraClientProbeServicesNewClientFailure(t *testing.T) { svc.Type = "antani" // should really not be supported for a long time } client, err := sess.newOrchestraClient(context.Background()) - if !errors.Is(err, probeservices.ErrUnsupportedService) { + if !errors.Is(err, probeservices.ErrUnsupportedServiceType) { t.Fatal("not the error we expected") } if client != nil { diff --git a/internal/probeservices/probeservices.go b/internal/probeservices/probeservices.go index 2e85bcdf04..c7313610c8 100644 --- a/internal/probeservices/probeservices.go +++ b/internal/probeservices/probeservices.go @@ -32,8 +32,8 @@ import ( ) var ( - // ErrUnsupportedService indicates that we don't support this service type. - ErrUnsupportedService = errors.New("probe services: unsupported service type") + // ErrUnsupportedServiceType indicates that we don't support this service type. + ErrUnsupportedServiceType = errors.New("probe services: unsupported service type") // ErrUnsupportedCloudFrontAddress indicates that we don't support this // cloudfront address (e.g. wrong scheme, presence of port). @@ -126,6 +126,6 @@ func NewClient(sess Session, service model.OOAPIService) (*Client, error) { } return client, nil default: - return nil, ErrUnsupportedService + return nil, ErrUnsupportedServiceType } } diff --git a/internal/probeservices/probeservices_test.go b/internal/probeservices/probeservices_test.go index 40063a485a..5f31d75c8d 100644 --- a/internal/probeservices/probeservices_test.go +++ b/internal/probeservices/probeservices_test.go @@ -53,7 +53,7 @@ func TestNewClientUnsupportedService(t *testing.T) { Address: "https://x.org", Type: "onion", }) - if !errors.Is(err, ErrUnsupportedService) { + if !errors.Is(err, ErrUnsupportedServiceType) { t.Fatal("not the error we expected") } if client != nil { @@ -381,7 +381,7 @@ func TestTryAllCanceledContext(t *testing.T) { // and so we don't basically do anything. But it also may be nonzero since // we also run tests in the cloud, which is slower than my desktop. So, I // have not written a specific test concerning out[4].Duration. - if !errors.Is(out[4].Err, ErrUnsupportedService) { + if !errors.Is(out[4].Err, ErrUnsupportedServiceType) { t.Fatal("invalid error") } if out[4].Service.Type != "onion" { From f1abe402eafe4e44fcb4fabfe813993a996d5f75 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 15:55:34 +0200 Subject: [PATCH 6/8] x --- internal/engine/experiment_integration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/engine/experiment_integration_test.go b/internal/engine/experiment_integration_test.go index 08bd8a191e..26a37509b4 100644 --- a/internal/engine/experiment_integration_test.go +++ b/internal/engine/experiment_integration_test.go @@ -3,6 +3,7 @@ package engine import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "os" @@ -11,6 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/probeservices" "github.com/ooni/probe-cli/v3/internal/registry" ) @@ -384,7 +386,7 @@ func TestOpenReportNewClientFailure(t *testing.T) { Type: "antani", } err = exp.OpenReportContext(context.Background()) - if err.Error() != "probe services: unsupported service type" { + if !errors.Is(err, probeservices.ErrUnsupportedServiceType) { t.Fatal(err) } } From 9fa65dc91661134b263232c6459cf250d1ebbd6b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 16:25:35 +0200 Subject: [PATCH 7/8] Revert "x" This reverts commit a8de19cef465e73485d588728635ab55453daa6f. --- internal/httpclientx/baseurl.go | 8 +++--- internal/httpclientx/baseurl_test.go | 12 ++++----- internal/httpclientx/getjson.go | 10 +++---- internal/httpclientx/getraw.go | 12 ++++----- internal/httpclientx/getxml.go | 10 +++---- internal/httpclientx/httpclientx.go | 4 +-- internal/httpclientx/overlapped.go | 40 ++++++++++++++-------------- internal/httpclientx/postjson.go | 14 +++++----- 8 files changed, 55 insertions(+), 55 deletions(-) diff --git a/internal/httpclientx/baseurl.go b/internal/httpclientx/baseurl.go index 6836bdcafe..c977bdc185 100644 --- a/internal/httpclientx/baseurl.go +++ b/internal/httpclientx/baseurl.go @@ -32,15 +32,15 @@ func (e *BaseURL) WithHostOverride(host string) *BaseURL { // NewBaseURLsFromModelOOAPIServices constructs new [*BaseURL] instances from the // given [model.OOAPIService] instances, assigning the host header if "cloudfront", and // skipping all the entries that are neither "https" not "cloudfront". -func NewBaseURLsFromModelOOAPIServices(svcs ...model.OOAPIService) (bases []*BaseURL) { +func NewBaseURLsFromModelOOAPIServices(svcs ...model.OOAPIService) (epnts []*BaseURL) { for _, svc := range svcs { - base := NewBaseURL(svc.Address) + epnt := NewBaseURL(svc.Address) switch svc.Type { case "cloudfront": - base = base.WithHostOverride(svc.Front) + epnt = epnt.WithHostOverride(svc.Front) fallthrough case "https": - bases = append(bases, base) + epnts = append(epnts, epnt) default: // skip entry } diff --git a/internal/httpclientx/baseurl_test.go b/internal/httpclientx/baseurl_test.go index ab0c4ef416..0643680fb7 100644 --- a/internal/httpclientx/baseurl_test.go +++ b/internal/httpclientx/baseurl_test.go @@ -9,21 +9,21 @@ import ( func TestBaseURL(t *testing.T) { t.Run("the constructor only assigns the URL", func(t *testing.T) { - base := NewBaseURL("https://www.example.com/") - if base.Value != "https://www.example.com/" { + epnt := NewBaseURL("https://www.example.com/") + if epnt.Value != "https://www.example.com/" { t.Fatal("unexpected URL") } - if base.HostOverride != "" { + if epnt.HostOverride != "" { t.Fatal("unexpected host") } }) t.Run("we can optionally get a copy with an assigned host header", func(t *testing.T) { - base := NewBaseURL("https://www.example.com/").WithHostOverride("www.cloudfront.com") - if base.Value != "https://www.example.com/" { + epnt := NewBaseURL("https://www.example.com/").WithHostOverride("www.cloudfront.com") + if epnt.Value != "https://www.example.com/" { t.Fatal("unexpected URL") } - if base.HostOverride != "www.cloudfront.com" { + if epnt.HostOverride != "www.cloudfront.com" { t.Fatal("unexpected host") } }) diff --git a/internal/httpclientx/getjson.go b/internal/httpclientx/getjson.go index ffdc6405c7..587582e128 100644 --- a/internal/httpclientx/getjson.go +++ b/internal/httpclientx/getjson.go @@ -15,18 +15,18 @@ import ( // // - ctx is the cancellable context; // -// - base is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - config contains the config. // // This function either returns an error or a valid Output. -func GetJSON[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { - return OverlappedIgnoreIndex(NewOverlappedGetJSON[Output](config).Run(ctx, base)) +func GetJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { + return OverlappedIgnoreIndex(NewOverlappedGetJSON[Output](config).Run(ctx, epnt)) } -func getJSON[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { +func getJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { // read the raw body - rawrespbody, err := GetRaw(ctx, base, config) + rawrespbody, err := GetRaw(ctx, epnt, config) // handle the case of error if err != nil { diff --git a/internal/httpclientx/getraw.go b/internal/httpclientx/getraw.go index f7c0788f62..d3bb46d316 100644 --- a/internal/httpclientx/getraw.go +++ b/internal/httpclientx/getraw.go @@ -15,22 +15,22 @@ import ( // // - ctx is the cancellable context; // -// - base is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetRaw(ctx context.Context, base *BaseURL, config *Config) ([]byte, error) { - return OverlappedIgnoreIndex(NewOverlappedGetRaw(config).Run(ctx, base)) +func GetRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { + return OverlappedIgnoreIndex(NewOverlappedGetRaw(config).Run(ctx, epnt)) } -func getRaw(ctx context.Context, base *BaseURL, config *Config) ([]byte, error) { +func getRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { // construct the request to use - req, err := http.NewRequestWithContext(ctx, "GET", base.Value, nil) + req, err := http.NewRequestWithContext(ctx, "GET", epnt.Value, nil) if err != nil { return nil, err } // get raw response body - return do(ctx, req, base, config) + return do(ctx, req, epnt, config) } diff --git a/internal/httpclientx/getxml.go b/internal/httpclientx/getxml.go index f3f1cedf0d..fe5e6dc857 100644 --- a/internal/httpclientx/getxml.go +++ b/internal/httpclientx/getxml.go @@ -15,18 +15,18 @@ import ( // // - ctx is the cancellable context; // -// - base is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetXML[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { - return OverlappedIgnoreIndex(NewOverlappedGetXML[Output](config).Run(ctx, base)) +func GetXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { + return OverlappedIgnoreIndex(NewOverlappedGetXML[Output](config).Run(ctx, epnt)) } -func getXML[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) { +func getXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { // read the raw body - rawrespbody, err := GetRaw(ctx, base, config) + rawrespbody, err := GetRaw(ctx, epnt, config) // handle the case of error if err != nil { diff --git a/internal/httpclientx/httpclientx.go b/internal/httpclientx/httpclientx.go index 45d2b5b143..b9c145fd70 100644 --- a/internal/httpclientx/httpclientx.go +++ b/internal/httpclientx/httpclientx.go @@ -42,7 +42,7 @@ func zeroValue[T any]() T { var ErrTruncated = errors.New("httpapi: truncated response body") // do is the internal function to finish preparing the request and getting a raw response. -func do(ctx context.Context, req *http.Request, base *BaseURL, config *Config) ([]byte, error) { +func do(ctx context.Context, req *http.Request, epnt *BaseURL, config *Config) ([]byte, error) { // optionally assign authorization if value := config.Authorization; value != "" { req.Header.Set("Authorization", value) @@ -56,7 +56,7 @@ func do(ctx context.Context, req *http.Request, base *BaseURL, config *Config) ( // OPTIONALLY allow for cloudfronting (the default in net/http is for // the req.Host to be empty and to use req.URL.Host) - req.Host = base.HostOverride + req.Host = epnt.HostOverride // get the response resp, err := config.Client.Do(req) diff --git a/internal/httpclientx/overlapped.go b/internal/httpclientx/overlapped.go index 0abee96458..242df1a9e8 100644 --- a/internal/httpclientx/overlapped.go +++ b/internal/httpclientx/overlapped.go @@ -35,7 +35,7 @@ type Overlapped[Output any] struct { // makes sense for the operation that you requested with the constructor. // // If you set it manually, you MUST modify it before calling [*Overlapped.Run]. - RunFunc func(ctx context.Context, base *BaseURL) (Output, error) + RunFunc func(ctx context.Context, epnt *BaseURL) (Output, error) // ScheduleInterval is the MANDATORY scheduling interval. // @@ -65,29 +65,29 @@ func newOverlappedWithFunc[Output any](fx func(context.Context, *BaseURL) (Outpu // NewOverlappedGetJSON constructs a [*Overlapped] for calling [GetJSON] with multiple URLs. func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) (Output, error) { - return getJSON[Output](ctx, base, config) + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { + return getJSON[Output](ctx, epnt, config) }) } // NewOverlappedGetRaw constructs a [*Overlapped] for calling [GetRaw] with multiple URLs. func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] { - return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) ([]byte, error) { - return getRaw(ctx, base, config) + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) ([]byte, error) { + return getRaw(ctx, epnt, config) }) } // NewOverlappedGetXML constructs a [*Overlapped] for calling [GetXML] with multiple URLs. func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) (Output, error) { - return getXML[Output](ctx, base, config) + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { + return getXML[Output](ctx, epnt, config) }) } // NewOverlappedPostJSON constructs a [*Overlapped] for calling [PostJSON] with multiple URLs. func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, base *BaseURL) (Output, error) { - return postJSON[Input, Output](ctx, base, input, config) + return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { + return postJSON[Input, Output](ctx, epnt, input, config) }) } @@ -96,8 +96,8 @@ var ErrGenericOverlappedFailure = errors.New("overlapped: generic failure") // Run runs the overlapped operations, returning the result of the first operation // that succeeds and its base URL index, or the error that occurred. -func (ovx *Overlapped[Output]) Run(ctx context.Context, bases ...*BaseURL) (Output, int, error) { - return OverlappedReduce[Output](ovx.Map(ctx, bases...)) +func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*BaseURL) (Output, int, error) { + return OverlappedReduce[Output](ovx.Map(ctx, epnts...)) } // OverlappedErrorOr combines error information, result information and the base URL index. @@ -112,15 +112,15 @@ type OverlappedErrorOr[Output any] struct { Value Output } -// Map applies the [*Overlapped.RunFunc] function to each bases entry, thus producing +// Map applies the [*Overlapped.RunFunc] function to each epnts entry, thus producing // a result for each entry. This function will cancel subsequent operations until there // is a success: subsequent results will be [context.Canceled] errors. // // Note that you SHOULD use [*Overlapped.Run] unless you want to observe the result // of each operation, which is mostly useful when running unit tests. // -// Note that this function will return a zero length slice if bases lenth is also zero. -func (ovx *Overlapped[Output]) Map(ctx context.Context, bases ...*BaseURL) []*OverlappedErrorOr[Output] { +// Note that this function will return a zero length slice if epnts lenth is also zero. +func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*OverlappedErrorOr[Output] { // create cancellable context for early cancellation and also apply the // watchdog timeout so that eventually this code returns. // @@ -153,12 +153,12 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, bases ...*BaseURL) []*Ov results := []*OverlappedErrorOr[Output]{} // keep looping until we have results for each base URLs - for len(results) < len(bases) { + for len(results) < len(epnts) { // if there are more base URLs to try, spawn a goroutine to try, // and, otherwise, we can safely stop ticking - if idx < len(bases) { - go ovx.transact(ctx, idx, bases[idx], output) + if idx < len(epnts) { + go ovx.transact(ctx, idx, epnts[idx], output) idx++ } else { ticker.Stop() @@ -170,7 +170,7 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, bases ...*BaseURL) []*Ov // background goroutines and stop ticking // // note that we MUST continue reading until we have - // exactly `len(bases)` results because the inner + // exactly `len(epnts)` results because the inner // goroutine performs blocking writes on the channel case res := <-output: results = append(results, res) @@ -222,9 +222,9 @@ func OverlappedReduce[Output any](results []*OverlappedErrorOr[Output]) (Output, // transact performs an HTTP transaction with the given URL and writes results to the output channel. func (ovx *Overlapped[Output]) transact( - ctx context.Context, idx int, base *BaseURL, output chan<- *OverlappedErrorOr[Output]) { + ctx context.Context, idx int, epnt *BaseURL, output chan<- *OverlappedErrorOr[Output]) { // obtain the results - value, err := ovx.RunFunc(ctx, base) + value, err := ovx.RunFunc(ctx, epnt) // emit the results // diff --git a/internal/httpclientx/postjson.go b/internal/httpclientx/postjson.go index d6a8bcb677..a6f044e4dc 100644 --- a/internal/httpclientx/postjson.go +++ b/internal/httpclientx/postjson.go @@ -17,18 +17,18 @@ import ( // // - ctx is the cancellable context; // -// - base is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*BaseURL] to use; // // - input is the input structure to JSON serialize as the request body; // // - config is the config to use. // // This function either returns an error or a valid Output. -func PostJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input, config *Config) (Output, error) { - return OverlappedIgnoreIndex(NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, base)) +func PostJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { + return OverlappedIgnoreIndex(NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, epnt)) } -func postJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input, config *Config) (Output, error) { +func postJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { // ensure we're not sending a nil map, pointer, or slice if _, err := NilSafetyErrorIfNil(input); err != nil { return zeroValue[Output](), err @@ -41,10 +41,10 @@ func postJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input } // log the raw request body - config.Logger.Debugf("POST %s: raw request body: %s", base.Value, string(rawreqbody)) + config.Logger.Debugf("POST %s: raw request body: %s", epnt.Value, string(rawreqbody)) // construct the request to use - req, err := http.NewRequestWithContext(ctx, "POST", base.Value, bytes.NewReader(rawreqbody)) + req, err := http.NewRequestWithContext(ctx, "POST", epnt.Value, bytes.NewReader(rawreqbody)) if err != nil { return zeroValue[Output](), err } @@ -53,7 +53,7 @@ func postJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input req.Header.Set("Content-Type", "application/json") // get the raw response body - rawrespbody, err := do(ctx, req, base, config) + rawrespbody, err := do(ctx, req, epnt, config) // handle the case of error if err != nil { From 79d3b70d9d57e370f64a200bf4a38c169fd466bd Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 25 Jun 2024 16:25:52 +0200 Subject: [PATCH 8/8] Revert "refactor(httpclientx): use better naming" This reverts commit c369bb3ebb509756312e3be0f1d834e6ff1fc4e1. --- internal/enginelocate/cloudflare.go | 2 +- internal/enginelocate/ubuntu.go | 2 +- internal/httpclientx/DESIGN.md | 32 ++++++------ internal/httpclientx/baseurl.go | 49 ------------------- internal/httpclientx/endpoint.go | 49 +++++++++++++++++++ .../{baseurl_test.go => endpoint_test.go} | 24 ++++----- internal/httpclientx/getjson.go | 6 +-- internal/httpclientx/getjson_test.go | 16 +++--- internal/httpclientx/getraw.go | 8 +-- internal/httpclientx/getraw_test.go | 8 +-- internal/httpclientx/getxml.go | 6 +-- internal/httpclientx/getxml_test.go | 10 ++-- internal/httpclientx/httpclientx.go | 4 +- internal/httpclientx/httpclientx_test.go | 14 +++--- internal/httpclientx/overlapped.go | 38 +++++++------- internal/httpclientx/overlapped_test.go | 38 +++++++------- internal/httpclientx/postjson.go | 10 ++-- internal/httpclientx/postjson_test.go | 26 +++++----- internal/oonirun/v2.go | 2 +- internal/probeservices/bouncer.go | 2 +- internal/probeservices/checkin.go | 2 +- internal/probeservices/collector.go | 4 +- internal/probeservices/login.go | 2 +- internal/probeservices/measurementmeta.go | 2 +- internal/probeservices/openvpn.go | 2 +- internal/probeservices/psiphon.go | 2 +- internal/probeservices/register.go | 2 +- internal/probeservices/tor.go | 2 +- .../webconnectivityalgo/calltesthelpers.go | 2 +- 29 files changed, 183 insertions(+), 183 deletions(-) delete mode 100644 internal/httpclientx/baseurl.go create mode 100644 internal/httpclientx/endpoint.go rename internal/httpclientx/{baseurl_test.go => endpoint_test.go} (63%) diff --git a/internal/enginelocate/cloudflare.go b/internal/enginelocate/cloudflare.go index 13f02cd807..66884eb32a 100644 --- a/internal/enginelocate/cloudflare.go +++ b/internal/enginelocate/cloudflare.go @@ -20,7 +20,7 @@ func cloudflareIPLookup( // get the raw response body data, err := httpclientx.GetRaw( ctx, - httpclientx.NewBaseURL("https://www.cloudflare.com/cdn-cgi/trace"), + httpclientx.NewEndpoint("https://www.cloudflare.com/cdn-cgi/trace"), &httpclientx.Config{ Authorization: "", // not needed Client: httpClient, diff --git a/internal/enginelocate/ubuntu.go b/internal/enginelocate/ubuntu.go index 84257800c0..61393a90a1 100644 --- a/internal/enginelocate/ubuntu.go +++ b/internal/enginelocate/ubuntu.go @@ -24,7 +24,7 @@ func ubuntuIPLookup( // read the HTTP response and parse as XML v, err := httpclientx.GetXML[*ubuntuResponse]( ctx, - httpclientx.NewBaseURL("https://geoip.ubuntu.com/lookup"), + httpclientx.NewEndpoint("https://geoip.ubuntu.com/lookup"), &httpclientx.Config{ Authorization: "", // not needed Client: httpClient, diff --git a/internal/httpclientx/DESIGN.md b/internal/httpclientx/DESIGN.md index 96efc78555..dd3c396f43 100644 --- a/internal/httpclientx/DESIGN.md +++ b/internal/httpclientx/DESIGN.md @@ -76,18 +76,18 @@ type Config struct { UserAgent string } -type BaseURL struct { - Value string // mandatory base-URL value - HostOverride string // optional for cloudfronting +type Endpoint struct { + URL string + Host string // optional for cloudfronting } -func GetJSON[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) +func GetJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) -func GetRaw(ctx context.Context, base *BaseURL, config *Config) ([]byte, error) +func GetRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) -func GetXML[Output any](ctx context.Context, base *BaseURL, config *Config) (Output, error) +func GetXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) -func PostJSON[Input, Output any](ctx context.Context, base *BaseURL, input Input, config *Config) (Output, error) +func PostJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) ``` (The `*Config` is the last argument because it is handy to create it inline when calling @@ -124,9 +124,9 @@ To avoid logging bodies, one just needs to pass `model.DiscardLogger` as the The code at `./internal/httpapi` performs sequential function calls. This design does not interact well with the `enginenetx` package and its dial tactics. A better strategy is to allow calls to be overlapped. This means that, if the `enginenetx` -is busy trying tactics for a given API base URL, we eventually try to use the -subsequent (semantically-equivalent) base URL after a given time, without waiting -for the first base URL to complete. +is busy trying tactics for a given API endpoint, we eventually try to use the +subsequent (semantically-equivalent) endpoint after a given time, without waiting +for the first endpoint to complete. We allow for overlapped operations by defining these constructors: @@ -144,7 +144,7 @@ They all construct the same `*Overlapped` struct, which looks like this: ```Go type Overlapped[Output any] struct { - RunFunc func(ctx context.Context, base *BaseURL) (Output, error) + RunFunc func(ctx context.Context, epnt *Endpoint) (Output, error) ScheduleInterval time.Duration } @@ -156,15 +156,15 @@ name (i.e., `NewOverlappedGetXML` configures `RunFunc` to run `GetXML`). Then, we define the following method: ```Go -func (ovx *Overlapped[Output]) Run(ctx context.Context, bases ...*BaseURL) (Output, error) +func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*Endpoint) (Output, error) ``` -This method starts N goroutines to issue the API calls with each base URL. (A classic example +This method starts N goroutines to issue the API calls with each endpoint URL. (A classic example is for the URLs to be `https://0.th.ooni.org/`, `https://1.th.ooni.org/` and so on.) -By default, `ScheduleInterval` is 15 seconds. If the first base URL does not provide a result +By default, `ScheduleInterval` is 15 seconds. If the first endpoint URL does not provide a result within 15 seconds, we try the second one. That is, every 15 seconds, we will attempt using -another base URL, until there's a successful response or we run out of URLs. +another endpoint URL, until there's a successful response or we run out of URLs. As soon as we have a successful response, we cancel all the other pending operations that may exist. Once all operations have terminated, we return to the caller. @@ -329,7 +329,7 @@ that is part of the same package, while in this package we marshal in `PostJSON` Consider the following code snippet: ```Go -resp, err := httpclientx.GetJSON[*APIResponse](ctx, base, config) +resp, err := httpclientx.GetJSON[*APIResponse](ctx, epnt, config) runtimex.Assert((resp == nil && err != nil) || (resp != nil && err == nil), "ouch") ``` diff --git a/internal/httpclientx/baseurl.go b/internal/httpclientx/baseurl.go deleted file mode 100644 index c977bdc185..0000000000 --- a/internal/httpclientx/baseurl.go +++ /dev/null @@ -1,49 +0,0 @@ -package httpclientx - -import "github.com/ooni/probe-cli/v3/internal/model" - -// BaseURL is an HTTP-endpoint base URL. -// -// The zero value is invalid; construct using [NewBaseURL]. -type BaseURL struct { - // Value is the MANDATORY base-URL Value. - Value string - - // HostOverride is the OPTIONAL host header to use for cloudfronting. - HostOverride string -} - -// NewBaseURL constructs a new [*BaseURL] instance using the given URL. -func NewBaseURL(URL string) *BaseURL { - return &BaseURL{ - Value: URL, - HostOverride: "", - } -} - -// WithHostOverride returns a copy of the [*BaseURL] using the given host header override. -func (e *BaseURL) WithHostOverride(host string) *BaseURL { - return &BaseURL{ - Value: e.Value, - HostOverride: host, - } -} - -// NewBaseURLsFromModelOOAPIServices constructs new [*BaseURL] instances from the -// given [model.OOAPIService] instances, assigning the host header if "cloudfront", and -// skipping all the entries that are neither "https" not "cloudfront". -func NewBaseURLsFromModelOOAPIServices(svcs ...model.OOAPIService) (epnts []*BaseURL) { - for _, svc := range svcs { - epnt := NewBaseURL(svc.Address) - switch svc.Type { - case "cloudfront": - epnt = epnt.WithHostOverride(svc.Front) - fallthrough - case "https": - epnts = append(epnts, epnt) - default: - // skip entry - } - } - return -} diff --git a/internal/httpclientx/endpoint.go b/internal/httpclientx/endpoint.go new file mode 100644 index 0000000000..abe5d86895 --- /dev/null +++ b/internal/httpclientx/endpoint.go @@ -0,0 +1,49 @@ +package httpclientx + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Endpoint is an HTTP endpoint. +// +// The zero value is invalid; construct using [NewEndpoint]. +type Endpoint struct { + // URL is the MANDATORY endpoint URL. + URL string + + // Host is the OPTIONAL host header to use for cloudfronting. + Host string +} + +// NewEndpoint constructs a new [*Endpoint] instance using the given URL. +func NewEndpoint(URL string) *Endpoint { + return &Endpoint{ + URL: URL, + Host: "", + } +} + +// WithHostOverride returns a copy of the [*Endpoint] using the given host header override. +func (e *Endpoint) WithHostOverride(host string) *Endpoint { + return &Endpoint{ + URL: e.URL, + Host: host, + } +} + +// NewEndpointFromModelOOAPIServices constructs new [*Endpoint] instances from the +// given [model.OOAPIService] instances, assigning the host header if "cloudfront", and +// skipping all the entries that are neither "https" not "cloudfront". +func NewEndpointFromModelOOAPIServices(svcs ...model.OOAPIService) (epnts []*Endpoint) { + for _, svc := range svcs { + epnt := NewEndpoint(svc.Address) + switch svc.Type { + case "cloudfront": + epnt = epnt.WithHostOverride(svc.Front) + fallthrough + case "https": + epnts = append(epnts, epnt) + default: + // skip entry + } + } + return +} diff --git a/internal/httpclientx/baseurl_test.go b/internal/httpclientx/endpoint_test.go similarity index 63% rename from internal/httpclientx/baseurl_test.go rename to internal/httpclientx/endpoint_test.go index 0643680fb7..e3ee6ed406 100644 --- a/internal/httpclientx/baseurl_test.go +++ b/internal/httpclientx/endpoint_test.go @@ -7,23 +7,23 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -func TestBaseURL(t *testing.T) { +func TestEndpoint(t *testing.T) { t.Run("the constructor only assigns the URL", func(t *testing.T) { - epnt := NewBaseURL("https://www.example.com/") - if epnt.Value != "https://www.example.com/" { + epnt := NewEndpoint("https://www.example.com/") + if epnt.URL != "https://www.example.com/" { t.Fatal("unexpected URL") } - if epnt.HostOverride != "" { + if epnt.Host != "" { t.Fatal("unexpected host") } }) t.Run("we can optionally get a copy with an assigned host header", func(t *testing.T) { - epnt := NewBaseURL("https://www.example.com/").WithHostOverride("www.cloudfront.com") - if epnt.Value != "https://www.example.com/" { + epnt := NewEndpoint("https://www.example.com/").WithHostOverride("www.cloudfront.com") + if epnt.URL != "https://www.example.com/" { t.Fatal("unexpected URL") } - if epnt.HostOverride != "www.cloudfront.com" { + if epnt.Host != "www.cloudfront.com" { t.Fatal("unexpected host") } }) @@ -50,14 +50,14 @@ func TestBaseURL(t *testing.T) { Front: "", }} - expect := []*BaseURL{{ - Value: "https://www.example.com/", + expect := []*Endpoint{{ + URL: "https://www.example.com/", }, { - Value: "https://www.example.com/", - HostOverride: "www.cloudfront.com", + URL: "https://www.example.com/", + Host: "www.cloudfront.com", }} - got := NewBaseURLsFromModelOOAPIServices(services...) + got := NewEndpointFromModelOOAPIServices(services...) if diff := cmp.Diff(expect, got); diff != "" { t.Fatal(diff) } diff --git a/internal/httpclientx/getjson.go b/internal/httpclientx/getjson.go index 587582e128..346c524238 100644 --- a/internal/httpclientx/getjson.go +++ b/internal/httpclientx/getjson.go @@ -15,16 +15,16 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*Endpoint] to use; // // - config contains the config. // // This function either returns an error or a valid Output. -func GetJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { +func GetJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { return OverlappedIgnoreIndex(NewOverlappedGetJSON[Output](config).Run(ctx, epnt)) } -func getJSON[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { +func getJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { // read the raw body rawrespbody, err := GetRaw(ctx, epnt, config) diff --git a/internal/httpclientx/getjson_test.go b/internal/httpclientx/getjson_test.go index ac6fd31a8a..9cc2dc3a79 100644 --- a/internal/httpclientx/getjson_test.go +++ b/internal/httpclientx/getjson_test.go @@ -28,7 +28,7 @@ func TestGetJSON(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -59,7 +59,7 @@ func TestGetJSON(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -90,7 +90,7 @@ func TestGetJSON(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -137,7 +137,7 @@ func TestGetJSONHeadersOkay(t *testing.T) { // send the request and receive the response apiresp, err := GetJSON[*apiResponse]( context.Background(), - NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), + NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), &Config{ Authorization: "scribai", Client: http.DefaultClient, @@ -194,7 +194,7 @@ func TestGetJSONLoggingOkay(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: logger, @@ -238,7 +238,7 @@ func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := GetJSON[map[string]string]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -268,7 +268,7 @@ func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := GetJSON[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -298,7 +298,7 @@ func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := GetJSON[[]string]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, diff --git a/internal/httpclientx/getraw.go b/internal/httpclientx/getraw.go index d3bb46d316..da155215aa 100644 --- a/internal/httpclientx/getraw.go +++ b/internal/httpclientx/getraw.go @@ -15,18 +15,18 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*Endpoint] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { +func GetRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) { return OverlappedIgnoreIndex(NewOverlappedGetRaw(config).Run(ctx, epnt)) } -func getRaw(ctx context.Context, epnt *BaseURL, config *Config) ([]byte, error) { +func getRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) { // construct the request to use - req, err := http.NewRequestWithContext(ctx, "GET", epnt.Value, nil) + req, err := http.NewRequestWithContext(ctx, "GET", epnt.URL, nil) if err != nil { return nil, err } diff --git a/internal/httpclientx/getraw_test.go b/internal/httpclientx/getraw_test.go index 0965805428..e9616ea7a7 100644 --- a/internal/httpclientx/getraw_test.go +++ b/internal/httpclientx/getraw_test.go @@ -18,7 +18,7 @@ func TestGetRaw(t *testing.T) { rawrespbody, err := GetRaw( context.Background(), - NewBaseURL("\t"), // <- invalid URL that we cannot parse + NewEndpoint("\t"), // <- invalid URL that we cannot parse &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -49,7 +49,7 @@ func TestGetRaw(t *testing.T) { rawrespbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -93,7 +93,7 @@ func TestGetRawHeadersOkay(t *testing.T) { // send the request and receive the response rawresp, err := GetRaw( context.Background(), - NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), + NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), &Config{ Authorization: "scribai", Client: http.DefaultClient, @@ -151,7 +151,7 @@ func TestGetRawLoggingOkay(t *testing.T) { rawrespbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: logger, diff --git a/internal/httpclientx/getxml.go b/internal/httpclientx/getxml.go index fe5e6dc857..8163f3df36 100644 --- a/internal/httpclientx/getxml.go +++ b/internal/httpclientx/getxml.go @@ -15,16 +15,16 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*Endpoint] to use; // // - config is the config to use. // // This function either returns an error or a valid Output. -func GetXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { +func GetXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { return OverlappedIgnoreIndex(NewOverlappedGetXML[Output](config).Run(ctx, epnt)) } -func getXML[Output any](ctx context.Context, epnt *BaseURL, config *Config) (Output, error) { +func getXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) { // read the raw body rawrespbody, err := GetRaw(ctx, epnt, config) diff --git a/internal/httpclientx/getxml_test.go b/internal/httpclientx/getxml_test.go index 91012f7cdd..519bda2056 100644 --- a/internal/httpclientx/getxml_test.go +++ b/internal/httpclientx/getxml_test.go @@ -24,7 +24,7 @@ func TestGetXML(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -55,7 +55,7 @@ func TestGetXML(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -86,7 +86,7 @@ func TestGetXML(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -133,7 +133,7 @@ func TestGetXMLHeadersOkay(t *testing.T) { // send the request and receive the response apiresp, err := GetXML[*apiResponse]( context.Background(), - NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), + NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), &Config{ Authorization: "scribai", Client: http.DefaultClient, @@ -190,7 +190,7 @@ func TestGetXMLLoggingOkay(t *testing.T) { // invoke the API resp, err := GetXML[*apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: logger, diff --git a/internal/httpclientx/httpclientx.go b/internal/httpclientx/httpclientx.go index b9c145fd70..97514b3536 100644 --- a/internal/httpclientx/httpclientx.go +++ b/internal/httpclientx/httpclientx.go @@ -42,7 +42,7 @@ func zeroValue[T any]() T { var ErrTruncated = errors.New("httpapi: truncated response body") // do is the internal function to finish preparing the request and getting a raw response. -func do(ctx context.Context, req *http.Request, epnt *BaseURL, config *Config) ([]byte, error) { +func do(ctx context.Context, req *http.Request, epnt *Endpoint, config *Config) ([]byte, error) { // optionally assign authorization if value := config.Authorization; value != "" { req.Header.Set("Authorization", value) @@ -56,7 +56,7 @@ func do(ctx context.Context, req *http.Request, epnt *BaseURL, config *Config) ( // OPTIONALLY allow for cloudfronting (the default in net/http is for // the req.Host to be empty and to use req.URL.Host) - req.Host = epnt.HostOverride + req.Host = epnt.Host // get the response resp, err := config.Client.Do(req) diff --git a/internal/httpclientx/httpclientx_test.go b/internal/httpclientx/httpclientx_test.go index a7a19385c3..41c5faca21 100644 --- a/internal/httpclientx/httpclientx_test.go +++ b/internal/httpclientx/httpclientx_test.go @@ -47,7 +47,7 @@ func TestGzipDecompression(t *testing.T) { // make sure we can read it respbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -79,7 +79,7 @@ func TestGzipDecompression(t *testing.T) { // attempt to get a response body respbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -109,7 +109,7 @@ func TestGzipDecompression(t *testing.T) { // make sure we can read it respbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -135,7 +135,7 @@ func TestHTTPStatusCodeHandling(t *testing.T) { respbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -168,7 +168,7 @@ func TestHTTPReadBodyErrorsHandling(t *testing.T) { respbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -200,7 +200,7 @@ func TestLimitMaximumBodySize(t *testing.T) { // note: here we're using a small max body size, definitely smaller than what we send respbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, @@ -233,7 +233,7 @@ func TestLimitMaximumBodySize(t *testing.T) { // note: here we're using a small max body size, definitely smaller than the gzip bomb respbody, err := GetRaw( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), &Config{ Client: http.DefaultClient, Logger: model.DiscardLogger, diff --git a/internal/httpclientx/overlapped.go b/internal/httpclientx/overlapped.go index 242df1a9e8..8577e119f8 100644 --- a/internal/httpclientx/overlapped.go +++ b/internal/httpclientx/overlapped.go @@ -28,14 +28,14 @@ const OverlappedDefaultWatchdogTimeout = 5 * time.Minute // call to start while the previous one is still in progress and very slowly downloading // a response. A future implementation MIGHT want to account for this possibility. type Overlapped[Output any] struct { - // RunFunc is the MANDATORY function that fetches the given [*BaseURL]. + // RunFunc is the MANDATORY function that fetches the given [*Endpoint]. // // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw], // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be the proper function that // makes sense for the operation that you requested with the constructor. // // If you set it manually, you MUST modify it before calling [*Overlapped.Run]. - RunFunc func(ctx context.Context, epnt *BaseURL) (Output, error) + RunFunc func(ctx context.Context, epnt *Endpoint) (Output, error) // ScheduleInterval is the MANDATORY scheduling interval. // @@ -55,7 +55,7 @@ type Overlapped[Output any] struct { WatchdogTimeout time.Duration } -func newOverlappedWithFunc[Output any](fx func(context.Context, *BaseURL) (Output, error)) *Overlapped[Output] { +func newOverlappedWithFunc[Output any](fx func(context.Context, *Endpoint) (Output, error)) *Overlapped[Output] { return &Overlapped[Output]{ RunFunc: fx, ScheduleInterval: OverlappedDefaultScheduleInterval, @@ -65,28 +65,28 @@ func newOverlappedWithFunc[Output any](fx func(context.Context, *BaseURL) (Outpu // NewOverlappedGetJSON constructs a [*Overlapped] for calling [GetJSON] with multiple URLs. func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) { return getJSON[Output](ctx, epnt, config) }) } // NewOverlappedGetRaw constructs a [*Overlapped] for calling [GetRaw] with multiple URLs. func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) ([]byte, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) ([]byte, error) { return getRaw(ctx, epnt, config) }) } // NewOverlappedGetXML constructs a [*Overlapped] for calling [GetXML] with multiple URLs. func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) { return getXML[Output](ctx, epnt, config) }) } // NewOverlappedPostJSON constructs a [*Overlapped] for calling [PostJSON] with multiple URLs. func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] { - return newOverlappedWithFunc(func(ctx context.Context, epnt *BaseURL) (Output, error) { + return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) { return postJSON[Input, Output](ctx, epnt, input, config) }) } @@ -95,17 +95,17 @@ func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Over var ErrGenericOverlappedFailure = errors.New("overlapped: generic failure") // Run runs the overlapped operations, returning the result of the first operation -// that succeeds and its base URL index, or the error that occurred. -func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*BaseURL) (Output, int, error) { +// that succeeds and its endpoint index, or the error that occurred. +func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*Endpoint) (Output, int, error) { return OverlappedReduce[Output](ovx.Map(ctx, epnts...)) } -// OverlappedErrorOr combines error information, result information and the base URL index. +// OverlappedErrorOr combines error information, result information and the endpoint index. type OverlappedErrorOr[Output any] struct { // Err is the error or nil. Err error - // Index is the base URL index. + // Index is the endpoint index. Index int // Value is the result. @@ -120,7 +120,7 @@ type OverlappedErrorOr[Output any] struct { // of each operation, which is mostly useful when running unit tests. // // Note that this function will return a zero length slice if epnts lenth is also zero. -func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*OverlappedErrorOr[Output] { +func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*Endpoint) []*OverlappedErrorOr[Output] { // create cancellable context for early cancellation and also apply the // watchdog timeout so that eventually this code returns. // @@ -142,7 +142,7 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*Ov ticker := time.NewTicker(ovx.ScheduleInterval) defer ticker.Stop() - // create index for the next base URL to try + // create index for the next endpoint to try idx := 0 // create vector for collecting results @@ -152,10 +152,10 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*Ov // then we're going to filter the results and produce a final result results := []*OverlappedErrorOr[Output]{} - // keep looping until we have results for each base URLs + // keep looping until we have results for each endpoints for len(results) < len(epnts) { - // if there are more base URLs to try, spawn a goroutine to try, + // if there are more endpoints to try, spawn a goroutine to try, // and, otherwise, we can safely stop ticking if idx < len(epnts) { go ovx.transact(ctx, idx, epnts[idx], output) @@ -180,7 +180,7 @@ func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*BaseURL) []*Ov } // this means the ticker ticked, so we should loop again and - // attempt another base URL because it's time to do that + // attempt another endpoint because it's time to do that case <-ticker.C: } } @@ -208,7 +208,7 @@ func OverlappedReduce[Output any](results []*OverlappedErrorOr[Output]) (Output, // handle the case where there's no error // - // this happens if the user provided no base URLs to measure + // this happens if the user provided no endpoints to measure if len(errorv) <= 0 { errorv = append(errorv, ErrGenericOverlappedFailure) } @@ -222,14 +222,14 @@ func OverlappedReduce[Output any](results []*OverlappedErrorOr[Output]) (Output, // transact performs an HTTP transaction with the given URL and writes results to the output channel. func (ovx *Overlapped[Output]) transact( - ctx context.Context, idx int, epnt *BaseURL, output chan<- *OverlappedErrorOr[Output]) { + ctx context.Context, idx int, epnt *Endpoint, output chan<- *OverlappedErrorOr[Output]) { // obtain the results value, err := ovx.RunFunc(ctx, epnt) // emit the results // // note that this unconditional channel write REQUIRES that we keep reading from - // the results channel in Run until we have a result per input base URL + // the results channel in Run until we have a result per input endpoint output <- &OverlappedErrorOr[Output]{ Err: err, Index: idx, diff --git a/internal/httpclientx/overlapped_test.go b/internal/httpclientx/overlapped_test.go index a0a2912300..e09d4c95da 100644 --- a/internal/httpclientx/overlapped_test.go +++ b/internal/httpclientx/overlapped_test.go @@ -34,7 +34,7 @@ func TestNewOverlappedPostJSONFastRecoverFromEarlyErrors(t *testing.T) { // // Because the first three THs fail fast but the schedule interval is the default (i.e., // 15 seconds), we're testing whether the algorithm allows us to recover quickly from - // failure and check the other base URLs without waiting for too much time. + // failure and check the other endpoints without waiting for too much time. // // Note: before changing the algorithm, this test ran for 45 seconds. Now it runs // for 1s because a previous goroutine terminating with error causes the next @@ -81,10 +81,10 @@ func TestNewOverlappedPostJSONFastRecoverFromEarlyErrors(t *testing.T) { results := overlapped.Map( context.Background(), - NewBaseURL(zeroTh.URL), - NewBaseURL(oneTh.URL), - NewBaseURL(twoTh.URL), - NewBaseURL(threeTh.URL), + NewEndpoint(zeroTh.URL), + NewEndpoint(oneTh.URL), + NewEndpoint(twoTh.URL), + NewEndpoint(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -195,10 +195,10 @@ func TestNewOverlappedPostJSONFirstCallSucceeds(t *testing.T) { results := overlapped.Map( context.Background(), - NewBaseURL(zeroTh.URL), - NewBaseURL(oneTh.URL), - NewBaseURL(twoTh.URL), - NewBaseURL(threeTh.URL), + NewEndpoint(zeroTh.URL), + NewEndpoint(oneTh.URL), + NewEndpoint(twoTh.URL), + NewEndpoint(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -254,7 +254,7 @@ func TestNewOverlappedPostJSONHandlesAllTimeouts(t *testing.T) { // - 2.th.ooni.org causes timeout // - 3.th.ooni.org causes timeout // - // We expect to loop for all base URLs and then discover that all of them + // We expect to loop for all endpoints and then discover that all of them // failed. To make the test ~quick, we reduce the scheduling interval, and // the watchdog timeout. // @@ -312,10 +312,10 @@ func TestNewOverlappedPostJSONHandlesAllTimeouts(t *testing.T) { results := overlapped.Map( context.Background(), - NewBaseURL(zeroTh.URL), - NewBaseURL(oneTh.URL), - NewBaseURL(twoTh.URL), - NewBaseURL(threeTh.URL), + NewEndpoint(zeroTh.URL), + NewEndpoint(oneTh.URL), + NewEndpoint(twoTh.URL), + NewEndpoint(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -422,10 +422,10 @@ func TestNewOverlappedPostJSONResetTimeoutSuccessCanceled(t *testing.T) { results := overlapped.Map( context.Background(), - NewBaseURL(zeroTh.URL), - NewBaseURL(oneTh.URL), - NewBaseURL(twoTh.URL), - NewBaseURL(threeTh.URL), + NewEndpoint(zeroTh.URL), + NewEndpoint(oneTh.URL), + NewEndpoint(twoTh.URL), + NewEndpoint(threeTh.URL), ) runtimex.Assert(len(results) == 4, "unexpected number of results") @@ -517,7 +517,7 @@ func TestNewOverlappedPostJSONWithNoURLs(t *testing.T) { } func TestNewOverlappedWithFuncDefaultsAreCorrect(t *testing.T) { - overlapped := newOverlappedWithFunc(func(ctx context.Context, e *BaseURL) (int, error) { + overlapped := newOverlappedWithFunc(func(ctx context.Context, e *Endpoint) (int, error) { return 1, nil }) if overlapped.ScheduleInterval != 15*time.Second { diff --git a/internal/httpclientx/postjson.go b/internal/httpclientx/postjson.go index a6f044e4dc..1dd2efe67d 100644 --- a/internal/httpclientx/postjson.go +++ b/internal/httpclientx/postjson.go @@ -17,18 +17,18 @@ import ( // // - ctx is the cancellable context; // -// - epnt is the HTTP [*BaseURL] to use; +// - epnt is the HTTP [*Endpoint] to use; // // - input is the input structure to JSON serialize as the request body; // // - config is the config to use. // // This function either returns an error or a valid Output. -func PostJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { +func PostJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) { return OverlappedIgnoreIndex(NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, epnt)) } -func postJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input, config *Config) (Output, error) { +func postJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) { // ensure we're not sending a nil map, pointer, or slice if _, err := NilSafetyErrorIfNil(input); err != nil { return zeroValue[Output](), err @@ -41,10 +41,10 @@ func postJSON[Input, Output any](ctx context.Context, epnt *BaseURL, input Input } // log the raw request body - config.Logger.Debugf("POST %s: raw request body: %s", epnt.Value, string(rawreqbody)) + config.Logger.Debugf("POST %s: raw request body: %s", epnt.URL, string(rawreqbody)) // construct the request to use - req, err := http.NewRequestWithContext(ctx, "POST", epnt.Value, bytes.NewReader(rawreqbody)) + req, err := http.NewRequestWithContext(ctx, "POST", epnt.URL, bytes.NewReader(rawreqbody)) if err != nil { return zeroValue[Output](), err } diff --git a/internal/httpclientx/postjson_test.go b/internal/httpclientx/postjson_test.go index b4e80a55de..7d4a4957d0 100644 --- a/internal/httpclientx/postjson_test.go +++ b/internal/httpclientx/postjson_test.go @@ -28,7 +28,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[chan int, *apiResponse]( context.Background(), - NewBaseURL(""), + NewEndpoint(""), req, &Config{ Client: http.DefaultClient, @@ -53,7 +53,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL("\t"), // <- invalid URL that we cannot parse + NewEndpoint("\t"), // <- invalid URL that we cannot parse req, &Config{ Client: http.DefaultClient, @@ -82,7 +82,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), req, &Config{ Client: http.DefaultClient, @@ -112,7 +112,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), req, &Config{ Client: http.DefaultClient, @@ -153,7 +153,7 @@ func TestPostJSON(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), req, &Config{ Client: http.DefaultClient, @@ -211,7 +211,7 @@ func TestPostJSONCommunicationOkay(t *testing.T) { // send the request and receive the response apiresp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL(server.URL).WithHostOverride("www.cloudfront.com"), + NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"), apireq, &Config{ Authorization: "scribai", @@ -288,7 +288,7 @@ func TestPostJSONLoggingOkay(t *testing.T) { resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), req, &Config{ Client: http.DefaultClient, @@ -335,7 +335,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[map[string]string, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), nil, &Config{ Client: http.DefaultClient, @@ -366,7 +366,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), nil, &Config{ Client: http.DefaultClient, @@ -397,7 +397,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[[]string, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), nil, &Config{ Client: http.DefaultClient, @@ -431,7 +431,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, map[string]string]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), apireq, &Config{ Client: http.DefaultClient, @@ -465,7 +465,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, *apiResponse]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), apireq, &Config{ Client: http.DefaultClient, @@ -499,7 +499,7 @@ func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { // invoke the API resp, err := PostJSON[*apiRequest, []string]( context.Background(), - NewBaseURL(server.URL), + NewEndpoint(server.URL), apireq, &Config{ Client: http.DefaultClient, diff --git a/internal/oonirun/v2.go b/internal/oonirun/v2.go index fe93f663dd..4330b07b76 100644 --- a/internal/oonirun/v2.go +++ b/internal/oonirun/v2.go @@ -66,7 +66,7 @@ func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient, logger model.Logger, URL string) (*V2Descriptor, error) { return httpclientx.GetJSON[*V2Descriptor]( ctx, - httpclientx.NewBaseURL(URL), + httpclientx.NewEndpoint(URL), &httpclientx.Config{ Authorization: "", // not needed Client: client, diff --git a/internal/probeservices/bouncer.go b/internal/probeservices/bouncer.go index 4fe41abf93..392c78c839 100644 --- a/internal/probeservices/bouncer.go +++ b/internal/probeservices/bouncer.go @@ -23,7 +23,7 @@ func (c *Client) GetTestHelpers(ctx context.Context) (map[string][]model.OOAPISe // get the response return httpclientx.GetJSON[map[string][]model.OOAPIService]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), &httpclientx.Config{ Client: c.HTTPClient, Logger: c.Logger, diff --git a/internal/probeservices/checkin.go b/internal/probeservices/checkin.go index 625e17f042..3b766c5a81 100644 --- a/internal/probeservices/checkin.go +++ b/internal/probeservices/checkin.go @@ -26,7 +26,7 @@ func (c Client) CheckIn( // issue the API call resp, err := httpclientx.PostJSON[*model.OOAPICheckInConfig, *model.OOAPICheckInResult]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), &config, &httpclientx.Config{ Authorization: "", // not needed diff --git a/internal/probeservices/collector.go b/internal/probeservices/collector.go index 5d28c83bba..907c749b01 100644 --- a/internal/probeservices/collector.go +++ b/internal/probeservices/collector.go @@ -68,7 +68,7 @@ func (c Client) OpenReport(ctx context.Context, rt model.OOAPIReportTemplate) (R cor, err := httpclientx.PostJSON[model.OOAPIReportTemplate, *model.OOAPICollectorOpenResponse]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), rt, &httpclientx.Config{ Client: c.HTTPClient, @@ -120,7 +120,7 @@ func (r reportChan) SubmitMeasurement(ctx context.Context, m *model.Measurement) updateResponse, err := httpclientx.PostJSON[ model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(r.client.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(r.client.Host), apiReq, &httpclientx.Config{ Client: r.client.HTTPClient, diff --git a/internal/probeservices/login.go b/internal/probeservices/login.go index cc08e8ec47..2b42339547 100644 --- a/internal/probeservices/login.go +++ b/internal/probeservices/login.go @@ -27,7 +27,7 @@ func (c Client) MaybeLogin(ctx context.Context) error { auth, err := httpclientx.PostJSON[*model.OOAPILoginCredentials, *model.OOAPILoginAuth]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), creds, &httpclientx.Config{ Client: c.HTTPClient, diff --git a/internal/probeservices/measurementmeta.go b/internal/probeservices/measurementmeta.go index 121a4b5e1d..4f9bea5416 100644 --- a/internal/probeservices/measurementmeta.go +++ b/internal/probeservices/measurementmeta.go @@ -35,7 +35,7 @@ func (c Client) GetMeasurementMeta( // get the response return httpclientx.GetJSON[*model.OOAPIMeasurementMeta]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), &httpclientx.Config{ Client: c.HTTPClient, Logger: c.Logger, diff --git a/internal/probeservices/openvpn.go b/internal/probeservices/openvpn.go index f2bab0556b..326cb70cb7 100644 --- a/internal/probeservices/openvpn.go +++ b/internal/probeservices/openvpn.go @@ -36,7 +36,7 @@ func (c Client) FetchOpenVPNConfig(ctx context.Context, provider, cc string) (re // use a model.DiscardLogger to avoid logging config return httpclientx.GetJSON[model.OOAPIVPNProviderConfig]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), &httpclientx.Config{ Client: c.HTTPClient, Logger: model.DiscardLogger, diff --git a/internal/probeservices/psiphon.go b/internal/probeservices/psiphon.go index 2c519b2b5e..7c5faac3d5 100644 --- a/internal/probeservices/psiphon.go +++ b/internal/probeservices/psiphon.go @@ -31,7 +31,7 @@ func (c Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { // use a model.DiscardLogger to avoid logging config return httpclientx.GetRaw( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), &httpclientx.Config{ Authorization: s, Client: c.HTTPClient, diff --git a/internal/probeservices/register.go b/internal/probeservices/register.go index c358fc45f4..de2ad472fb 100644 --- a/internal/probeservices/register.go +++ b/internal/probeservices/register.go @@ -39,7 +39,7 @@ func (c Client) MaybeRegister(ctx context.Context, metadata model.OOAPIProbeMeta resp, err := httpclientx.PostJSON[*model.OOAPIRegisterRequest, *model.OOAPIRegisterResponse]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), req, &httpclientx.Config{ Client: c.HTTPClient, diff --git a/internal/probeservices/tor.go b/internal/probeservices/tor.go index b46a6f25f4..262cb05d1f 100644 --- a/internal/probeservices/tor.go +++ b/internal/probeservices/tor.go @@ -36,7 +36,7 @@ func (c Client) FetchTorTargets(ctx context.Context, cc string) (map[string]mode // use a model.DiscardLogger to avoid logging bridges return httpclientx.GetJSON[map[string]model.OOAPITorTarget]( ctx, - httpclientx.NewBaseURL(URL).WithHostOverride(c.Host), + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), &httpclientx.Config{ Authorization: s, Client: c.HTTPClient, diff --git a/internal/webconnectivityalgo/calltesthelpers.go b/internal/webconnectivityalgo/calltesthelpers.go index d28bc6f165..49a8379771 100644 --- a/internal/webconnectivityalgo/calltesthelpers.go +++ b/internal/webconnectivityalgo/calltesthelpers.go @@ -37,7 +37,7 @@ func CallWebConnectivityTestHelper(ctx context.Context, creq *model.THRequest, ) // perform the overlapped HTTP API calls - cresp, idx, err := overlapped.Run(ctx, httpclientx.NewBaseURLsFromModelOOAPIServices(testhelpers...)...) + cresp, idx, err := overlapped.Run(ctx, httpclientx.NewEndpointFromModelOOAPIServices(testhelpers...)...) // handle the case where all test helpers failed if err != nil {