From 71175a0cc43e40e742e1989e5e8e314d8a20bd59 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 8 Aug 2022 20:11:59 +0200 Subject: [PATCH 01/83] refactor: move Engine to the model pkg This mechanical refactoring moves model.Engine to the model package. This changce is functional to subsequent changes to more easily generate and maintain existing experiments. --- internal/engine/experiment.go | 70 ------------------- .../engine/experiment_integration_test.go | 2 +- internal/engine/experimentbuilder.go | 4 +- internal/model/experiment.go | 70 +++++++++++++++++++ internal/oonirun/experiment.go | 4 +- pkg/oonimkall/tasksession.go | 2 +- 6 files changed, 76 insertions(+), 76 deletions(-) diff --git a/internal/engine/experiment.go b/internal/engine/experiment.go index be5a21a023..069e5b6d97 100644 --- a/internal/engine/experiment.go +++ b/internal/engine/experiment.go @@ -26,76 +26,6 @@ func formatTimeNowUTC() string { return time.Now().UTC().Format(dateFormat) } -// Experiment is an experiment instance. -type Experiment interface { - // KibiBytesReceived accounts for the KibiBytes received by the experiment. - KibiBytesReceived() float64 - - // KibiBytesSent is like KibiBytesReceived but for the bytes sent. - KibiBytesSent() float64 - - // Name returns the experiment name. - Name() string - - // GetSummaryKeys returns a data structure containing a - // summary of the test keys for ooniprobe. - GetSummaryKeys(m *model.Measurement) (any, error) - - // ReportID returns the open report's ID, if we have opened a report - // successfully before, or an empty string, otherwise. - // - // Deprecated: new code should use a Submitter. - ReportID() string - - // MeasureAsync runs an async measurement. This operation could post - // one or more measurements onto the returned channel. We'll close the - // channel when we've emitted all the measurements. - // - // Arguments: - // - // - ctx is the context for deadline/cancellation/timeout; - // - // - input is the input (typically a URL but it could also be - // just an endpoint or an empty string for input-less experiments - // such as, e.g., ndt7 and dash). - // - // Return value: - // - // - on success, channel where to post measurements (the channel - // will be closed when done) and nil error; - // - // - on failure, nil channel and non-nil error. - MeasureAsync(ctx context.Context, input string) (<-chan *model.Measurement, error) - - // MeasureWithContext performs a synchronous measurement. - // - // Return value: strictly either a non-nil measurement and - // a nil error or a nil measurement and a non-nil error. - // - // CAVEAT: while this API is perfectly fine for experiments that - // return a single measurement, it will only return the first measurement - // when used with an asynchronous experiment. - MeasureWithContext(ctx context.Context, input string) (measurement *model.Measurement, err error) - - // SaveMeasurement saves a measurement on the specified file path. - // - // Deprecated: new code should use a Saver. - SaveMeasurement(measurement *model.Measurement, filePath string) error - - // SubmitAndUpdateMeasurementContext submits a measurement and updates the - // fields whose value has changed as part of the submission. - // - // Deprecated: new code should use a Submitter. - SubmitAndUpdateMeasurementContext( - ctx context.Context, measurement *model.Measurement) error - - // OpenReportContext will open a report using the given context - // to possibly limit the lifetime of this operation. - // - // Deprecated: new code should use a Submitter. - OpenReportContext(ctx context.Context) error -} - // experiment implements Experiment. type experiment struct { byteCounter *bytecounter.Counter diff --git a/internal/engine/experiment_integration_test.go b/internal/engine/experiment_integration_test.go index dd91b7ffbc..70dfe9e1b7 100644 --- a/internal/engine/experiment_integration_test.go +++ b/internal/engine/experiment_integration_test.go @@ -313,7 +313,7 @@ func TestRunHHFM(t *testing.T) { runexperimentflow(t, builder.NewExperiment(), "") } -func runexperimentflow(t *testing.T, experiment Experiment, input string) { +func runexperimentflow(t *testing.T, experiment model.Experiment, input string) { ctx := context.Background() err := experiment.OpenReportContext(ctx) if err != nil { diff --git a/internal/engine/experimentbuilder.go b/internal/engine/experimentbuilder.go index 64dc7c43d3..af73a99428 100644 --- a/internal/engine/experimentbuilder.go +++ b/internal/engine/experimentbuilder.go @@ -72,7 +72,7 @@ type ExperimentBuilder interface { SetCallbacks(callbacks model.ExperimentCallbacks) // NewExperiment creates the experiment instance. - NewExperiment() Experiment + NewExperiment() model.Experiment } // experimentBuilder implements ExperimentBuilder. @@ -268,7 +268,7 @@ func (b *experimentBuilder) fieldbyname(v interface{}, key string) (reflect.Valu } // NewExperiment creates the experiment -func (b *experimentBuilder) NewExperiment() Experiment { +func (b *experimentBuilder) NewExperiment() model.Experiment { experiment := b.build(b.config) experiment.callbacks = b.callbacks return experiment diff --git a/internal/model/experiment.go b/internal/model/experiment.go index 5c605eff8e..d954e11431 100644 --- a/internal/model/experiment.go +++ b/internal/model/experiment.go @@ -141,3 +141,73 @@ type ExperimentMeasurer interface { // GetSummaryKeys returns summary keys expected by ooni/probe-cli. GetSummaryKeys(*Measurement) (interface{}, error) } + +// Experiment is an experiment instance. +type Experiment interface { + // KibiBytesReceived accounts for the KibiBytes received by the experiment. + KibiBytesReceived() float64 + + // KibiBytesSent is like KibiBytesReceived but for the bytes sent. + KibiBytesSent() float64 + + // Name returns the experiment name. + Name() string + + // GetSummaryKeys returns a data structure containing a + // summary of the test keys for ooniprobe. + GetSummaryKeys(m *Measurement) (any, error) + + // ReportID returns the open report's ID, if we have opened a report + // successfully before, or an empty string, otherwise. + // + // Deprecated: new code should use a Submitter. + ReportID() string + + // MeasureAsync runs an async measurement. This operation could post + // one or more measurements onto the returned channel. We'll close the + // channel when we've emitted all the measurements. + // + // Arguments: + // + // - ctx is the context for deadline/cancellation/timeout; + // + // - input is the input (typically a URL but it could also be + // just an endpoint or an empty string for input-less experiments + // such as, e.g., ndt7 and dash). + // + // Return value: + // + // - on success, channel where to post measurements (the channel + // will be closed when done) and nil error; + // + // - on failure, nil channel and non-nil error. + MeasureAsync(ctx context.Context, input string) (<-chan *Measurement, error) + + // MeasureWithContext performs a synchronous measurement. + // + // Return value: strictly either a non-nil measurement and + // a nil error or a nil measurement and a non-nil error. + // + // CAVEAT: while this API is perfectly fine for experiments that + // return a single measurement, it will only return the first measurement + // when used with an asynchronous experiment. + MeasureWithContext(ctx context.Context, input string) (measurement *Measurement, err error) + + // SaveMeasurement saves a measurement on the specified file path. + // + // Deprecated: new code should use a Saver. + SaveMeasurement(measurement *Measurement, filePath string) error + + // SubmitAndUpdateMeasurementContext submits a measurement and updates the + // fields whose value has changed as part of the submission. + // + // Deprecated: new code should use a Submitter. + SubmitAndUpdateMeasurementContext( + ctx context.Context, measurement *Measurement) error + + // OpenReportContext will open a report using the given context + // to possibly limit the lifetime of this operation. + // + // Deprecated: new code should use a Submitter. + OpenReportContext(ctx context.Context) error +} diff --git a/internal/oonirun/experiment.go b/internal/oonirun/experiment.go index 10cfdfa96e..a8f7a0fa3d 100644 --- a/internal/oonirun/experiment.go +++ b/internal/oonirun/experiment.go @@ -117,7 +117,7 @@ type inputProcessor interface { } // newInputProcessor creates a new inputProcessor instance. -func (ed *Experiment) newInputProcessor(experiment engine.Experiment, +func (ed *Experiment) newInputProcessor(experiment model.Experiment, inputList []model.OOAPIURLInfo, saver engine.Saver, submitter engine.Submitter) inputProcessor { return &engine.InputProcessor{ Annotations: ed.Annotations, @@ -138,7 +138,7 @@ func (ed *Experiment) newInputProcessor(experiment engine.Experiment, } // newSaver creates a new engine.Saver instance. -func (ed *Experiment) newSaver(experiment engine.Experiment) (engine.Saver, error) { +func (ed *Experiment) newSaver(experiment model.Experiment) (engine.Saver, error) { return engine.NewSaver(engine.SaverConfig{ Enabled: !ed.NoJSON, Experiment: experiment, diff --git a/pkg/oonimkall/tasksession.go b/pkg/oonimkall/tasksession.go index 86d2e08434..320258cdb9 100644 --- a/pkg/oonimkall/tasksession.go +++ b/pkg/oonimkall/tasksession.go @@ -72,7 +72,7 @@ func (b *taskExperimentBuilderEngine) NewExperimentInstance() taskExperiment { // taskExperimentEngine wraps ./internal/engine's Experiment. type taskExperimentEngine struct { - engine.Experiment + model.Experiment } var _ taskExperiment = &taskExperimentEngine{} From b94527871a246d3831e89b3f16cf9eb21cf44cde Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 8 Aug 2022 20:24:43 +0200 Subject: [PATCH 02/83] refactor: move ExperimentBuild into the model package Like the previous commit, here the idea is to move things around in a mechanical fashion before working on more substantial changes. --- cmd/ooniprobe/internal/nettests/dnscheck.go | 2 +- cmd/ooniprobe/internal/nettests/nettests.go | 2 +- .../internal/nettests/stunreachability.go | 2 +- .../internal/nettests/web_connectivity.go | 2 +- internal/engine/allexperiments.go | 55 ++++++------- .../engine/experiment_integration_test.go | 2 +- internal/engine/experimentbuilder.go | 80 ++----------------- internal/engine/inputloader.go | 10 +-- internal/engine/inputloader_network_test.go | 2 +- internal/engine/inputloader_test.go | 36 ++++----- internal/engine/session.go | 2 +- internal/model/experiment.go | 70 ++++++++++++++++ internal/oonirun/experiment.go | 4 +- internal/oonirun/session.go | 2 +- pkg/oonimkall/experiment.go | 3 +- pkg/oonimkall/taskmocks_test.go | 4 +- pkg/oonimkall/taskmodel.go | 2 +- pkg/oonimkall/taskrunner.go | 8 +- pkg/oonimkall/taskrunner_test.go | 60 +++++++------- pkg/oonimkall/tasksession.go | 2 +- 20 files changed, 175 insertions(+), 175 deletions(-) diff --git a/cmd/ooniprobe/internal/nettests/dnscheck.go b/cmd/ooniprobe/internal/nettests/dnscheck.go index 7fed35fcb0..54bf283910 100644 --- a/cmd/ooniprobe/internal/nettests/dnscheck.go +++ b/cmd/ooniprobe/internal/nettests/dnscheck.go @@ -16,7 +16,7 @@ func (n DNSCheck) lookupURLs(ctl *Controller) ([]string, error) { // not needed because we have default static input in the engine }, ExperimentName: "dnscheck", - InputPolicy: engine.InputOrStaticDefault, + InputPolicy: model.InputOrStaticDefault, Session: ctl.Session, SourceFiles: ctl.InputFiles, StaticInputs: ctl.Inputs, diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index 2a0a2755a8..3809f8b611 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -123,7 +123,7 @@ func (c *Controller) SetNettestIndex(i, n int) { // // This function will continue to run in most cases but will // immediately halt if something's wrong with the file system. -func (c *Controller) Run(builder engine.ExperimentBuilder, inputs []string) error { +func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error { // This will configure the controller as handler for the callbacks // called by ooni/probe-engine/experiment.Experiment. builder.SetCallbacks(model.ExperimentCallbacks(c)) diff --git a/cmd/ooniprobe/internal/nettests/stunreachability.go b/cmd/ooniprobe/internal/nettests/stunreachability.go index 76baa29478..6b8fe1ca96 100644 --- a/cmd/ooniprobe/internal/nettests/stunreachability.go +++ b/cmd/ooniprobe/internal/nettests/stunreachability.go @@ -16,7 +16,7 @@ func (n STUNReachability) lookupURLs(ctl *Controller) ([]string, error) { // not needed because we have default static input in the engine }, ExperimentName: "stunreachability", - InputPolicy: engine.InputOrStaticDefault, + InputPolicy: model.InputOrStaticDefault, Session: ctl.Session, SourceFiles: ctl.InputFiles, StaticInputs: ctl.Inputs, diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index 9a89b5d558..fd681c3ef7 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -22,7 +22,7 @@ func (n WebConnectivity) lookupURLs(ctl *Controller, categories []string) ([]str }, }, ExperimentName: "web_connectivity", - InputPolicy: engine.InputOrQueryBackend, + InputPolicy: model.InputOrQueryBackend, Session: ctl.Session, SourceFiles: ctl.InputFiles, StaticInputs: ctl.Inputs, diff --git a/internal/engine/allexperiments.go b/internal/engine/allexperiments.go index d3471d7261..68df83371b 100644 --- a/internal/engine/allexperiments.go +++ b/internal/engine/allexperiments.go @@ -34,6 +34,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" + "github.com/ooni/probe-cli/v3/internal/model" ) var experimentsByName = map[string]func(*Session) *experimentBuilder{ @@ -46,7 +47,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ }, config: &dash.Config{}, interruptible: true, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -58,7 +59,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &dnscheck.Config{}, - inputPolicy: InputOrStaticDefault, + inputPolicy: model.InputOrStaticDefault, } }, @@ -70,7 +71,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &dnsping.Config{}, - inputPolicy: InputOrStaticDefault, + inputPolicy: model.InputOrStaticDefault, } }, @@ -86,7 +87,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ SleepTime: int64(time.Second), }, interruptible: true, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -98,7 +99,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &fbmessenger.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -110,7 +111,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &hhfm.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -122,7 +123,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &httphostheader.Config{}, - inputPolicy: InputOrQueryBackend, + inputPolicy: model.InputOrQueryBackend, } }, @@ -134,7 +135,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &hirl.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -147,7 +148,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ }, config: &ndt7.Config{}, interruptible: true, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -159,7 +160,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &psiphon.Config{}, - inputPolicy: InputOptional, + inputPolicy: model.InputOptional, } }, @@ -171,7 +172,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &quicping.Config{}, - inputPolicy: InputStrictlyRequired, + inputPolicy: model.InputStrictlyRequired, } }, @@ -183,7 +184,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &riseupvpn.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -195,7 +196,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &run.Config{}, - inputPolicy: InputStrictlyRequired, + inputPolicy: model.InputStrictlyRequired, } }, @@ -207,7 +208,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &simplequicping.Config{}, - inputPolicy: InputStrictlyRequired, + inputPolicy: model.InputStrictlyRequired, } }, @@ -219,7 +220,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &signal.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -231,7 +232,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &sniblocking.Config{}, - inputPolicy: InputOrQueryBackend, + inputPolicy: model.InputOrQueryBackend, } }, @@ -243,7 +244,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &stunreachability.Config{}, - inputPolicy: InputOrStaticDefault, + inputPolicy: model.InputOrStaticDefault, } }, @@ -255,7 +256,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &tcpping.Config{}, - inputPolicy: InputStrictlyRequired, + inputPolicy: model.InputStrictlyRequired, } }, @@ -267,7 +268,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &tlsping.Config{}, - inputPolicy: InputStrictlyRequired, + inputPolicy: model.InputStrictlyRequired, } }, @@ -279,7 +280,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &telegram.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -291,7 +292,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &tlstool.Config{}, - inputPolicy: InputOrQueryBackend, + inputPolicy: model.InputOrQueryBackend, } }, @@ -303,7 +304,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &tor.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -315,7 +316,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &torsf.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -327,7 +328,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &urlgetter.Config{}, - inputPolicy: InputStrictlyRequired, + inputPolicy: model.InputStrictlyRequired, } }, @@ -339,7 +340,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &vanillator.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, @@ -351,7 +352,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &webconnectivity.Config{}, - inputPolicy: InputOrQueryBackend, + inputPolicy: model.InputOrQueryBackend, } }, @@ -363,7 +364,7 @@ var experimentsByName = map[string]func(*Session) *experimentBuilder{ )) }, config: &whatsapp.Config{}, - inputPolicy: InputNone, + inputPolicy: model.InputNone, } }, } diff --git a/internal/engine/experiment_integration_test.go b/internal/engine/experiment_integration_test.go index 70dfe9e1b7..252140646d 100644 --- a/internal/engine/experiment_integration_test.go +++ b/internal/engine/experiment_integration_test.go @@ -142,7 +142,7 @@ func TestNeedsInput(t *testing.T) { if err != nil { t.Fatal(err) } - if builder.InputPolicy() != InputOrQueryBackend { + if builder.InputPolicy() != model.InputOrQueryBackend { t.Fatal("web_connectivity certainly needs input") } } diff --git a/internal/engine/experimentbuilder.go b/internal/engine/experimentbuilder.go index af73a99428..07492199a7 100644 --- a/internal/engine/experimentbuilder.go +++ b/internal/engine/experimentbuilder.go @@ -14,67 +14,6 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -// InputPolicy describes the experiment policy with respect to input. That is -// whether it requires input, optionally accepts input, does not want input. -type InputPolicy string - -const ( - // InputOrQueryBackend indicates that the experiment requires - // external input to run and that this kind of input is URLs - // from the citizenlab/test-lists repository. If this input - // not provided to the experiment, then the code that runs the - // experiment is supposed to fetch from URLs from OONI's backends. - InputOrQueryBackend = InputPolicy("or_query_backend") - - // InputStrictlyRequired indicates that the experiment - // requires input and we currently don't have an API for - // fetching such input. Therefore, either the user specifies - // input or the experiment will fail for the lack of input. - InputStrictlyRequired = InputPolicy("strictly_required") - - // InputOptional indicates that the experiment handles input, - // if any; otherwise it fetchs input/uses a default. - InputOptional = InputPolicy("optional") - - // InputNone indicates that the experiment does not want any - // input and ignores the input if provided with it. - InputNone = InputPolicy("none") - - // We gather input from StaticInput and SourceFiles. If there is - // input, we return it. Otherwise, we return an internal static - // list of inputs to be used with this experiment. - InputOrStaticDefault = InputPolicy("or_static_default") -) - -// ExperimentBuilder builds an experiment. -type ExperimentBuilder interface { - // Interruptible tells you whether this is an interruptible experiment. This kind - // of experiments (e.g. ndt7) may be interrupted mid way. - Interruptible() bool - - // InputPolicy returns the experiment input policy. - InputPolicy() InputPolicy - - // Options returns information about the experiment's options. - Options() (map[string]OptionInfo, error) - - // SetOptionAny sets an option whose value is an any value. We will use reasonable - // heuristics to convert the any value to the proper type of the field whose name is - // contained by the key variable. If we cannot convert the provided any value to - // the proper type, then this function returns an error. - SetOptionAny(key string, value any) error - - // SetOptionsAny sets options from a map[string]any. See the documentation of - // the SetOptionAny method for more information. - SetOptionsAny(options map[string]any) error - - // SetCallbacks sets the experiment's interactive callbacks. - SetCallbacks(callbacks model.ExperimentCallbacks) - - // NewExperiment creates the experiment instance. - NewExperiment() model.Experiment -} - // experimentBuilder implements ExperimentBuilder. type experimentBuilder struct { // build is the constructor that build an experiment with the given config. @@ -87,7 +26,7 @@ type experimentBuilder struct { config interface{} // inputPolicy contains the experiment's InputPolicy. - inputPolicy InputPolicy + inputPolicy model.InputPolicy // interruptible indicates whether the experiment is interruptible. interruptible bool @@ -99,19 +38,10 @@ func (b *experimentBuilder) Interruptible() bool { } // InputPolicy implements ExperimentBuilder.InputPolicy. -func (b *experimentBuilder) InputPolicy() InputPolicy { +func (b *experimentBuilder) InputPolicy() model.InputPolicy { return b.inputPolicy } -// OptionInfo contains info about an option. -type OptionInfo struct { - // Doc contains the documentation. - Doc string - - // Type contains the type. - Type string -} - var ( // ErrConfigIsNotAStructPointer indicates we expected a pointer to struct. ErrConfigIsNotAStructPointer = errors.New("config is not a struct pointer") @@ -138,8 +68,8 @@ var ( ) // Options implements ExperimentBuilder.Options. -func (b *experimentBuilder) Options() (map[string]OptionInfo, error) { - result := make(map[string]OptionInfo) +func (b *experimentBuilder) Options() (map[string]model.ExperimentOptionInfo, error) { + result := make(map[string]model.ExperimentOptionInfo) ptrinfo := reflect.ValueOf(b.config) if ptrinfo.Kind() != reflect.Ptr { return nil, ErrConfigIsNotAStructPointer @@ -150,7 +80,7 @@ func (b *experimentBuilder) Options() (map[string]OptionInfo, error) { } for i := 0; i < structinfo.NumField(); i++ { field := structinfo.Field(i) - result[field.Name] = OptionInfo{ + result[field.Name] = model.ExperimentOptionInfo{ Doc: field.Tag.Get("ooni"), Type: field.Type.String(), } diff --git a/internal/engine/inputloader.go b/internal/engine/inputloader.go index b6792bfaeb..d519a99c21 100644 --- a/internal/engine/inputloader.go +++ b/internal/engine/inputloader.go @@ -86,7 +86,7 @@ type InputLoader struct { // current experiment. We will not load any input if // the policy says we should not. You MUST fill in // this field. - InputPolicy InputPolicy + InputPolicy model.InputPolicy // Logger is the optional logger that the InputLoader // should be using. If not set, we will use the default @@ -112,13 +112,13 @@ type InputLoader struct { // return a list of URLs because this is the only input we support. func (il *InputLoader) Load(ctx context.Context) ([]model.OOAPIURLInfo, error) { switch il.InputPolicy { - case InputOptional: + case model.InputOptional: return il.loadOptional() - case InputOrQueryBackend: + case model.InputOrQueryBackend: return il.loadOrQueryBackend(ctx) - case InputStrictlyRequired: + case model.InputStrictlyRequired: return il.loadStrictlyRequired(ctx) - case InputOrStaticDefault: + case model.InputOrStaticDefault: return il.loadOrStaticDefault(ctx) default: return il.loadNone() diff --git a/internal/engine/inputloader_network_test.go b/internal/engine/inputloader_network_test.go index 5e82aa51c8..f696a01666 100644 --- a/internal/engine/inputloader_network_test.go +++ b/internal/engine/inputloader_network_test.go @@ -30,7 +30,7 @@ func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { } defer sess.Close() il := &engine.InputLoader{ - InputPolicy: engine.InputOrQueryBackend, + InputPolicy: model.InputOrQueryBackend, Session: sess, } ctx := context.Background() diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index bd0c6ce714..58dfc39445 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -19,7 +19,7 @@ import ( func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) { il := &InputLoader{ StaticInputs: []string{"https://www.google.com/"}, - InputPolicy: InputNone, + InputPolicy: model.InputNone, } ctx := context.Background() out, err := il.Load(ctx) @@ -37,7 +37,7 @@ func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) { "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, - InputPolicy: InputNone, + InputPolicy: model.InputNone, } ctx := context.Background() out, err := il.Load(ctx) @@ -56,7 +56,7 @@ func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, - InputPolicy: InputNone, + InputPolicy: model.InputNone, } ctx := context.Background() out, err := il.Load(ctx) @@ -70,7 +70,7 @@ func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { func TestInputLoaderInputNoneWithNoInput(t *testing.T) { il := &InputLoader{ - InputPolicy: InputNone, + InputPolicy: model.InputNone, } ctx := context.Background() out, err := il.Load(ctx) @@ -84,7 +84,7 @@ func TestInputLoaderInputNoneWithNoInput(t *testing.T) { func TestInputLoaderInputOptionalWithNoInput(t *testing.T) { il := &InputLoader{ - InputPolicy: InputOptional, + InputPolicy: model.InputOptional, } ctx := context.Background() out, err := il.Load(ctx) @@ -103,7 +103,7 @@ func TestInputLoaderInputOptionalWithInput(t *testing.T) { "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, - InputPolicy: InputOptional, + InputPolicy: model.InputOptional, } ctx := context.Background() out, err := il.Load(ctx) @@ -133,7 +133,7 @@ func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { "/nonexistent", "testdata/inputloader2.txt", }, - InputPolicy: InputOptional, + InputPolicy: model.InputOptional, } ctx := context.Background() out, err := il.Load(ctx) @@ -152,7 +152,7 @@ func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, - InputPolicy: InputStrictlyRequired, + InputPolicy: model.InputStrictlyRequired, } ctx := context.Background() out, err := il.Load(ctx) @@ -176,7 +176,7 @@ func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { il := &InputLoader{ - InputPolicy: InputStrictlyRequired, + InputPolicy: model.InputStrictlyRequired, } ctx := context.Background() out, err := il.Load(ctx) @@ -190,7 +190,7 @@ func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { il := &InputLoader{ - InputPolicy: InputStrictlyRequired, + InputPolicy: model.InputStrictlyRequired, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader3.txt", // we want it before inputloader2.txt @@ -215,7 +215,7 @@ func TestInputLoaderInputOrStaticDefaultWithInput(t *testing.T) { "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, - InputPolicy: InputOrStaticDefault, + InputPolicy: model.InputOrStaticDefault, } ctx := context.Background() out, err := il.Load(ctx) @@ -240,7 +240,7 @@ func TestInputLoaderInputOrStaticDefaultWithInput(t *testing.T) { func TestInputLoaderInputOrStaticDefaultWithEmptyFile(t *testing.T) { il := &InputLoader{ ExperimentName: "dnscheck", - InputPolicy: InputOrStaticDefault, + InputPolicy: model.InputOrStaticDefault, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader3.txt", // we want it before inputloader2.txt @@ -260,7 +260,7 @@ func TestInputLoaderInputOrStaticDefaultWithEmptyFile(t *testing.T) { func TestInputLoaderInputOrStaticDefaultWithoutInputDNSCheck(t *testing.T) { il := &InputLoader{ ExperimentName: "dnscheck", - InputPolicy: InputOrStaticDefault, + InputPolicy: model.InputOrStaticDefault, } ctx := context.Background() out, err := il.Load(ctx) @@ -287,7 +287,7 @@ func TestInputLoaderInputOrStaticDefaultWithoutInputDNSCheck(t *testing.T) { func TestInputLoaderInputOrStaticDefaultWithoutInputStunReachability(t *testing.T) { il := &InputLoader{ ExperimentName: "stunreachability", - InputPolicy: InputOrStaticDefault, + InputPolicy: model.InputOrStaticDefault, } ctx := context.Background() out, err := il.Load(ctx) @@ -323,7 +323,7 @@ func TestStaticBareInputForExperimentWorksWithNonCanonicalNames(t *testing.T) { func TestInputLoaderInputOrStaticDefaultWithoutInputOtherName(t *testing.T) { il := &InputLoader{ ExperimentName: "xx", - InputPolicy: InputOrStaticDefault, + InputPolicy: model.InputOrStaticDefault, } ctx := context.Background() out, err := il.Load(ctx) @@ -342,7 +342,7 @@ func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, - InputPolicy: InputOrQueryBackend, + InputPolicy: model.InputOrQueryBackend, } ctx := context.Background() out, err := il.Load(ctx) @@ -377,7 +377,7 @@ func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing } defer sess.Close() il := &InputLoader{ - InputPolicy: InputOrQueryBackend, + InputPolicy: model.InputOrQueryBackend, Session: sess, } ctx, cancel := context.WithCancel(context.Background()) @@ -393,7 +393,7 @@ func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { il := &InputLoader{ - InputPolicy: InputOrQueryBackend, + InputPolicy: model.InputOrQueryBackend, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader3.txt", // we want it before inputloader2.txt diff --git a/internal/engine/session.go b/internal/engine/session.go index bf0ef7d10d..13ec0d0252 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -390,7 +390,7 @@ var ErrAlreadyUsingProxy = errors.New( // NewExperimentBuilder returns a new experiment builder // for the experiment with the given name, or an error if // there's no such experiment with the given name -func (s *Session) NewExperimentBuilder(name string) (ExperimentBuilder, error) { +func (s *Session) NewExperimentBuilder(name string) (model.ExperimentBuilder, error) { eb, err := newExperimentBuilder(s, name) if err != nil { return nil, err diff --git a/internal/model/experiment.go b/internal/model/experiment.go index d954e11431..70aceb6d36 100644 --- a/internal/model/experiment.go +++ b/internal/model/experiment.go @@ -211,3 +211,73 @@ type Experiment interface { // Deprecated: new code should use a Submitter. OpenReportContext(ctx context.Context) error } + +// InputPolicy describes the experiment policy with respect to input. That is +// whether it requires input, optionally accepts input, does not want input. +type InputPolicy string + +const ( + // InputOrQueryBackend indicates that the experiment requires + // external input to run and that this kind of input is URLs + // from the citizenlab/test-lists repository. If this input + // not provided to the experiment, then the code that runs the + // experiment is supposed to fetch from URLs from OONI's backends. + InputOrQueryBackend = InputPolicy("or_query_backend") + + // InputStrictlyRequired indicates that the experiment + // requires input and we currently don't have an API for + // fetching such input. Therefore, either the user specifies + // input or the experiment will fail for the lack of input. + InputStrictlyRequired = InputPolicy("strictly_required") + + // InputOptional indicates that the experiment handles input, + // if any; otherwise it fetchs input/uses a default. + InputOptional = InputPolicy("optional") + + // InputNone indicates that the experiment does not want any + // input and ignores the input if provided with it. + InputNone = InputPolicy("none") + + // We gather input from StaticInput and SourceFiles. If there is + // input, we return it. Otherwise, we return an internal static + // list of inputs to be used with this experiment. + InputOrStaticDefault = InputPolicy("or_static_default") +) + +// ExperimentBuilder builds an experiment. +type ExperimentBuilder interface { + // Interruptible tells you whether this is an interruptible experiment. This kind + // of experiments (e.g. ndt7) may be interrupted mid way. + Interruptible() bool + + // InputPolicy returns the experiment input policy. + InputPolicy() InputPolicy + + // Options returns information about the experiment's options. + Options() (map[string]ExperimentOptionInfo, error) + + // SetOptionAny sets an option whose value is an any value. We will use reasonable + // heuristics to convert the any value to the proper type of the field whose name is + // contained by the key variable. If we cannot convert the provided any value to + // the proper type, then this function returns an error. + SetOptionAny(key string, value any) error + + // SetOptionsAny sets options from a map[string]any. See the documentation of + // the SetOptionAny method for more information. + SetOptionsAny(options map[string]any) error + + // SetCallbacks sets the experiment's interactive callbacks. + SetCallbacks(callbacks ExperimentCallbacks) + + // NewExperiment creates the experiment instance. + NewExperiment() Experiment +} + +// ExperimentOptionInfo contains info about an experiment option. +type ExperimentOptionInfo struct { + // Doc contains the documentation. + Doc string + + // Type contains the type. + Type string +} diff --git a/internal/oonirun/experiment.go b/internal/oonirun/experiment.go index a8f7a0fa3d..25ee39657f 100644 --- a/internal/oonirun/experiment.go +++ b/internal/oonirun/experiment.go @@ -157,7 +157,7 @@ func (ed *Experiment) newSubmitter(ctx context.Context) (engine.Submitter, error } // newExperimentBuilder creates a new engine.ExperimentBuilder for the given experimentName. -func (ed *Experiment) newExperimentBuilder(experimentName string) (engine.ExperimentBuilder, error) { +func (ed *Experiment) newExperimentBuilder(experimentName string) (model.ExperimentBuilder, error) { return ed.Session.NewExperimentBuilder(ed.Name) } @@ -167,7 +167,7 @@ type inputLoader interface { } // newInputLoader creates a new inputLoader. -func (ed *Experiment) newInputLoader(inputPolicy engine.InputPolicy) inputLoader { +func (ed *Experiment) newInputLoader(inputPolicy model.InputPolicy) inputLoader { return &engine.InputLoader{ CheckInConfig: &model.OOAPICheckInConfig{ RunType: model.RunTypeManual, diff --git a/internal/oonirun/session.go b/internal/oonirun/session.go index 9346c5ced9..6075c8db78 100644 --- a/internal/oonirun/session.go +++ b/internal/oonirun/session.go @@ -29,5 +29,5 @@ type Session interface { Logger() model.Logger // NewExperimentBuilder creates a new engine.ExperimentBuilder. - NewExperimentBuilder(name string) (engine.ExperimentBuilder, error) + NewExperimentBuilder(name string) (model.ExperimentBuilder, error) } diff --git a/pkg/oonimkall/experiment.go b/pkg/oonimkall/experiment.go index 4f7077cf17..b8c58f5a20 100644 --- a/pkg/oonimkall/experiment.go +++ b/pkg/oonimkall/experiment.go @@ -3,7 +3,6 @@ package oonimkall import ( "context" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -67,7 +66,7 @@ type experimentBuilder interface { // experimentBuilderWrapper wraps *ExperimentBuilder type experimentBuilderWrapper struct { - eb engine.ExperimentBuilder + eb model.ExperimentBuilder } // newExperiment implements experimentBuilder.newExperiment diff --git a/pkg/oonimkall/taskmocks_test.go b/pkg/oonimkall/taskmocks_test.go index 8b551f6a06..af58a82dd9 100644 --- a/pkg/oonimkall/taskmocks_test.go +++ b/pkg/oonimkall/taskmocks_test.go @@ -87,7 +87,7 @@ type MockableTaskRunnerDependencies struct { // taskExperimentBuilder: MockableSetCallbacks func(callbacks model.ExperimentCallbacks) - MockableInputPolicy func() engine.InputPolicy + MockableInputPolicy func() model.InputPolicy MockableNewExperimentInstance func() taskExperiment MockableInterruptible func() bool @@ -169,7 +169,7 @@ func (dep *MockableTaskRunnerDependencies) SetCallbacks(callbacks model.Experime dep.MockableSetCallbacks(callbacks) } -func (dep *MockableTaskRunnerDependencies) InputPolicy() engine.InputPolicy { +func (dep *MockableTaskRunnerDependencies) InputPolicy() model.InputPolicy { return dep.MockableInputPolicy() } diff --git a/pkg/oonimkall/taskmodel.go b/pkg/oonimkall/taskmodel.go index 12dfdeebcc..cd60ee9b4f 100644 --- a/pkg/oonimkall/taskmodel.go +++ b/pkg/oonimkall/taskmodel.go @@ -227,7 +227,7 @@ type taskExperimentBuilder interface { SetCallbacks(callbacks model.ExperimentCallbacks) // InputPolicy returns the experiment's input policy. - InputPolicy() engine.InputPolicy + InputPolicy() model.InputPolicy // NewExperiment creates the new experiment. NewExperimentInstance() taskExperiment diff --git a/pkg/oonimkall/taskrunner.go b/pkg/oonimkall/taskrunner.go index f63eda23c2..7ec5eb12d9 100644 --- a/pkg/oonimkall/taskrunner.go +++ b/pkg/oonimkall/taskrunner.go @@ -184,12 +184,12 @@ func (r *runnerForTask) Run(rootCtx context.Context) { // In fact, our current app assumes that it's its // responsibility to load the inputs, not oonimkall's. switch builder.InputPolicy() { - case engine.InputOrQueryBackend, engine.InputStrictlyRequired: + case model.InputOrQueryBackend, model.InputStrictlyRequired: if len(r.settings.Inputs) <= 0 { r.emitter.EmitFailureStartup("no input provided") return } - case engine.InputOrStaticDefault: + case model.InputOrStaticDefault: if len(r.settings.Inputs) <= 0 { inputs, err := engine.StaticBareInputForExperiment(r.settings.Name) if err != nil { @@ -198,7 +198,7 @@ func (r *runnerForTask) Run(rootCtx context.Context) { } r.settings.Inputs = inputs } - case engine.InputOptional: + case model.InputOptional: if len(r.settings.Inputs) <= 0 { r.settings.Inputs = append(r.settings.Inputs, "") } @@ -240,7 +240,7 @@ func (r *runnerForTask) Run(rootCtx context.Context) { // this policy in the future, but for now this covers in a // reasonable way web connectivity, so we should be ok. switch builder.InputPolicy() { - case engine.InputOrQueryBackend, engine.InputStrictlyRequired: + case model.InputOrQueryBackend, model.InputStrictlyRequired: var ( cancelMeas context.CancelFunc cancelSubmit context.CancelFunc diff --git a/pkg/oonimkall/taskrunner_test.go b/pkg/oonimkall/taskrunner_test.go index 5e61621ffb..06d8fd66ab 100644 --- a/pkg/oonimkall/taskrunner_test.go +++ b/pkg/oonimkall/taskrunner_test.go @@ -204,8 +204,8 @@ func TestTaskRunnerRun(t *testing.T) { }, MockableSetCallbacks: func(callbacks model.ExperimentCallbacks) { }, - MockableInputPolicy: func() engine.InputPolicy { - return engine.InputNone + MockableInputPolicy: func() model.InputPolicy { + return model.InputNone }, MockableInterruptible: func() bool { return false @@ -310,8 +310,8 @@ func TestTaskRunnerRun(t *testing.T) { t.Run("with missing input and InputOrQueryBackend policy", func(t *testing.T) { runner, emitter := newRunnerForTesting() fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputOrQueryBackend + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputOrQueryBackend } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -331,8 +331,8 @@ func TestTaskRunnerRun(t *testing.T) { t.Run("with missing input and InputStrictlyRequired policy", func(t *testing.T) { runner, emitter := newRunnerForTesting() fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputStrictlyRequired + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputStrictlyRequired } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -355,8 +355,8 @@ func TestTaskRunnerRun(t *testing.T) { runner, emitter := newRunnerForTesting() runner.settings.Name = "Antani" // no input for this experiment fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputOrStaticDefault + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputOrStaticDefault } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -377,8 +377,8 @@ func TestTaskRunnerRun(t *testing.T) { runner, emitter := newRunnerForTesting() runner.settings.Inputs = append(runner.settings.Inputs, "https://x.org/") fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputNone + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputNone } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -419,8 +419,8 @@ func TestTaskRunnerRun(t *testing.T) { t.Run("with success and InputNone policy", func(t *testing.T) { runner, emitter := newRunnerForTesting() fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputNone + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputNone } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -445,8 +445,8 @@ func TestTaskRunnerRun(t *testing.T) { t.Run("with measurement failure and InputNone policy", func(t *testing.T) { runner, emitter := newRunnerForTesting() fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputNone + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputNone } fake.MockableMeasureWithContext = func(ctx context.Context, input string) (measurement *model.Measurement, err error) { return nil, errors.New("preconditions error") @@ -475,8 +475,8 @@ func TestTaskRunnerRun(t *testing.T) { // which is what was happening in the above referenced issue. runner, emitter := newRunnerForTesting() fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputNone + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputNone } fake.MockableMeasureWithContext = func(ctx context.Context, input string) (measurement *model.Measurement, err error) { return nil, errors.New("preconditions error") @@ -506,8 +506,8 @@ func TestTaskRunnerRun(t *testing.T) { runner, emitter := newRunnerForTesting() runner.settings.Inputs = []string{"a", "b", "c", "d"} fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputStrictlyRequired + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputStrictlyRequired } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -554,8 +554,8 @@ func TestTaskRunnerRun(t *testing.T) { runner, emitter := newRunnerForTesting() runner.settings.Inputs = []string{"a", "b", "c", "d"} fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputOptional + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputOptional } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -601,8 +601,8 @@ func TestTaskRunnerRun(t *testing.T) { t.Run("with success and InputOptional and no input", func(t *testing.T) { runner, emitter := newRunnerForTesting() fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputOptional + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputOptional } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -631,8 +631,8 @@ func TestTaskRunnerRun(t *testing.T) { runner, emitter := newRunnerForTesting() runner.settings.Name = experimentName fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputOrStaticDefault + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputOrStaticDefault } runner.sessionBuilder = fake events := runAndCollect(runner, emitter) @@ -667,8 +667,8 @@ func TestTaskRunnerRun(t *testing.T) { runner.settings.Inputs = []string{"a", "b", "c", "d"} runner.settings.Options.MaxRuntime = 2 fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputStrictlyRequired + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputStrictlyRequired } fake.MockableMeasureWithContext = func(ctx context.Context, input string) (measurement *model.Measurement, err error) { time.Sleep(1 * time.Second) @@ -708,8 +708,8 @@ func TestTaskRunnerRun(t *testing.T) { runner.settings.Inputs = []string{"a", "b", "c", "d"} runner.settings.Options.MaxRuntime = 2 fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputStrictlyRequired + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputStrictlyRequired } fake.MockableInterruptible = func() bool { return true @@ -743,8 +743,8 @@ func TestTaskRunnerRun(t *testing.T) { runner, emitter := newRunnerForTesting() runner.settings.Inputs = []string{"a"} fake := fakeSuccessfulRun() - fake.MockableInputPolicy = func() engine.InputPolicy { - return engine.InputStrictlyRequired + fake.MockableInputPolicy = func() model.InputPolicy { + return model.InputStrictlyRequired } fake.MockableSubmitAndUpdateMeasurementContext = func(ctx context.Context, measurement *model.Measurement) error { return errors.New("cannot submit") diff --git a/pkg/oonimkall/tasksession.go b/pkg/oonimkall/tasksession.go index 320258cdb9..27b1dd6192 100644 --- a/pkg/oonimkall/tasksession.go +++ b/pkg/oonimkall/tasksession.go @@ -59,7 +59,7 @@ func (sess *taskSessionEngine) NewExperimentBuilderByName( // taskExperimentBuilderEngine wraps ./internal/engine's // ExperimentBuilder type. type taskExperimentBuilderEngine struct { - engine.ExperimentBuilder + model.ExperimentBuilder } var _ taskExperimentBuilder = &taskExperimentBuilderEngine{} From ce04ee2f2d6bf95c779ce030e306eae7cbde3e28 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 8 Aug 2022 21:37:02 +0200 Subject: [PATCH 03/83] refactor: unentangle experiment builder from engine Here the idea is that the logic for creating an experiment lives in internal/registry along with a list of experiments. In turn, this design will allow us soon to make it very easy to add a new experiment, by making each experiment register itself. However, before doing that, I need to refactor the code so that each experiment and the engine can both import from registry. +---------------+ +--------+ | An experiment | _| engine | +---------------+ | +--------+ | | | | | +----------+ | `----->| registry |<--. imports imports +----------+ Once the above is possible, then each experiment can register itself. --- internal/engine/allexperiments.go | 373 +---------------- .../engine/experiment_integration_test.go | 83 ---- internal/engine/experimentbuilder.go | 202 +--------- internal/engine/experimentbuilder_test.go | 346 ---------------- internal/engine/inputloader.go | 3 +- internal/registry/allexperiments.go | 378 ++++++++++++++++++ internal/registry/doc.go | 2 + internal/registry/factory.go | 224 +++++++++++ internal/registry/factory_test.go | 347 ++++++++++++++++ 9 files changed, 977 insertions(+), 981 deletions(-) create mode 100644 internal/registry/allexperiments.go create mode 100644 internal/registry/doc.go create mode 100644 internal/registry/factory.go create mode 100644 internal/registry/factory_test.go diff --git a/internal/engine/allexperiments.go b/internal/engine/allexperiments.go index 68df83371b..2efacf9db2 100644 --- a/internal/engine/allexperiments.go +++ b/internal/engine/allexperiments.go @@ -1,379 +1,12 @@ package engine // -// List of all implemented experiments +// List of all implemented experiments. // -import ( - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dash" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnsping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/example" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/httphostheader" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/ndt7" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/quicping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/run" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/signal" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/simplequicping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/sniblocking" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/stunreachability" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tcpping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlsping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tor" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" - "github.com/ooni/probe-cli/v3/internal/model" -) - -var experimentsByName = map[string]func(*Session) *experimentBuilder{ - "dash": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, dash.NewExperimentMeasurer( - *config.(*dash.Config), - )) - }, - config: &dash.Config{}, - interruptible: true, - inputPolicy: model.InputNone, - } - }, - - "dnscheck": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, dnscheck.NewExperimentMeasurer( - *config.(*dnscheck.Config), - )) - }, - config: &dnscheck.Config{}, - inputPolicy: model.InputOrStaticDefault, - } - }, - - "dnsping": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, dnsping.NewExperimentMeasurer( - *config.(*dnsping.Config), - )) - }, - config: &dnsping.Config{}, - inputPolicy: model.InputOrStaticDefault, - } - }, - - "example": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, example.NewExperimentMeasurer( - *config.(*example.Config), "example", - )) - }, - config: &example.Config{ - Message: "Good day from the example experiment!", - SleepTime: int64(time.Second), - }, - interruptible: true, - inputPolicy: model.InputNone, - } - }, - - "facebook_messenger": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, fbmessenger.NewExperimentMeasurer( - *config.(*fbmessenger.Config), - )) - }, - config: &fbmessenger.Config{}, - inputPolicy: model.InputNone, - } - }, - - "http_header_field_manipulation": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, hhfm.NewExperimentMeasurer( - *config.(*hhfm.Config), - )) - }, - config: &hhfm.Config{}, - inputPolicy: model.InputNone, - } - }, - - "http_host_header": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, httphostheader.NewExperimentMeasurer( - *config.(*httphostheader.Config), - )) - }, - config: &httphostheader.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "http_invalid_request_line": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, hirl.NewExperimentMeasurer( - *config.(*hirl.Config), - )) - }, - config: &hirl.Config{}, - inputPolicy: model.InputNone, - } - }, - - "ndt": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, ndt7.NewExperimentMeasurer( - *config.(*ndt7.Config), - )) - }, - config: &ndt7.Config{}, - interruptible: true, - inputPolicy: model.InputNone, - } - }, - - "psiphon": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, psiphon.NewExperimentMeasurer( - *config.(*psiphon.Config), - )) - }, - config: &psiphon.Config{}, - inputPolicy: model.InputOptional, - } - }, - - "quicping": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, quicping.NewExperimentMeasurer( - *config.(*quicping.Config), - )) - }, - config: &quicping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "riseupvpn": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, riseupvpn.NewExperimentMeasurer( - *config.(*riseupvpn.Config), - )) - }, - config: &riseupvpn.Config{}, - inputPolicy: model.InputNone, - } - }, - - "run": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, run.NewExperimentMeasurer( - *config.(*run.Config), - )) - }, - config: &run.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "simplequicping": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, simplequicping.NewExperimentMeasurer( - *config.(*simplequicping.Config), - )) - }, - config: &simplequicping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "signal": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, signal.NewExperimentMeasurer( - *config.(*signal.Config), - )) - }, - config: &signal.Config{}, - inputPolicy: model.InputNone, - } - }, - - "sni_blocking": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, sniblocking.NewExperimentMeasurer( - *config.(*sniblocking.Config), - )) - }, - config: &sniblocking.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "stunreachability": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, stunreachability.NewExperimentMeasurer( - *config.(*stunreachability.Config), - )) - }, - config: &stunreachability.Config{}, - inputPolicy: model.InputOrStaticDefault, - } - }, - - "tcpping": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, tcpping.NewExperimentMeasurer( - *config.(*tcpping.Config), - )) - }, - config: &tcpping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "tlsping": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, tlsping.NewExperimentMeasurer( - *config.(*tlsping.Config), - )) - }, - config: &tlsping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "telegram": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, telegram.NewExperimentMeasurer( - *config.(*telegram.Config), - )) - }, - config: &telegram.Config{}, - inputPolicy: model.InputNone, - } - }, - - "tlstool": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, tlstool.NewExperimentMeasurer( - *config.(*tlstool.Config), - )) - }, - config: &tlstool.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "tor": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, tor.NewExperimentMeasurer( - *config.(*tor.Config), - )) - }, - config: &tor.Config{}, - inputPolicy: model.InputNone, - } - }, - - "torsf": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, torsf.NewExperimentMeasurer( - *config.(*torsf.Config), - )) - }, - config: &torsf.Config{}, - inputPolicy: model.InputNone, - } - }, - - "urlgetter": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, urlgetter.NewExperimentMeasurer( - *config.(*urlgetter.Config), - )) - }, - config: &urlgetter.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "vanilla_tor": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, vanillator.NewExperimentMeasurer( - *config.(*vanillator.Config), - )) - }, - config: &vanillator.Config{}, - inputPolicy: model.InputNone, - } - }, - - "web_connectivity": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, webconnectivity.NewExperimentMeasurer( - *config.(*webconnectivity.Config), - )) - }, - config: &webconnectivity.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "whatsapp": func(session *Session) *experimentBuilder { - return &experimentBuilder{ - build: func(config interface{}) *experiment { - return newExperiment(session, whatsapp.NewExperimentMeasurer( - *config.(*whatsapp.Config), - )) - }, - config: &whatsapp.Config{}, - inputPolicy: model.InputNone, - } - }, -} +import "github.com/ooni/probe-cli/v3/internal/registry" // AllExperiments returns the name of all experiments func AllExperiments() []string { - var names []string - for key := range experimentsByName { - names = append(names, key) - } - return names + return registry.ExperimentNames() } diff --git a/internal/engine/experiment_integration_test.go b/internal/engine/experiment_integration_test.go index 252140646d..1ba6431017 100644 --- a/internal/engine/experiment_integration_test.go +++ b/internal/engine/experiment_integration_test.go @@ -12,7 +12,6 @@ import ( "strings" "testing" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/example" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -218,88 +217,6 @@ func TestMeasurementFailure(t *testing.T) { } } -func TestUseOptions(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - sess := newSessionForTesting(t) - defer sess.Close() - builder, err := sess.NewExperimentBuilder("example") - if err != nil { - t.Fatal(err) - } - options, err := builder.Options() - if err != nil { - t.Fatal(err) - } - var ( - returnError bool - message bool - sleepTime bool - other int64 - ) - for name, option := range options { - if name == "ReturnError" { - returnError = true - if option.Type != "bool" { - t.Fatal("ReturnError is not a bool") - } - if option.Doc != "Toogle to return a mocked error" { - t.Fatal("ReturnError doc is wrong") - } - } else if name == "Message" { - message = true - if option.Type != "string" { - t.Fatal("Message is not a string") - } - if option.Doc != "Message to emit at test completion" { - t.Fatal("Message doc is wrong") - } - } else if name == "SleepTime" { - sleepTime = true - if option.Type != "int64" { - t.Fatal("SleepTime is not an int64") - } - if option.Doc != "Amount of time to sleep for" { - t.Fatal("SleepTime doc is wrong") - } - } else { - other++ - } - } - if other != 0 { - t.Fatal("found unexpected option") - } - if !returnError { - t.Fatal("did not find ReturnError option") - } - if !message { - t.Fatal("did not find Message option") - } - if !sleepTime { - t.Fatal("did not find SleepTime option") - } - if err := builder.SetOptionAny("ReturnError", true); err != nil { - t.Fatal("cannot set ReturnError field") - } - if err := builder.SetOptionAny("SleepTime", 10); err != nil { - t.Fatal("cannot set SleepTime field") - } - if err := builder.SetOptionAny("Message", "antani"); err != nil { - t.Fatal("cannot set Message field") - } - config := builder.(*experimentBuilder).config.(*example.Config) - if config.ReturnError != true { - t.Fatal("config.ReturnError was not changed") - } - if config.SleepTime != 10 { - t.Fatal("config.SleepTime was not changed") - } - if config.Message != "antani" { - t.Fatal("config.Message was not changed") - } -} - func TestRunHHFM(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") diff --git a/internal/engine/experimentbuilder.go b/internal/engine/experimentbuilder.go index 07492199a7..3cbd46f218 100644 --- a/internal/engine/experimentbuilder.go +++ b/internal/engine/experimentbuilder.go @@ -5,173 +5,46 @@ package engine // import ( - "errors" - "fmt" - "reflect" - "strconv" - - "github.com/iancoleman/strcase" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/registry" ) // experimentBuilder implements ExperimentBuilder. +// +// This type is now just a tiny wrapper around registry.Factory. type experimentBuilder struct { - // build is the constructor that build an experiment with the given config. - build func(config interface{}) *experiment + factory *registry.Factory // callbacks contains callbacks for the new experiment. callbacks model.ExperimentCallbacks - // config contains the experiment's config. - config interface{} - - // inputPolicy contains the experiment's InputPolicy. - inputPolicy model.InputPolicy - - // interruptible indicates whether the experiment is interruptible. - interruptible bool + // session is the session + session *Session } // Interruptible implements ExperimentBuilder.Interruptible. func (b *experimentBuilder) Interruptible() bool { - return b.interruptible + return b.factory.Interruptible() } // InputPolicy implements ExperimentBuilder.InputPolicy. func (b *experimentBuilder) InputPolicy() model.InputPolicy { - return b.inputPolicy + return b.factory.InputPolicy() } -var ( - // ErrConfigIsNotAStructPointer indicates we expected a pointer to struct. - ErrConfigIsNotAStructPointer = errors.New("config is not a struct pointer") - - // ErrNoSuchField indicates there's no field with the given name. - ErrNoSuchField = errors.New("no such field") - - // ErrCannotSetIntegerOption means SetOptionAny couldn't set an integer option. - ErrCannotSetIntegerOption = errors.New("cannot set integer option") - - // ErrInvalidStringRepresentationOfBool indicates the string you passed - // to SetOptionaAny is not a valid string representation of a bool. - ErrInvalidStringRepresentationOfBool = errors.New("invalid string representation of bool") - - // ErrCannotSetBoolOption means SetOptionAny couldn't set a bool option. - ErrCannotSetBoolOption = errors.New("cannot set bool option") - - // ErrCannotSetStringOption means SetOptionAny couldn't set a string option. - ErrCannotSetStringOption = errors.New("cannot set string option") - - // ErrUnsupportedOptionType means we don't support the type passed to - // the SetOptionAny method as an opaque any type. - ErrUnsupportedOptionType = errors.New("unsupported option type") -) - // Options implements ExperimentBuilder.Options. func (b *experimentBuilder) Options() (map[string]model.ExperimentOptionInfo, error) { - result := make(map[string]model.ExperimentOptionInfo) - ptrinfo := reflect.ValueOf(b.config) - if ptrinfo.Kind() != reflect.Ptr { - return nil, ErrConfigIsNotAStructPointer - } - structinfo := ptrinfo.Elem().Type() - if structinfo.Kind() != reflect.Struct { - return nil, ErrConfigIsNotAStructPointer - } - for i := 0; i < structinfo.NumField(); i++ { - field := structinfo.Field(i) - result[field.Name] = model.ExperimentOptionInfo{ - Doc: field.Tag.Get("ooni"), - Type: field.Type.String(), - } - } - return result, nil -} - -// setOptionBool sets a bool option. -func (b *experimentBuilder) setOptionBool(field reflect.Value, value any) error { - switch v := value.(type) { - case bool: - field.SetBool(v) - return nil - case string: - if v != "true" && v != "false" { - return fmt.Errorf("%w: %s", ErrInvalidStringRepresentationOfBool, v) - } - field.SetBool(v == "true") - return nil - default: - return fmt.Errorf("%w from a value of type %T", ErrCannotSetBoolOption, value) - } -} - -// setOptionInt sets an int option -func (b *experimentBuilder) setOptionInt(field reflect.Value, value any) error { - switch v := value.(type) { - case int64: - field.SetInt(v) - return nil - case int32: - field.SetInt(int64(v)) - return nil - case int16: - field.SetInt(int64(v)) - return nil - case int8: - field.SetInt(int64(v)) - return nil - case int: - field.SetInt(int64(v)) - return nil - case string: - number, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return fmt.Errorf("%w: %s", ErrCannotSetIntegerOption, err.Error()) - } - field.SetInt(number) - return nil - default: - return fmt.Errorf("%w from a value of type %T", ErrCannotSetIntegerOption, value) - } -} - -// setOptionString sets a string option -func (b *experimentBuilder) setOptionString(field reflect.Value, value any) error { - switch v := value.(type) { - case string: - field.SetString(v) - return nil - default: - return fmt.Errorf("%w from a value of type %T", ErrCannotSetStringOption, value) - } + return b.factory.Options() } // SetOptionAny implements ExperimentBuilder.SetOptionAny. func (b *experimentBuilder) SetOptionAny(key string, value any) error { - field, err := b.fieldbyname(b.config, key) - if err != nil { - return err - } - switch field.Kind() { - case reflect.Int64: - return b.setOptionInt(field, value) - case reflect.Bool: - return b.setOptionBool(field, value) - case reflect.String: - return b.setOptionString(field, value) - default: - return fmt.Errorf("%w: %T", ErrUnsupportedOptionType, value) - } + return b.factory.SetOptionAny(key, value) } // SetOptionsAny implements ExperimentBuilder.SetOptionsAny. func (b *experimentBuilder) SetOptionsAny(options map[string]any) error { - for key, value := range options { - if err := b.SetOptionAny(key, value); err != nil { - return err - } - } - return nil + return b.factory.SetOptionsAny(options) } // SetCallbacks implements ExperimentBuilder.SetCallbacks. @@ -179,57 +52,24 @@ func (b *experimentBuilder) SetCallbacks(callbacks model.ExperimentCallbacks) { b.callbacks = callbacks } -// fieldbyname return v's field whose name is equal to the given key. -func (b *experimentBuilder) fieldbyname(v interface{}, key string) (reflect.Value, error) { - // See https://stackoverflow.com/a/6396678/4354461 - ptrinfo := reflect.ValueOf(v) - if ptrinfo.Kind() != reflect.Ptr { - return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) - } - structinfo := ptrinfo.Elem() - if structinfo.Kind() != reflect.Struct { - return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) - } - field := structinfo.FieldByName(key) - if !field.IsValid() || !field.CanSet() { - return reflect.Value{}, fmt.Errorf("%w: %s", ErrNoSuchField, key) - } - return field, nil -} - // NewExperiment creates the experiment func (b *experimentBuilder) NewExperiment() model.Experiment { - experiment := b.build(b.config) + measurer := b.factory.NewExperimentMeasurer() + experiment := newExperiment(b.session, measurer) experiment.callbacks = b.callbacks return experiment } -// canonicalizeExperimentName allows code to provide experiment names -// in a more flexible way, where we have aliases. -// -// Because we allow for uppercase experiment names for backwards -// compatibility with MK, we need to add some exceptions here when -// mapping (e.g., DNSCheck => dnscheck). -func canonicalizeExperimentName(name string) string { - switch name = strcase.ToSnake(name); name { - case "ndt_7": - name = "ndt" // since 2020-03-18, we use ndt7 to implement ndt by default - case "dns_check": - name = "dnscheck" - case "stun_reachability": - name = "stunreachability" - default: - } - return name -} - // newExperimentBuilder creates a new experimentBuilder instance. func newExperimentBuilder(session *Session, name string) (*experimentBuilder, error) { - factory := experimentsByName[canonicalizeExperimentName(name)] - if factory == nil { - return nil, fmt.Errorf("no such experiment: %s", name) + factory, err := registry.NewFactory(name) + if err != nil { + return nil, err + } + builder := &experimentBuilder{ + factory: factory, + callbacks: model.NewPrinterCallbacks(session.Logger()), + session: session, } - builder := factory(session) - builder.callbacks = model.NewPrinterCallbacks(session.Logger()) return builder, nil } diff --git a/internal/engine/experimentbuilder_test.go b/internal/engine/experimentbuilder_test.go index 9197032785..00a22ef62d 100644 --- a/internal/engine/experimentbuilder_test.go +++ b/internal/engine/experimentbuilder_test.go @@ -1,347 +1 @@ package engine - -import ( - "errors" - "testing" - - "github.com/google/go-cmp/cmp" -) - -type fakeExperimentConfig struct { - Chan chan any `ooni:"we cannot set this"` - String string `ooni:"a string"` - Truth bool `ooni:"something that no-one knows"` - Value int64 `ooni:"a number"` -} - -func TestExperimentBuilderOptions(t *testing.T) { - t.Run("when config is not a pointer", func(t *testing.T) { - b := &experimentBuilder{ - config: 17, - } - options, err := b.Options() - if !errors.Is(err, ErrConfigIsNotAStructPointer) { - t.Fatal("expected an error here") - } - if options != nil { - t.Fatal("expected nil here") - } - }) - - t.Run("when config is not a struct", func(t *testing.T) { - number := 17 - b := &experimentBuilder{ - config: &number, - } - options, err := b.Options() - if !errors.Is(err, ErrConfigIsNotAStructPointer) { - t.Fatal("expected an error here") - } - if options != nil { - t.Fatal("expected nil here") - } - }) - - t.Run("when config is a pointer to struct", func(t *testing.T) { - config := &fakeExperimentConfig{} - b := &experimentBuilder{ - config: config, - } - options, err := b.Options() - if err != nil { - t.Fatal(err) - } - for name, value := range options { - switch name { - case "Chan": - if value.Doc != "we cannot set this" { - t.Fatal("invalid doc") - } - if value.Type != "chan interface {}" { - t.Fatal("invalid type", value.Type) - } - case "String": - if value.Doc != "a string" { - t.Fatal("invalid doc") - } - if value.Type != "string" { - t.Fatal("invalid type", value.Type) - } - case "Truth": - if value.Doc != "something that no-one knows" { - t.Fatal("invalid doc") - } - if value.Type != "bool" { - t.Fatal("invalid type", value.Type) - } - case "Value": - if value.Doc != "a number" { - t.Fatal("invalid doc") - } - if value.Type != "int64" { - t.Fatal("invalid type", value.Type) - } - default: - t.Fatal("unknown name", name) - } - } - }) -} - -func TestExperimentBuilderSetOptionAny(t *testing.T) { - var inputs = []struct { - TestCaseName string - InitialConfig any - FieldName string - FieldValue any - ExpectErr error - ExpectConfig any - }{{ - TestCaseName: "config is not a pointer", - InitialConfig: fakeExperimentConfig{}, - FieldName: "Antani", - FieldValue: true, - ExpectErr: ErrConfigIsNotAStructPointer, - ExpectConfig: fakeExperimentConfig{}, - }, { - TestCaseName: "config is not a pointer to struct", - InitialConfig: func() *int { - v := 17 - return &v - }(), - FieldName: "Antani", - FieldValue: true, - ExpectErr: ErrConfigIsNotAStructPointer, - ExpectConfig: func() *int { - v := 17 - return &v - }(), - }, { - TestCaseName: "for missing field", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Antani", - FieldValue: true, - ExpectErr: ErrNoSuchField, - ExpectConfig: &fakeExperimentConfig{}, - }, { - TestCaseName: "[bool] for true value represented as string", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Truth", - FieldValue: "true", - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Truth: true, - }, - }, { - TestCaseName: "[bool] for false value represented as string", - InitialConfig: &fakeExperimentConfig{ - Truth: true, - }, - FieldName: "Truth", - FieldValue: "false", - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Truth: false, // must have been flipped - }, - }, { - TestCaseName: "[bool] for true value", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Truth", - FieldValue: true, - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Truth: true, - }, - }, { - TestCaseName: "[bool] for false value", - InitialConfig: &fakeExperimentConfig{ - Truth: true, - }, - FieldName: "Truth", - FieldValue: false, - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Truth: false, // must have been flipped - }, - }, { - TestCaseName: "[bool] for invalid string representation of bool", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Truth", - FieldValue: "xxx", - ExpectErr: ErrInvalidStringRepresentationOfBool, - ExpectConfig: &fakeExperimentConfig{}, - }, { - TestCaseName: "[bool] for value we don't know how to convert to bool", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Truth", - FieldValue: make(chan any), - ExpectErr: ErrCannotSetBoolOption, - ExpectConfig: &fakeExperimentConfig{}, - }, { - TestCaseName: "[int] for int", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: 17, - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Value: 17, - }, - }, { - TestCaseName: "[int] for int64", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: int64(17), - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Value: 17, - }, - }, { - TestCaseName: "[int] for int32", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: int32(17), - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Value: 17, - }, - }, { - TestCaseName: "[int] for int16", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: int16(17), - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Value: 17, - }, - }, { - TestCaseName: "[int] for int8", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: int8(17), - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Value: 17, - }, - }, { - TestCaseName: "[int] for string representation of int", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: "17", - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - Value: 17, - }, - }, { - TestCaseName: "[int] for invalid string representation of int", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: "xx", - ExpectErr: ErrCannotSetIntegerOption, - ExpectConfig: &fakeExperimentConfig{}, - }, { - TestCaseName: "[int] for type we don't know how to convert to int", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Value", - FieldValue: make(chan any), - ExpectErr: ErrCannotSetIntegerOption, - ExpectConfig: &fakeExperimentConfig{}, - }, { - TestCaseName: "[string] for serialized bool value while setting a string value", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "String", - FieldValue: "true", - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - String: "true", - }, - }, { - TestCaseName: "[string] for serialized int value while setting a string value", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "String", - FieldValue: "155", - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - String: "155", - }, - }, { - TestCaseName: "[string] for any other string", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "String", - FieldValue: "xxx", - ExpectErr: nil, - ExpectConfig: &fakeExperimentConfig{ - String: "xxx", - }, - }, { - TestCaseName: "[string] for type we don't know how to convert to string", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "String", - FieldValue: make(chan any), - ExpectErr: ErrCannotSetStringOption, - ExpectConfig: &fakeExperimentConfig{}, - }, { - TestCaseName: "for a field that we don't know how to set", - InitialConfig: &fakeExperimentConfig{}, - FieldName: "Chan", - FieldValue: make(chan any), - ExpectErr: ErrUnsupportedOptionType, - ExpectConfig: &fakeExperimentConfig{}, - }} - - for _, input := range inputs { - t.Run(input.TestCaseName, func(t *testing.T) { - ec := input.InitialConfig - b := &experimentBuilder{config: ec} - err := b.SetOptionAny(input.FieldName, input.FieldValue) - if !errors.Is(err, input.ExpectErr) { - t.Fatal(err) - } - if diff := cmp.Diff(input.ExpectConfig, ec); diff != "" { - t.Fatal(diff) - } - }) - } -} - -func TestExperimentBuilderSetOptionsAny(t *testing.T) { - b := &experimentBuilder{config: &fakeExperimentConfig{}} - - t.Run("we correctly handle an empty map", func(t *testing.T) { - if err := b.SetOptionsAny(nil); err != nil { - t.Fatal(err) - } - }) - - t.Run("we correctly handle a map containing options", func(t *testing.T) { - f := &fakeExperimentConfig{} - privateb := &experimentBuilder{config: f} - opts := map[string]any{ - "String": "yoloyolo", - "Value": "174", - "Truth": "true", - } - if err := privateb.SetOptionsAny(opts); err != nil { - t.Fatal(err) - } - if f.String != "yoloyolo" { - t.Fatal("cannot set string value") - } - if f.Value != 174 { - t.Fatal("cannot set integer value") - } - if f.Truth != true { - t.Fatal("cannot set bool value") - } - }) - - t.Run("we handle mistakes in a map containing string options", func(t *testing.T) { - opts := map[string]any{ - "String": "yoloyolo", - "Value": "xx", - "Truth": "true", - } - if err := b.SetOptionsAny(opts); !errors.Is(err, ErrCannotSetIntegerOption) { - t.Fatal("unexpected err", err) - } - }) -} diff --git a/internal/engine/inputloader.go b/internal/engine/inputloader.go index d519a99c21..172800c62c 100644 --- a/internal/engine/inputloader.go +++ b/internal/engine/inputloader.go @@ -11,6 +11,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/fsx" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/registry" "github.com/ooni/probe-cli/v3/internal/stuninput" ) @@ -299,7 +300,7 @@ func StaticBareInputForExperiment(name string) ([]string, error) { // Implementation note: we may be called from pkg/oonimkall // with a non-canonical experiment name, so we need to convert // the experiment name to be canonical before proceeding. - switch canonicalizeExperimentName(name) { + switch registry.CanonicalizeExperimentName(name) { case "dnscheck": return dnsCheckDefaultInput, nil case "stunreachability": diff --git a/internal/registry/allexperiments.go b/internal/registry/allexperiments.go new file mode 100644 index 0000000000..5448524079 --- /dev/null +++ b/internal/registry/allexperiments.go @@ -0,0 +1,378 @@ +package registry + +// +// List of all implemented experiments. +// + +import ( + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dash" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnsping" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/example" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/httphostheader" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/ndt7" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/quicping" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/run" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/signal" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/simplequicping" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/sniblocking" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/stunreachability" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tcpping" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlsping" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tor" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" + "github.com/ooni/probe-cli/v3/internal/model" +) + +var experimentsByName = map[string]func() *Factory{ + "dash": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dash.NewExperimentMeasurer( + *config.(*dash.Config), + ) + }, + config: &dash.Config{}, + interruptible: true, + inputPolicy: model.InputNone, + } + }, + + "dnscheck": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dnscheck.NewExperimentMeasurer( + *config.(*dnscheck.Config), + ) + }, + config: &dnscheck.Config{}, + inputPolicy: model.InputOrStaticDefault, + } + }, + + "dnsping": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dnsping.NewExperimentMeasurer( + *config.(*dnsping.Config), + ) + }, + config: &dnsping.Config{}, + inputPolicy: model.InputOrStaticDefault, + } + }, + + "example": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return example.NewExperimentMeasurer( + *config.(*example.Config), "example", + ) + }, + config: &example.Config{ + Message: "Good day from the example experiment!", + SleepTime: int64(time.Second), + }, + interruptible: true, + inputPolicy: model.InputNone, + } + }, + + "facebook_messenger": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return fbmessenger.NewExperimentMeasurer( + *config.(*fbmessenger.Config), + ) + }, + config: &fbmessenger.Config{}, + inputPolicy: model.InputNone, + } + }, + + "http_header_field_manipulation": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return hhfm.NewExperimentMeasurer( + *config.(*hhfm.Config), + ) + }, + config: &hhfm.Config{}, + inputPolicy: model.InputNone, + } + }, + + "http_host_header": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return httphostheader.NewExperimentMeasurer( + *config.(*httphostheader.Config), + ) + }, + config: &httphostheader.Config{}, + inputPolicy: model.InputOrQueryBackend, + } + }, + + "http_invalid_request_line": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return hirl.NewExperimentMeasurer( + *config.(*hirl.Config), + ) + }, + config: &hirl.Config{}, + inputPolicy: model.InputNone, + } + }, + + "ndt": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return ndt7.NewExperimentMeasurer( + *config.(*ndt7.Config), + ) + }, + config: &ndt7.Config{}, + interruptible: true, + inputPolicy: model.InputNone, + } + }, + + "psiphon": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return psiphon.NewExperimentMeasurer( + *config.(*psiphon.Config), + ) + }, + config: &psiphon.Config{}, + inputPolicy: model.InputOptional, + } + }, + + "quicping": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return quicping.NewExperimentMeasurer( + *config.(*quicping.Config), + ) + }, + config: &quicping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } + }, + + "riseupvpn": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return riseupvpn.NewExperimentMeasurer( + *config.(*riseupvpn.Config), + ) + }, + config: &riseupvpn.Config{}, + inputPolicy: model.InputNone, + } + }, + + "run": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return run.NewExperimentMeasurer( + *config.(*run.Config), + ) + }, + config: &run.Config{}, + inputPolicy: model.InputStrictlyRequired, + } + }, + + "simplequicping": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return simplequicping.NewExperimentMeasurer( + *config.(*simplequicping.Config), + ) + }, + config: &simplequicping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } + }, + + "signal": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return signal.NewExperimentMeasurer( + *config.(*signal.Config), + ) + }, + config: &signal.Config{}, + inputPolicy: model.InputNone, + } + }, + + "sni_blocking": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return sniblocking.NewExperimentMeasurer( + *config.(*sniblocking.Config), + ) + }, + config: &sniblocking.Config{}, + inputPolicy: model.InputOrQueryBackend, + } + }, + + "stunreachability": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return stunreachability.NewExperimentMeasurer( + *config.(*stunreachability.Config), + ) + }, + config: &stunreachability.Config{}, + inputPolicy: model.InputOrStaticDefault, + } + }, + + "tcpping": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tcpping.NewExperimentMeasurer( + *config.(*tcpping.Config), + ) + }, + config: &tcpping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } + }, + + "tlsping": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tlsping.NewExperimentMeasurer( + *config.(*tlsping.Config), + ) + }, + config: &tlsping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } + }, + + "telegram": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return telegram.NewExperimentMeasurer( + *config.(*telegram.Config), + ) + }, + config: &telegram.Config{}, + inputPolicy: model.InputNone, + } + }, + + "tlstool": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tlstool.NewExperimentMeasurer( + *config.(*tlstool.Config), + ) + }, + config: &tlstool.Config{}, + inputPolicy: model.InputOrQueryBackend, + } + }, + + "tor": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tor.NewExperimentMeasurer( + *config.(*tor.Config), + ) + }, + config: &tor.Config{}, + inputPolicy: model.InputNone, + } + }, + + "torsf": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return torsf.NewExperimentMeasurer( + *config.(*torsf.Config), + ) + }, + config: &torsf.Config{}, + inputPolicy: model.InputNone, + } + }, + + "urlgetter": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return urlgetter.NewExperimentMeasurer( + *config.(*urlgetter.Config), + ) + }, + config: &urlgetter.Config{}, + inputPolicy: model.InputStrictlyRequired, + } + }, + + "vanilla_tor": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return vanillator.NewExperimentMeasurer( + *config.(*vanillator.Config), + ) + }, + config: &vanillator.Config{}, + inputPolicy: model.InputNone, + } + }, + + "web_connectivity": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return webconnectivity.NewExperimentMeasurer( + *config.(*webconnectivity.Config), + ) + }, + config: &webconnectivity.Config{}, + inputPolicy: model.InputOrQueryBackend, + } + }, + + "whatsapp": func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return whatsapp.NewExperimentMeasurer( + *config.(*whatsapp.Config), + ) + }, + config: &whatsapp.Config{}, + inputPolicy: model.InputNone, + } + }, +} + +// ExperimentNames returns the name of all experiments +func ExperimentNames() (names []string) { + for key := range experimentsByName { + names = append(names, key) + } + return +} diff --git a/internal/registry/doc.go b/internal/registry/doc.go new file mode 100644 index 0000000000..6929580df1 --- /dev/null +++ b/internal/registry/doc.go @@ -0,0 +1,2 @@ +// Package registry contains a registry of all the available experiments. +package registry diff --git a/internal/registry/factory.go b/internal/registry/factory.go new file mode 100644 index 0000000000..60d61d27b1 --- /dev/null +++ b/internal/registry/factory.go @@ -0,0 +1,224 @@ +package registry + +// +// Factory for constructing experiments. +// + +import ( + "errors" + "fmt" + "reflect" + "strconv" + + "github.com/iancoleman/strcase" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Factory allows to construct an experiment measurer. +type Factory struct { + // build is the constructor that build an experiment with the given config. + build func(config interface{}) model.ExperimentMeasurer + + // config contains the experiment's config. + config any + + // inputPolicy contains the experiment's InputPolicy. + inputPolicy model.InputPolicy + + // interruptible indicates whether the experiment is interruptible. + interruptible bool +} + +// Interruptible returns whether the experiment is interruptible. +func (b *Factory) Interruptible() bool { + return b.interruptible +} + +// InputPolicy returns the experiment's InputPolicy. +func (b *Factory) InputPolicy() model.InputPolicy { + return b.inputPolicy +} + +var ( + // ErrConfigIsNotAStructPointer indicates we expected a pointer to struct. + ErrConfigIsNotAStructPointer = errors.New("config is not a struct pointer") + + // ErrNoSuchField indicates there's no field with the given name. + ErrNoSuchField = errors.New("no such field") + + // ErrCannotSetIntegerOption means SetOptionAny couldn't set an integer option. + ErrCannotSetIntegerOption = errors.New("cannot set integer option") + + // ErrInvalidStringRepresentationOfBool indicates the string you passed + // to SetOptionaAny is not a valid string representation of a bool. + ErrInvalidStringRepresentationOfBool = errors.New("invalid string representation of bool") + + // ErrCannotSetBoolOption means SetOptionAny couldn't set a bool option. + ErrCannotSetBoolOption = errors.New("cannot set bool option") + + // ErrCannotSetStringOption means SetOptionAny couldn't set a string option. + ErrCannotSetStringOption = errors.New("cannot set string option") + + // ErrUnsupportedOptionType means we don't support the type passed to + // the SetOptionAny method as an opaque any type. + ErrUnsupportedOptionType = errors.New("unsupported option type") +) + +// Options returns the options exposed by this experiment. +func (b *Factory) Options() (map[string]model.ExperimentOptionInfo, error) { + result := make(map[string]model.ExperimentOptionInfo) + ptrinfo := reflect.ValueOf(b.config) + if ptrinfo.Kind() != reflect.Ptr { + return nil, ErrConfigIsNotAStructPointer + } + structinfo := ptrinfo.Elem().Type() + if structinfo.Kind() != reflect.Struct { + return nil, ErrConfigIsNotAStructPointer + } + for i := 0; i < structinfo.NumField(); i++ { + field := structinfo.Field(i) + result[field.Name] = model.ExperimentOptionInfo{ + Doc: field.Tag.Get("ooni"), + Type: field.Type.String(), + } + } + return result, nil +} + +// setOptionBool sets a bool option. +func (b *Factory) setOptionBool(field reflect.Value, value any) error { + switch v := value.(type) { + case bool: + field.SetBool(v) + return nil + case string: + if v != "true" && v != "false" { + return fmt.Errorf("%w: %s", ErrInvalidStringRepresentationOfBool, v) + } + field.SetBool(v == "true") + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetBoolOption, value) + } +} + +// setOptionInt sets an int option +func (b *Factory) setOptionInt(field reflect.Value, value any) error { + switch v := value.(type) { + case int64: + field.SetInt(v) + return nil + case int32: + field.SetInt(int64(v)) + return nil + case int16: + field.SetInt(int64(v)) + return nil + case int8: + field.SetInt(int64(v)) + return nil + case int: + field.SetInt(int64(v)) + return nil + case string: + number, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return fmt.Errorf("%w: %s", ErrCannotSetIntegerOption, err.Error()) + } + field.SetInt(number) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetIntegerOption, value) + } +} + +// setOptionString sets a string option +func (b *Factory) setOptionString(field reflect.Value, value any) error { + switch v := value.(type) { + case string: + field.SetString(v) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetStringOption, value) + } +} + +// SetOptionAny sets an option given any value. +func (b *Factory) SetOptionAny(key string, value any) error { + field, err := b.fieldbyname(b.config, key) + if err != nil { + return err + } + switch field.Kind() { + case reflect.Int64: + return b.setOptionInt(field, value) + case reflect.Bool: + return b.setOptionBool(field, value) + case reflect.String: + return b.setOptionString(field, value) + default: + return fmt.Errorf("%w: %T", ErrUnsupportedOptionType, value) + } +} + +// SetOptionsAny calls SetOptionAny for each entry inside [options]. +func (b *Factory) SetOptionsAny(options map[string]any) error { + for key, value := range options { + if err := b.SetOptionAny(key, value); err != nil { + return err + } + } + return nil +} + +// fieldbyname return v's field whose name is equal to the given key. +func (b *Factory) fieldbyname(v interface{}, key string) (reflect.Value, error) { + // See https://stackoverflow.com/a/6396678/4354461 + ptrinfo := reflect.ValueOf(v) + if ptrinfo.Kind() != reflect.Ptr { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + structinfo := ptrinfo.Elem() + if structinfo.Kind() != reflect.Struct { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + field := structinfo.FieldByName(key) + if !field.IsValid() || !field.CanSet() { + return reflect.Value{}, fmt.Errorf("%w: %s", ErrNoSuchField, key) + } + return field, nil +} + +// NewExperimentMeasurer creates the experiment +func (b *Factory) NewExperimentMeasurer() model.ExperimentMeasurer { + return b.build(b.config) +} + +// CanonicalizeExperimentName allows code to provide experiment names +// in a more flexible way, where we have aliases. +// +// Because we allow for uppercase experiment names for backwards +// compatibility with MK, we need to add some exceptions here when +// mapping (e.g., DNSCheck => dnscheck). +func CanonicalizeExperimentName(name string) string { + switch name = strcase.ToSnake(name); name { + case "ndt_7": + name = "ndt" // since 2020-03-18, we use ndt7 to implement ndt by default + case "dns_check": + name = "dnscheck" + case "stun_reachability": + name = "stunreachability" + default: + } + return name +} + +// NewFactory creates a new Factory instance. +func NewFactory(name string) (*Factory, error) { + factory := experimentsByName[CanonicalizeExperimentName(name)] + if factory == nil { + return nil, fmt.Errorf("no such experiment: %s", name) + } + builder := factory() + return builder, nil +} diff --git a/internal/registry/factory_test.go b/internal/registry/factory_test.go new file mode 100644 index 0000000000..f881ffdd81 --- /dev/null +++ b/internal/registry/factory_test.go @@ -0,0 +1,347 @@ +package registry + +import ( + "errors" + "testing" + + "github.com/google/go-cmp/cmp" +) + +type fakeExperimentConfig struct { + Chan chan any `ooni:"we cannot set this"` + String string `ooni:"a string"` + Truth bool `ooni:"something that no-one knows"` + Value int64 `ooni:"a number"` +} + +func TestExperimentBuilderOptions(t *testing.T) { + t.Run("when config is not a pointer", func(t *testing.T) { + b := &Factory{ + config: 17, + } + options, err := b.Options() + if !errors.Is(err, ErrConfigIsNotAStructPointer) { + t.Fatal("expected an error here") + } + if options != nil { + t.Fatal("expected nil here") + } + }) + + t.Run("when config is not a struct", func(t *testing.T) { + number := 17 + b := &Factory{ + config: &number, + } + options, err := b.Options() + if !errors.Is(err, ErrConfigIsNotAStructPointer) { + t.Fatal("expected an error here") + } + if options != nil { + t.Fatal("expected nil here") + } + }) + + t.Run("when config is a pointer to struct", func(t *testing.T) { + config := &fakeExperimentConfig{} + b := &Factory{ + config: config, + } + options, err := b.Options() + if err != nil { + t.Fatal(err) + } + for name, value := range options { + switch name { + case "Chan": + if value.Doc != "we cannot set this" { + t.Fatal("invalid doc") + } + if value.Type != "chan interface {}" { + t.Fatal("invalid type", value.Type) + } + case "String": + if value.Doc != "a string" { + t.Fatal("invalid doc") + } + if value.Type != "string" { + t.Fatal("invalid type", value.Type) + } + case "Truth": + if value.Doc != "something that no-one knows" { + t.Fatal("invalid doc") + } + if value.Type != "bool" { + t.Fatal("invalid type", value.Type) + } + case "Value": + if value.Doc != "a number" { + t.Fatal("invalid doc") + } + if value.Type != "int64" { + t.Fatal("invalid type", value.Type) + } + default: + t.Fatal("unknown name", name) + } + } + }) +} + +func TestExperimentBuilderSetOptionAny(t *testing.T) { + var inputs = []struct { + TestCaseName string + InitialConfig any + FieldName string + FieldValue any + ExpectErr error + ExpectConfig any + }{{ + TestCaseName: "config is not a pointer", + InitialConfig: fakeExperimentConfig{}, + FieldName: "Antani", + FieldValue: true, + ExpectErr: ErrConfigIsNotAStructPointer, + ExpectConfig: fakeExperimentConfig{}, + }, { + TestCaseName: "config is not a pointer to struct", + InitialConfig: func() *int { + v := 17 + return &v + }(), + FieldName: "Antani", + FieldValue: true, + ExpectErr: ErrConfigIsNotAStructPointer, + ExpectConfig: func() *int { + v := 17 + return &v + }(), + }, { + TestCaseName: "for missing field", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Antani", + FieldValue: true, + ExpectErr: ErrNoSuchField, + ExpectConfig: &fakeExperimentConfig{}, + }, { + TestCaseName: "[bool] for true value represented as string", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Truth", + FieldValue: "true", + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Truth: true, + }, + }, { + TestCaseName: "[bool] for false value represented as string", + InitialConfig: &fakeExperimentConfig{ + Truth: true, + }, + FieldName: "Truth", + FieldValue: "false", + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Truth: false, // must have been flipped + }, + }, { + TestCaseName: "[bool] for true value", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Truth", + FieldValue: true, + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Truth: true, + }, + }, { + TestCaseName: "[bool] for false value", + InitialConfig: &fakeExperimentConfig{ + Truth: true, + }, + FieldName: "Truth", + FieldValue: false, + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Truth: false, // must have been flipped + }, + }, { + TestCaseName: "[bool] for invalid string representation of bool", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Truth", + FieldValue: "xxx", + ExpectErr: ErrInvalidStringRepresentationOfBool, + ExpectConfig: &fakeExperimentConfig{}, + }, { + TestCaseName: "[bool] for value we don't know how to convert to bool", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Truth", + FieldValue: make(chan any), + ExpectErr: ErrCannotSetBoolOption, + ExpectConfig: &fakeExperimentConfig{}, + }, { + TestCaseName: "[int] for int", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: 17, + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Value: 17, + }, + }, { + TestCaseName: "[int] for int64", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: int64(17), + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Value: 17, + }, + }, { + TestCaseName: "[int] for int32", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: int32(17), + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Value: 17, + }, + }, { + TestCaseName: "[int] for int16", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: int16(17), + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Value: 17, + }, + }, { + TestCaseName: "[int] for int8", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: int8(17), + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Value: 17, + }, + }, { + TestCaseName: "[int] for string representation of int", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: "17", + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Value: 17, + }, + }, { + TestCaseName: "[int] for invalid string representation of int", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: "xx", + ExpectErr: ErrCannotSetIntegerOption, + ExpectConfig: &fakeExperimentConfig{}, + }, { + TestCaseName: "[int] for type we don't know how to convert to int", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: make(chan any), + ExpectErr: ErrCannotSetIntegerOption, + ExpectConfig: &fakeExperimentConfig{}, + }, { + TestCaseName: "[string] for serialized bool value while setting a string value", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "String", + FieldValue: "true", + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + String: "true", + }, + }, { + TestCaseName: "[string] for serialized int value while setting a string value", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "String", + FieldValue: "155", + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + String: "155", + }, + }, { + TestCaseName: "[string] for any other string", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "String", + FieldValue: "xxx", + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + String: "xxx", + }, + }, { + TestCaseName: "[string] for type we don't know how to convert to string", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "String", + FieldValue: make(chan any), + ExpectErr: ErrCannotSetStringOption, + ExpectConfig: &fakeExperimentConfig{}, + }, { + TestCaseName: "for a field that we don't know how to set", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Chan", + FieldValue: make(chan any), + ExpectErr: ErrUnsupportedOptionType, + ExpectConfig: &fakeExperimentConfig{}, + }} + + for _, input := range inputs { + t.Run(input.TestCaseName, func(t *testing.T) { + ec := input.InitialConfig + b := &Factory{config: ec} + err := b.SetOptionAny(input.FieldName, input.FieldValue) + if !errors.Is(err, input.ExpectErr) { + t.Fatal(err) + } + if diff := cmp.Diff(input.ExpectConfig, ec); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestExperimentBuilderSetOptionsAny(t *testing.T) { + b := &Factory{config: &fakeExperimentConfig{}} + + t.Run("we correctly handle an empty map", func(t *testing.T) { + if err := b.SetOptionsAny(nil); err != nil { + t.Fatal(err) + } + }) + + t.Run("we correctly handle a map containing options", func(t *testing.T) { + f := &fakeExperimentConfig{} + privateb := &Factory{config: f} + opts := map[string]any{ + "String": "yoloyolo", + "Value": "174", + "Truth": "true", + } + if err := privateb.SetOptionsAny(opts); err != nil { + t.Fatal(err) + } + if f.String != "yoloyolo" { + t.Fatal("cannot set string value") + } + if f.Value != 174 { + t.Fatal("cannot set integer value") + } + if f.Truth != true { + t.Fatal("cannot set bool value") + } + }) + + t.Run("we handle mistakes in a map containing string options", func(t *testing.T) { + opts := map[string]any{ + "String": "yoloyolo", + "Value": "xx", + "Truth": "true", + } + if err := b.SetOptionsAny(opts); !errors.Is(err, ErrCannotSetIntegerOption) { + t.Fatal("unexpected err", err) + } + }) +} From 9c9936d3bd97b4f1862329bb4a96bcc74e8952b1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 8 Aug 2022 22:14:39 +0200 Subject: [PATCH 04/83] refactor(registry): one file for each experiment This design allows us to automatically add new experiments by writing files in the proper package. --- internal/engine/allexperiments.go | 5 + internal/registry/allexperiments.go | 372 +------------------------- internal/registry/dash.go | 23 ++ internal/registry/dnscheck.go | 22 ++ internal/registry/dnsping.go | 22 ++ internal/registry/example.go | 28 ++ internal/registry/factory.go | 5 +- internal/registry/fbmessenger.go | 22 ++ internal/registry/hhfm.go | 22 ++ internal/registry/hirl.go | 22 ++ internal/registry/httphostheader.go | 22 ++ internal/registry/ndt.go | 23 ++ internal/registry/psiphon.go | 22 ++ internal/registry/quicping.go | 22 ++ internal/registry/riseupvpn.go | 22 ++ internal/registry/run.go | 22 ++ internal/registry/signal.go | 22 ++ internal/registry/simplequicping.go | 22 ++ internal/registry/sniblocking.go | 22 ++ internal/registry/stunreachability.go | 22 ++ internal/registry/tcpping.go | 22 ++ internal/registry/telegram.go | 22 ++ internal/registry/tlsping.go | 22 ++ internal/registry/tlstool.go | 22 ++ internal/registry/tor.go | 22 ++ internal/registry/torsf.go | 22 ++ internal/registry/urlgetter.go | 22 ++ internal/registry/vanillator.go | 22 ++ internal/registry/webconnectivity.go | 22 ++ internal/registry/whatsapp.go | 22 ++ 30 files changed, 612 insertions(+), 372 deletions(-) create mode 100644 internal/registry/dash.go create mode 100644 internal/registry/dnscheck.go create mode 100644 internal/registry/dnsping.go create mode 100644 internal/registry/example.go create mode 100644 internal/registry/fbmessenger.go create mode 100644 internal/registry/hhfm.go create mode 100644 internal/registry/hirl.go create mode 100644 internal/registry/httphostheader.go create mode 100644 internal/registry/ndt.go create mode 100644 internal/registry/psiphon.go create mode 100644 internal/registry/quicping.go create mode 100644 internal/registry/riseupvpn.go create mode 100644 internal/registry/run.go create mode 100644 internal/registry/signal.go create mode 100644 internal/registry/simplequicping.go create mode 100644 internal/registry/sniblocking.go create mode 100644 internal/registry/stunreachability.go create mode 100644 internal/registry/tcpping.go create mode 100644 internal/registry/telegram.go create mode 100644 internal/registry/tlsping.go create mode 100644 internal/registry/tlstool.go create mode 100644 internal/registry/tor.go create mode 100644 internal/registry/torsf.go create mode 100644 internal/registry/urlgetter.go create mode 100644 internal/registry/vanillator.go create mode 100644 internal/registry/webconnectivity.go create mode 100644 internal/registry/whatsapp.go diff --git a/internal/engine/allexperiments.go b/internal/engine/allexperiments.go index 2efacf9db2..ae4cb5641e 100644 --- a/internal/engine/allexperiments.go +++ b/internal/engine/allexperiments.go @@ -3,6 +3,11 @@ package engine // // List of all implemented experiments. // +// Note: if you're looking for a way to register a new experiment, we +// now use the internal/registry package for this purpose. +// +// (This comment will eventually autodestruct.) +// import "github.com/ooni/probe-cli/v3/internal/registry" diff --git a/internal/registry/allexperiments.go b/internal/registry/allexperiments.go index 5448524079..4cb28ac17e 100644 --- a/internal/registry/allexperiments.go +++ b/internal/registry/allexperiments.go @@ -1,377 +1,11 @@ package registry -// -// List of all implemented experiments. -// - -import ( - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dash" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnsping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/example" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/httphostheader" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/ndt7" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/quicping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/run" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/signal" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/simplequicping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/sniblocking" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/stunreachability" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tcpping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlsping" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tor" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" - "github.com/ooni/probe-cli/v3/internal/model" -) - -var experimentsByName = map[string]func() *Factory{ - "dash": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return dash.NewExperimentMeasurer( - *config.(*dash.Config), - ) - }, - config: &dash.Config{}, - interruptible: true, - inputPolicy: model.InputNone, - } - }, - - "dnscheck": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return dnscheck.NewExperimentMeasurer( - *config.(*dnscheck.Config), - ) - }, - config: &dnscheck.Config{}, - inputPolicy: model.InputOrStaticDefault, - } - }, - - "dnsping": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return dnsping.NewExperimentMeasurer( - *config.(*dnsping.Config), - ) - }, - config: &dnsping.Config{}, - inputPolicy: model.InputOrStaticDefault, - } - }, - - "example": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return example.NewExperimentMeasurer( - *config.(*example.Config), "example", - ) - }, - config: &example.Config{ - Message: "Good day from the example experiment!", - SleepTime: int64(time.Second), - }, - interruptible: true, - inputPolicy: model.InputNone, - } - }, - - "facebook_messenger": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return fbmessenger.NewExperimentMeasurer( - *config.(*fbmessenger.Config), - ) - }, - config: &fbmessenger.Config{}, - inputPolicy: model.InputNone, - } - }, - - "http_header_field_manipulation": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return hhfm.NewExperimentMeasurer( - *config.(*hhfm.Config), - ) - }, - config: &hhfm.Config{}, - inputPolicy: model.InputNone, - } - }, - - "http_host_header": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return httphostheader.NewExperimentMeasurer( - *config.(*httphostheader.Config), - ) - }, - config: &httphostheader.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "http_invalid_request_line": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return hirl.NewExperimentMeasurer( - *config.(*hirl.Config), - ) - }, - config: &hirl.Config{}, - inputPolicy: model.InputNone, - } - }, - - "ndt": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return ndt7.NewExperimentMeasurer( - *config.(*ndt7.Config), - ) - }, - config: &ndt7.Config{}, - interruptible: true, - inputPolicy: model.InputNone, - } - }, - - "psiphon": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return psiphon.NewExperimentMeasurer( - *config.(*psiphon.Config), - ) - }, - config: &psiphon.Config{}, - inputPolicy: model.InputOptional, - } - }, - - "quicping": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return quicping.NewExperimentMeasurer( - *config.(*quicping.Config), - ) - }, - config: &quicping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "riseupvpn": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return riseupvpn.NewExperimentMeasurer( - *config.(*riseupvpn.Config), - ) - }, - config: &riseupvpn.Config{}, - inputPolicy: model.InputNone, - } - }, - - "run": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return run.NewExperimentMeasurer( - *config.(*run.Config), - ) - }, - config: &run.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "simplequicping": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return simplequicping.NewExperimentMeasurer( - *config.(*simplequicping.Config), - ) - }, - config: &simplequicping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "signal": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return signal.NewExperimentMeasurer( - *config.(*signal.Config), - ) - }, - config: &signal.Config{}, - inputPolicy: model.InputNone, - } - }, - - "sni_blocking": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return sniblocking.NewExperimentMeasurer( - *config.(*sniblocking.Config), - ) - }, - config: &sniblocking.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "stunreachability": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return stunreachability.NewExperimentMeasurer( - *config.(*stunreachability.Config), - ) - }, - config: &stunreachability.Config{}, - inputPolicy: model.InputOrStaticDefault, - } - }, - - "tcpping": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tcpping.NewExperimentMeasurer( - *config.(*tcpping.Config), - ) - }, - config: &tcpping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "tlsping": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tlsping.NewExperimentMeasurer( - *config.(*tlsping.Config), - ) - }, - config: &tlsping.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "telegram": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return telegram.NewExperimentMeasurer( - *config.(*telegram.Config), - ) - }, - config: &telegram.Config{}, - inputPolicy: model.InputNone, - } - }, - - "tlstool": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tlstool.NewExperimentMeasurer( - *config.(*tlstool.Config), - ) - }, - config: &tlstool.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "tor": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tor.NewExperimentMeasurer( - *config.(*tor.Config), - ) - }, - config: &tor.Config{}, - inputPolicy: model.InputNone, - } - }, - - "torsf": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return torsf.NewExperimentMeasurer( - *config.(*torsf.Config), - ) - }, - config: &torsf.Config{}, - inputPolicy: model.InputNone, - } - }, - - "urlgetter": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return urlgetter.NewExperimentMeasurer( - *config.(*urlgetter.Config), - ) - }, - config: &urlgetter.Config{}, - inputPolicy: model.InputStrictlyRequired, - } - }, - - "vanilla_tor": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return vanillator.NewExperimentMeasurer( - *config.(*vanillator.Config), - ) - }, - config: &vanillator.Config{}, - inputPolicy: model.InputNone, - } - }, - - "web_connectivity": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return webconnectivity.NewExperimentMeasurer( - *config.(*webconnectivity.Config), - ) - }, - config: &webconnectivity.Config{}, - inputPolicy: model.InputOrQueryBackend, - } - }, - - "whatsapp": func() *Factory { - return &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return whatsapp.NewExperimentMeasurer( - *config.(*whatsapp.Config), - ) - }, - config: &whatsapp.Config{}, - inputPolicy: model.InputNone, - } - }, -} +// Where we register all the available experiments. +var allexperiments = map[string]*Factory{} // ExperimentNames returns the name of all experiments func ExperimentNames() (names []string) { - for key := range experimentsByName { + for key := range allexperiments { names = append(names, key) } return diff --git a/internal/registry/dash.go b/internal/registry/dash.go new file mode 100644 index 0000000000..f56bced240 --- /dev/null +++ b/internal/registry/dash.go @@ -0,0 +1,23 @@ +package registry + +// +// Registers the `dash' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dash" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["dash"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dash.NewExperimentMeasurer( + *config.(*dash.Config), + ) + }, + config: &dash.Config{}, + interruptible: true, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/dnscheck.go b/internal/registry/dnscheck.go new file mode 100644 index 0000000000..b4c264df17 --- /dev/null +++ b/internal/registry/dnscheck.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `dnscheck' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["dnscheck"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dnscheck.NewExperimentMeasurer( + *config.(*dnscheck.Config), + ) + }, + config: &dnscheck.Config{}, + inputPolicy: model.InputOrStaticDefault, + } +} diff --git a/internal/registry/dnsping.go b/internal/registry/dnsping.go new file mode 100644 index 0000000000..6ced01b75c --- /dev/null +++ b/internal/registry/dnsping.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `dnsping' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnsping" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["dnsping"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dnsping.NewExperimentMeasurer( + *config.(*dnsping.Config), + ) + }, + config: &dnsping.Config{}, + inputPolicy: model.InputOrStaticDefault, + } +} diff --git a/internal/registry/example.go b/internal/registry/example.go new file mode 100644 index 0000000000..947715cfe9 --- /dev/null +++ b/internal/registry/example.go @@ -0,0 +1,28 @@ +package registry + +// +// Registers the `example' experiment. +// + +import ( + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/experiment/example" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["example"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return example.NewExperimentMeasurer( + *config.(*example.Config), "example", + ) + }, + config: &example.Config{ + Message: "Good day from the example experiment!", + SleepTime: int64(time.Second), + }, + interruptible: true, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/factory.go b/internal/registry/factory.go index 60d61d27b1..b0b441670f 100644 --- a/internal/registry/factory.go +++ b/internal/registry/factory.go @@ -215,10 +215,9 @@ func CanonicalizeExperimentName(name string) string { // NewFactory creates a new Factory instance. func NewFactory(name string) (*Factory, error) { - factory := experimentsByName[CanonicalizeExperimentName(name)] + factory := allexperiments[CanonicalizeExperimentName(name)] if factory == nil { return nil, fmt.Errorf("no such experiment: %s", name) } - builder := factory() - return builder, nil + return factory, nil } diff --git a/internal/registry/fbmessenger.go b/internal/registry/fbmessenger.go new file mode 100644 index 0000000000..6d2cda4c1d --- /dev/null +++ b/internal/registry/fbmessenger.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `fbmessenger' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["facebook_messenger"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return fbmessenger.NewExperimentMeasurer( + *config.(*fbmessenger.Config), + ) + }, + config: &fbmessenger.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/hhfm.go b/internal/registry/hhfm.go new file mode 100644 index 0000000000..9a2bdafe58 --- /dev/null +++ b/internal/registry/hhfm.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `hhfm' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["http_header_field_manipulation"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return hhfm.NewExperimentMeasurer( + *config.(*hhfm.Config), + ) + }, + config: &hhfm.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/hirl.go b/internal/registry/hirl.go new file mode 100644 index 0000000000..63cb6339bd --- /dev/null +++ b/internal/registry/hirl.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `hirl' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["http_invalid_request_line"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return hirl.NewExperimentMeasurer( + *config.(*hirl.Config), + ) + }, + config: &hirl.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/httphostheader.go b/internal/registry/httphostheader.go new file mode 100644 index 0000000000..7f346145cd --- /dev/null +++ b/internal/registry/httphostheader.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `httphostheader' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/httphostheader" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["http_host_header"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return httphostheader.NewExperimentMeasurer( + *config.(*httphostheader.Config), + ) + }, + config: &httphostheader.Config{}, + inputPolicy: model.InputOrQueryBackend, + } +} diff --git a/internal/registry/ndt.go b/internal/registry/ndt.go new file mode 100644 index 0000000000..2a1a2c5beb --- /dev/null +++ b/internal/registry/ndt.go @@ -0,0 +1,23 @@ +package registry + +// +// Registers the `ndt' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/ndt7" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["ndt"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return ndt7.NewExperimentMeasurer( + *config.(*ndt7.Config), + ) + }, + config: &ndt7.Config{}, + interruptible: true, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/psiphon.go b/internal/registry/psiphon.go new file mode 100644 index 0000000000..7702b31270 --- /dev/null +++ b/internal/registry/psiphon.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `psiphon' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["psiphon"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return psiphon.NewExperimentMeasurer( + *config.(*psiphon.Config), + ) + }, + config: &psiphon.Config{}, + inputPolicy: model.InputOptional, + } +} diff --git a/internal/registry/quicping.go b/internal/registry/quicping.go new file mode 100644 index 0000000000..cc59ff008e --- /dev/null +++ b/internal/registry/quicping.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `quicping' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/quicping" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["quicping"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return quicping.NewExperimentMeasurer( + *config.(*quicping.Config), + ) + }, + config: &quicping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } +} diff --git a/internal/registry/riseupvpn.go b/internal/registry/riseupvpn.go new file mode 100644 index 0000000000..de2c88d768 --- /dev/null +++ b/internal/registry/riseupvpn.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `riseupvpn' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["riseupvpn"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return riseupvpn.NewExperimentMeasurer( + *config.(*riseupvpn.Config), + ) + }, + config: &riseupvpn.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/run.go b/internal/registry/run.go new file mode 100644 index 0000000000..120578ca81 --- /dev/null +++ b/internal/registry/run.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `run' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/run" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["run"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return run.NewExperimentMeasurer( + *config.(*run.Config), + ) + }, + config: &run.Config{}, + inputPolicy: model.InputStrictlyRequired, + } +} diff --git a/internal/registry/signal.go b/internal/registry/signal.go new file mode 100644 index 0000000000..71a919b3b1 --- /dev/null +++ b/internal/registry/signal.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `signal' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/signal" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["signal"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return signal.NewExperimentMeasurer( + *config.(*signal.Config), + ) + }, + config: &signal.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/simplequicping.go b/internal/registry/simplequicping.go new file mode 100644 index 0000000000..22230b3d4a --- /dev/null +++ b/internal/registry/simplequicping.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `simplequicping' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/simplequicping" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["simplequicping"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return simplequicping.NewExperimentMeasurer( + *config.(*simplequicping.Config), + ) + }, + config: &simplequicping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } +} diff --git a/internal/registry/sniblocking.go b/internal/registry/sniblocking.go new file mode 100644 index 0000000000..cd3409f17f --- /dev/null +++ b/internal/registry/sniblocking.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `sniblocking' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/sniblocking" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["sni_blocking"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return sniblocking.NewExperimentMeasurer( + *config.(*sniblocking.Config), + ) + }, + config: &sniblocking.Config{}, + inputPolicy: model.InputOrQueryBackend, + } +} diff --git a/internal/registry/stunreachability.go b/internal/registry/stunreachability.go new file mode 100644 index 0000000000..bf6f6a4c50 --- /dev/null +++ b/internal/registry/stunreachability.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `stunreachability' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/stunreachability" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["stunreachability"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return stunreachability.NewExperimentMeasurer( + *config.(*stunreachability.Config), + ) + }, + config: &stunreachability.Config{}, + inputPolicy: model.InputOrStaticDefault, + } +} diff --git a/internal/registry/tcpping.go b/internal/registry/tcpping.go new file mode 100644 index 0000000000..5ff79ea417 --- /dev/null +++ b/internal/registry/tcpping.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `tcpping' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tcpping" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["tcpping"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tcpping.NewExperimentMeasurer( + *config.(*tcpping.Config), + ) + }, + config: &tcpping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } +} diff --git a/internal/registry/telegram.go b/internal/registry/telegram.go new file mode 100644 index 0000000000..43f1db031c --- /dev/null +++ b/internal/registry/telegram.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `telegram' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["telegram"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return telegram.NewExperimentMeasurer( + *config.(*telegram.Config), + ) + }, + config: &telegram.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/tlsping.go b/internal/registry/tlsping.go new file mode 100644 index 0000000000..0f944809ec --- /dev/null +++ b/internal/registry/tlsping.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `tlsping' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlsping" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["tlsping"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tlsping.NewExperimentMeasurer( + *config.(*tlsping.Config), + ) + }, + config: &tlsping.Config{}, + inputPolicy: model.InputStrictlyRequired, + } +} diff --git a/internal/registry/tlstool.go b/internal/registry/tlstool.go new file mode 100644 index 0000000000..434351601b --- /dev/null +++ b/internal/registry/tlstool.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `tlstool' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["tlstool"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tlstool.NewExperimentMeasurer( + *config.(*tlstool.Config), + ) + }, + config: &tlstool.Config{}, + inputPolicy: model.InputOrQueryBackend, + } +} diff --git a/internal/registry/tor.go b/internal/registry/tor.go new file mode 100644 index 0000000000..5a2a2ba08f --- /dev/null +++ b/internal/registry/tor.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `tor' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tor" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["tor"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tor.NewExperimentMeasurer( + *config.(*tor.Config), + ) + }, + config: &tor.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/torsf.go b/internal/registry/torsf.go new file mode 100644 index 0000000000..b31888fda2 --- /dev/null +++ b/internal/registry/torsf.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `torsf' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["torsf"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return torsf.NewExperimentMeasurer( + *config.(*torsf.Config), + ) + }, + config: &torsf.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/urlgetter.go b/internal/registry/urlgetter.go new file mode 100644 index 0000000000..84762e052d --- /dev/null +++ b/internal/registry/urlgetter.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `urlgetter' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["urlgetter"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return urlgetter.NewExperimentMeasurer( + *config.(*urlgetter.Config), + ) + }, + config: &urlgetter.Config{}, + inputPolicy: model.InputStrictlyRequired, + } +} diff --git a/internal/registry/vanillator.go b/internal/registry/vanillator.go new file mode 100644 index 0000000000..042ae549ed --- /dev/null +++ b/internal/registry/vanillator.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `vanilla_tor' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["vanilla_tor"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return vanillator.NewExperimentMeasurer( + *config.(*vanillator.Config), + ) + }, + config: &vanillator.Config{}, + inputPolicy: model.InputNone, + } +} diff --git a/internal/registry/webconnectivity.go b/internal/registry/webconnectivity.go new file mode 100644 index 0000000000..850542033d --- /dev/null +++ b/internal/registry/webconnectivity.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `web_connectivity' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["web_connectivity"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return webconnectivity.NewExperimentMeasurer( + *config.(*webconnectivity.Config), + ) + }, + config: &webconnectivity.Config{}, + inputPolicy: model.InputOrQueryBackend, + } +} diff --git a/internal/registry/whatsapp.go b/internal/registry/whatsapp.go new file mode 100644 index 0000000000..9fd8d0f96d --- /dev/null +++ b/internal/registry/whatsapp.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `whatsapp' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["whatsapp"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return whatsapp.NewExperimentMeasurer( + *config.(*whatsapp.Config), + ) + }, + config: &whatsapp.Config{}, + inputPolicy: model.InputNone, + } +} From ddf4f0e10dde0fa8bd1b570ae840c897ad8004ea Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 8 Aug 2022 22:50:41 +0200 Subject: [PATCH 05/83] feat: add script to generate experiment's boilerplate --- go.mod | 6 + go.sum | 6 + internal/cmd/boilerplate/experiment.go | 221 ++++++++++++++++++ .../cmd/boilerplate/experiment/doc.go.txt | 4 + .../boilerplate/experiment/maintask.go.txt | 50 ++++ .../boilerplate/experiment/measurer.go.txt | 138 +++++++++++ .../cmd/boilerplate/experiment/model.go.txt | 75 ++++++ .../boilerplate/experiment/registry.go.txt | 23 ++ .../cmd/boilerplate/experiment/tasks.go.txt | 138 +++++++++++ internal/cmd/boilerplate/main.go | 37 +++ internal/cmd/boilerplate/task.go | 127 ++++++++++ .../boilerplate/task/systemresolver.go.txt | 77 ++++++ .../cmd/boilerplate/task/tcpconnect.go.txt | 77 ++++++ .../cmd/boilerplate/task/tlshandshake.go.txt | 101 ++++++++ internal/cmd/boilerplate/utils.go | 53 +++++ 15 files changed, 1133 insertions(+) create mode 100644 internal/cmd/boilerplate/experiment.go create mode 100644 internal/cmd/boilerplate/experiment/doc.go.txt create mode 100644 internal/cmd/boilerplate/experiment/maintask.go.txt create mode 100644 internal/cmd/boilerplate/experiment/measurer.go.txt create mode 100644 internal/cmd/boilerplate/experiment/model.go.txt create mode 100644 internal/cmd/boilerplate/experiment/registry.go.txt create mode 100644 internal/cmd/boilerplate/experiment/tasks.go.txt create mode 100644 internal/cmd/boilerplate/main.go create mode 100644 internal/cmd/boilerplate/task.go create mode 100644 internal/cmd/boilerplate/task/systemresolver.go.txt create mode 100644 internal/cmd/boilerplate/task/tcpconnect.go.txt create mode 100644 internal/cmd/boilerplate/task/tlshandshake.go.txt create mode 100644 internal/cmd/boilerplate/utils.go diff --git a/go.mod b/go.mod index a6be194053..5ced355d44 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,11 @@ require ( golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a ) +require ( + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) + require ( filippo.io/edwards25519 v1.0.0-rc.1.0.20210721174708-390f27c3be20 // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect @@ -99,6 +104,7 @@ require ( github.com/refraction-networking/utls v1.1.0 // indirect github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spf13/cobra v1.5.0 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/templexxx/cpu v0.0.9 // indirect github.com/templexxx/xorsimd v0.4.1 // indirect diff --git a/go.sum b/go.sum index bd7170ccd2..f1d1762c73 100644 --- a/go.sum +++ b/go.sum @@ -150,6 +150,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -390,6 +391,7 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -767,6 +769,7 @@ github.com/rubenv/sql-migrate v1.1.1 h1:haR5Hn8hbW9/SpAICrXoZqXnywS7Q5WijwkQENPe github.com/rubenv/sql-migrate v1.1.1/go.mod h1:/7TZymwxN8VWumcIxw1jjHEcR1djpdkMHQPT4FWdnbQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -827,10 +830,13 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go new file mode 100644 index 0000000000..44a153e830 --- /dev/null +++ b/internal/cmd/boilerplate/experiment.go @@ -0,0 +1,221 @@ +package main + +// +// Code to generate a new experiment. +// + +import ( + _ "embed" + "path/filepath" + "strconv" + "text/template" + + "github.com/AlecAivazis/survey/v2" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// Implements interactively generating a new experiment. +type NewExperimentCommand struct{} + +// Information about the experiment to create. +type ExperimentInfo struct { + // Experiment name + Name string + + // Experiment version + Version string + + // Experiment spec URL + SpecURL string + + // Experiment input policy + InputPolicy string + + // Overall experiment timeout + Timeout int64 + + // Whether we'll run parallel tasks. + Parallel bool + + // Whether this experimenti is interuptible. + Interruptible bool +} + +// Called by the CLI parser +func (c *NewExperimentCommand) Run(*cobra.Command, []string) { + printf("\n") + printf("Welcome! This command will help you to automatically generate code\n") + printf("implementing a new OONI network experiment!\n") + print("\n") + + info := getExperimentInfo() + info.Interruptible = info.InputPolicy == "InputNone" + + makeExperimentDirectory(info) + generateDocGo(info) + generateMeasurerGo(info) + generateModelGo(info) + if info.Parallel { + generateTasksGo(info) + generateMainTaskGo(info) + } + generateRegistryEntryGo(info) +} + +// Obtains the experiment info +func getExperimentInfo() *ExperimentInfo { + return &ExperimentInfo{ + Name: getExperimentName(), + Version: getExperimentVersion(), + SpecURL: getExperimentSpecURL(), + InputPolicy: getExperimentInputPolicy(), + Timeout: getExperimentTimeout(), + Parallel: getExperimentParallel(), + Interruptible: false, + } +} + +// Obtains the experiment name +func getExperimentName() string { + prompt := &survey.Input{ + Message: "Experiment's name:", + } + var experiment string + err := survey.AskOne(prompt, &experiment) + runtimex.PanicOnError(err, "survey.AskOne failed") + return experiment +} + +// Obtains the experiment version +func getExperimentVersion() string { + prompt := &survey.Input{ + Message: "Experiment's version:", + } + var version string + err := survey.AskOne(prompt, &version) + runtimex.PanicOnError(err, "survey.AskOne failed") + return version +} + +// Obtains the experiment spec URL +func getExperimentSpecURL() string { + prompt := &survey.Input{ + Message: "Experiment's spec URL:", + } + var specURL string + err := survey.AskOne(prompt, &specURL) + runtimex.PanicOnError(err, "survey.AskOne failed") + return specURL +} + +// Obtains the experiment input policy. +func getExperimentInputPolicy() string { + var inputPolicy string + prompt := &survey.Select{ + Message: "Choose an experiment input policy:", + Options: []string{ + "InputOptional", + "InputOrQueryBackend", + "InputOrStaticDefault", + "InputStrictlyRequired", + "InputNone", + }, + } + err := survey.AskOne(prompt, &inputPolicy) + runtimex.PanicOnError(err, "survey.AskOne failed") + return inputPolicy +} + +// Obtains the experiment timeout. +func getExperimentTimeout() int64 { + prompt := &survey.Input{ + Message: "Experiment's _overall_ timeout in seconds (just hit enter for no timeout):", + } + var value string + err := survey.AskOne(prompt, &value) + runtimex.PanicOnError(err, "survey.AskOne failed") + if value == "" { + return 0 + } + timeout, err := strconv.ParseInt(value, 10, 64) + runtimex.PanicOnError(err, "strconv.ParseInt failed") + return timeout +} + +// Obtains the experiment parallel setting. +func getExperimentParallel() bool { + var parallel bool + prompt := &survey.Confirm{ + Message: "Do you want to generate code for running tasks in parallel?", + } + err := survey.AskOne(prompt, ¶llel) + runtimex.PanicOnError(err, "survey.AskOne failed") + return parallel +} + +// Creates a directory for the new experiment. +func makeExperimentDirectory(info *ExperimentInfo) { + fulldir := filepath.Join("internal", "experiment", info.Name) + mkdirP(fulldir) +} + +//go:embed "experiment/doc.go.txt" +var experimentDocGoTemplate string + +// Generates the doc.go file +func generateDocGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Name, "doc.go") + tmpl := template.Must(template.New("doc.go").Parse(experimentDocGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/measurer.go.txt" +var experimentMeasurerGoTemplate string + +// Generates the measurer.go file +func generateMeasurerGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Name, "measurer.go") + tmpl := template.Must(template.New("measurer.go").Parse(experimentMeasurerGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/model.go.txt" +var experimentModelGoTemplate string + +// Generates the model.go file +func generateModelGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Name, "model.go") + tmpl := template.Must(template.New("model.go").Parse(experimentModelGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/tasks.go.txt" +var experimentTasksGoTemplate string + +// Generates the tasks.go file +func generateTasksGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Name, "tasks.go") + tmpl := template.Must(template.New("tasks.go").Parse(experimentTasksGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/maintask.go.txt" +var experimentMainTaskGoTemplate string + +// Generates the maintask.go file +func generateMainTaskGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Name, "maintask.go") + tmpl := template.Must(template.New("maintask.go").Parse(experimentMainTaskGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/registry.go.txt" +var experimentRegistryEntryGoTemplate string + +// Generates the experiment's entry inside ./internal/registry +func generateRegistryEntryGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "registry", info.Name+".go") + tmpl := template.Must(template.New("registryentry.go").Parse(experimentRegistryEntryGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} diff --git a/internal/cmd/boilerplate/experiment/doc.go.txt b/internal/cmd/boilerplate/experiment/doc.go.txt new file mode 100644 index 0000000000..5b45a1a372 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/doc.go.txt @@ -0,0 +1,4 @@ +// Package {{ .Name }} implements the {{ .Name }} experiment. +// +// See {{ .SpecURL }}. +package {{ .Name }} diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt new file mode 100644 index 0000000000..adf0887a16 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -0,0 +1,50 @@ +package {{ .Name }} + +// +// The main task. +// +// Note: the autogenerated code contains a very basic task +// that does nothing. You should, of course, edit it! +// + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// MainTask is the main task. +type MainTask struct { + // IDGenerator is the TaskIDGenerator to use. + IDGenerator *TaskIDGenerator + + // Logger is the logger to use. + Logger model.Logger + + // TODO: add fields. + // + // Please, keep in mind that, with multiple tasks you should + // most like only share fields as read-only here! +} + +// NewMainTask creates a new MainTask instance. +// +// TODO: add the required parameters. +func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger) *MainTask { + return &MainTask{ + IDGenerator: idGenerator, + Logger: logger, + // TODO: fill fields. + } +} + +// Run implements Task +func (t *MainTask) Run(ctx context.Context, sched TaskScheduler) error { + t.Logger.Infof("Hello, world") + return nil +} + +// Repr implements Task +func (t *MainTask) Repr() string { + return "{{ .Name }}Task" +} diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt new file mode 100644 index 0000000000..4cb7a22501 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -0,0 +1,138 @@ +package {{ .Name }} + +// +// Measurer for {{ .Name }} +// + +import ( + "context" + {{ if ne .InputPolicy "InputOptional" }} + "errors" + {{ end }} + {{ if ne .Timeout 0 }} + "time" + {{ end }} + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Measurer implements {{ .Name }}. +type Measurer struct{ + // Config contains the experiment's config. + Config *Config +} + +// NewExperimentMeasurer creates a new model.ExperimentMeasurer for {{ .Name }}. +func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { + return &Measurer{ + Config: config, + } +} + +// ExperimentName implements ExperimentMeasurer.ExperimentName. +func (m *Measurer) ExperimentName() string { + return "{{ .Name }}" +} + +// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion. +func (m *Measurer) ExperimentVersion() string { + return "{{ .Version }}" +} + +// Run implements ExperimentMeasurer.Run. +func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { + // Reminder: When this function returns an error, the measurement result + // WILL NOT be submitted to the OONI backend. You SHOULD only return an error + // for fundamental errors (e.g., the input is invalid or missing). + + {{ if eq .InputPolicy "InputNone" }} + // honour {{ .InputPolicy }} + if measurement.Input != "" { + sess.Logger().Warn("BUG: {{ .Name }} got input but has {{ .InputPolicy }}!") + return errors.New("{{ .Name }}: this experiment does not take any input") + } + + {{ else if eq .InputPolicy "InputOptional" }} + // honour {{ .InputPolicy }} + input := measurement.Input + if input == "" { + input = m.optionalInput() + } + + {{ else if eq .InputPolicy "InputOrQueryBackend" }} + // honour {{ .InputPolicy }} + input := measurement.Input + if input == "" { + sess.Logger().Warn("BUG: {{ .Name }} got no input but has {{ .InputPolicy }}!") + return errors.New("{{ .Name }}: no input provided") + } + + {{ else if eq .InputPolicy "InputOrStaticDefault" }} + // honour {{ .InputPolicy }} + input := measurement.Input + if input == "" { + sess.Logger().Warn("BUG: {{ .Name }} got no input but has {{ .InputPolicy }}!") + return errors.New("{{ .Name }}: no input provided") + } + + {{ else if eq .InputPolicy "InputStrictlyRequired" }} + // honour {{ .InputPolicy }} + input := measurement.Input + if input == "" { + return errors.New("{{ .Name }}: no input provided") + } + + {{ else }} + THIS SHOULD NOT HAPPEN! SOMETHING'S WRONG WITH THE AUTO-GENERATOR! + {{ end }} + + // initialize the experiment's test keys + tk := NewTestKeys() + measurement.TestKeys = tk + + {{ if ne .Timeout 0 }} + // ensure there's an overall timeout + ctx, cancel := context.WithTimeout(ctx, {{.Timeout}} * time.Second) + defer cancel() + {{ end }} + +{{ if eq .Parallel false }} + // defer the rest of the work to run() + return m.run(ctx, sess, measurement, callbacks) +{{ else }} + // defer the rest of the work to the main task + const parallelism = 10 // TODO: adjust depending on your needs + idGenerator := NewTaskIDGenerator() + main := NewMainTask(idGenerator, sess.Logger()) + ts := NewTaskSchedulerWaiter(sess.Logger(), parallelism) + ts.Schedule(ctx, main) + ts.Wait() + return nil +{{ end }} +} + +{{ if eq .Parallel false }} + +// run implements Run. +func (m *Measurer) run(ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { + // TODO: implement + sess.Logger().Infof("Hello, world") + return nil +} + +{{ end }} + +{{ if eq .InputPolicy "InputOptional" }} +func (m *Measurer) optionalInput() model.MeasurementTarget { + panic("not implemented") +} +{{ end }} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + sk := SummaryKeys{isAnomaly: false} + // TODO: implement + return sk, nil +} diff --git a/internal/cmd/boilerplate/experiment/model.go.txt b/internal/cmd/boilerplate/experiment/model.go.txt new file mode 100644 index 0000000000..a510188a03 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/model.go.txt @@ -0,0 +1,75 @@ +package {{ .Name }} + +// +// Data model for {{ .Name }} +// + +{{ if eq .Parallel true }} +import "sync" +{{ end }} + +// Config contains configuration for the {{ .Name }} experiment. +type Config struct { + // TODO: add fields here if you need any config +} + +{{ if eq .Parallel true }} + +// testKeys contains the experiment test keys. +type testKeys struct { + // mu provides mutual exclusion for accessing the test keys + mu sync.Mutex + + // TODO: add here fields produced by this experiment + // + // For example: + // + // Failure *string +} + +// TestKeys is the interface with which you access the real test keys. +type TestKeys interface { + // TODO: define thread-safe accessors for the real test keys. + // + // For example: + // + // SetFailure(failure *string) +} + +// TODO: implement thread-safe accessors for the real test keys. +// +// For example: +// +// func (tk *testKeys) SetFailure(failure *string) { +// tk.mu.Lock() +// tk.Failure = failure +// tk.mu.Unlock() +// } + +// NewTestKeys creates a new instance of TestKeys. +func NewTestKeys() TestKeys { + return &testKeys{} +} + +{{ else }} + +// TestKeys contains the results produced by the experiment. +type TestKeys struct{ + // TODO: add here fields produced by this experiment +} + +// NewTestKeys creates a new instance of TestKeys. +func NewTestKeys() *TestKeys { + return &TestKeys{} +} + +{{ end }} + +// Summary contains the summary results. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + isAnomaly bool + // TODO: add here additional summary fields. +} diff --git a/internal/cmd/boilerplate/experiment/registry.go.txt b/internal/cmd/boilerplate/experiment/registry.go.txt new file mode 100644 index 0000000000..2ebf60ce6c --- /dev/null +++ b/internal/cmd/boilerplate/experiment/registry.go.txt @@ -0,0 +1,23 @@ +package registry + +// +// Registers the `{{ .Name }}' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/experiment/{{ .Name }}" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["{{ .Name }}"] = &Factory{ + build: func(config any) model.ExperimentMeasurer { + return {{ .Name }}.NewExperimentMeasurer( + config.(*{{ .Name }}.Config), + ) + }, + config: &{{ .Name }}.Config{}, + interruptible: {{ .Interruptible }}, + inputPolicy: model.{{ .InputPolicy }}, + } +} diff --git a/internal/cmd/boilerplate/experiment/tasks.go.txt b/internal/cmd/boilerplate/experiment/tasks.go.txt new file mode 100644 index 0000000000..9183ea2605 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/tasks.go.txt @@ -0,0 +1,138 @@ +package {{ .Name }} + +// +// Library for running background tasks. +// +// Note: the autogenerated code contains a generic library but you +// may want to further customize what we do depending on specific +// needs that your experiment may actually have. +// + +import ( + "context" + "sync" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// TaskIDGenerator generates unique IDs for tasks. +type TaskIDGenerator struct { + c *atomicx.Int64 +} + +// NewTaskIDGenerator creates a new TaskIDGenerator instance. +func NewTaskIDGenerator() *TaskIDGenerator { + return &TaskIDGenerator{ + c: &atomicx.Int64{}, + } +} + +// Next returns the next unique task ID. +func (ig *TaskIDGenerator) Next() int64 { + return ig.c.Add(1) +} + +// TaskScheduler schedules new tasks. +type TaskScheduler interface { + // Schedule scheduless a task for executing ASAP. + Schedule(ctx context.Context, task Task) +} + +// TaskSchedulerWaiter schedules new tasks and waits for them. +type TaskSchedulerWaiter interface { + // We inherit from the TaskScheduler. + TaskScheduler + + // Waits for all tasks to finish. + Wait() +} + +// taskScheduler implements TaskScheduler. +type taskScheduler struct { + // logger is the logger to use. + logger model.Logger + + // sema contains a semaphore for throttling concurrent tasks. + sema chan bool + + // wg is the wait group to wait for completion. + wg *sync.WaitGroup +} + +// NewTaskSchedulerWaiter creates a new TaskSchedulerWaiter instance. The [maxTasks] +// argument controls the parallelism. A zero or negative argument implies +// only a single task at a time could run. The [logger] argument is the +// logger we should use to print information about tasks completion. You +// should use model.DiscardLogger to disable this functionality. +func NewTaskSchedulerWaiter(logger model.Logger, maxTasks int) TaskSchedulerWaiter { + if maxTasks < 1 { + maxTasks = 1 // as documented + } + tl := &taskScheduler{ + logger: logger, + sema: make(chan bool, maxTasks), + wg: &sync.WaitGroup{}, + } + return tl +} + +// Task is one of the tasks run by this experiment. +type Task interface { + // Repr returns a printable representation of the task. + Repr() string + + // Run runs the task and returns the result. + Run(ctx context.Context, sched TaskScheduler) error +} + +// Schedule implements TaskScheduler. +func (ts *taskScheduler) Schedule(ctx context.Context, task Task) { + wrapper := &taskWrapper{ + logger: ts.logger, + sema: ts.sema, + task: task, + wg: ts.wg, + } + go wrapper.run(ctx, ts) +} + +// Wait implements TaskScheduler. +func (ts *taskScheduler) Wait() { + ts.wg.Wait() +} + +// taskWrapper wraps a task encapsulating scheduling logic. +type taskWrapper struct { + // logger is the logger to use. + logger model.Logger + + // sema is the controlling semaphore. + sema chan bool + + // task is the actual task. + task Task + + // wg allows the parent to wait for jobs to terminate. + wg *sync.WaitGroup +} + +// Run runs the task which in turn runs the task's job. +func (tw *taskWrapper) run(ctx context.Context, sched TaskScheduler) { + // Ensure we don't have too many concurrent tasks. + tw.sema <- true + defer func() { <-tw.sema }() + + // Synchronize with the parent task. + defer tw.wg.Done() + + // Create operation logger. + ol := measurexlite.NewOperationLogger(tw.logger, tw.task.Repr()) + + // Run task. + err := tw.task.Run(ctx, sched) + + // Task completed. + ol.Stop(err) +} diff --git a/internal/cmd/boilerplate/main.go b/internal/cmd/boilerplate/main.go new file mode 100644 index 0000000000..bc8add9b3b --- /dev/null +++ b/internal/cmd/boilerplate/main.go @@ -0,0 +1,37 @@ +// Command boilerplate assists you in generating code for new experiments. +// +// We will generate experiments under the ./internal/experiment folder rather +// than under ./internal/engine/experiment because we are moving away from the +// experiment folder. +package main + +import ( + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +func main() { + root := &cobra.Command{ + Use: "boilerplate", + Short: "Helps to auto-generate code for new experiments", + } + + newExperiment := &cobra.Command{ + Use: "new-experiment", + Args: cobra.NoArgs, + Short: "Interactively generate a new experiment", + Run: (&NewExperimentCommand{}).Run, + } + root.AddCommand(newExperiment) + + newflow := &cobra.Command{ + Use: "new-task", + Args: cobra.NoArgs, + Short: "Interactively generate a new task for an experiment", + Run: (&NewTaskCommand{}).Run, + } + root.AddCommand(newflow) + + err := root.Execute() + runtimex.PanicOnError(err, "root.Execute failed") +} diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go new file mode 100644 index 0000000000..b0738c2e7b --- /dev/null +++ b/internal/cmd/boilerplate/task.go @@ -0,0 +1,127 @@ +package main + +// +// Code to generate a new experiment flow. +// + +import ( + _ "embed" + "path/filepath" + "strings" + "text/template" + + "github.com/AlecAivazis/survey/v2" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// Implements interactively generating a new experiment. +type NewTaskCommand struct{} + +// Information about a task to autogenerate. +type TaskInfo struct { + // The task struct's name. + StructName string + + // Description contains the description. + Description string + + // The task template. + Template string +} + +// Called by the CLI parser +func (c *NewTaskCommand) Run(*cobra.Command, []string) { + printf("\n") + printf("Welcome! This command will help you to automatically generate a task\n") + printf("to include it into an existing OONI experiment!\n") + print("\n") + + experimentName := getExperimentName() + info := getTaskInfo() + + generateTaskGo(experimentName, info) +} + +// Obtains information about the task to generate. +func getTaskInfo() *TaskInfo { + return &TaskInfo{ + StructName: getTaskStructName(), + Description: getTaskDescription(), + Template: getTaskTemplate(), + } +} + +// Returns the name of the task struct. +func getTaskStructName() string { + prompt := &survey.Input{ + Message: "Task struct name (e.g., 'Datacenter'):", + } + var name string + err := survey.AskOne(prompt, &name) + runtimex.PanicOnError(err, "survey.AskOne failed") + return name +} + +// Returns the task's description +func getTaskDescription() string { + prompt := &survey.Input{ + Message: "Short documentation for this task:", + } + var docs string + err := survey.AskOne(prompt, &docs) + runtimex.PanicOnError(err, "survey.AskOne failed") + return docs +} + +//go:embed "task/systemresolver.go.txt" +var systemResolverTemplate string + +//go:embed "task/tcpconnect.go.txt" +var tcpConnectTemplate string + +//go:embed "task/tlshandshake.go.txt" +var tlsHandshakeTemplate string + +// The list of known tasks +var knownTasks = map[string]string{ + "system-resolver": systemResolverTemplate, + "tcp-connect": tcpConnectTemplate, + "tls-handshake": tlsHandshakeTemplate, +} + +// Names of known tasks +var knownTaskNames []string + +// Autogenerates the names of the tasks. +func init() { + for name := range knownTasks { + knownTaskNames = append(knownTaskNames, name) + } +} + +// Returns the task template to use. +func getTaskTemplate() string { + var name string + prompt := &survey.Select{ + Message: "Choose a task you would like to generate:", + Options: knownTaskNames, + } + err := survey.AskOne(prompt, &name) + runtimex.PanicOnError(err, "survey.AskOne failed") + return name +} + +// Generates code for the new task. +func generateTaskGo(experiment string, info *TaskInfo) { + name := "task" + strings.ToLower(info.StructName) + ".go" + fullpath := filepath.Join("internal", "experiment", experiment, name) + tmpl := template.Must(template.New("T1").Parse(knownTasks[info.Template])) + mapping := map[string]string{ + "Experiment": experiment, + "StructName": info.StructName, + "Template": info.Template, + "Description": info.Description, + } + writeTemplate(fullpath, tmpl, mapping) +} diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt new file mode 100644 index 0000000000..c19a8d44ff --- /dev/null +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -0,0 +1,77 @@ +package {{ .Experiment }} + +// +// {{ .StructName }}: {{ .Description }} +// +// This task implements the {{ .Template }} template. +// + +import ( + "context" + "fmt" + "time" + + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// {{ .Description }} +type {{ .StructName }}Task struct { + // Domain is the domain to resolve. + Domain string + + // IDGenerator is the TaskIDgenerator to use. + IDGenerator TaskIDGenerator + + // Logger is the logger to use. + Logger model.Logger + + // ZeroTime is the measurement's zero time. + ZeroTime time.Time +} + +// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. +// +// Arguments: +// +// - domain is the domain to resolve; +// +// - idGenerator is the TaskIDGenerator to use; +// +// - logger is the logger to use; +// +// - zeroTime is the zero time of the measurement. +func New{{ .StructName }}Task(domain string, idGenerator TaskIDGenerator, + logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + return &{{ .StructName }}Task{ + Domain: domain, + IDGenerator: idGenerator, + Logger: logger, + ZeroTime: zeroTime, + } +} + +var _ Task = &{{ .StructName }}Task{} + +// Run implements Task +func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { + const defaultTimeout = 4 * time.Second // TODO: you may want to change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + reso := netxlite.NewStdlibResolver(t.Logger) + addresses, err := reso.LookupHost(ctx, t.Domain) + if err != nil { + // TODO: write something into the test keys + return err + } + // TODO: write something into the test keys + for range addresses { + // TODO: fan out a number of async child tasks for each IP address + } + return nil +} + +// Repr implements Task +func (t *{{ .StructName }}Task) Repr() string { + return fmt.Sprintf("{{ .StructName}}Task%+v", t) +} diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt new file mode 100644 index 0000000000..f2b69373b8 --- /dev/null +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -0,0 +1,77 @@ +package {{ .Experiment }} + +// +// {{ .StructName }}: {{ .Description }} +// +// This task implements the {{ .Template }} template. +// + +import ( + "context" + "fmt" + "time" + + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// {{ .Description }} +type {{ .StructName }}Task struct { + // Address is the address to connect to. + Address string + + // IDGenerator is the TaskIDgenerator to use. + IDGenerator TaskIDGenerator + + // Logger is the logger to use. + Logger model.Logger + + // ZeroTime is the measurement's zero time. + ZeroTime time.Time +} + +// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. +// +// Arguments: +// +// - address is the address to connect to; +// +// - idGenerator is the TaskIDGenerator to use; +// +// - logger is the logger to use; +// +// - zeroTime is the zero time of the measurement. +func New{{ .StructName }}Task(address string, idGenerator TaskIDGenerator, + logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + return &{{ .StructName }}Task{ + Address: address, + IDGenerator: idGenerator, + Logger: logger, + ZeroTime: zeroTime, + } +} + +var _ Task = &{{ .StructName }}Task{} + +// Run implements Task +func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { + const defaultTimeout = 10 * time.Second // TODO: you may want to change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) + dialer := trace.NewDialerWithoutResolver(t.Logger) + conn, err := dialer.DialContext(ctx, "tcp", t.Address) + _ = <-trace.TCPConnect // TODO: use this result + if err != nil { + // TODO: write something into the test keys + return err + } + conn.Close() + // TODO: write something into the test keys + return nil +} + +// Repr implements Task +func (t *{{ .StructName }}Task) Repr() string { + return fmt.Sprintf("{{ .StructName}}Task%+v", t) +} diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt new file mode 100644 index 0000000000..1c783df401 --- /dev/null +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -0,0 +1,101 @@ +package {{ .Experiment }} + +// +// {{ .StructName }}: {{ .Description }} +// +// This task implements the {{ .Template }} template. +// + +import ( + "context" + "crypto/tls" + "fmt" + "time" + + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// {{ .Description }} +type {{ .StructName }}Task struct { + // ALPN is the ALPN to use. + ALPN []string + + // Address is the address to connect to. + Address string + + // IDGenerator is the TaskIDgenerator to use. + IDGenerator TaskIDGenerator + + // Logger is the logger to use. + Logger model.Logger + + // SNI is the SNI to use. + SNI string + + // ZeroTime is the measurement's zero time. + ZeroTime time.Time +} + +// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. +// +// Arguments: +// +// - address is the address to connect to; +// +// - alpn is the ALPN to use; +// +// - sni is the SNI to use; +// +// - idGenerator is the TaskIDGenerator to use; +// +// - logger is the logger to use; +// +// - zeroTime is the zero time of the measurement. +func New{{ .StructName }}Task( + address string, alpn []string, sni string, idGenerator TaskIDGenerator, + logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + return &{{ .StructName }}Task{ + Address: address, + ALPN: alpn, + IDGenerator: idGenerator, + Logger: logger, + SNI: sni, + ZeroTime: zeroTime, + } +} + +var _ Task = &{{ .StructName }}Task{} + +// Run implements Task +func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { + const defaultTimeout = 15 * time.Second // TODO: you may want to change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) + dialer := trace.NewDialerWithoutResolver(t.Logger) + conn, err := dialer.DialContext(ctx, "tcp", t.Address) + _ = <-trace.TCPConnect // TODO: use this result + if err != nil { + // TODO: write something into the test keys + return err + } + defer conn.Close() + conn = trace.WrapNetConn(conn) + thx := trace.NewTLSHandshakerStdlib(t.Logger) + config := &tls.Config{ + NextProtos: t.ALPN, + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: t.SNI, + } + _, _, err = thx.Handshake(ctx, conn, config) + _ = <-trace.TLSHandshake // TODO: use this result + _ = trace.NetworkEvents() // TODO: use this result + return err +} + +// Repr implements Task +func (t *{{ .StructName }}Task) Repr() string { + return fmt.Sprintf("{{ .StructName}}Task%+v", t) +} diff --git a/internal/cmd/boilerplate/utils.go b/internal/cmd/boilerplate/utils.go new file mode 100644 index 0000000000..7bc4a053b3 --- /dev/null +++ b/internal/cmd/boilerplate/utils.go @@ -0,0 +1,53 @@ +package main + +// +// Utility functions +// + +import ( + "fmt" + "os" + "text/template" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// Permissions with which we create new directories +const newDirPermissions = 0755 + +// Helper to write less when printing to stdout +func printf(format string, args ...any) { + fmt.Fprintf(os.Stdout, format, args...) +} + +// Permissions with which we create new files +const newFilePermissions = 0644 + +// Creates a file for writing +func openForWriting(filepath string) *os.File { + filep, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, newFilePermissions) + runtimex.PanicOnError(err, "os.OpenFile failed") + return filep +} + +// Ensures that we close a file without I/O errors +func closeFile(fp *os.File) { + err := fp.Close() + runtimex.PanicOnError(err, "fp.Close failed") +} + +// Generic function for writing a template. +func writeTemplate(fullpath string, tmpl *template.Template, info any) { + printf("🚧 generating %s...\n", fullpath) + filep := openForWriting(fullpath) + defer closeFile(filep) + err := tmpl.Execute(filep, info) + runtimex.PanicOnError(err, "cannot execute a text/template") +} + +// Creates directories recursively +func mkdirP(fulldir string) { + printf("🐚 mkdir -p %s\n", fulldir) + err := os.MkdirAll(fulldir, newDirPermissions) + runtimex.PanicOnError(err, "o}s.MkdirAll failed") +} From 9211251fa95f3b161e10ea7741eedf022215b599 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 8 Aug 2022 23:00:32 +0200 Subject: [PATCH 06/83] fix(boilerplate): run go fmt at the end --- internal/cmd/boilerplate/experiment.go | 3 +++ internal/cmd/boilerplate/task.go | 1 + internal/cmd/boilerplate/utils.go | 13 ++++++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index 44a153e830..c06d5318e7 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -61,6 +61,9 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { generateMainTaskGo(info) } generateRegistryEntryGo(info) + + pkg := filepath.Join("internal", "experiment", info.Name, "/...") + gofmt(pkg) } // Obtains the experiment info diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index b0738c2e7b..666846fd69 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -124,4 +124,5 @@ func generateTaskGo(experiment string, info *TaskInfo) { "Description": info.Description, } writeTemplate(fullpath, tmpl, mapping) + gofmt(fullpath) } diff --git a/internal/cmd/boilerplate/utils.go b/internal/cmd/boilerplate/utils.go index 7bc4a053b3..f12db05432 100644 --- a/internal/cmd/boilerplate/utils.go +++ b/internal/cmd/boilerplate/utils.go @@ -7,9 +7,11 @@ package main import ( "fmt" "os" + "path/filepath" "text/template" "github.com/ooni/probe-cli/v3/internal/runtimex" + "golang.org/x/sys/execabs" ) // Permissions with which we create new directories @@ -49,5 +51,14 @@ func writeTemplate(fullpath string, tmpl *template.Template, info any) { func mkdirP(fulldir string) { printf("🐚 mkdir -p %s\n", fulldir) err := os.MkdirAll(fulldir, newDirPermissions) - runtimex.PanicOnError(err, "o}s.MkdirAll failed") + runtimex.PanicOnError(err, "os.MkdirAll failed") +} + +// Formats a package using go fmt. +func gofmt(packagepath string) { + cmd := execabs.Command("go", "fmt", "."+string(filepath.Separator)+packagepath) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + printf("🐚 %s\n", cmd.String()) + err := cmd.Run() + runtimex.PanicOnError(err, "cmd.Run failed") } From 339057c58ab752567b8c672ad00b781af7ddba3e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 8 Aug 2022 23:14:40 +0200 Subject: [PATCH 07/83] fix(boilerplate): minor fixes --- internal/cmd/boilerplate/experiment.go | 11 +++++++++ .../boilerplate/experiment/maintask.go.txt | 23 +++++++++++++++++-- .../cmd/boilerplate/experiment/tasks.go.txt | 1 + .../boilerplate/task/systemresolver.go.txt | 2 +- .../cmd/boilerplate/task/tcpconnect.go.txt | 2 +- .../cmd/boilerplate/task/tlshandshake.go.txt | 2 +- 6 files changed, 36 insertions(+), 5 deletions(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index c06d5318e7..06d0d079ad 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -64,6 +64,17 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { pkg := filepath.Join("internal", "experiment", info.Name, "/...") gofmt(pkg) + + printf("\n") + printf("🏁 All done! Now you can run:\n") + printf("\n") + printf("* `go build -v ./internal/cmd/miniooni` to build `miniooni`;\n") + printf("\n") + printf("* `./miniooni -n [options] %s` to test your new experiment;\n", info.Name) + printf("\n") + printf("* `go run ./internal/cmd/boilerplate new-task` to add tasks\n") + printf(" to you new experiment (e.g., a TLS handshake).\n") + printf("\n") } // Obtains the experiment info diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index adf0887a16..7ae679d821 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -1,7 +1,18 @@ package {{ .Name }} // -// The main task. +// The main task of the {{ .Name }} experiment. +// +// You autogenerated code for a multi-task experiment. This +// means that the "main" function of the experiment, i.e., +// Measurer.Run will instantiate the MainTask, execute it and +// wait for it, and its child tasks, to complete. So, this +// file is the one that deserves most of your attention. +// +// You _may_ want to autogenerate specific tasks (e.g., TLS +// handshake or DNS lookup) using boilerplate's `new-task` +// subcommand. This subcommand will give you a good starting +// point for performing specific submeasurements. // // Note: the autogenerated code contains a very basic task // that does nothing. You should, of course, edit it! @@ -9,6 +20,7 @@ package {{ .Name }} import ( "context" + "fmt" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -29,6 +41,13 @@ type MainTask struct { // NewMainTask creates a new MainTask instance. // +// Arguments: +// +// - idGenerator allows us to generate unique IDs for each measurement, such +// that we can recognize which observation originates from which submeasurement; +// +// - logger is the logger to use; +// // TODO: add the required parameters. func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger) *MainTask { return &MainTask{ @@ -46,5 +65,5 @@ func (t *MainTask) Run(ctx context.Context, sched TaskScheduler) error { // Repr implements Task func (t *MainTask) Repr() string { - return "{{ .Name }}Task" + return fmt.Sprintf("{{ .Name }}Task: %+v", t) } diff --git a/internal/cmd/boilerplate/experiment/tasks.go.txt b/internal/cmd/boilerplate/experiment/tasks.go.txt index 9183ea2605..08384c57a5 100644 --- a/internal/cmd/boilerplate/experiment/tasks.go.txt +++ b/internal/cmd/boilerplate/experiment/tasks.go.txt @@ -95,6 +95,7 @@ func (ts *taskScheduler) Schedule(ctx context.Context, task Task) { task: task, wg: ts.wg, } + ts.wg.Add(1) go wrapper.run(ctx, ts) } diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index c19a8d44ff..8381068a6b 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -73,5 +73,5 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task%+v", t) + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) } diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index f2b69373b8..aa0ae11316 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -73,5 +73,5 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task%+v", t) + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) } diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt index 1c783df401..33e4bf3b2c 100644 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -97,5 +97,5 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task%+v", t) + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) } From 2027528c2e98cfdf84bce14ee078fa1e35bbd70f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 00:18:04 +0200 Subject: [PATCH 08/83] boilerplate: more fixes and improvements --- .../boilerplate/experiment/maintask.go.txt | 4 + .../boilerplate/experiment/measurer.go.txt | 7 +- .../cmd/boilerplate/experiment/model.go.txt | 51 +++-- .../cmd/boilerplate/experiment/tasks.go.txt | 12 +- internal/cmd/boilerplate/task.go | 4 + internal/cmd/boilerplate/task/httpget.go.txt | 174 ++++++++++++++++++ .../boilerplate/task/systemresolver.go.txt | 12 +- .../cmd/boilerplate/task/tcpconnect.go.txt | 7 + .../cmd/boilerplate/task/tlshandshake.go.txt | 9 + 9 files changed, 259 insertions(+), 21 deletions(-) create mode 100644 internal/cmd/boilerplate/task/httpget.go.txt diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index 7ae679d821..212c341d4b 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -59,6 +59,10 @@ func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger) *MainTask { // Run implements Task func (t *MainTask) Run(ctx context.Context, sched TaskScheduler) error { + // TODO: here you should start child tasks. + // + // You can use `boilerplate new-task` to automatically generate code for + // predefined tasks (e.g., HTTP GET, TLS handshake). t.Logger.Infof("Hello, world") return nil } diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index 4cb7a22501..3c30766f27 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -106,9 +106,12 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, idGenerator := NewTaskIDGenerator() main := NewMainTask(idGenerator, sess.Logger()) ts := NewTaskSchedulerWaiter(sess.Logger(), parallelism) - ts.Schedule(ctx, main) + ts.Start(ctx, main) ts.Wait() - return nil + + // return whether there was a fundamental failure, which would prevent + // the measurement from being submitted to the OONI collector. + return tk.FundamentalFailure() {{ end }} } diff --git a/internal/cmd/boilerplate/experiment/model.go.txt b/internal/cmd/boilerplate/experiment/model.go.txt index a510188a03..a184ce5b47 100644 --- a/internal/cmd/boilerplate/experiment/model.go.txt +++ b/internal/cmd/boilerplate/experiment/model.go.txt @@ -15,34 +15,61 @@ type Config struct { {{ if eq .Parallel true }} +// TestKeys is the interface with which you access the real test keys. +type TestKeys interface { + // SetFundamentalFailure sets testKeys.fundamentalFailure. + SetFundamentalFailure(err error) + + // FundamentalFailure gets testKeys.fundamentalFailure. + FundamentalFailure() error + + // TODO: define thread-safe accessors for the real test keys. +} + // testKeys contains the experiment test keys. type testKeys struct { // mu provides mutual exclusion for accessing the test keys mu sync.Mutex - // TODO: add here fields produced by this experiment + // fundamentalFailure indicates that some fundamental error occurred + // in a background task and, as such, you don't want to submit the + // measuremnt to the OONI collector. + fundamentalFailure error + + // TODO: add here fields produced by this experiment. They should + // be public such that we can JSON serialize them. // // For example: // - // Failure *string + // // Blocked indicates that the resource is censored. + // Blocked bool `json:"blocked" } -// TestKeys is the interface with which you access the real test keys. -type TestKeys interface { - // TODO: define thread-safe accessors for the real test keys. - // - // For example: - // - // SetFailure(failure *string) +var _ TestKeys = &testKeys{} + +// SetFundamentalFailure implements TestKeys. +func (tk *testKeys) SetFundamentalFailure(err error) { + tk.mu.Lock() + tk.fundamentalFailure = err + tk.mu.Unlock() +} + +// FundamentalFailure implements TestKeys. +func (tk *testKeys) FundamentalFailure() error { + tk.mu.Lock() + err := tk.fundamentalFailure + tk.mu.Unlock() + return err } -// TODO: implement thread-safe accessors for the real test keys. +// TODO: implement thread-safe setters for the real test keys. This allows +// tasks to write directly into the TestKeys. // // For example: // -// func (tk *testKeys) SetFailure(failure *string) { +// func (tk *testKeys) SetBlocked(blocked bool) { // tk.mu.Lock() -// tk.Failure = failure +// tk.Blocked = blocked // tk.mu.Unlock() // } diff --git a/internal/cmd/boilerplate/experiment/tasks.go.txt b/internal/cmd/boilerplate/experiment/tasks.go.txt index 08384c57a5..f0ca94a912 100644 --- a/internal/cmd/boilerplate/experiment/tasks.go.txt +++ b/internal/cmd/boilerplate/experiment/tasks.go.txt @@ -36,8 +36,8 @@ func (ig *TaskIDGenerator) Next() int64 { // TaskScheduler schedules new tasks. type TaskScheduler interface { - // Schedule scheduless a task for executing ASAP. - Schedule(ctx context.Context, task Task) + // Start schedules a task for executing ASAP. + Start(ctx context.Context, task Task) } // TaskSchedulerWaiter schedules new tasks and waits for them. @@ -65,7 +65,7 @@ type taskScheduler struct { // argument controls the parallelism. A zero or negative argument implies // only a single task at a time could run. The [logger] argument is the // logger we should use to print information about tasks completion. You -// should use model.DiscardLogger to disable this functionality. +// should use the [model.DiscardLogger] singleton to disable this functionality. func NewTaskSchedulerWaiter(logger model.Logger, maxTasks int) TaskSchedulerWaiter { if maxTasks < 1 { maxTasks = 1 // as documented @@ -87,8 +87,8 @@ type Task interface { Run(ctx context.Context, sched TaskScheduler) error } -// Schedule implements TaskScheduler. -func (ts *taskScheduler) Schedule(ctx context.Context, task Task) { +// Start implements TaskScheduler. +func (ts *taskScheduler) Start(ctx context.Context, task Task) { wrapper := &taskWrapper{ logger: ts.logger, sema: ts.sema, @@ -119,7 +119,7 @@ type taskWrapper struct { wg *sync.WaitGroup } -// Run runs the task which in turn runs the task's job. +// Runs the task. func (tw *taskWrapper) run(ctx context.Context, sched TaskScheduler) { // Ensure we don't have too many concurrent tasks. tw.sema <- true diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index 666846fd69..346717ce29 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -74,6 +74,9 @@ func getTaskDescription() string { return docs } +//go:embed "task/httpget.go.txt" +var httpGetTemplate string + //go:embed "task/systemresolver.go.txt" var systemResolverTemplate string @@ -85,6 +88,7 @@ var tlsHandshakeTemplate string // The list of known tasks var knownTasks = map[string]string{ + "http-get": httpGetTemplate, "system-resolver": systemResolverTemplate, "tcp-connect": tcpConnectTemplate, "tls-handshake": tlsHandshakeTemplate, diff --git a/internal/cmd/boilerplate/task/httpget.go.txt b/internal/cmd/boilerplate/task/httpget.go.txt new file mode 100644 index 0000000000..25d84d5838 --- /dev/null +++ b/internal/cmd/boilerplate/task/httpget.go.txt @@ -0,0 +1,174 @@ +package {{ .Experiment }} + +// +// {{ .StructName }}: {{ .Description }} +// +// This task implements the {{ .Template }} template. +// + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// {{ .Description }} +type {{ .StructName }}Task struct { + // Address is the address to connect to. + Address string + + // IDGenerator is the TaskIDgenerator to use. + IDGenerator TaskIDGenerator + + // Logger is the logger to use. + Logger model.Logger + + // ZeroTime is the measurement's zero time. + ZeroTime time.Time +} + +// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. +// +// Arguments: +// +// - address is the address to connect to; +// +// - idGenerator is the TaskIDGenerator to use; +// +// - logger is the logger to use; +// +// - zeroTime is the zero time of the measurement. +func New{{ .StructName }}Task(address string, idGenerator TaskIDGenerator, + logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + return &{{ .StructName }}Task{ + Address: address, + IDGenerator: idGenerator, + Logger: logger, + ZeroTime: zeroTime, + } +} + +var _ Task = &{{ .StructName }}Task{} + +// Run implements Task +func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { + // 1. configure timeout for this task + const defaultTimeout = 15 * time.Second // TODO: you may want to change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + // 2. create a trace + trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) + + // 3. perform the TCP connect + dialer := trace.NewDialerWithoutResolver(t.Logger) + conn, err := dialer.DialContext(ctx, "tcp", t.Address) + t.onTCPConnectResult(err, <-trace.TCPConnect) + if err != nil { + return err + } + defer conn.Close() + + // 4. perform the plaintext HTTP GET + URL := &url.URL{ + Scheme: "http", + Host: t.hostHeader(), + } + // TODO(bassosimone): we should expose this function from measurexlite + req, err := measurex.NewHTTPRequestWithContext(ctx, "GET", URL.String(), nil) + if err != nil { + t.onNewHTTPRequestFailure(err) + return err + } + // TODO(bassosimone): we should implement HTTP tracing in measurexlite + txp := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(conn), + netxlite.NewNullTLSDialer(), + ) + defer txp.CloseIdleConnections() + resp, err := txp.RoundTrip(req) + if err != nil { + t.onHTTPTransactionResult(resp, nil, err) + return err + } + defer resp.Body.Close() + + // 5. read the response body + const maxResponseSize = 1 << 22 // TODO: you may want to change this default + reader := io.LimitReader(resp.Body, maxResponseSize) + data, err := netxlite.ReadAllContext(ctx, reader) + if err != nil { + t.onHTTPTransactionResult(resp, nil, err) + return err + } + + // 4. we're done! + return t.onHTTPTransactionResult(resp, data, nil) +} + +// Called when the result of the TCP connect becomes available. +// +// Arguments: +// +// - err is the possibly nil error; +// +// - tcpTrace contains the never-nil archival results. +func (t *{{ .StructName }}Task) onTCPConnectResult( + err error, tcpTrace *model.ArchivalTCPConnectResult) { + // TODO: implement this method. Typically here you want to + // save into the test keys using thread safe code. +} + +// Called when we fail to create a new HTTP request. +// +// This typically indicates that something was off in the experiment's +// settings and could be caused by an implementation error. +func (t *{{ .StructName}}Task) onNewHTTPRequestFailure(err error) { + // TODO: implement this method. Typically here you want to + // save into the test keys using thread safe code. +} + +// Called when the results of the HTTP transaction become available. +// +// Arguments: +// +// - resp is the possibly-nil HTTP response; +// +// - respBody is the possibly-nil-or-empty HTTP body; +// +// - err is the possibly-nil error. +func (t *{{ .StructName }}Task) onHTTPTransactionResult( + resp *http.Response, respBody []byte, err error) error { + // TODO: implement this method. Typically here you want to + // save into the test keys using thread safe code. + return nil +} + +// Returns the host header without the port if we're using the default +// port. Otherwise, it returns the host header with the port. +func (t *{{ .StructName }}Task) hostHeader() string { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return t.Address + } + if port == "80" { + return addr + } + return t.Address // there was no need to parse after all 😬 +} + +// Repr implements Task +func (t *{{ .StructName }}Task) Repr() string { + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) +} diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 8381068a6b..98f36613ca 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -55,9 +55,15 @@ var _ Task = &{{ .StructName }}Task{} // Run implements Task func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { + // 1. configure timeout for this task const defaultTimeout = 4 * time.Second // TODO: you may want to change this default ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() + + // 2. construct a system resolver. + // + // TODO(bassosimone): we need to implement trace support for the system + // resolver such that it's possible to use a trace here. reso := netxlite.NewStdlibResolver(t.Logger) addresses, err := reso.LookupHost(ctx, t.Domain) if err != nil { @@ -65,9 +71,13 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er return err } // TODO: write something into the test keys + + // 3. (typically) fan out a number of child async tasks to use the IP addrs for range addresses { - // TODO: fan out a number of async child tasks for each IP address + // TODO: implement } + + // 4. we're done! return nil } diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index aa0ae11316..849e6c4331 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -55,10 +55,15 @@ var _ Task = &{{ .StructName }}Task{} // Run implements Task func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { + // 1. configure timeout for this task const defaultTimeout = 10 * time.Second // TODO: you may want to change this default ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() + + // 2. create a trace trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) + + // 3. perform the TCP connect dialer := trace.NewDialerWithoutResolver(t.Logger) conn, err := dialer.DialContext(ctx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: use this result @@ -68,6 +73,8 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er } conn.Close() // TODO: write something into the test keys + + // 4. we're done! return nil } diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt index 33e4bf3b2c..71a335aa53 100644 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -70,10 +70,15 @@ var _ Task = &{{ .StructName }}Task{} // Run implements Task func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { + // 1. configure timeout for this task const defaultTimeout = 15 * time.Second // TODO: you may want to change this default ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() + + // 2. create a trace trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) + + // 3. perform the TCP connect dialer := trace.NewDialerWithoutResolver(t.Logger) conn, err := dialer.DialContext(ctx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: use this result @@ -82,6 +87,8 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er return err } defer conn.Close() + + // 4. TLS handshake conn = trace.WrapNetConn(conn) thx := trace.NewTLSHandshakerStdlib(t.Logger) config := &tls.Config{ @@ -92,6 +99,8 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er _, _, err = thx.Handshake(ctx, conn, config) _ = <-trace.TLSHandshake // TODO: use this result _ = trace.NetworkEvents() // TODO: use this result + + // 5. we're done! return err } From cccb41cad9f256b8e504c74964f364362bf18764 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 00:28:36 +0200 Subject: [PATCH 09/83] boilerplate: some extra changes --- internal/cmd/boilerplate/experiment/maintask.go.txt | 8 +++++++- internal/cmd/boilerplate/experiment/measurer.go.txt | 3 ++- internal/cmd/boilerplate/task/httpget.go.txt | 8 +++++++- internal/cmd/boilerplate/task/systemresolver.go.txt | 8 +++++++- internal/cmd/boilerplate/task/tcpconnect.go.txt | 8 +++++++- internal/cmd/boilerplate/task/tlshandshake.go.txt | 8 +++++++- 6 files changed, 37 insertions(+), 6 deletions(-) diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index 212c341d4b..eeb4f9935a 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -33,6 +33,9 @@ type MainTask struct { // Logger is the logger to use. Logger model.Logger + // TestKeys contains the TestKeys. + TestKeys TestKeys + // TODO: add fields. // // Please, keep in mind that, with multiple tasks you should @@ -48,11 +51,14 @@ type MainTask struct { // // - logger is the logger to use; // +// - tk contains the TestKeys were to write results. +// // TODO: add the required parameters. -func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger) *MainTask { +func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys) *MainTask { return &MainTask{ IDGenerator: idGenerator, Logger: logger, + TestKeys: tk, // TODO: fill fields. } } diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index 3c30766f27..8a1c250102 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -104,7 +104,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // defer the rest of the work to the main task const parallelism = 10 // TODO: adjust depending on your needs idGenerator := NewTaskIDGenerator() - main := NewMainTask(idGenerator, sess.Logger()) + main := NewMainTask(idGenerator, sess.Logger(), tk) ts := NewTaskSchedulerWaiter(sess.Logger(), parallelism) ts.Start(ctx, main) ts.Wait() @@ -129,6 +129,7 @@ func (m *Measurer) run(ctx context.Context, sess model.ExperimentSession, {{ if eq .InputPolicy "InputOptional" }} func (m *Measurer) optionalInput() model.MeasurementTarget { + // TODO: implement panic("not implemented") } {{ end }} diff --git a/internal/cmd/boilerplate/task/httpget.go.txt b/internal/cmd/boilerplate/task/httpget.go.txt index 25d84d5838..15b8a0bb11 100644 --- a/internal/cmd/boilerplate/task/httpget.go.txt +++ b/internal/cmd/boilerplate/task/httpget.go.txt @@ -32,6 +32,9 @@ type {{ .StructName }}Task struct { // Logger is the logger to use. Logger model.Logger + // TestKeys contains the TestKeys. + TestKeys TestKeys + // ZeroTime is the measurement's zero time. ZeroTime time.Time } @@ -46,13 +49,16 @@ type {{ .StructName }}Task struct { // // - logger is the logger to use; // +// - tk contains the TestKeys; +// // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task(address string, idGenerator TaskIDGenerator, - logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, IDGenerator: idGenerator, Logger: logger, + TestKeys: tk, ZeroTime: zeroTime, } } diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 98f36613ca..afafa532c1 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -26,6 +26,9 @@ type {{ .StructName }}Task struct { // Logger is the logger to use. Logger model.Logger + // TestKeys contains the TestKeys. + TestKeys TestKeys + // ZeroTime is the measurement's zero time. ZeroTime time.Time } @@ -40,13 +43,16 @@ type {{ .StructName }}Task struct { // // - logger is the logger to use; // +// - tk contains the TestKeys; +// // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task(domain string, idGenerator TaskIDGenerator, - logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Domain: domain, IDGenerator: idGenerator, Logger: logger, + TestKeys: tk, ZeroTime: zeroTime, } } diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index 849e6c4331..69afa81af4 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -26,6 +26,9 @@ type {{ .StructName }}Task struct { // Logger is the logger to use. Logger model.Logger + // TestKeys contains the TestKeys. + TestKeys TestKeys + // ZeroTime is the measurement's zero time. ZeroTime time.Time } @@ -40,13 +43,16 @@ type {{ .StructName }}Task struct { // // - logger is the logger to use; // +// - tk contains the TestKeys; +// // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task(address string, idGenerator TaskIDGenerator, - logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, IDGenerator: idGenerator, Logger: logger, + TestKeys: tk, ZeroTime: zeroTime, } } diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt index 71a335aa53..0ea6f66b30 100644 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -31,6 +31,9 @@ type {{ .StructName }}Task struct { // Logger is the logger to use. Logger model.Logger + // TestKeys contains the TestKeys. + TestKeys TestKeys + // SNI is the SNI to use. SNI string @@ -52,16 +55,19 @@ type {{ .StructName }}Task struct { // // - logger is the logger to use; // +// - tk contains the TestKeys; +// // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task( address string, alpn []string, sni string, idGenerator TaskIDGenerator, - logger model.Logger, zeroTime time.Time) *{{ .StructName }}Task { + logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, ALPN: alpn, IDGenerator: idGenerator, Logger: logger, SNI: sni, + TestKeys: tk, ZeroTime: zeroTime, } } From eb397020b804134e0bb7adb5ab25ed0fc7301430 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 00:30:26 +0200 Subject: [PATCH 10/83] boilerplate: record one more TODO --- internal/cmd/boilerplate/experiment.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index 06d0d079ad..dfff3f1777 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -50,7 +50,8 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { print("\n") info := getExperimentInfo() - info.Interruptible = info.InputPolicy == "InputNone" + // TODO(bassosimone): which is the condition for interruptible? It seems + // to me only performance experiments are interruptible?! makeExperimentDirectory(info) generateDocGo(info) From 2b992032b85577036da8d1ba3f665b05f659ddbb Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 01:47:03 +0200 Subject: [PATCH 11/83] fix(boilerplate): a bunch of minor fixes --- internal/cmd/boilerplate/experiment.go | 2 +- internal/cmd/boilerplate/experiment/maintask.go.txt | 2 +- internal/cmd/boilerplate/experiment/model.go.txt | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index dfff3f1777..d113ec72d1 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -74,7 +74,7 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { printf("* `./miniooni -n [options] %s` to test your new experiment;\n", info.Name) printf("\n") printf("* `go run ./internal/cmd/boilerplate new-task` to add tasks\n") - printf(" to you new experiment (e.g., a TLS handshake).\n") + printf(" to your new experiment (e.g., a TLS handshake).\n") printf("\n") } diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index eeb4f9935a..467961f02e 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -51,7 +51,7 @@ type MainTask struct { // // - logger is the logger to use; // -// - tk contains the TestKeys were to write results. +// - tk contains the TestKeys where to write results. // // TODO: add the required parameters. func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys) *MainTask { diff --git a/internal/cmd/boilerplate/experiment/model.go.txt b/internal/cmd/boilerplate/experiment/model.go.txt index a184ce5b47..1ac000753b 100644 --- a/internal/cmd/boilerplate/experiment/model.go.txt +++ b/internal/cmd/boilerplate/experiment/model.go.txt @@ -32,8 +32,8 @@ type testKeys struct { mu sync.Mutex // fundamentalFailure indicates that some fundamental error occurred - // in a background task and, as such, you don't want to submit the - // measuremnt to the OONI collector. + // in a background task. As such, you don't want to submit this + // measurement to the OONI collector. fundamentalFailure error // TODO: add here fields produced by this experiment. They should From 1cf7f0f61cb36e1d01863cc0337768876b8658b1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 07:21:20 +0200 Subject: [PATCH 12/83] feat: new boilerplate improvements --- internal/cmd/boilerplate/experiment/maintask.go.txt | 13 ++++++++++--- internal/cmd/boilerplate/experiment/measurer.go.txt | 2 +- internal/cmd/boilerplate/experiment/model.go.txt | 2 +- internal/cmd/boilerplate/task/httpget.go.txt | 12 +++++++++--- internal/cmd/boilerplate/task/systemresolver.go.txt | 4 ++-- internal/cmd/boilerplate/task/tcpconnect.go.txt | 7 ++++--- internal/cmd/boilerplate/task/tlshandshake.go.txt | 4 ++-- 7 files changed, 29 insertions(+), 15 deletions(-) diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index 467961f02e..67fb7ba4e4 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -36,6 +36,9 @@ type MainTask struct { // TestKeys contains the TestKeys. TestKeys TestKeys + // ZeroTime is the zero time of the measurement. + ZeroTime time.Time + // TODO: add fields. // // Please, keep in mind that, with multiple tasks you should @@ -51,15 +54,19 @@ type MainTask struct { // // - logger is the logger to use; // -// - tk contains the TestKeys where to write results. +// - tk contains the TestKeys where to write results; +// +// - zeroTime is the measurement's zero time. // // TODO: add the required parameters. -func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys) *MainTask { +func NewMainTask( + idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys, zeroTime time.Time) *MainTask { return &MainTask{ IDGenerator: idGenerator, Logger: logger, TestKeys: tk, - // TODO: fill fields. + ZeroTime: zeroTime, + // TODO: fill newly-added fields. } } diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index 8a1c250102..26b1978cd3 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -104,7 +104,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // defer the rest of the work to the main task const parallelism = 10 // TODO: adjust depending on your needs idGenerator := NewTaskIDGenerator() - main := NewMainTask(idGenerator, sess.Logger(), tk) + main := NewMainTask(idGenerator, sess.Logger(), tk, measurement.MeasurementStartTimeSaved) ts := NewTaskSchedulerWaiter(sess.Logger(), parallelism) ts.Start(ctx, main) ts.Wait() diff --git a/internal/cmd/boilerplate/experiment/model.go.txt b/internal/cmd/boilerplate/experiment/model.go.txt index 1ac000753b..5c0d6461e0 100644 --- a/internal/cmd/boilerplate/experiment/model.go.txt +++ b/internal/cmd/boilerplate/experiment/model.go.txt @@ -29,7 +29,7 @@ type TestKeys interface { // testKeys contains the experiment test keys. type testKeys struct { // mu provides mutual exclusion for accessing the test keys - mu sync.Mutex + mu *sync.Mutex // fundamentalFailure indicates that some fundamental error occurred // in a background task. As such, you don't want to submit this diff --git a/internal/cmd/boilerplate/task/httpget.go.txt b/internal/cmd/boilerplate/task/httpget.go.txt index 15b8a0bb11..8d1de15fda 100644 --- a/internal/cmd/boilerplate/task/httpget.go.txt +++ b/internal/cmd/boilerplate/task/httpget.go.txt @@ -27,7 +27,7 @@ type {{ .StructName }}Task struct { Address string // IDGenerator is the TaskIDgenerator to use. - IDGenerator TaskIDGenerator + IDGenerator *TaskIDGenerator // Logger is the logger to use. Logger model.Logger @@ -35,6 +35,9 @@ type {{ .StructName }}Task struct { // TestKeys contains the TestKeys. TestKeys TestKeys + // URLPath is the URL's path. + URLPath string + // ZeroTime is the measurement's zero time. ZeroTime time.Time } @@ -51,14 +54,17 @@ type {{ .StructName }}Task struct { // // - tk contains the TestKeys; // +// - urlPath is the URL path to use; +// // - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task(address string, idGenerator TaskIDGenerator, - logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { +func New{{ .StructName }}Task(address string, idGenerator *TaskIDGenerator, + logger model.Logger, tk TestKeys, urlPath string, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, IDGenerator: idGenerator, Logger: logger, TestKeys: tk, + URLPath: urlPath, ZeroTime: zeroTime, } } diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index afafa532c1..4e580f6b8b 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -21,7 +21,7 @@ type {{ .StructName }}Task struct { Domain string // IDGenerator is the TaskIDgenerator to use. - IDGenerator TaskIDGenerator + IDGenerator *TaskIDGenerator // Logger is the logger to use. Logger model.Logger @@ -46,7 +46,7 @@ type {{ .StructName }}Task struct { // - tk contains the TestKeys; // // - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task(domain string, idGenerator TaskIDGenerator, +func New{{ .StructName }}Task(domain string, idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Domain: domain, diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index 69afa81af4..defd46bee2 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -21,7 +21,7 @@ type {{ .StructName }}Task struct { Address string // IDGenerator is the TaskIDgenerator to use. - IDGenerator TaskIDGenerator + IDGenerator *TaskIDGenerator // Logger is the logger to use. Logger model.Logger @@ -46,8 +46,9 @@ type {{ .StructName }}Task struct { // - tk contains the TestKeys; // // - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task(address string, idGenerator TaskIDGenerator, - logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { +func New{{ .StructName }}Task( + address string, idGenerator *TaskIDGenerator, logger model.Logger, + tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, IDGenerator: idGenerator, diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt index 0ea6f66b30..45d3d2760d 100644 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -26,7 +26,7 @@ type {{ .StructName }}Task struct { Address string // IDGenerator is the TaskIDgenerator to use. - IDGenerator TaskIDGenerator + IDGenerator *TaskIDGenerator // Logger is the logger to use. Logger model.Logger @@ -59,7 +59,7 @@ type {{ .StructName }}Task struct { // // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task( - address string, alpn []string, sni string, idGenerator TaskIDGenerator, + address string, alpn []string, sni string, idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, From ba7a9c8f78c1f263f7818f16fbddd0f8f4484dea Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 08:02:45 +0200 Subject: [PATCH 13/83] boilerplate: split by type more aggressively --- internal/cmd/boilerplate/experiment.go | 76 +++++-------- .../boilerplate/experiment/maintask.go.txt | 13 ++- .../boilerplate/experiment/measurer.go.txt | 66 +++++------- .../cmd/boilerplate/experiment/model.go.txt | 102 ------------------ .../cmd/boilerplate/experiment/tasks.go.txt | 2 +- internal/cmd/boilerplate/task/httpget.go.txt | 2 +- .../boilerplate/task/systemresolver.go.txt | 2 +- .../cmd/boilerplate/task/tcpconnect.go.txt | 2 +- .../cmd/boilerplate/task/tlshandshake.go.txt | 2 +- 9 files changed, 65 insertions(+), 202 deletions(-) delete mode 100644 internal/cmd/boilerplate/experiment/model.go.txt diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index d113ec72d1..d65012dfc1 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -7,7 +7,6 @@ package main import ( _ "embed" "path/filepath" - "strconv" "text/template" "github.com/AlecAivazis/survey/v2" @@ -32,13 +31,7 @@ type ExperimentInfo struct { // Experiment input policy InputPolicy string - // Overall experiment timeout - Timeout int64 - - // Whether we'll run parallel tasks. - Parallel bool - - // Whether this experimenti is interuptible. + // Whether this experimenti is interruptible. Interruptible bool } @@ -56,11 +49,9 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { makeExperimentDirectory(info) generateDocGo(info) generateMeasurerGo(info) - generateModelGo(info) - if info.Parallel { - generateTasksGo(info) - generateMainTaskGo(info) - } + generateModelsGo(info) + generateTasksGo(info) + generateMainTaskGo(info) generateRegistryEntryGo(info) pkg := filepath.Join("internal", "experiment", info.Name, "/...") @@ -85,8 +76,6 @@ func getExperimentInfo() *ExperimentInfo { Version: getExperimentVersion(), SpecURL: getExperimentSpecURL(), InputPolicy: getExperimentInputPolicy(), - Timeout: getExperimentTimeout(), - Parallel: getExperimentParallel(), Interruptible: false, } } @@ -142,33 +131,6 @@ func getExperimentInputPolicy() string { return inputPolicy } -// Obtains the experiment timeout. -func getExperimentTimeout() int64 { - prompt := &survey.Input{ - Message: "Experiment's _overall_ timeout in seconds (just hit enter for no timeout):", - } - var value string - err := survey.AskOne(prompt, &value) - runtimex.PanicOnError(err, "survey.AskOne failed") - if value == "" { - return 0 - } - timeout, err := strconv.ParseInt(value, 10, 64) - runtimex.PanicOnError(err, "strconv.ParseInt failed") - return timeout -} - -// Obtains the experiment parallel setting. -func getExperimentParallel() bool { - var parallel bool - prompt := &survey.Confirm{ - Message: "Do you want to generate code for running tasks in parallel?", - } - err := survey.AskOne(prompt, ¶llel) - runtimex.PanicOnError(err, "survey.AskOne failed") - return parallel -} - // Creates a directory for the new experiment. func makeExperimentDirectory(info *ExperimentInfo) { fulldir := filepath.Join("internal", "experiment", info.Name) @@ -195,14 +157,32 @@ func generateMeasurerGo(info *ExperimentInfo) { writeTemplate(fullpath, tmpl, info) } -//go:embed "experiment/model.go.txt" -var experimentModelGoTemplate string +//go:embed "experiment/config.go.txt" +var experimentConfigGoTemplate string + +//go:embed "experiment/summary.go.txt" +var experimentSummaryGoTemplate string + +//go:embed "experiment/testkeys.go.txt" +var experimentTestkeysGoTemplate string // Generates the model.go file -func generateModelGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Name, "model.go") - tmpl := template.Must(template.New("model.go").Parse(experimentModelGoTemplate)) - writeTemplate(fullpath, tmpl, info) +func generateModelsGo(info *ExperimentInfo) { + { + fullpath := filepath.Join("internal", "experiment", info.Name, "config.go") + tmpl := template.Must(template.New("config.go").Parse(experimentConfigGoTemplate)) + writeTemplate(fullpath, tmpl, info) + } + { + fullpath := filepath.Join("internal", "experiment", info.Name, "summary.go") + tmpl := template.Must(template.New("model.go").Parse(experimentSummaryGoTemplate)) + writeTemplate(fullpath, tmpl, info) + } + { + fullpath := filepath.Join("internal", "experiment", info.Name, "testkeys.go") + tmpl := template.Must(template.New("model.go").Parse(experimentTestkeysGoTemplate)) + writeTemplate(fullpath, tmpl, info) + } } //go:embed "experiment/tasks.go.txt" diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index 67fb7ba4e4..5e976e157a 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -3,7 +3,7 @@ package {{ .Name }} // // The main task of the {{ .Name }} experiment. // -// You autogenerated code for a multi-task experiment. This +// This autogenerated code is for a multi-task experiment. This // means that the "main" function of the experiment, i.e., // Measurer.Run will instantiate the MainTask, execute it and // wait for it, and its child tasks, to complete. So, this @@ -15,12 +15,13 @@ package {{ .Name }} // point for performing specific submeasurements. // // Note: the autogenerated code contains a very basic task -// that does nothing. You should, of course, edit it! +// that does ~nothing. You should, of course, edit it! // import ( "context" "fmt" + "time" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -43,6 +44,8 @@ type MainTask struct { // // Please, keep in mind that, with multiple tasks you should // most like only share fields as read-only here! + // + // Please, keep the field names sorted alphabetically! } // NewMainTask creates a new MainTask instance. @@ -58,7 +61,7 @@ type MainTask struct { // // - zeroTime is the measurement's zero time. // -// TODO: add the required parameters. +// TODO: add the required parameters and document them. func NewMainTask( idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys, zeroTime time.Time) *MainTask { return &MainTask{ @@ -72,7 +75,7 @@ func NewMainTask( // Run implements Task func (t *MainTask) Run(ctx context.Context, sched TaskScheduler) error { - // TODO: here you should start child tasks. + // TODO: here you should start you child tasks. // // You can use `boilerplate new-task` to automatically generate code for // predefined tasks (e.g., HTTP GET, TLS handshake). @@ -82,5 +85,5 @@ func (t *MainTask) Run(ctx context.Context, sched TaskScheduler) error { // Repr implements Task func (t *MainTask) Repr() string { - return fmt.Sprintf("{{ .Name }}Task: %+v", t) + return fmt.Sprintf("{{ .Name }}Task: %+v", t) // TODO: make this prettier } diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index 26b1978cd3..f55d9fe72b 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -1,7 +1,12 @@ package {{ .Name }} // -// Measurer for {{ .Name }} +// Measurer for {{ .Name }}. +// +// The Measurer implements performing a single measurement. Because this +// autogenerated code is optimized to run background tasks, this code's +// main job is to parse inputs and then schedule the "main" task implementing +// the experiment. As such, there is little to change for you here. // import ( @@ -9,37 +14,37 @@ import ( {{ if ne .InputPolicy "InputOptional" }} "errors" {{ end }} - {{ if ne .Timeout 0 }} - "time" - {{ end }} "github.com/ooni/probe-cli/v3/internal/model" ) -// Measurer implements {{ .Name }}. +// Measurer implements the {{ .Name }} experiment. type Measurer struct{ // Config contains the experiment's config. Config *Config } // NewExperimentMeasurer creates a new model.ExperimentMeasurer for {{ .Name }}. +// +// Code in the internal/registry package binds this function with an ExperimentBuilder +// which is responsible of instantiating a new experiment. func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { return &Measurer{ Config: config, } } -// ExperimentName implements ExperimentMeasurer.ExperimentName. +// ExperimentName implements model.ExperimentMeasurer. func (m *Measurer) ExperimentName() string { return "{{ .Name }}" } -// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion. +// ExperimentVersion implements model.ExperimentMeasurer. func (m *Measurer) ExperimentVersion() string { return "{{ .Version }}" } -// Run implements ExperimentMeasurer.Run. +// Run implements model.ExperimentMeasurer. func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { // Reminder: When this function returns an error, the measurement result @@ -57,7 +62,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // honour {{ .InputPolicy }} input := measurement.Input if input == "" { - input = m.optionalInput() + input = m.defaultInput() } {{ else if eq .InputPolicy "InputOrQueryBackend" }} @@ -91,20 +96,16 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, tk := NewTestKeys() measurement.TestKeys = tk - {{ if ne .Timeout 0 }} - // ensure there's an overall timeout - ctx, cancel := context.WithTimeout(ctx, {{.Timeout}} * time.Second) - defer cancel() - {{ end }} - -{{ if eq .Parallel false }} - // defer the rest of the work to run() - return m.run(ctx, sess, measurement, callbacks) -{{ else }} // defer the rest of the work to the main task - const parallelism = 10 // TODO: adjust depending on your needs idGenerator := NewTaskIDGenerator() - main := NewMainTask(idGenerator, sess.Logger(), tk, measurement.MeasurementStartTimeSaved) + startTime := measurement.MeasurementStartTimeSaved + main := NewMainTask( + idGenerator, + sess.Logger(), + tk, + startTime, + ) + const parallelism = 10 // TODO: adjust depending on your needs ts := NewTaskSchedulerWaiter(sess.Logger(), parallelism) ts.Start(ctx, main) ts.Wait() @@ -112,31 +113,12 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. return tk.FundamentalFailure() -{{ end }} -} - -{{ if eq .Parallel false }} - -// run implements Run. -func (m *Measurer) run(ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { - // TODO: implement - sess.Logger().Infof("Hello, world") - return nil } -{{ end }} - {{ if eq .InputPolicy "InputOptional" }} -func (m *Measurer) optionalInput() model.MeasurementTarget { +// defaultInput returns the default input value when input is not present +func (m *Measurer) defaultInput() model.MeasurementTarget { // TODO: implement panic("not implemented") } {{ end }} - -// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. -func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { - sk := SummaryKeys{isAnomaly: false} - // TODO: implement - return sk, nil -} diff --git a/internal/cmd/boilerplate/experiment/model.go.txt b/internal/cmd/boilerplate/experiment/model.go.txt deleted file mode 100644 index 5c0d6461e0..0000000000 --- a/internal/cmd/boilerplate/experiment/model.go.txt +++ /dev/null @@ -1,102 +0,0 @@ -package {{ .Name }} - -// -// Data model for {{ .Name }} -// - -{{ if eq .Parallel true }} -import "sync" -{{ end }} - -// Config contains configuration for the {{ .Name }} experiment. -type Config struct { - // TODO: add fields here if you need any config -} - -{{ if eq .Parallel true }} - -// TestKeys is the interface with which you access the real test keys. -type TestKeys interface { - // SetFundamentalFailure sets testKeys.fundamentalFailure. - SetFundamentalFailure(err error) - - // FundamentalFailure gets testKeys.fundamentalFailure. - FundamentalFailure() error - - // TODO: define thread-safe accessors for the real test keys. -} - -// testKeys contains the experiment test keys. -type testKeys struct { - // mu provides mutual exclusion for accessing the test keys - mu *sync.Mutex - - // fundamentalFailure indicates that some fundamental error occurred - // in a background task. As such, you don't want to submit this - // measurement to the OONI collector. - fundamentalFailure error - - // TODO: add here fields produced by this experiment. They should - // be public such that we can JSON serialize them. - // - // For example: - // - // // Blocked indicates that the resource is censored. - // Blocked bool `json:"blocked" -} - -var _ TestKeys = &testKeys{} - -// SetFundamentalFailure implements TestKeys. -func (tk *testKeys) SetFundamentalFailure(err error) { - tk.mu.Lock() - tk.fundamentalFailure = err - tk.mu.Unlock() -} - -// FundamentalFailure implements TestKeys. -func (tk *testKeys) FundamentalFailure() error { - tk.mu.Lock() - err := tk.fundamentalFailure - tk.mu.Unlock() - return err -} - -// TODO: implement thread-safe setters for the real test keys. This allows -// tasks to write directly into the TestKeys. -// -// For example: -// -// func (tk *testKeys) SetBlocked(blocked bool) { -// tk.mu.Lock() -// tk.Blocked = blocked -// tk.mu.Unlock() -// } - -// NewTestKeys creates a new instance of TestKeys. -func NewTestKeys() TestKeys { - return &testKeys{} -} - -{{ else }} - -// TestKeys contains the results produced by the experiment. -type TestKeys struct{ - // TODO: add here fields produced by this experiment -} - -// NewTestKeys creates a new instance of TestKeys. -func NewTestKeys() *TestKeys { - return &TestKeys{} -} - -{{ end }} - -// Summary contains the summary results. -// -// Note that this structure is part of the ABI contract with ooniprobe -// therefore we should be careful when changing it. -type SummaryKeys struct { - isAnomaly bool - // TODO: add here additional summary fields. -} diff --git a/internal/cmd/boilerplate/experiment/tasks.go.txt b/internal/cmd/boilerplate/experiment/tasks.go.txt index f0ca94a912..7509e41537 100644 --- a/internal/cmd/boilerplate/experiment/tasks.go.txt +++ b/internal/cmd/boilerplate/experiment/tasks.go.txt @@ -121,7 +121,7 @@ type taskWrapper struct { // Runs the task. func (tw *taskWrapper) run(ctx context.Context, sched TaskScheduler) { - // Ensure we don't have too many concurrent tasks. + // Ensure we do not have too many concurrent tasks. tw.sema <- true defer func() { <-tw.sema }() diff --git a/internal/cmd/boilerplate/task/httpget.go.txt b/internal/cmd/boilerplate/task/httpget.go.txt index 8d1de15fda..cf026a1807 100644 --- a/internal/cmd/boilerplate/task/httpget.go.txt +++ b/internal/cmd/boilerplate/task/httpget.go.txt @@ -182,5 +182,5 @@ func (t *{{ .StructName }}Task) hostHeader() string { // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier } diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 4e580f6b8b..1033e0fe7d 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -89,5 +89,5 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier } diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index defd46bee2..2839797d89 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -87,5 +87,5 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier } diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt index 45d3d2760d..056d4827a1 100644 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -112,5 +112,5 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) + return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier } From 848f2aa24350b6131dd5c3fda6f79b5f7c95e2fa Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 08:03:25 +0200 Subject: [PATCH 14/83] fix: add files I forgot to commit --- .../cmd/boilerplate/experiment/config.go.txt | 12 +++ .../cmd/boilerplate/experiment/summary.go.txt | 23 +++++ .../boilerplate/experiment/testkeys.go.txt | 87 +++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 internal/cmd/boilerplate/experiment/config.go.txt create mode 100644 internal/cmd/boilerplate/experiment/summary.go.txt create mode 100644 internal/cmd/boilerplate/experiment/testkeys.go.txt diff --git a/internal/cmd/boilerplate/experiment/config.go.txt b/internal/cmd/boilerplate/experiment/config.go.txt new file mode 100644 index 0000000000..75965123d6 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/config.go.txt @@ -0,0 +1,12 @@ +package {{ .Name }} + +// +// Config for {{ .Name }} and associated functions. +// + +// Config contains {{ .Name }} experiment configuration. +type Config struct { + // TODO: add fields here if you need any config +} + +// TODO: implement convenience accessors you may need for Config diff --git a/internal/cmd/boilerplate/experiment/summary.go.txt b/internal/cmd/boilerplate/experiment/summary.go.txt new file mode 100644 index 0000000000..7d3d8f6644 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/summary.go.txt @@ -0,0 +1,23 @@ +package {{ .Name }} + +// +// Experiment summary result returned to ooniprobe. +// + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Summary contains the summary results. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + isAnomaly bool + // TODO: add here additional summary fields. +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + sk := SummaryKeys{isAnomaly: false} + // TODO: implement + return sk, nil +} diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt new file mode 100644 index 0000000000..cf9b4d75df --- /dev/null +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -0,0 +1,87 @@ +package {{ .Name }} + +// +// TestKeys for {{ .Name }}. +// +// Note: for historical reasons, we call TestKeys the JSON object +// containing the results produced by OONI experiments. +// + +import "sync" + +// TestKeys is the thread-safe interface with which you access the real +// content of the test keys, contained by [testKeys]. +type TestKeys interface { + // TODO: define thread-safe setters for the real test keys. You may also + // want to create getters _if_ you need to access specific content inside + // the Measurer.Run func. For example, we do that for fundamentalFailure + // because we need to know whether the overall experiment failed. + + // SetFundamentalFailure sets [testKeys.fundamentalFailure]. + SetFundamentalFailure(err error) + + // FundamentalFailure gets [testKeys.fundamentalFailure]. + FundamentalFailure() error +} + +// testKeys contains the results produced by {{ .Name }}. +type testKeys struct { + // TODO: add here fields produced by this experiment. They should + // be public such that we can JSON serialize them. + // + // Ideally, try to keep the field names alphabetically sorted. + // + // For example: + // + // // Blocked indicates that the resource is censored. + // Blocked bool `json:"blocked" + + // fundamentalFailure indicates that some fundamental error occurred + // in a background task. A fundamental error is something like a programmer + // such as a failure to parse a URL that was hardcoded in the codebase. When + // this class of errors happens, you certainly don't want to submit the + // resulting measurement to the OONI collector. + fundamentalFailure error + + // mu provides mutual exclusion for accessing the test keys. + mu *sync.Mutex +} + +var _ TestKeys = &testKeys{} + +// TODO: implement thread-safe setters for the real test keys. This allows +// tasks to write directly into the TestKeys. +// +// For example: +// +// func (tk *testKeys) SetBlocked(blocked bool) { +// tk.mu.Lock() +// tk.Blocked = blocked +// tk.mu.Unlock() +// } +// +// In some cases, you may also need to write thread-safe getters. For example, +// below we also define a getter for fundamentalFailure because we need to +// read its value inside the autogenerated Runner.Main func. + +// SetFundamentalFailure implements TestKeys. +func (tk *testKeys) SetFundamentalFailure(err error) { + tk.mu.Lock() + tk.fundamentalFailure = err + tk.mu.Unlock() +} + +// FundamentalFailure implements TestKeys. +func (tk *testKeys) FundamentalFailure() error { + tk.mu.Lock() + err := tk.fundamentalFailure + tk.mu.Unlock() + return err +} + +// NewTestKeys creates a new instance of TestKeys. +func NewTestKeys() TestKeys { + return &testKeys{ + // TODO: here you should initialize all the fields + } +} From 30524163876aee6df175f1e217af11aad824ad18 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 08:08:31 +0200 Subject: [PATCH 15/83] feat(boilerplate): allow interrupting experiments --- internal/cmd/boilerplate/experiment.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index d65012dfc1..250f78f028 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -43,8 +43,6 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { print("\n") info := getExperimentInfo() - // TODO(bassosimone): which is the condition for interruptible? It seems - // to me only performance experiments are interruptible?! makeExperimentDirectory(info) generateDocGo(info) @@ -76,7 +74,7 @@ func getExperimentInfo() *ExperimentInfo { Version: getExperimentVersion(), SpecURL: getExperimentSpecURL(), InputPolicy: getExperimentInputPolicy(), - Interruptible: false, + Interruptible: getExperimentInterruptible(), } } @@ -131,6 +129,17 @@ func getExperimentInputPolicy() string { return inputPolicy } +// Returns whether we can interrupt experiments midway. +func getExperimentInterruptible() bool { + var interruptible bool + prompt := &survey.Confirm{ + Message: "Should the engine be able to abruptly interrupt a measurement?", + } + err := survey.AskOne(prompt, &interruptible) + runtimex.PanicOnError(err, "survey.AskOne failed") + return interruptible +} + // Creates a directory for the new experiment. func makeExperimentDirectory(info *ExperimentInfo) { fulldir := filepath.Join("internal", "experiment", info.Name) From 5d41081675f80c641fd2bbcf7a99afd46ff66282 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 08:23:04 +0200 Subject: [PATCH 16/83] boilerplate: add more explanatory code --- internal/cmd/boilerplate/experiment.go | 42 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index 250f78f028..fab768856a 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -44,6 +44,10 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { info := getExperimentInfo() + printf("\n") + printf("Thank you! Now I'm going to generate boilerplate code for the new experiment!\n") + printf("\n") + makeExperimentDirectory(info) generateDocGo(info) generateMeasurerGo(info) @@ -56,7 +60,7 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { gofmt(pkg) printf("\n") - printf("🏁 All done! Now you can run:\n") + printf("🏁 All done! Now you can try:\n") printf("\n") printf("* `go build -v ./internal/cmd/miniooni` to build `miniooni`;\n") printf("\n") @@ -80,6 +84,10 @@ func getExperimentInfo() *ExperimentInfo { // Obtains the experiment name func getExperimentName() string { + printf("Each OONI experiment has a name, which should match [a-z]+. The experiment\n") + printf("name determines the Go package name and the name with which you're calling the\n") + printf("experiment by name from the command line.\n") + print("\n") prompt := &survey.Input{ Message: "Experiment's name:", } @@ -91,6 +99,9 @@ func getExperimentName() string { // Obtains the experiment version func getExperimentVersion() string { + print("\n") + printf("Each OONI experiment has a .. version number.\n") + print("\n") prompt := &survey.Input{ Message: "Experiment's version:", } @@ -102,6 +113,12 @@ func getExperimentVersion() string { // Obtains the experiment spec URL func getExperimentSpecURL() string { + print("\n") + printf("Any OONI experiment should be associated with a public specification URL\n") + printf("describing the experiment design and implementation.\n") + print("\n") + printf("Typically, specs live at https://github.com/ooni/spec/tree/master/nettests.\n") + print("\n") prompt := &survey.Input{ Message: "Experiment's spec URL:", } @@ -113,11 +130,24 @@ func getExperimentSpecURL() string { // Obtains the experiment input policy. func getExperimentInputPolicy() string { + print("\n") + printf("Each OONI experiment has a specific policy regarding input, which is one of:\n") + print("\n") + printf("* InputOrQueryBackend: the user can specify input using --input or --input-file, but, if\n") + printf(" input is missing, the experiment will query the OONI backend to obtain input.\n") + print("\n") + printf("* InputOrStaticDefault: the user can specify input using --input or --input-file, but, if\n") + printf(" input is missing, the experiment will a static list bundled with the probe.\n") + print("\n") + printf("* InputStrictlyRequired: the user can specify input using --input or --input-file, and, if\n") + printf(" input is missing, the experiment will emit an error and refuse to run.\n") + print("\n") + printf("* InputStrictlyRequired: the user cannot specify any input.\n") + print("\n") var inputPolicy string prompt := &survey.Select{ Message: "Choose an experiment input policy:", Options: []string{ - "InputOptional", "InputOrQueryBackend", "InputOrStaticDefault", "InputStrictlyRequired", @@ -131,6 +161,14 @@ func getExperimentInputPolicy() string { // Returns whether we can interrupt experiments midway. func getExperimentInterruptible() bool { + print("\n") + printf("Most OONI experiments runs short measurements. For such experiments, we do not\n") + printf("want the engine to be able to interrupt a measurement. Rather, we have well defined\n") + printf("interruption points between measuring an input and measuring the next one.\n") + print("\n") + printf("Though, network performance experiments and, generally, all the experiments whose\n") + printf("measurements could last for dozens of seconds, are interruptible.\n") + print("\n") var interruptible bool prompt := &survey.Confirm{ Message: "Should the engine be able to abruptly interrupt a measurement?", From e6756dc6318b377953b61d71ccd9519a829abd10 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 9 Aug 2022 10:28:59 +0200 Subject: [PATCH 17/83] boilerplate: last changes before asking for feedback --- internal/cmd/boilerplate/experiment.go | 17 ++++- .../boilerplate/experiment/inputparser.go.txt | 63 +++++++++++++++++++ .../boilerplate/experiment/maintask.go.txt | 17 ++++- .../boilerplate/experiment/measurer.go.txt | 40 +++++------- .../cmd/boilerplate/task/tcpconnect.go.txt | 19 ++++-- 5 files changed, 121 insertions(+), 35 deletions(-) create mode 100644 internal/cmd/boilerplate/experiment/inputparser.go.txt diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index fab768856a..1583dd06b6 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -31,7 +31,7 @@ type ExperimentInfo struct { // Experiment input policy InputPolicy string - // Whether this experimenti is interruptible. + // Whether this experiment is interruptible. Interruptible bool } @@ -55,6 +55,9 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { generateTasksGo(info) generateMainTaskGo(info) generateRegistryEntryGo(info) + if info.InputPolicy != "InputNone" { + generateInputParserGo(info) + } pkg := filepath.Join("internal", "experiment", info.Name, "/...") gofmt(pkg) @@ -64,7 +67,7 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { printf("\n") printf("* `go build -v ./internal/cmd/miniooni` to build `miniooni`;\n") printf("\n") - printf("* `./miniooni -n [options] %s` to test your new experiment;\n", info.Name) + printf("* `./miniooni -n %s` to test your new experiment;\n", info.Name) printf("\n") printf("* `go run ./internal/cmd/boilerplate new-task` to add tasks\n") printf(" to your new experiment (e.g., a TLS handshake).\n") @@ -261,3 +264,13 @@ func generateRegistryEntryGo(info *ExperimentInfo) { tmpl := template.Must(template.New("registryentry.go").Parse(experimentRegistryEntryGoTemplate)) writeTemplate(fullpath, tmpl, info) } + +//go:embed "experiment/inputparser.go.txt" +var experimentInputParserGoTemplate string + +// Generates the experiment's entry inside ./internal/registry +func generateInputParserGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Name, "inputparser.go") + tmpl := template.Must(template.New("inputparser.go").Parse(experimentInputParserGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} diff --git a/internal/cmd/boilerplate/experiment/inputparser.go.txt b/internal/cmd/boilerplate/experiment/inputparser.go.txt new file mode 100644 index 0000000000..185de819b2 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/inputparser.go.txt @@ -0,0 +1,63 @@ +package {{ .Name }} + +// +// Routines for parsing the experiment's input. +// + +import ( + "errors" + "net" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// InputParser helps to print the experiment's input. +type InputParser struct { + // List of accepted URL schemes. + AcceptedSchemes []string + + // Whether to allow endpoints in input. + AllowEndpoints bool + + // The default scheme to use if AllowEndpoints == true. + DefaultScheme string +} + +// Parse parses the experiment input and return the resulting URL. +func (ip *InputParser) Parse(input string) (*url.URL, error) { + // put this check at top-level such that we always see the crash if needed + runtimex.PanicIfTrue( + ip.AllowEndpoints && ip.DefaultScheme == "", + "invalid configuration for InputParser.AllowEndpoints == true", + ) + URL, err := url.Parse(input) + if err != nil { + return ip.maybeAllowEndpoints(URL, err) + } + for _, scheme := range ip.AcceptedSchemes { + if URL.Scheme == scheme { + // TODO: here you may want to perform additional parsing + return URL, nil + } + } + return nil, errors.New("cannot parse input") +} + +// Logic to conditionally allow endpoints if the user is fine with that. +func (ip *InputParser) maybeAllowEndpoints(URL *url.URL, err error) (*url.URL, error) { + runtimex.PanicIfNil(err, "expected to be called with a non-nil error") + if ip.AllowEndpoints && URL.Scheme != "" && URL.Opaque != "" && URL.User == nil && + URL.Host == "" && URL.Path == "" && URL.RawPath == "" && + URL.RawQuery == "" && URL.Fragment == "" && URL.RawFragment == "" { + // See https://go.dev/play/p/Rk5pS_zGY5U + // + // Note that we know that `ip.DefaultScheme != ""` from the above runtime check. + out := &url.URL{ + Scheme: ip.DefaultScheme, + Host: net.JoinHostPort(URL.Scheme, URL.Opaque), + } + return out, nil + } + return nil, err +} diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index 5e976e157a..0ad236a581 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -21,6 +21,7 @@ package {{ .Name }} import ( "context" "fmt" + {{ if ne .InputPolicy "InputNone" }}"net/url"{{ end }} "time" "github.com/ooni/probe-cli/v3/internal/model" @@ -40,6 +41,11 @@ type MainTask struct { // ZeroTime is the zero time of the measurement. ZeroTime time.Time + {{ if ne .InputPolicy "InputNone" }} + // Input contains the parsed experiment input. + Input *url.URL + {{ end }} + // TODO: add fields. // // Please, keep in mind that, with multiple tasks you should @@ -59,16 +65,21 @@ type MainTask struct { // // - tk contains the TestKeys where to write results; // -// - zeroTime is the measurement's zero time. +// - zeroTime is the measurement's zero time;{{ if ne .InputPolicy "InputNone" }} +// +// - input contains the experiment's input;{{ end }} // // TODO: add the required parameters and document them. -func NewMainTask( - idGenerator *TaskIDGenerator, logger model.Logger, tk TestKeys, zeroTime time.Time) *MainTask { +func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger, + tk TestKeys, zeroTime time.Time, {{ if ne .InputPolicy "InputNone" }}input *url.URL{{ end }}) *MainTask { return &MainTask{ IDGenerator: idGenerator, Logger: logger, TestKeys: tk, ZeroTime: zeroTime, + {{ if ne .InputPolicy "InputNone" }} + Input: input, + {{ end }} // TODO: fill newly-added fields. } } diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index f55d9fe72b..b0ec659c6b 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -54,8 +54,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, {{ if eq .InputPolicy "InputNone" }} // honour {{ .InputPolicy }} if measurement.Input != "" { - sess.Logger().Warn("BUG: {{ .Name }} got input but has {{ .InputPolicy }}!") - return errors.New("{{ .Name }}: this experiment does not take any input") + return errors.New("this experiment does not take any input") } {{ else if eq .InputPolicy "InputOptional" }} @@ -65,31 +64,25 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, input = m.defaultInput() } - {{ else if eq .InputPolicy "InputOrQueryBackend" }} + {{ else }} // honour {{ .InputPolicy }} input := measurement.Input if input == "" { - sess.Logger().Warn("BUG: {{ .Name }} got no input but has {{ .InputPolicy }}!") - return errors.New("{{ .Name }}: no input provided") + return errors.New("no input provided") } + {{ end }} - {{ else if eq .InputPolicy "InputOrStaticDefault" }} - // honour {{ .InputPolicy }} - input := measurement.Input - if input == "" { - sess.Logger().Warn("BUG: {{ .Name }} got no input but has {{ .InputPolicy }}!") - return errors.New("{{ .Name }}: no input provided") + {{ if ne .InputPolicy "InputNone" }} + // convert the input string to a URL + inputParser := &InputParser{ + AcceptedSchemes: []string{"http", "https"}, // TODO: you may want to change this + AllowEndpoints: false, // TODO: you may want to change this + DefaultScheme: "", // TODO: you may want to change this } - - {{ else if eq .InputPolicy "InputStrictlyRequired" }} - // honour {{ .InputPolicy }} - input := measurement.Input - if input == "" { - return errors.New("{{ .Name }}: no input provided") + URL, err := inputParser.Parse(string(measurement.Input)) + if err != nil { + return err } - - {{ else }} - THIS SHOULD NOT HAPPEN! SOMETHING'S WRONG WITH THE AUTO-GENERATOR! {{ end }} // initialize the experiment's test keys @@ -99,12 +92,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // defer the rest of the work to the main task idGenerator := NewTaskIDGenerator() startTime := measurement.MeasurementStartTimeSaved - main := NewMainTask( - idGenerator, - sess.Logger(), - tk, - startTime, - ) + main := NewMainTask(idGenerator, sess.Logger(), tk, startTime, {{ if ne .InputPolicy "InputNone" }}URL{{end}}) const parallelism = 10 // TODO: adjust depending on your needs ts := NewTaskSchedulerWaiter(sess.Logger(), parallelism) ts.Start(ctx, main) diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index 2839797d89..70025219e1 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -73,18 +73,29 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // 3. perform the TCP connect dialer := trace.NewDialerWithoutResolver(t.Logger) conn, err := dialer.DialContext(ctx, "tcp", t.Address) - _ = <-trace.TCPConnect // TODO: use this result + t.onTCPConnectResult(err, <-trace.TCPConnect) if err != nil { - // TODO: write something into the test keys return err } - conn.Close() - // TODO: write something into the test keys + defer conn.Close() // 4. we're done! return nil } +// Called when the result of the TCP connect becomes available. +// +// Arguments: +// +// - err is the possibly nil error; +// +// - tcpTrace contains the never-nil archival results. +func (t *{{ .StructName }}Task) onTCPConnectResult( + err error, tcpTrace *model.ArchivalTCPConnectResult) { + // TODO: implement this method. Typically here you want to + // save into the test keys using thread safe code. +} + // Repr implements Task func (t *{{ .StructName }}Task) Repr() string { return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier From 04c759fe7cf204847506b95f9745274c3b98948d Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 11 Aug 2022 11:09:38 +0200 Subject: [PATCH 18/83] fix(boilerplate): allow experiment names to contain underscore --- internal/cmd/boilerplate/experiment.go | 30 +++++++++++-------- .../cmd/boilerplate/experiment/config.go.txt | 6 ++-- .../cmd/boilerplate/experiment/doc.go.txt | 4 +-- .../boilerplate/experiment/inputparser.go.txt | 2 +- .../boilerplate/experiment/maintask.go.txt | 4 +-- .../boilerplate/experiment/measurer.go.txt | 2 +- .../boilerplate/experiment/registry.go.txt | 8 ++--- .../cmd/boilerplate/experiment/summary.go.txt | 2 +- .../cmd/boilerplate/experiment/tasks.go.txt | 2 +- .../boilerplate/experiment/testkeys.go.txt | 2 +- internal/cmd/boilerplate/task.go | 18 +++++++++-- internal/cmd/boilerplate/task/httpget.go.txt | 2 +- .../boilerplate/task/systemresolver.go.txt | 2 +- .../cmd/boilerplate/task/tcpconnect.go.txt | 2 +- .../cmd/boilerplate/task/tlshandshake.go.txt | 2 +- 15 files changed, 54 insertions(+), 34 deletions(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index 1583dd06b6..3df2237de5 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -7,6 +7,7 @@ package main import ( _ "embed" "path/filepath" + "strings" "text/template" "github.com/AlecAivazis/survey/v2" @@ -35,6 +36,11 @@ type ExperimentInfo struct { Interruptible bool } +// Package returns the package name. +func (info *ExperimentInfo) Package() string { + return strings.ReplaceAll(strings.ToLower(info.Name), "_", "") +} + // Called by the CLI parser func (c *NewExperimentCommand) Run(*cobra.Command, []string) { printf("\n") @@ -59,7 +65,7 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { generateInputParserGo(info) } - pkg := filepath.Join("internal", "experiment", info.Name, "/...") + pkg := filepath.Join("internal", "experiment", info.Package(), "/...") gofmt(pkg) printf("\n") @@ -67,7 +73,7 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { printf("\n") printf("* `go build -v ./internal/cmd/miniooni` to build `miniooni`;\n") printf("\n") - printf("* `./miniooni -n %s` to test your new experiment;\n", info.Name) + printf("* `./miniooni -n %s` to test your new experiment;\n", info.Package()) printf("\n") printf("* `go run ./internal/cmd/boilerplate new-task` to add tasks\n") printf(" to your new experiment (e.g., a TLS handshake).\n") @@ -183,7 +189,7 @@ func getExperimentInterruptible() bool { // Creates a directory for the new experiment. func makeExperimentDirectory(info *ExperimentInfo) { - fulldir := filepath.Join("internal", "experiment", info.Name) + fulldir := filepath.Join("internal", "experiment", info.Package()) mkdirP(fulldir) } @@ -192,7 +198,7 @@ var experimentDocGoTemplate string // Generates the doc.go file func generateDocGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Name, "doc.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "doc.go") tmpl := template.Must(template.New("doc.go").Parse(experimentDocGoTemplate)) writeTemplate(fullpath, tmpl, info) } @@ -202,7 +208,7 @@ var experimentMeasurerGoTemplate string // Generates the measurer.go file func generateMeasurerGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Name, "measurer.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "measurer.go") tmpl := template.Must(template.New("measurer.go").Parse(experimentMeasurerGoTemplate)) writeTemplate(fullpath, tmpl, info) } @@ -219,17 +225,17 @@ var experimentTestkeysGoTemplate string // Generates the model.go file func generateModelsGo(info *ExperimentInfo) { { - fullpath := filepath.Join("internal", "experiment", info.Name, "config.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "config.go") tmpl := template.Must(template.New("config.go").Parse(experimentConfigGoTemplate)) writeTemplate(fullpath, tmpl, info) } { - fullpath := filepath.Join("internal", "experiment", info.Name, "summary.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "summary.go") tmpl := template.Must(template.New("model.go").Parse(experimentSummaryGoTemplate)) writeTemplate(fullpath, tmpl, info) } { - fullpath := filepath.Join("internal", "experiment", info.Name, "testkeys.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "testkeys.go") tmpl := template.Must(template.New("model.go").Parse(experimentTestkeysGoTemplate)) writeTemplate(fullpath, tmpl, info) } @@ -240,7 +246,7 @@ var experimentTasksGoTemplate string // Generates the tasks.go file func generateTasksGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Name, "tasks.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "tasks.go") tmpl := template.Must(template.New("tasks.go").Parse(experimentTasksGoTemplate)) writeTemplate(fullpath, tmpl, info) } @@ -250,7 +256,7 @@ var experimentMainTaskGoTemplate string // Generates the maintask.go file func generateMainTaskGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Name, "maintask.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "maintask.go") tmpl := template.Must(template.New("maintask.go").Parse(experimentMainTaskGoTemplate)) writeTemplate(fullpath, tmpl, info) } @@ -260,7 +266,7 @@ var experimentRegistryEntryGoTemplate string // Generates the experiment's entry inside ./internal/registry func generateRegistryEntryGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "registry", info.Name+".go") + fullpath := filepath.Join("internal", "registry", info.Package()+".go") tmpl := template.Must(template.New("registryentry.go").Parse(experimentRegistryEntryGoTemplate)) writeTemplate(fullpath, tmpl, info) } @@ -270,7 +276,7 @@ var experimentInputParserGoTemplate string // Generates the experiment's entry inside ./internal/registry func generateInputParserGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Name, "inputparser.go") + fullpath := filepath.Join("internal", "experiment", info.Package(), "inputparser.go") tmpl := template.Must(template.New("inputparser.go").Parse(experimentInputParserGoTemplate)) writeTemplate(fullpath, tmpl, info) } diff --git a/internal/cmd/boilerplate/experiment/config.go.txt b/internal/cmd/boilerplate/experiment/config.go.txt index 75965123d6..c39944072e 100644 --- a/internal/cmd/boilerplate/experiment/config.go.txt +++ b/internal/cmd/boilerplate/experiment/config.go.txt @@ -1,10 +1,10 @@ -package {{ .Name }} +package {{ .Package }} // -// Config for {{ .Name }} and associated functions. +// Config for {{ .Package }} and associated functions. // -// Config contains {{ .Name }} experiment configuration. +// Config contains {{ .Package }} experiment configuration. type Config struct { // TODO: add fields here if you need any config } diff --git a/internal/cmd/boilerplate/experiment/doc.go.txt b/internal/cmd/boilerplate/experiment/doc.go.txt index 5b45a1a372..e63bc27281 100644 --- a/internal/cmd/boilerplate/experiment/doc.go.txt +++ b/internal/cmd/boilerplate/experiment/doc.go.txt @@ -1,4 +1,4 @@ -// Package {{ .Name }} implements the {{ .Name }} experiment. +// Package {{ .Package }} implements the {{ .Name }} experiment. // // See {{ .SpecURL }}. -package {{ .Name }} +package {{ .Package }} diff --git a/internal/cmd/boilerplate/experiment/inputparser.go.txt b/internal/cmd/boilerplate/experiment/inputparser.go.txt index 185de819b2..80f6cd7f6b 100644 --- a/internal/cmd/boilerplate/experiment/inputparser.go.txt +++ b/internal/cmd/boilerplate/experiment/inputparser.go.txt @@ -1,4 +1,4 @@ -package {{ .Name }} +package {{ .Package }} // // Routines for parsing the experiment's input. diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index 0ad236a581..576c2e0079 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -1,4 +1,4 @@ -package {{ .Name }} +package {{ .Package }} // // The main task of the {{ .Name }} experiment. @@ -96,5 +96,5 @@ func (t *MainTask) Run(ctx context.Context, sched TaskScheduler) error { // Repr implements Task func (t *MainTask) Repr() string { - return fmt.Sprintf("{{ .Name }}Task: %+v", t) // TODO: make this prettier + return fmt.Sprintf("{{ .Package }}Task: %+v", t) // TODO: make this prettier } diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index b0ec659c6b..0015395a88 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -1,4 +1,4 @@ -package {{ .Name }} +package {{ .Package }} // // Measurer for {{ .Name }}. diff --git a/internal/cmd/boilerplate/experiment/registry.go.txt b/internal/cmd/boilerplate/experiment/registry.go.txt index 2ebf60ce6c..1cc64977ab 100644 --- a/internal/cmd/boilerplate/experiment/registry.go.txt +++ b/internal/cmd/boilerplate/experiment/registry.go.txt @@ -5,18 +5,18 @@ package registry // import ( - "github.com/ooni/probe-cli/v3/internal/experiment/{{ .Name }}" + "github.com/ooni/probe-cli/v3/internal/experiment/{{ .Package }}" "github.com/ooni/probe-cli/v3/internal/model" ) func init() { allexperiments["{{ .Name }}"] = &Factory{ build: func(config any) model.ExperimentMeasurer { - return {{ .Name }}.NewExperimentMeasurer( - config.(*{{ .Name }}.Config), + return {{ .Package }}.NewExperimentMeasurer( + config.(*{{ .Package }}.Config), ) }, - config: &{{ .Name }}.Config{}, + config: &{{ .Package }}.Config{}, interruptible: {{ .Interruptible }}, inputPolicy: model.{{ .InputPolicy }}, } diff --git a/internal/cmd/boilerplate/experiment/summary.go.txt b/internal/cmd/boilerplate/experiment/summary.go.txt index 7d3d8f6644..39e4e07f55 100644 --- a/internal/cmd/boilerplate/experiment/summary.go.txt +++ b/internal/cmd/boilerplate/experiment/summary.go.txt @@ -1,4 +1,4 @@ -package {{ .Name }} +package {{ .Package }} // // Experiment summary result returned to ooniprobe. diff --git a/internal/cmd/boilerplate/experiment/tasks.go.txt b/internal/cmd/boilerplate/experiment/tasks.go.txt index 7509e41537..8a86194dac 100644 --- a/internal/cmd/boilerplate/experiment/tasks.go.txt +++ b/internal/cmd/boilerplate/experiment/tasks.go.txt @@ -1,4 +1,4 @@ -package {{ .Name }} +package {{ .Package }} // // Library for running background tasks. diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt index cf9b4d75df..84b36ba528 100644 --- a/internal/cmd/boilerplate/experiment/testkeys.go.txt +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -1,4 +1,4 @@ -package {{ .Name }} +package {{ .Package }} // // TestKeys for {{ .Name }}. diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index 346717ce29..eb0e0530d0 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -37,12 +37,26 @@ func (c *NewTaskCommand) Run(*cobra.Command, []string) { printf("to include it into an existing OONI experiment!\n") print("\n") - experimentName := getExperimentName() + experimentName := getExperimentPackageName() info := getTaskInfo() generateTaskGo(experimentName, info) } +// Obtains the experiment's package name +func getExperimentPackageName() string { + printf("Please, enter the name of the Go package under ./internal/experiment` for\n") + printf("which you want to autogenerate a new task.\n") + print("\n") + prompt := &survey.Input{ + Message: "Experiment's package name:", + } + var experiment string + err := survey.AskOne(prompt, &experiment) + runtimex.PanicOnError(err, "survey.AskOne failed") + return experiment +} + // Obtains information about the task to generate. func getTaskInfo() *TaskInfo { return &TaskInfo{ @@ -122,7 +136,7 @@ func generateTaskGo(experiment string, info *TaskInfo) { fullpath := filepath.Join("internal", "experiment", experiment, name) tmpl := template.Must(template.New("T1").Parse(knownTasks[info.Template])) mapping := map[string]string{ - "Experiment": experiment, + "Package": experiment, "StructName": info.StructName, "Template": info.Template, "Description": info.Description, diff --git a/internal/cmd/boilerplate/task/httpget.go.txt b/internal/cmd/boilerplate/task/httpget.go.txt index cf026a1807..f1190f9519 100644 --- a/internal/cmd/boilerplate/task/httpget.go.txt +++ b/internal/cmd/boilerplate/task/httpget.go.txt @@ -1,4 +1,4 @@ -package {{ .Experiment }} +package {{ .Package }} // // {{ .StructName }}: {{ .Description }} diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 1033e0fe7d..bafa91f7d9 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -1,4 +1,4 @@ -package {{ .Experiment }} +package {{ .Package }} // // {{ .StructName }}: {{ .Description }} diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index 70025219e1..e98b1b791c 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -1,4 +1,4 @@ -package {{ .Experiment }} +package {{ .Package }} // // {{ .StructName }}: {{ .Description }} diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt index 056d4827a1..c7400f0aa6 100644 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -1,4 +1,4 @@ -package {{ .Experiment }} +package {{ .Package }} // // {{ .StructName }}: {{ .Description }} From 6c9018d5637dc59ebbb91c81fe0d5d470ca9491c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 11 Aug 2022 11:10:47 +0200 Subject: [PATCH 19/83] fix docs --- internal/cmd/boilerplate/experiment.go | 2 +- internal/cmd/boilerplate/task.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index 3df2237de5..79d67a16e2 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -93,7 +93,7 @@ func getExperimentInfo() *ExperimentInfo { // Obtains the experiment name func getExperimentName() string { - printf("Each OONI experiment has a name, which should match [a-z]+. The experiment\n") + printf("Each OONI experiment has a name, which should match [a-z_]+. The experiment\n") printf("name determines the Go package name and the name with which you're calling the\n") printf("experiment by name from the command line.\n") print("\n") diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index eb0e0530d0..7192eddccc 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -45,8 +45,7 @@ func (c *NewTaskCommand) Run(*cobra.Command, []string) { // Obtains the experiment's package name func getExperimentPackageName() string { - printf("Please, enter the name of the Go package under ./internal/experiment` for\n") - printf("which you want to autogenerate a new task.\n") + printf("Each experiment is a Go package under ./internal/experiment`.\n") print("\n") prompt := &survey.Input{ Message: "Experiment's package name:", From 5f33f434211967158068073e581b6a5e2a114146 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 12:15:42 +0200 Subject: [PATCH 20/83] progress --- internal/cmd/boilerplate/experiment.go | 44 +--------------- .../boilerplate/experiment/maintask.go.txt | 4 +- .../cmd/boilerplate/experiment/summary.go.txt | 2 +- .../boilerplate/experiment/testkeys.go.txt | 41 +++------------ internal/cmd/boilerplate/task.go | 6 +-- internal/cmd/boilerplate/task/httpget.go.txt | 5 +- .../boilerplate/task/systemresolver.go.txt | 4 +- .../cmd/boilerplate/task/tcpconnect.go.txt | 6 +-- .../cmd/boilerplate/task/tlshandshake.go.txt | 52 +++++++++++++++---- 9 files changed, 62 insertions(+), 102 deletions(-) diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index 79d67a16e2..6b7940a5eb 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -69,14 +69,7 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { gofmt(pkg) printf("\n") - printf("🏁 All done! Now you can try:\n") - printf("\n") - printf("* `go build -v ./internal/cmd/miniooni` to build `miniooni`;\n") - printf("\n") - printf("* `./miniooni -n %s` to test your new experiment;\n", info.Package()) - printf("\n") - printf("* `go run ./internal/cmd/boilerplate new-task` to add tasks\n") - printf(" to your new experiment (e.g., a TLS handshake).\n") + printf("🏁 All done!\n") printf("\n") } @@ -93,10 +86,6 @@ func getExperimentInfo() *ExperimentInfo { // Obtains the experiment name func getExperimentName() string { - printf("Each OONI experiment has a name, which should match [a-z_]+. The experiment\n") - printf("name determines the Go package name and the name with which you're calling the\n") - printf("experiment by name from the command line.\n") - print("\n") prompt := &survey.Input{ Message: "Experiment's name:", } @@ -108,9 +97,6 @@ func getExperimentName() string { // Obtains the experiment version func getExperimentVersion() string { - print("\n") - printf("Each OONI experiment has a .. version number.\n") - print("\n") prompt := &survey.Input{ Message: "Experiment's version:", } @@ -122,12 +108,6 @@ func getExperimentVersion() string { // Obtains the experiment spec URL func getExperimentSpecURL() string { - print("\n") - printf("Any OONI experiment should be associated with a public specification URL\n") - printf("describing the experiment design and implementation.\n") - print("\n") - printf("Typically, specs live at https://github.com/ooni/spec/tree/master/nettests.\n") - print("\n") prompt := &survey.Input{ Message: "Experiment's spec URL:", } @@ -139,20 +119,6 @@ func getExperimentSpecURL() string { // Obtains the experiment input policy. func getExperimentInputPolicy() string { - print("\n") - printf("Each OONI experiment has a specific policy regarding input, which is one of:\n") - print("\n") - printf("* InputOrQueryBackend: the user can specify input using --input or --input-file, but, if\n") - printf(" input is missing, the experiment will query the OONI backend to obtain input.\n") - print("\n") - printf("* InputOrStaticDefault: the user can specify input using --input or --input-file, but, if\n") - printf(" input is missing, the experiment will a static list bundled with the probe.\n") - print("\n") - printf("* InputStrictlyRequired: the user can specify input using --input or --input-file, and, if\n") - printf(" input is missing, the experiment will emit an error and refuse to run.\n") - print("\n") - printf("* InputStrictlyRequired: the user cannot specify any input.\n") - print("\n") var inputPolicy string prompt := &survey.Select{ Message: "Choose an experiment input policy:", @@ -170,14 +136,6 @@ func getExperimentInputPolicy() string { // Returns whether we can interrupt experiments midway. func getExperimentInterruptible() bool { - print("\n") - printf("Most OONI experiments runs short measurements. For such experiments, we do not\n") - printf("want the engine to be able to interrupt a measurement. Rather, we have well defined\n") - printf("interruption points between measuring an input and measuring the next one.\n") - print("\n") - printf("Though, network performance experiments and, generally, all the experiments whose\n") - printf("measurements could last for dozens of seconds, are interruptible.\n") - print("\n") var interruptible bool prompt := &survey.Confirm{ Message: "Should the engine be able to abruptly interrupt a measurement?", diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt index 576c2e0079..900216f7d4 100644 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ b/internal/cmd/boilerplate/experiment/maintask.go.txt @@ -36,7 +36,7 @@ type MainTask struct { Logger model.Logger // TestKeys contains the TestKeys. - TestKeys TestKeys + TestKeys *TestKeys // ZeroTime is the zero time of the measurement. ZeroTime time.Time @@ -71,7 +71,7 @@ type MainTask struct { // // TODO: add the required parameters and document them. func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger, - tk TestKeys, zeroTime time.Time, {{ if ne .InputPolicy "InputNone" }}input *url.URL{{ end }}) *MainTask { + tk *TestKeys, zeroTime time.Time, {{ if ne .InputPolicy "InputNone" }}input *url.URL{{ end }}) *MainTask { return &MainTask{ IDGenerator: idGenerator, Logger: logger, diff --git a/internal/cmd/boilerplate/experiment/summary.go.txt b/internal/cmd/boilerplate/experiment/summary.go.txt index 39e4e07f55..7b885a754b 100644 --- a/internal/cmd/boilerplate/experiment/summary.go.txt +++ b/internal/cmd/boilerplate/experiment/summary.go.txt @@ -11,8 +11,8 @@ import "github.com/ooni/probe-cli/v3/internal/model" // Note that this structure is part of the ABI contract with ooniprobe // therefore we should be careful when changing it. type SummaryKeys struct { - isAnomaly bool // TODO: add here additional summary fields. + isAnomaly bool } // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt index 84b36ba528..a10cb1d01d 100644 --- a/internal/cmd/boilerplate/experiment/testkeys.go.txt +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -9,23 +9,8 @@ package {{ .Package }} import "sync" -// TestKeys is the thread-safe interface with which you access the real -// content of the test keys, contained by [testKeys]. -type TestKeys interface { - // TODO: define thread-safe setters for the real test keys. You may also - // want to create getters _if_ you need to access specific content inside - // the Measurer.Run func. For example, we do that for fundamentalFailure - // because we need to know whether the overall experiment failed. - - // SetFundamentalFailure sets [testKeys.fundamentalFailure]. - SetFundamentalFailure(err error) - - // FundamentalFailure gets [testKeys.fundamentalFailure]. - FundamentalFailure() error -} - -// testKeys contains the results produced by {{ .Name }}. -type testKeys struct { +// TestKeys contains the results produced by {{ .Name }}. +type TestKeys struct { // TODO: add here fields produced by this experiment. They should // be public such that we can JSON serialize them. // @@ -34,7 +19,7 @@ type testKeys struct { // For example: // // // Blocked indicates that the resource is censored. - // Blocked bool `json:"blocked" + // Blocked bool `json:"blocked"` // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer @@ -47,32 +32,22 @@ type testKeys struct { mu *sync.Mutex } -var _ TestKeys = &testKeys{} - -// TODO: implement thread-safe setters for the real test keys. This allows +// TODO: implement more thread-safe setters for the real test keys. This allows // tasks to write directly into the TestKeys. // -// For example: -// -// func (tk *testKeys) SetBlocked(blocked bool) { -// tk.mu.Lock() -// tk.Blocked = blocked -// tk.mu.Unlock() -// } -// // In some cases, you may also need to write thread-safe getters. For example, // below we also define a getter for fundamentalFailure because we need to // read its value inside the autogenerated Runner.Main func. // SetFundamentalFailure implements TestKeys. -func (tk *testKeys) SetFundamentalFailure(err error) { +func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Lock() tk.fundamentalFailure = err tk.mu.Unlock() } // FundamentalFailure implements TestKeys. -func (tk *testKeys) FundamentalFailure() error { +func (tk *TestKeys) FundamentalFailure() error { tk.mu.Lock() err := tk.fundamentalFailure tk.mu.Unlock() @@ -80,8 +55,8 @@ func (tk *testKeys) FundamentalFailure() error { } // NewTestKeys creates a new instance of TestKeys. -func NewTestKeys() TestKeys { - return &testKeys{ +func NewTestKeys() *TestKeys { + return &TestKeys{ // TODO: here you should initialize all the fields } } diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index 7192eddccc..995500dcf1 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -45,8 +45,6 @@ func (c *NewTaskCommand) Run(*cobra.Command, []string) { // Obtains the experiment's package name func getExperimentPackageName() string { - printf("Each experiment is a Go package under ./internal/experiment`.\n") - print("\n") prompt := &survey.Input{ Message: "Experiment's package name:", } @@ -68,7 +66,7 @@ func getTaskInfo() *TaskInfo { // Returns the name of the task struct. func getTaskStructName() string { prompt := &survey.Input{ - Message: "Task struct name (e.g., 'Datacenter'):", + Message: "Task struct name:", } var name string err := survey.AskOne(prompt, &name) @@ -131,7 +129,7 @@ func getTaskTemplate() string { // Generates code for the new task. func generateTaskGo(experiment string, info *TaskInfo) { - name := "task" + strings.ToLower(info.StructName) + ".go" + name := strings.ToLower(info.StructName) + ".go" fullpath := filepath.Join("internal", "experiment", experiment, name) tmpl := template.Must(template.New("T1").Parse(knownTasks[info.Template])) mapping := map[string]string{ diff --git a/internal/cmd/boilerplate/task/httpget.go.txt b/internal/cmd/boilerplate/task/httpget.go.txt index f1190f9519..4d54c3bd25 100644 --- a/internal/cmd/boilerplate/task/httpget.go.txt +++ b/internal/cmd/boilerplate/task/httpget.go.txt @@ -33,7 +33,7 @@ type {{ .StructName }}Task struct { Logger model.Logger // TestKeys contains the TestKeys. - TestKeys TestKeys + TestKeys *TestKeys // URLPath is the URL's path. URLPath string @@ -58,7 +58,7 @@ type {{ .StructName }}Task struct { // // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task(address string, idGenerator *TaskIDGenerator, - logger model.Logger, tk TestKeys, urlPath string, zeroTime time.Time) *{{ .StructName }}Task { + logger model.Logger, tk *TestKeys, urlPath string, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, IDGenerator: idGenerator, @@ -107,7 +107,6 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er netxlite.NewSingleUseDialer(conn), netxlite.NewNullTLSDialer(), ) - defer txp.CloseIdleConnections() resp, err := txp.RoundTrip(req) if err != nil { t.onHTTPTransactionResult(resp, nil, err) diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index bafa91f7d9..09506b2aa0 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -27,7 +27,7 @@ type {{ .StructName }}Task struct { Logger model.Logger // TestKeys contains the TestKeys. - TestKeys TestKeys + TestKeys *TestKeys // ZeroTime is the measurement's zero time. ZeroTime time.Time @@ -47,7 +47,7 @@ type {{ .StructName }}Task struct { // // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task(domain string, idGenerator *TaskIDGenerator, - logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { + logger model.Logger, tk *TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Domain: domain, IDGenerator: idGenerator, diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt index e98b1b791c..cd77bd0fe2 100644 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ b/internal/cmd/boilerplate/task/tcpconnect.go.txt @@ -27,7 +27,7 @@ type {{ .StructName }}Task struct { Logger model.Logger // TestKeys contains the TestKeys. - TestKeys TestKeys + TestKeys *TestKeys // ZeroTime is the measurement's zero time. ZeroTime time.Time @@ -48,7 +48,7 @@ type {{ .StructName }}Task struct { // - zeroTime is the zero time of the measurement. func New{{ .StructName }}Task( address string, idGenerator *TaskIDGenerator, logger model.Logger, - tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { + tk *TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, IDGenerator: idGenerator, @@ -87,7 +87,7 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // // Arguments: // -// - err is the possibly nil error; +// - err is the possibly-nil error; // // - tcpTrace contains the never-nil archival results. func (t *{{ .StructName }}Task) onTCPConnectResult( diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt index c7400f0aa6..518ba6ba76 100644 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ b/internal/cmd/boilerplate/task/tlshandshake.go.txt @@ -32,7 +32,7 @@ type {{ .StructName }}Task struct { Logger model.Logger // TestKeys contains the TestKeys. - TestKeys TestKeys + TestKeys *TestKeys // SNI is the SNI to use. SNI string @@ -58,9 +58,8 @@ type {{ .StructName }}Task struct { // - tk contains the TestKeys; // // - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task( - address string, alpn []string, sni string, idGenerator *TaskIDGenerator, - logger model.Logger, tk TestKeys, zeroTime time.Time) *{{ .StructName }}Task { +func New{{ .StructName }}Task(address string, alpn []string, sni string, idGenerator *TaskIDGenerator, + logger model.Logger, tk *TestKeys, zeroTime time.Time) *{{ .StructName }}Task { return &{{ .StructName }}Task{ Address: address, ALPN: alpn, @@ -87,27 +86,58 @@ func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) er // 3. perform the TCP connect dialer := trace.NewDialerWithoutResolver(t.Logger) conn, err := dialer.DialContext(ctx, "tcp", t.Address) - _ = <-trace.TCPConnect // TODO: use this result + t.onTCPConnectResult(err, <-trace.TCPConnect) if err != nil { - // TODO: write something into the test keys return err } + conn = trace.WrapNetConn(conn) defer conn.Close() // 4. TLS handshake - conn = trace.WrapNetConn(conn) thx := trace.NewTLSHandshakerStdlib(t.Logger) config := &tls.Config{ NextProtos: t.ALPN, RootCAs: netxlite.NewDefaultCertPool(), ServerName: t.SNI, } - _, _, err = thx.Handshake(ctx, conn, config) - _ = <-trace.TLSHandshake // TODO: use this result - _ = trace.NetworkEvents() // TODO: use this result + tlsConn, _, err := thx.Handshake(ctx, conn, config) + t.onTLSHandshakeResult(err, <-trace.TLSHandshake, trace.NetworkEvents()) + if err != nil { + return err + } + tlsConn.Close() // 5. we're done! - return err + return nil +} + +// Called when the result of the TCP connect becomes available. +// +// Arguments: +// +// - err is the possibly nil error; +// +// - tcpTrace contains the never-nil archival results. +func (t *{{ .StructName }}Task) onTCPConnectResult( + err error, tcpTrace *model.ArchivalTCPConnectResult) { + // TODO: implement this method. Typically here you want to + // save into the test keys using thread safe code. +} + +// Called when the result of the TLS handshake becomes available. +// +// Arguments: +// +// - err is the possibly-nil error; +// +// - tlsTrace contains the never-nil archival results; +// +// - netEvents contains the possibly empty network events. +func (t *{{ .StructName }}Task) onTLSHandshakeResult( + err error, tlsTrace *model.ArchivalTLSOrQUICHandshakeResult, + netEvents []*model.ArchivalNetworkEvent) { + // TODO: implement this method. Typically here you want to + // save into the test keys using thread safe code. } // Repr implements Task From 3845fbfc9d6664ad924ba948ba76e612c9ef281f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 18:35:36 +0200 Subject: [PATCH 21/83] subsequent changes to make it easier to manage --- internal/cmd/boilerplate/experiment.go | 22 -- .../boilerplate/experiment/maintask.go.txt | 100 ------- .../boilerplate/experiment/measurer.go.txt | 30 +- .../cmd/boilerplate/experiment/tasks.go.txt | 139 ---------- internal/cmd/boilerplate/task.go | 19 +- internal/cmd/boilerplate/task/endpoint.go.txt | 259 ++++++++++++++++++ internal/cmd/boilerplate/task/httpget.go.txt | 185 ------------- .../boilerplate/task/systemresolver.go.txt | 96 +++---- .../cmd/boilerplate/task/tcpconnect.go.txt | 102 ------- .../cmd/boilerplate/task/tlshandshake.go.txt | 146 ---------- internal/measurexlite/http.go | 93 +++++++ 11 files changed, 421 insertions(+), 770 deletions(-) delete mode 100644 internal/cmd/boilerplate/experiment/maintask.go.txt delete mode 100644 internal/cmd/boilerplate/experiment/tasks.go.txt create mode 100644 internal/cmd/boilerplate/task/endpoint.go.txt delete mode 100644 internal/cmd/boilerplate/task/httpget.go.txt delete mode 100644 internal/cmd/boilerplate/task/tcpconnect.go.txt delete mode 100644 internal/cmd/boilerplate/task/tlshandshake.go.txt create mode 100644 internal/measurexlite/http.go diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go index 6b7940a5eb..845275c7ed 100644 --- a/internal/cmd/boilerplate/experiment.go +++ b/internal/cmd/boilerplate/experiment.go @@ -58,8 +58,6 @@ func (c *NewExperimentCommand) Run(*cobra.Command, []string) { generateDocGo(info) generateMeasurerGo(info) generateModelsGo(info) - generateTasksGo(info) - generateMainTaskGo(info) generateRegistryEntryGo(info) if info.InputPolicy != "InputNone" { generateInputParserGo(info) @@ -199,26 +197,6 @@ func generateModelsGo(info *ExperimentInfo) { } } -//go:embed "experiment/tasks.go.txt" -var experimentTasksGoTemplate string - -// Generates the tasks.go file -func generateTasksGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Package(), "tasks.go") - tmpl := template.Must(template.New("tasks.go").Parse(experimentTasksGoTemplate)) - writeTemplate(fullpath, tmpl, info) -} - -//go:embed "experiment/maintask.go.txt" -var experimentMainTaskGoTemplate string - -// Generates the maintask.go file -func generateMainTaskGo(info *ExperimentInfo) { - fullpath := filepath.Join("internal", "experiment", info.Package(), "maintask.go") - tmpl := template.Must(template.New("maintask.go").Parse(experimentMainTaskGoTemplate)) - writeTemplate(fullpath, tmpl, info) -} - //go:embed "experiment/registry.go.txt" var experimentRegistryEntryGoTemplate string diff --git a/internal/cmd/boilerplate/experiment/maintask.go.txt b/internal/cmd/boilerplate/experiment/maintask.go.txt deleted file mode 100644 index 900216f7d4..0000000000 --- a/internal/cmd/boilerplate/experiment/maintask.go.txt +++ /dev/null @@ -1,100 +0,0 @@ -package {{ .Package }} - -// -// The main task of the {{ .Name }} experiment. -// -// This autogenerated code is for a multi-task experiment. This -// means that the "main" function of the experiment, i.e., -// Measurer.Run will instantiate the MainTask, execute it and -// wait for it, and its child tasks, to complete. So, this -// file is the one that deserves most of your attention. -// -// You _may_ want to autogenerate specific tasks (e.g., TLS -// handshake or DNS lookup) using boilerplate's `new-task` -// subcommand. This subcommand will give you a good starting -// point for performing specific submeasurements. -// -// Note: the autogenerated code contains a very basic task -// that does ~nothing. You should, of course, edit it! -// - -import ( - "context" - "fmt" - {{ if ne .InputPolicy "InputNone" }}"net/url"{{ end }} - "time" - - "github.com/ooni/probe-cli/v3/internal/model" -) - -// MainTask is the main task. -type MainTask struct { - // IDGenerator is the TaskIDGenerator to use. - IDGenerator *TaskIDGenerator - - // Logger is the logger to use. - Logger model.Logger - - // TestKeys contains the TestKeys. - TestKeys *TestKeys - - // ZeroTime is the zero time of the measurement. - ZeroTime time.Time - - {{ if ne .InputPolicy "InputNone" }} - // Input contains the parsed experiment input. - Input *url.URL - {{ end }} - - // TODO: add fields. - // - // Please, keep in mind that, with multiple tasks you should - // most like only share fields as read-only here! - // - // Please, keep the field names sorted alphabetically! -} - -// NewMainTask creates a new MainTask instance. -// -// Arguments: -// -// - idGenerator allows us to generate unique IDs for each measurement, such -// that we can recognize which observation originates from which submeasurement; -// -// - logger is the logger to use; -// -// - tk contains the TestKeys where to write results; -// -// - zeroTime is the measurement's zero time;{{ if ne .InputPolicy "InputNone" }} -// -// - input contains the experiment's input;{{ end }} -// -// TODO: add the required parameters and document them. -func NewMainTask(idGenerator *TaskIDGenerator, logger model.Logger, - tk *TestKeys, zeroTime time.Time, {{ if ne .InputPolicy "InputNone" }}input *url.URL{{ end }}) *MainTask { - return &MainTask{ - IDGenerator: idGenerator, - Logger: logger, - TestKeys: tk, - ZeroTime: zeroTime, - {{ if ne .InputPolicy "InputNone" }} - Input: input, - {{ end }} - // TODO: fill newly-added fields. - } -} - -// Run implements Task -func (t *MainTask) Run(ctx context.Context, sched TaskScheduler) error { - // TODO: here you should start you child tasks. - // - // You can use `boilerplate new-task` to automatically generate code for - // predefined tasks (e.g., HTTP GET, TLS handshake). - t.Logger.Infof("Hello, world") - return nil -} - -// Repr implements Task -func (t *MainTask) Repr() string { - return fmt.Sprintf("{{ .Package }}Task: %+v", t) // TODO: make this prettier -} diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index 0015395a88..18d2662172 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -11,10 +11,11 @@ package {{ .Package }} import ( "context" - {{ if ne .InputPolicy "InputOptional" }} "errors" - {{ end }} + "sync" + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -89,14 +90,23 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, tk := NewTestKeys() measurement.TestKeys = tk - // defer the rest of the work to the main task - idGenerator := NewTaskIDGenerator() - startTime := measurement.MeasurementStartTimeSaved - main := NewMainTask(idGenerator, sess.Logger(), tk, startTime, {{ if ne .InputPolicy "InputNone" }}URL{{end}}) - const parallelism = 10 // TODO: adjust depending on your needs - ts := NewTaskSchedulerWaiter(sess.Logger(), parallelism) - ts.Start(ctx, main) - ts.Wait() + // create variables required to run parallel tasks + idGenerator := &atomicx.Int64{} + wg := &sync.WaitGroup{} + + // start background tasks + // TODO: replace this code with code for running your background tasks + {{ if ne .InputPolicy "InputNone" }}_ = URL{{ end }} + wg.Add(1) + go func(logger model.Logger, idx int64) { + defer wg.Done() + ol := measurexlite.NewOperationLogger(logger, "{{ .Package }}#%d", idx) + tk.SetFundamentalFailure(errors.New("experiment not implemented")) + ol.Stop(nil) + }(sess.Logger(), idGenerator.Add(1)) + + // wait for background tasks to join + wg.Wait() // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. diff --git a/internal/cmd/boilerplate/experiment/tasks.go.txt b/internal/cmd/boilerplate/experiment/tasks.go.txt deleted file mode 100644 index 8a86194dac..0000000000 --- a/internal/cmd/boilerplate/experiment/tasks.go.txt +++ /dev/null @@ -1,139 +0,0 @@ -package {{ .Package }} - -// -// Library for running background tasks. -// -// Note: the autogenerated code contains a generic library but you -// may want to further customize what we do depending on specific -// needs that your experiment may actually have. -// - -import ( - "context" - "sync" - - "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" -) - -// TaskIDGenerator generates unique IDs for tasks. -type TaskIDGenerator struct { - c *atomicx.Int64 -} - -// NewTaskIDGenerator creates a new TaskIDGenerator instance. -func NewTaskIDGenerator() *TaskIDGenerator { - return &TaskIDGenerator{ - c: &atomicx.Int64{}, - } -} - -// Next returns the next unique task ID. -func (ig *TaskIDGenerator) Next() int64 { - return ig.c.Add(1) -} - -// TaskScheduler schedules new tasks. -type TaskScheduler interface { - // Start schedules a task for executing ASAP. - Start(ctx context.Context, task Task) -} - -// TaskSchedulerWaiter schedules new tasks and waits for them. -type TaskSchedulerWaiter interface { - // We inherit from the TaskScheduler. - TaskScheduler - - // Waits for all tasks to finish. - Wait() -} - -// taskScheduler implements TaskScheduler. -type taskScheduler struct { - // logger is the logger to use. - logger model.Logger - - // sema contains a semaphore for throttling concurrent tasks. - sema chan bool - - // wg is the wait group to wait for completion. - wg *sync.WaitGroup -} - -// NewTaskSchedulerWaiter creates a new TaskSchedulerWaiter instance. The [maxTasks] -// argument controls the parallelism. A zero or negative argument implies -// only a single task at a time could run. The [logger] argument is the -// logger we should use to print information about tasks completion. You -// should use the [model.DiscardLogger] singleton to disable this functionality. -func NewTaskSchedulerWaiter(logger model.Logger, maxTasks int) TaskSchedulerWaiter { - if maxTasks < 1 { - maxTasks = 1 // as documented - } - tl := &taskScheduler{ - logger: logger, - sema: make(chan bool, maxTasks), - wg: &sync.WaitGroup{}, - } - return tl -} - -// Task is one of the tasks run by this experiment. -type Task interface { - // Repr returns a printable representation of the task. - Repr() string - - // Run runs the task and returns the result. - Run(ctx context.Context, sched TaskScheduler) error -} - -// Start implements TaskScheduler. -func (ts *taskScheduler) Start(ctx context.Context, task Task) { - wrapper := &taskWrapper{ - logger: ts.logger, - sema: ts.sema, - task: task, - wg: ts.wg, - } - ts.wg.Add(1) - go wrapper.run(ctx, ts) -} - -// Wait implements TaskScheduler. -func (ts *taskScheduler) Wait() { - ts.wg.Wait() -} - -// taskWrapper wraps a task encapsulating scheduling logic. -type taskWrapper struct { - // logger is the logger to use. - logger model.Logger - - // sema is the controlling semaphore. - sema chan bool - - // task is the actual task. - task Task - - // wg allows the parent to wait for jobs to terminate. - wg *sync.WaitGroup -} - -// Runs the task. -func (tw *taskWrapper) run(ctx context.Context, sched TaskScheduler) { - // Ensure we do not have too many concurrent tasks. - tw.sema <- true - defer func() { <-tw.sema }() - - // Synchronize with the parent task. - defer tw.wg.Done() - - // Create operation logger. - ol := measurexlite.NewOperationLogger(tw.logger, tw.task.Repr()) - - // Run task. - err := tw.task.Run(ctx, sched) - - // Task completed. - ol.Stop(err) -} diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index 995500dcf1..c18e7a43e2 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -7,6 +7,7 @@ package main import ( _ "embed" "path/filepath" + "sort" "strings" "text/template" @@ -85,24 +86,19 @@ func getTaskDescription() string { return docs } -//go:embed "task/httpget.go.txt" -var httpGetTemplate string +//go:embed "task/endpoint.go.txt" +var endpointTemplate string //go:embed "task/systemresolver.go.txt" var systemResolverTemplate string -//go:embed "task/tcpconnect.go.txt" -var tcpConnectTemplate string - -//go:embed "task/tlshandshake.go.txt" -var tlsHandshakeTemplate string - // The list of known tasks var knownTasks = map[string]string{ - "http-get": httpGetTemplate, + "http": endpointTemplate, + "https": endpointTemplate, "system-resolver": systemResolverTemplate, - "tcp-connect": tcpConnectTemplate, - "tls-handshake": tlsHandshakeTemplate, + "tcp": endpointTemplate, + "tls": endpointTemplate, } // Names of known tasks @@ -113,6 +109,7 @@ func init() { for name := range knownTasks { knownTaskNames = append(knownTaskNames, name) } + sort.Strings(knownTaskNames) } // Returns the task template to use. diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt new file mode 100644 index 0000000000..8c43684b8a --- /dev/null +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -0,0 +1,259 @@ +package {{ .Package }} + +// +// {{ .StructName }}: {{ .Description }} +// + +import ( + "context" + {{ if or (eq .Template "tls") (eq .Template "https") -}} + "crypto/tls" + {{- end }} + {{ if or (eq .Template "http") (eq .Template "https") -}} + "io" + "net" + "net/http" + "net/url" + {{- end }} + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + {{ if or (eq .Template "tls") (eq .Template "https") (eq .Template "http") -}} + "github.com/ooni/probe-cli/v3/internal/netxlite" + {{- end }} +) + +// {{ .Description }} +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +// +// This task implements the {{ .Template }} template. +type {{ .StructName }}Task struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + {{ if or (eq .Template "tls") (eq .Template "https") }} + // ALPN is the OPTIONAL ALPN to use. + ALPN []string + + // SNI is the OPTIONAL SNI to use. + SNI string + {{ end }} + + {{ if or (eq .Template "http") (eq .Template "https") }} + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string + {{ end }} +} + +// Start starts this task in a background gorountine. +func (t *{{ .StructName }}Task) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go t.run(ctx, index) +} + +// run runs this task in the background. +func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { + // synchronize with wait group + defer t.WaitGroup.Done() + + // configure a timeout + const defaultTimeout = 15 * time.Second // TODO: change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}#%d", index) // TODO: edit + + {{ if or (eq .Template "tcp") (eq .Template "tls") (eq .Template "https") (eq .Template "http") }} + // perform the TCP connect + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + _ = <-trace.TCPConnect // TODO: save + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + _ = trace.NetworkEvents() // TODO: save + tcpConn.Close() + }() + {{ end }} + + {{ if or (eq .Template "tls") (eq .Template "https") }} + // perform TLS handshake + tlsSNI, err := t.sni() + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + tlsHandshaker := trace.NewTLSHandshakerStdlib(t.Logger) + tlsConfig := &tls.Config{ + NextProtos: t.alpn(), + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: tlsSNI, + } + tlsConn, _, err := tlsHandshaker.Handshake(ctx, tcpConn, tlsConfig) + _ = <-trace.TLSHandshake // TODO: save + if err != nil { + ol.Stop(err) + return + } + defer tlsConn.Close() + {{ end }} + + {{ if eq .Template "http" }} + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(tcpConn), + netxlite.NewNullTLSDialer(), + ) + {{ else if eq .Template "https" }} + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewNullDialer(), + // note: netxlite guarantees that here tlsConn is a netxlite.TLSConn + netxlite.NewSingleUseTLSDialer(tlsConn.(netxlite.TLSConn)), + ) + {{ end }} + + {{ if or (eq .Template "http") (eq .Template "https") }} + // create HTTP request + httpReq, err := t.newHTTPRequest(ctx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP round trip + httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + {{ end }} + + // completed successfully + ol.Stop(nil) +} + +{{ if or (eq .Template "tls") (eq .Template "https") }} +// alpn returns the user-configured ALPN or a reasonable default +func (t *{{ .StructName }}Task) alpn() []string { + if len(t.ALPN) > 0 { + return t.ALPN + } + return []string{"h2", "http/1.1"} +} + +// sni returns the user-configured SNI or a reasonable default +func (t *{{ .StructName }}Task) sni() (string, error) { + if t.SNI != "" { + return t.SNI, nil + } + addr, _, err := net.SplitHostPort(t.Address) + if err != nil { + return "", err + } + return addr, nil +} +{{ end }} + +{{ if or (eq .Template "http") (eq .Template "https") }} +// urlHost computes the host to include into the URL +func (t *{{ .StructName }}Task) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + {{ if eq .Template "http" -}} + if port == "80" && scheme == "http" { + return addr, nil + } + {{- else if eq .Template "https" -}} + if port == "443" && scheme == "https" { + return addr, nil + } + {{- end }} + return t.Address, nil // there was no need to parse after all 😬 +} + +// newHTTPRequest creates a new HTTP request. +func (t *{{ .StructName }}Task) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "{{ if eq .Template "http" }}http{{ else }}https{{ end }}" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *{{ .StructName }}Task) httpTransaction(ctx context.Context, txp model.HTTPTransport, + req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 22 // TODO: you may want to change this default + resp, err := txp.RoundTrip(req) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save + if err != nil { + return resp, []byte{}, err + } + defer resp.Body.Close() + reader := io.LimitReader(resp.Body, maxbody) + body, err := netxlite.ReadAllContext(ctx, reader) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + return resp, body, err +} +{{ end }} diff --git a/internal/cmd/boilerplate/task/httpget.go.txt b/internal/cmd/boilerplate/task/httpget.go.txt deleted file mode 100644 index 4d54c3bd25..0000000000 --- a/internal/cmd/boilerplate/task/httpget.go.txt +++ /dev/null @@ -1,185 +0,0 @@ -package {{ .Package }} - -// -// {{ .StructName }}: {{ .Description }} -// -// This task implements the {{ .Template }} template. -// - -import ( - "context" - "fmt" - "io" - "net" - "net/http" - "net/url" - "time" - - "github.com/ooni/probe-cli/v3/internal/measurex" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// {{ .Description }} -type {{ .StructName }}Task struct { - // Address is the address to connect to. - Address string - - // IDGenerator is the TaskIDgenerator to use. - IDGenerator *TaskIDGenerator - - // Logger is the logger to use. - Logger model.Logger - - // TestKeys contains the TestKeys. - TestKeys *TestKeys - - // URLPath is the URL's path. - URLPath string - - // ZeroTime is the measurement's zero time. - ZeroTime time.Time -} - -// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. -// -// Arguments: -// -// - address is the address to connect to; -// -// - idGenerator is the TaskIDGenerator to use; -// -// - logger is the logger to use; -// -// - tk contains the TestKeys; -// -// - urlPath is the URL path to use; -// -// - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task(address string, idGenerator *TaskIDGenerator, - logger model.Logger, tk *TestKeys, urlPath string, zeroTime time.Time) *{{ .StructName }}Task { - return &{{ .StructName }}Task{ - Address: address, - IDGenerator: idGenerator, - Logger: logger, - TestKeys: tk, - URLPath: urlPath, - ZeroTime: zeroTime, - } -} - -var _ Task = &{{ .StructName }}Task{} - -// Run implements Task -func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { - // 1. configure timeout for this task - const defaultTimeout = 15 * time.Second // TODO: you may want to change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) - defer cancel() - - // 2. create a trace - trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) - - // 3. perform the TCP connect - dialer := trace.NewDialerWithoutResolver(t.Logger) - conn, err := dialer.DialContext(ctx, "tcp", t.Address) - t.onTCPConnectResult(err, <-trace.TCPConnect) - if err != nil { - return err - } - defer conn.Close() - - // 4. perform the plaintext HTTP GET - URL := &url.URL{ - Scheme: "http", - Host: t.hostHeader(), - } - // TODO(bassosimone): we should expose this function from measurexlite - req, err := measurex.NewHTTPRequestWithContext(ctx, "GET", URL.String(), nil) - if err != nil { - t.onNewHTTPRequestFailure(err) - return err - } - // TODO(bassosimone): we should implement HTTP tracing in measurexlite - txp := netxlite.NewHTTPTransport( - t.Logger, - netxlite.NewSingleUseDialer(conn), - netxlite.NewNullTLSDialer(), - ) - resp, err := txp.RoundTrip(req) - if err != nil { - t.onHTTPTransactionResult(resp, nil, err) - return err - } - defer resp.Body.Close() - - // 5. read the response body - const maxResponseSize = 1 << 22 // TODO: you may want to change this default - reader := io.LimitReader(resp.Body, maxResponseSize) - data, err := netxlite.ReadAllContext(ctx, reader) - if err != nil { - t.onHTTPTransactionResult(resp, nil, err) - return err - } - - // 4. we're done! - return t.onHTTPTransactionResult(resp, data, nil) -} - -// Called when the result of the TCP connect becomes available. -// -// Arguments: -// -// - err is the possibly nil error; -// -// - tcpTrace contains the never-nil archival results. -func (t *{{ .StructName }}Task) onTCPConnectResult( - err error, tcpTrace *model.ArchivalTCPConnectResult) { - // TODO: implement this method. Typically here you want to - // save into the test keys using thread safe code. -} - -// Called when we fail to create a new HTTP request. -// -// This typically indicates that something was off in the experiment's -// settings and could be caused by an implementation error. -func (t *{{ .StructName}}Task) onNewHTTPRequestFailure(err error) { - // TODO: implement this method. Typically here you want to - // save into the test keys using thread safe code. -} - -// Called when the results of the HTTP transaction become available. -// -// Arguments: -// -// - resp is the possibly-nil HTTP response; -// -// - respBody is the possibly-nil-or-empty HTTP body; -// -// - err is the possibly-nil error. -func (t *{{ .StructName }}Task) onHTTPTransactionResult( - resp *http.Response, respBody []byte, err error) error { - // TODO: implement this method. Typically here you want to - // save into the test keys using thread safe code. - return nil -} - -// Returns the host header without the port if we're using the default -// port. Otherwise, it returns the host header with the port. -func (t *{{ .StructName }}Task) hostHeader() string { - addr, port, err := net.SplitHostPort(t.Address) - if err != nil { - t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) - return t.Address - } - if port == "80" { - return addr - } - return t.Address // there was no need to parse after all 😬 -} - -// Repr implements Task -func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier -} diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 09506b2aa0..7aa2bdb50d 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -3,91 +3,77 @@ package {{ .Package }} // // {{ .StructName }}: {{ .Description }} // -// This task implements the {{ .Template }} template. -// import ( "context" - "fmt" + "sync" "time" - "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" ) // {{ .Description }} +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +// +// This task implements the {{ .Template }} template. type {{ .StructName }}Task struct { - // Domain is the domain to resolve. + // Domain is the MANDATORY domain to resolve. Domain string - // IDGenerator is the TaskIDgenerator to use. - IDGenerator *TaskIDGenerator + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 - // Logger is the logger to use. + // Logger is the MANDATORY logger to use. Logger model.Logger - // TestKeys contains the TestKeys. + // TestKeys is MANDATORY and contains the TestKeys. TestKeys *TestKeys - // ZeroTime is the measurement's zero time. + // ZeroTime is the MANDATORY measurement's zero time. ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup } -// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. -// -// Arguments: -// -// - domain is the domain to resolve; -// -// - idGenerator is the TaskIDGenerator to use; -// -// - logger is the logger to use; -// -// - tk contains the TestKeys; -// -// - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task(domain string, idGenerator *TaskIDGenerator, - logger model.Logger, tk *TestKeys, zeroTime time.Time) *{{ .StructName }}Task { - return &{{ .StructName }}Task{ - Domain: domain, - IDGenerator: idGenerator, - Logger: logger, - TestKeys: tk, - ZeroTime: zeroTime, - } +// Start starts this task in a background gorountine. +func (t *{{ .StructName }}Task) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go t.run(ctx, index) } -var _ Task = &{{ .StructName }}Task{} +// run runs this task in the background. +func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { + // synchronize with wait group + defer t.WaitGroup.Done() -// Run implements Task -func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { - // 1. configure timeout for this task - const defaultTimeout = 4 * time.Second // TODO: you may want to change this default + // configure a timeout + const defaultTimeout = 4 * time.Second // TODO: change this default ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() - // 2. construct a system resolver. - // - // TODO(bassosimone): we need to implement trace support for the system - // resolver such that it's possible to use a trace here. - reso := netxlite.NewStdlibResolver(t.Logger) - addresses, err := reso.LookupHost(ctx, t.Domain) + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}#%d", index) // TODO: edit + + // construct a system resolver. + reso := measurexlite.NewStdlibResolver(t.Logger) + addrs, err := reso.LookupHost(ctx, t.Domain) + _ = reso.DNSLookupsFromRoundTrip() // TODO: save if err != nil { - // TODO: write something into the test keys + ol.Stop(err) return err } - // TODO: write something into the test keys - // 3. (typically) fan out a number of child async tasks to use the IP addrs - for range addresses { + // emit successful log message + ol.Stop(nil) + + // (typically) fan out a number of child async tasks to use the IP addrs + for range addrs { // TODO: implement } - - // 4. we're done! - return nil -} - -// Repr implements Task -func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier } diff --git a/internal/cmd/boilerplate/task/tcpconnect.go.txt b/internal/cmd/boilerplate/task/tcpconnect.go.txt deleted file mode 100644 index cd77bd0fe2..0000000000 --- a/internal/cmd/boilerplate/task/tcpconnect.go.txt +++ /dev/null @@ -1,102 +0,0 @@ -package {{ .Package }} - -// -// {{ .StructName }}: {{ .Description }} -// -// This task implements the {{ .Template }} template. -// - -import ( - "context" - "fmt" - "time" - - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" -) - -// {{ .Description }} -type {{ .StructName }}Task struct { - // Address is the address to connect to. - Address string - - // IDGenerator is the TaskIDgenerator to use. - IDGenerator *TaskIDGenerator - - // Logger is the logger to use. - Logger model.Logger - - // TestKeys contains the TestKeys. - TestKeys *TestKeys - - // ZeroTime is the measurement's zero time. - ZeroTime time.Time -} - -// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. -// -// Arguments: -// -// - address is the address to connect to; -// -// - idGenerator is the TaskIDGenerator to use; -// -// - logger is the logger to use; -// -// - tk contains the TestKeys; -// -// - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task( - address string, idGenerator *TaskIDGenerator, logger model.Logger, - tk *TestKeys, zeroTime time.Time) *{{ .StructName }}Task { - return &{{ .StructName }}Task{ - Address: address, - IDGenerator: idGenerator, - Logger: logger, - TestKeys: tk, - ZeroTime: zeroTime, - } -} - -var _ Task = &{{ .StructName }}Task{} - -// Run implements Task -func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { - // 1. configure timeout for this task - const defaultTimeout = 10 * time.Second // TODO: you may want to change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) - defer cancel() - - // 2. create a trace - trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) - - // 3. perform the TCP connect - dialer := trace.NewDialerWithoutResolver(t.Logger) - conn, err := dialer.DialContext(ctx, "tcp", t.Address) - t.onTCPConnectResult(err, <-trace.TCPConnect) - if err != nil { - return err - } - defer conn.Close() - - // 4. we're done! - return nil -} - -// Called when the result of the TCP connect becomes available. -// -// Arguments: -// -// - err is the possibly-nil error; -// -// - tcpTrace contains the never-nil archival results. -func (t *{{ .StructName }}Task) onTCPConnectResult( - err error, tcpTrace *model.ArchivalTCPConnectResult) { - // TODO: implement this method. Typically here you want to - // save into the test keys using thread safe code. -} - -// Repr implements Task -func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier -} diff --git a/internal/cmd/boilerplate/task/tlshandshake.go.txt b/internal/cmd/boilerplate/task/tlshandshake.go.txt deleted file mode 100644 index 518ba6ba76..0000000000 --- a/internal/cmd/boilerplate/task/tlshandshake.go.txt +++ /dev/null @@ -1,146 +0,0 @@ -package {{ .Package }} - -// -// {{ .StructName }}: {{ .Description }} -// -// This task implements the {{ .Template }} template. -// - -import ( - "context" - "crypto/tls" - "fmt" - "time" - - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// {{ .Description }} -type {{ .StructName }}Task struct { - // ALPN is the ALPN to use. - ALPN []string - - // Address is the address to connect to. - Address string - - // IDGenerator is the TaskIDgenerator to use. - IDGenerator *TaskIDGenerator - - // Logger is the logger to use. - Logger model.Logger - - // TestKeys contains the TestKeys. - TestKeys *TestKeys - - // SNI is the SNI to use. - SNI string - - // ZeroTime is the measurement's zero time. - ZeroTime time.Time -} - -// New{{ .StructName}}Task creates a new {{ .StructName }}Task instance. -// -// Arguments: -// -// - address is the address to connect to; -// -// - alpn is the ALPN to use; -// -// - sni is the SNI to use; -// -// - idGenerator is the TaskIDGenerator to use; -// -// - logger is the logger to use; -// -// - tk contains the TestKeys; -// -// - zeroTime is the zero time of the measurement. -func New{{ .StructName }}Task(address string, alpn []string, sni string, idGenerator *TaskIDGenerator, - logger model.Logger, tk *TestKeys, zeroTime time.Time) *{{ .StructName }}Task { - return &{{ .StructName }}Task{ - Address: address, - ALPN: alpn, - IDGenerator: idGenerator, - Logger: logger, - SNI: sni, - TestKeys: tk, - ZeroTime: zeroTime, - } -} - -var _ Task = &{{ .StructName }}Task{} - -// Run implements Task -func (t *{{ .StructName }}Task) Run(ctx context.Context, sched TaskScheduler) error { - // 1. configure timeout for this task - const defaultTimeout = 15 * time.Second // TODO: you may want to change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) - defer cancel() - - // 2. create a trace - trace := measurexlite.NewTrace(t.IDGenerator.Next(), t.ZeroTime) - - // 3. perform the TCP connect - dialer := trace.NewDialerWithoutResolver(t.Logger) - conn, err := dialer.DialContext(ctx, "tcp", t.Address) - t.onTCPConnectResult(err, <-trace.TCPConnect) - if err != nil { - return err - } - conn = trace.WrapNetConn(conn) - defer conn.Close() - - // 4. TLS handshake - thx := trace.NewTLSHandshakerStdlib(t.Logger) - config := &tls.Config{ - NextProtos: t.ALPN, - RootCAs: netxlite.NewDefaultCertPool(), - ServerName: t.SNI, - } - tlsConn, _, err := thx.Handshake(ctx, conn, config) - t.onTLSHandshakeResult(err, <-trace.TLSHandshake, trace.NetworkEvents()) - if err != nil { - return err - } - tlsConn.Close() - - // 5. we're done! - return nil -} - -// Called when the result of the TCP connect becomes available. -// -// Arguments: -// -// - err is the possibly nil error; -// -// - tcpTrace contains the never-nil archival results. -func (t *{{ .StructName }}Task) onTCPConnectResult( - err error, tcpTrace *model.ArchivalTCPConnectResult) { - // TODO: implement this method. Typically here you want to - // save into the test keys using thread safe code. -} - -// Called when the result of the TLS handshake becomes available. -// -// Arguments: -// -// - err is the possibly-nil error; -// -// - tlsTrace contains the never-nil archival results; -// -// - netEvents contains the possibly empty network events. -func (t *{{ .StructName }}Task) onTLSHandshakeResult( - err error, tlsTrace *model.ArchivalTLSOrQUICHandshakeResult, - netEvents []*model.ArchivalNetworkEvent) { - // TODO: implement this method. Typically here you want to - // save into the test keys using thread safe code. -} - -// Repr implements Task -func (t *{{ .StructName }}Task) Repr() string { - return fmt.Sprintf("{{ .StructName}}Task: %+v", t) // TODO: make this prettier -} diff --git a/internal/measurexlite/http.go b/internal/measurexlite/http.go new file mode 100644 index 0000000000..e71e610fb4 --- /dev/null +++ b/internal/measurexlite/http.go @@ -0,0 +1,93 @@ +package measurexlite + +import ( + "net/http" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tracex" +) + +// NewArchivalHTTPRequestResult creates a new model.ArchivalHTTPRequestResult. +// +// Arguments: +// +// - txp is the HTTP transport used for the HTTP transaction; +// +// - req is the certainly-non-nil HTTP request; +// +// - resp is the possibly-nil HTTP response; +// +// - maxRespBodySize is the maximum body snapshot size; +// +// - body is the possibly-nil HTTP response body; +// +// - err is the possibly-nil error that occurred during the HTTP transaction. +func (tx *Trace) NewArchivalHTTPRequestResult( + txp model.HTTPTransport, req *http.Request, resp *http.Response, maxRespBodySize int64, + body []byte, err error) *model.ArchivalHTTPRequestResult { + ev := &model.ArchivalHTTPRequestResult{ + Failure: tracex.NewFailure(err), + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalMaybeBinaryData{}, + BodyIsTruncated: false, + HeadersList: newHTTPHeaderList(req.Header), + Headers: newHTTPHeaderMap(req.Header), + Method: req.Method, + Tor: model.ArchivalHTTPTor{}, + Transport: txp.Network(), + URL: req.URL.String(), + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalMaybeBinaryData{}, + BodyIsTruncated: false, + Code: 0, + HeadersList: []model.ArchivalHTTPHeader{}, + Headers: map[string]model.ArchivalMaybeBinaryData{}, + Locations: []string{}, + }, + T: tx.TimeSince(tx.ZeroTime).Seconds(), + } + if resp != nil { + if body != nil { + ev.Response.Body.Value = string(body) + ev.Response.BodyIsTruncated = int64(len(body)) >= maxRespBodySize + } + ev.Response.Code = int64(resp.StatusCode) + ev.Response.HeadersList = newHTTPHeaderList(resp.Header) + ev.Response.Headers = newHTTPHeaderMap(resp.Header) + loc, err := resp.Location() + if err == nil { + ev.Response.Locations = append(ev.Response.Locations, loc.String()) + } + } + return ev +} + +// newHTTPHeaderList creates a list representation of HTTP headers +func newHTTPHeaderList(header http.Header) (out []model.ArchivalHTTPHeader) { + for key, values := range header { + for _, value := range values { + out = append(out, model.ArchivalHTTPHeader{ + Key: key, + Value: model.ArchivalMaybeBinaryData{ + Value: value, + }, + }) + } + } + return +} + +// newHTTPHeaderMap creates a map representation of HTTP headers +func newHTTPHeaderMap(header http.Header) (out map[string]model.ArchivalMaybeBinaryData) { + out = make(map[string]model.ArchivalMaybeBinaryData) + for key, values := range header { + for _, value := range values { + out[key] = model.ArchivalMaybeBinaryData{ + Value: value, + } + break + } + } + return +} From 186b159eab356436ae15f2b408ad3305b5302515 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 18:38:16 +0200 Subject: [PATCH 22/83] additional fixes --- internal/cmd/boilerplate/task/systemresolver.go.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 7aa2bdb50d..424b7085be 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -57,16 +57,19 @@ func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + // start the operation logger ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}#%d", index) // TODO: edit // construct a system resolver. - reso := measurexlite.NewStdlibResolver(t.Logger) + reso := trace.NewStdlibResolver(t.Logger) addrs, err := reso.LookupHost(ctx, t.Domain) - _ = reso.DNSLookupsFromRoundTrip() // TODO: save + _ = trace.DNSLookupsFromRoundTrip() // TODO: save if err != nil { ol.Stop(err) - return err + return } // emit successful log message From c55dabca040a96c171a9e2bd800a90ba3904f1ae Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 18:45:44 +0200 Subject: [PATCH 23/83] chore: autogenerate boilerplate for telegram experiment --- internal/experiment/telegram/config.go | 12 ++ internal/experiment/telegram/datacenter.go | 176 ++++++++++++++++ internal/experiment/telegram/doc.go | 4 + internal/experiment/telegram/measurer.go | 85 ++++++++ internal/experiment/telegram/summary.go | 23 +++ internal/experiment/telegram/systemdns.go | 82 ++++++++ internal/experiment/telegram/testkeys.go | 62 ++++++ internal/experiment/telegram/webhttp.go | 176 ++++++++++++++++ internal/experiment/telegram/webhttps.go | 224 +++++++++++++++++++++ internal/registry/telegram.go | 11 +- 10 files changed, 850 insertions(+), 5 deletions(-) create mode 100644 internal/experiment/telegram/config.go create mode 100644 internal/experiment/telegram/datacenter.go create mode 100644 internal/experiment/telegram/doc.go create mode 100644 internal/experiment/telegram/measurer.go create mode 100644 internal/experiment/telegram/summary.go create mode 100644 internal/experiment/telegram/systemdns.go create mode 100644 internal/experiment/telegram/testkeys.go create mode 100644 internal/experiment/telegram/webhttp.go create mode 100644 internal/experiment/telegram/webhttps.go diff --git a/internal/experiment/telegram/config.go b/internal/experiment/telegram/config.go new file mode 100644 index 0000000000..3734b7e2e9 --- /dev/null +++ b/internal/experiment/telegram/config.go @@ -0,0 +1,12 @@ +package telegram + +// +// Config for telegram and associated functions. +// + +// Config contains telegram experiment configuration. +type Config struct { + // TODO: add fields here if you need any config +} + +// TODO: implement convenience accessors you may need for Config diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go new file mode 100644 index 0000000000..68c6dea5f5 --- /dev/null +++ b/internal/experiment/telegram/datacenter.go @@ -0,0 +1,176 @@ +package telegram + +// +// Datacenter: Measures a Telegram data center (DC). +// + +import ( + "context" + + "io" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures a Telegram data center (DC). +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +// +// This task implements the http template. +type DatacenterTask struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background gorountine. +func (t *DatacenterTask) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go t.run(ctx, index) +} + +// run runs this task in the background. +func (t *DatacenterTask) run(ctx context.Context, index int64) { + // synchronize with wait group + defer t.WaitGroup.Done() + + // configure a timeout + const defaultTimeout = 15 * time.Second // TODO: change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "Datacenter#%d", index) // TODO: edit + + // perform the TCP connect + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + _ = <-trace.TCPConnect // TODO: save + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + _ = trace.NetworkEvents() // TODO: save + tcpConn.Close() + }() + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(tcpConn), + netxlite.NewNullTLSDialer(), + ) + + // create HTTP request + httpReq, err := t.newHTTPRequest(ctx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP round trip + httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + + // completed successfully + ol.Stop(nil) +} + +// urlHost computes the host to include into the URL +func (t *DatacenterTask) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + if port == "80" && scheme == "http" { + return addr, nil + } + return t.Address, nil // there was no need to parse after all 😬 +} + +// newHTTPRequest creates a new HTTP request. +func (t *DatacenterTask) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "http" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *DatacenterTask) httpTransaction(ctx context.Context, txp model.HTTPTransport, + req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 22 // TODO: you may want to change this default + resp, err := txp.RoundTrip(req) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save + if err != nil { + return resp, []byte{}, err + } + defer resp.Body.Close() + reader := io.LimitReader(resp.Body, maxbody) + body, err := netxlite.ReadAllContext(ctx, reader) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + return resp, body, err +} diff --git a/internal/experiment/telegram/doc.go b/internal/experiment/telegram/doc.go new file mode 100644 index 0000000000..09ea0ddb40 --- /dev/null +++ b/internal/experiment/telegram/doc.go @@ -0,0 +1,4 @@ +// Package telegram implements the telegram experiment. +// +// See https://github.com/ooni/spec/blob/master/nettests/ts-020-telegram.md. +package telegram diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go new file mode 100644 index 0000000000..e06ee30670 --- /dev/null +++ b/internal/experiment/telegram/measurer.go @@ -0,0 +1,85 @@ +package telegram + +// +// Measurer for telegram. +// +// The Measurer implements performing a single measurement. Because this +// autogenerated code is optimized to run background tasks, this code's +// main job is to parse inputs and then schedule the "main" task implementing +// the experiment. As such, there is little to change for you here. +// + +import ( + "context" + "errors" + "sync" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Measurer implements the telegram experiment. +type Measurer struct { + // Config contains the experiment's config. + Config *Config +} + +// NewExperimentMeasurer creates a new model.ExperimentMeasurer for telegram. +// +// Code in the internal/registry package binds this function with an ExperimentBuilder +// which is responsible of instantiating a new experiment. +func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { + return &Measurer{ + Config: config, + } +} + +// ExperimentName implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentName() string { + return "telegram" +} + +// ExperimentVersion implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentVersion() string { + return "0.3.0" +} + +// Run implements model.ExperimentMeasurer. +func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { + // Reminder: When this function returns an error, the measurement result + // WILL NOT be submitted to the OONI backend. You SHOULD only return an error + // for fundamental errors (e.g., the input is invalid or missing). + + // honour InputNone + if measurement.Input != "" { + return errors.New("this experiment does not take any input") + } + + // initialize the experiment's test keys + tk := NewTestKeys() + measurement.TestKeys = tk + + // create variables required to run parallel tasks + idGenerator := &atomicx.Int64{} + wg := &sync.WaitGroup{} + + // start background tasks + // TODO: replace this code with code for running your background tasks + + wg.Add(1) + go func(logger model.Logger, idx int64) { + defer wg.Done() + ol := measurexlite.NewOperationLogger(logger, "telegram#%d", idx) + tk.SetFundamentalFailure(errors.New("experiment not implemented")) + ol.Stop(nil) + }(sess.Logger(), idGenerator.Add(1)) + + // wait for background tasks to join + wg.Wait() + + // return whether there was a fundamental failure, which would prevent + // the measurement from being submitted to the OONI collector. + return tk.FundamentalFailure() +} diff --git a/internal/experiment/telegram/summary.go b/internal/experiment/telegram/summary.go new file mode 100644 index 0000000000..108d4ecf52 --- /dev/null +++ b/internal/experiment/telegram/summary.go @@ -0,0 +1,23 @@ +package telegram + +// +// Experiment summary result returned to ooniprobe. +// + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Summary contains the summary results. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + // TODO: add here additional summary fields. + isAnomaly bool +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + sk := SummaryKeys{isAnomaly: false} + // TODO: implement + return sk, nil +} diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go new file mode 100644 index 0000000000..432b851244 --- /dev/null +++ b/internal/experiment/telegram/systemdns.go @@ -0,0 +1,82 @@ +package telegram + +// +// SystemDNS: Resolves web.telegram.org using the system resolver. +// + +import ( + "context" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Resolves web.telegram.org using the system resolver. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +// +// This task implements the system-resolver template. +type SystemDNSTask struct { + // Domain is the MANDATORY domain to resolve. + Domain string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup +} + +// Start starts this task in a background gorountine. +func (t *SystemDNSTask) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go t.run(ctx, index) +} + +// run runs this task in the background. +func (t *SystemDNSTask) run(ctx context.Context, index int64) { + // synchronize with wait group + defer t.WaitGroup.Done() + + // configure a timeout + const defaultTimeout = 4 * time.Second // TODO: change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "SystemDNS#%d", index) // TODO: edit + + // construct a system resolver. + reso := trace.NewStdlibResolver(t.Logger) + addrs, err := reso.LookupHost(ctx, t.Domain) + _ = trace.DNSLookupsFromRoundTrip() // TODO: save + if err != nil { + ol.Stop(err) + return + } + + // emit successful log message + ol.Stop(nil) + + // (typically) fan out a number of child async tasks to use the IP addrs + for range addrs { + // TODO: implement + } +} diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go new file mode 100644 index 0000000000..4111a6f3fd --- /dev/null +++ b/internal/experiment/telegram/testkeys.go @@ -0,0 +1,62 @@ +package telegram + +// +// TestKeys for telegram. +// +// Note: for historical reasons, we call TestKeys the JSON object +// containing the results produced by OONI experiments. +// + +import "sync" + +// TestKeys contains the results produced by telegram. +type TestKeys struct { + // TODO: add here fields produced by this experiment. They should + // be public such that we can JSON serialize them. + // + // Ideally, try to keep the field names alphabetically sorted. + // + // For example: + // + // // Blocked indicates that the resource is censored. + // Blocked bool `json:"blocked"` + + // fundamentalFailure indicates that some fundamental error occurred + // in a background task. A fundamental error is something like a programmer + // such as a failure to parse a URL that was hardcoded in the codebase. When + // this class of errors happens, you certainly don't want to submit the + // resulting measurement to the OONI collector. + fundamentalFailure error + + // mu provides mutual exclusion for accessing the test keys. + mu *sync.Mutex +} + +// TODO: implement more thread-safe setters for the real test keys. This allows +// tasks to write directly into the TestKeys. +// +// In some cases, you may also need to write thread-safe getters. For example, +// below we also define a getter for fundamentalFailure because we need to +// read its value inside the autogenerated Runner.Main func. + +// SetFundamentalFailure implements TestKeys. +func (tk *TestKeys) SetFundamentalFailure(err error) { + tk.mu.Lock() + tk.fundamentalFailure = err + tk.mu.Unlock() +} + +// FundamentalFailure implements TestKeys. +func (tk *TestKeys) FundamentalFailure() error { + tk.mu.Lock() + err := tk.fundamentalFailure + tk.mu.Unlock() + return err +} + +// NewTestKeys creates a new instance of TestKeys. +func NewTestKeys() *TestKeys { + return &TestKeys{ + // TODO: here you should initialize all the fields + } +} diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go new file mode 100644 index 0000000000..5cc88398a9 --- /dev/null +++ b/internal/experiment/telegram/webhttp.go @@ -0,0 +1,176 @@ +package telegram + +// +// WebHTTP: Measures Telegram Web using HTTP. +// + +import ( + "context" + + "io" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures Telegram Web using HTTP. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +// +// This task implements the http template. +type WebHTTPTask struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background gorountine. +func (t *WebHTTPTask) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go t.run(ctx, index) +} + +// run runs this task in the background. +func (t *WebHTTPTask) run(ctx context.Context, index int64) { + // synchronize with wait group + defer t.WaitGroup.Done() + + // configure a timeout + const defaultTimeout = 15 * time.Second // TODO: change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTP#%d", index) // TODO: edit + + // perform the TCP connect + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + _ = <-trace.TCPConnect // TODO: save + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + _ = trace.NetworkEvents() // TODO: save + tcpConn.Close() + }() + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(tcpConn), + netxlite.NewNullTLSDialer(), + ) + + // create HTTP request + httpReq, err := t.newHTTPRequest(ctx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP round trip + httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + + // completed successfully + ol.Stop(nil) +} + +// urlHost computes the host to include into the URL +func (t *WebHTTPTask) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + if port == "80" && scheme == "http" { + return addr, nil + } + return t.Address, nil // there was no need to parse after all 😬 +} + +// newHTTPRequest creates a new HTTP request. +func (t *WebHTTPTask) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "http" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *WebHTTPTask) httpTransaction(ctx context.Context, txp model.HTTPTransport, + req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 22 // TODO: you may want to change this default + resp, err := txp.RoundTrip(req) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save + if err != nil { + return resp, []byte{}, err + } + defer resp.Body.Close() + reader := io.LimitReader(resp.Body, maxbody) + body, err := netxlite.ReadAllContext(ctx, reader) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + return resp, body, err +} diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go new file mode 100644 index 0000000000..8be57c2d79 --- /dev/null +++ b/internal/experiment/telegram/webhttps.go @@ -0,0 +1,224 @@ +package telegram + +// +// WebHTTPS: Measures Telegram Web using HTTPS. +// + +import ( + "context" + "crypto/tls" + "io" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures Telegram Web using HTTPS. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +// +// This task implements the https template. +type WebHTTPSTask struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // ALPN is the OPTIONAL ALPN to use. + ALPN []string + + // SNI is the OPTIONAL SNI to use. + SNI string + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background gorountine. +func (t *WebHTTPSTask) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go t.run(ctx, index) +} + +// run runs this task in the background. +func (t *WebHTTPSTask) run(ctx context.Context, index int64) { + // synchronize with wait group + defer t.WaitGroup.Done() + + // configure a timeout + const defaultTimeout = 15 * time.Second // TODO: change this default + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTPS#%d", index) // TODO: edit + + // perform the TCP connect + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + _ = <-trace.TCPConnect // TODO: save + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + _ = trace.NetworkEvents() // TODO: save + tcpConn.Close() + }() + + // perform TLS handshake + tlsSNI, err := t.sni() + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + tlsHandshaker := trace.NewTLSHandshakerStdlib(t.Logger) + tlsConfig := &tls.Config{ + NextProtos: t.alpn(), + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: tlsSNI, + } + tlsConn, _, err := tlsHandshaker.Handshake(ctx, tcpConn, tlsConfig) + _ = <-trace.TLSHandshake // TODO: save + if err != nil { + ol.Stop(err) + return + } + defer tlsConn.Close() + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewNullDialer(), + // note: netxlite guarantees that here tlsConn is a netxlite.TLSConn + netxlite.NewSingleUseTLSDialer(tlsConn.(netxlite.TLSConn)), + ) + + // create HTTP request + httpReq, err := t.newHTTPRequest(ctx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP round trip + httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + + // completed successfully + ol.Stop(nil) +} + +// alpn returns the user-configured ALPN or a reasonable default +func (t *WebHTTPSTask) alpn() []string { + if len(t.ALPN) > 0 { + return t.ALPN + } + return []string{"h2", "http/1.1"} +} + +// sni returns the user-configured SNI or a reasonable default +func (t *WebHTTPSTask) sni() (string, error) { + if t.SNI != "" { + return t.SNI, nil + } + addr, _, err := net.SplitHostPort(t.Address) + if err != nil { + return "", err + } + return addr, nil +} + +// urlHost computes the host to include into the URL +func (t *WebHTTPSTask) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + if port == "443" && scheme == "https" { + return addr, nil + } + return t.Address, nil // there was no need to parse after all 😬 +} + +// newHTTPRequest creates a new HTTP request. +func (t *WebHTTPSTask) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "https" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *WebHTTPSTask) httpTransaction(ctx context.Context, txp model.HTTPTransport, + req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 22 // TODO: you may want to change this default + resp, err := txp.RoundTrip(req) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save + if err != nil { + return resp, []byte{}, err + } + defer resp.Body.Close() + reader := io.LimitReader(resp.Body, maxbody) + body, err := netxlite.ReadAllContext(ctx, reader) + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + return resp, body, err +} diff --git a/internal/registry/telegram.go b/internal/registry/telegram.go index 43f1db031c..2cb03e7e11 100644 --- a/internal/registry/telegram.go +++ b/internal/registry/telegram.go @@ -5,18 +5,19 @@ package registry // import ( - "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/experiment/telegram" "github.com/ooni/probe-cli/v3/internal/model" ) func init() { allexperiments["telegram"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { + build: func(config any) model.ExperimentMeasurer { return telegram.NewExperimentMeasurer( - *config.(*telegram.Config), + config.(*telegram.Config), ) }, - config: &telegram.Config{}, - inputPolicy: model.InputNone, + config: &telegram.Config{}, + interruptible: false, + inputPolicy: model.InputNone, } } From 3dc30cba53c6cfddeb9deac71fe4f36979a773bc Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 19:04:04 +0200 Subject: [PATCH 24/83] fix: make sure we init the test keys --- internal/cmd/boilerplate/experiment/testkeys.go.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt index a10cb1d01d..e9d4356d75 100644 --- a/internal/cmd/boilerplate/experiment/testkeys.go.txt +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -56,7 +56,9 @@ func (tk *TestKeys) FundamentalFailure() error { // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { + // TODO: here you should initialize all the fields return &TestKeys{ - // TODO: here you should initialize all the fields + fundamentalFailure: nil, + mu: &sync.Mutex{}, } } From 9af1400ebe77f910a500e1fef9d0abc8c29e63c4 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 19:42:27 +0200 Subject: [PATCH 25/83] Start writing the telegram experiment --- internal/cmd/boilerplate/task/endpoint.go.txt | 29 +++++-- .../boilerplate/task/systemresolver.go.txt | 29 +++++-- internal/cmd/miniooni/main.go | 10 --- internal/experiment/telegram/datacenter.go | 22 ++++-- internal/experiment/telegram/measurer.go | 48 +++++++++--- internal/experiment/telegram/systemdns.go | 75 ++++++++++++++++--- internal/experiment/telegram/testkeys.go | 4 +- internal/experiment/telegram/webhttp.go | 20 +++-- internal/experiment/telegram/webhttps.go | 22 ++++-- 9 files changed, 189 insertions(+), 70 deletions(-) diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt index 8c43684b8a..546415854f 100644 --- a/internal/cmd/boilerplate/task/endpoint.go.txt +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -1,7 +1,7 @@ package {{ .Package }} // -// {{ .StructName }}: {{ .Description }} +// {{ .StructName }}Task // import ( @@ -26,6 +26,15 @@ import ( {{- end }} ) +// +// Autogenerated section +// +// Suggestion: keep changes in this section minimal to facilitate +// generating the code again next time. +// +// You should insert your own code at the bottom. +// + // {{ .Description }} // // The zero value of this structure IS NOT valid and you MUST initialize @@ -79,13 +88,13 @@ func (t *{{ .StructName }}Task) Start(ctx context.Context) { } // run runs this task in the background. -func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { +func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { // synchronize with wait group defer t.WaitGroup.Done() // configure a timeout const defaultTimeout = 15 * time.Second // TODO: change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) defer cancel() // create trace @@ -97,7 +106,7 @@ func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { {{ if or (eq .Template "tcp") (eq .Template "tls") (eq .Template "https") (eq .Template "http") }} // perform the TCP connect tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -124,7 +133,7 @@ func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { RootCAs: netxlite.NewDefaultCertPool(), ServerName: tlsSNI, } - tlsConn, _, err := tlsHandshaker.Handshake(ctx, tcpConn, tlsConfig) + tlsConn, _, err := tlsHandshaker.Handshake(opCtx, tcpConn, tlsConfig) _ = <-trace.TLSHandshake // TODO: save if err != nil { ol.Stop(err) @@ -152,7 +161,7 @@ func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { {{ if or (eq .Template "http") (eq .Template "https") }} // create HTTP request - httpReq, err := t.newHTTPRequest(ctx) + httpReq, err := t.newHTTPRequest(opCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) @@ -160,7 +169,7 @@ func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { } // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -257,3 +266,9 @@ func (t *{{ .StructName }}Task) httpTransaction(ctx context.Context, txp model.H return resp, body, err } {{ end }} + +// +// User section +// +// We suggest adding your custom methods and functions here. +// diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 424b7085be..41055074ee 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -1,7 +1,7 @@ package {{ .Package }} // -// {{ .StructName }}: {{ .Description }} +// {{ .StructName }}Task // import ( @@ -14,6 +14,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) +// +// Autogenerated section +// +// Suggestion: keep changes in this section minimal to facilitate +// generating the code again next time. +// +// You should insert your own code at the bottom. +// + // {{ .Description }} // // The zero value of this structure IS NOT valid and you MUST initialize @@ -48,13 +57,13 @@ func (t *{{ .StructName }}Task) Start(ctx context.Context) { } // run runs this task in the background. -func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { +func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { // synchronize with wait group defer t.WaitGroup.Done() - // configure a timeout - const defaultTimeout = 4 * time.Second // TODO: change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: change this default + opCtx, cancel := context.WithTimeout(parentCtx, timeout) defer cancel() // create trace @@ -63,9 +72,9 @@ func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { // start the operation logger ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}#%d", index) // TODO: edit - // construct a system resolver. + // runs the lookup reso := trace.NewStdlibResolver(t.Logger) - addrs, err := reso.LookupHost(ctx, t.Domain) + addrs, err := reso.LookupHost(opCtx, t.Domain) _ = trace.DNSLookupsFromRoundTrip() // TODO: save if err != nil { ol.Stop(err) @@ -80,3 +89,9 @@ func (t *{{ .StructName }}Task) run(ctx context.Context, index int64) { // TODO: implement } } + +// +// User section +// +// We suggest adding your custom methods and functions here. +// diff --git a/internal/cmd/miniooni/main.go b/internal/cmd/miniooni/main.go index 75b4e75e78..f65fc84fcc 100644 --- a/internal/cmd/miniooni/main.go +++ b/internal/cmd/miniooni/main.go @@ -6,16 +6,6 @@ package main // Main function // -import ( - "fmt" - "os" -) - func main() { - defer func() { - if s := recover(); s != nil { - fmt.Fprintf(os.Stderr, "FATAL: %s\n", s) - } - }() Main() } diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index 68c6dea5f5..12fdf70720 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -1,7 +1,7 @@ package telegram // -// Datacenter: Measures a Telegram data center (DC). +// DatacenterTask // import ( @@ -63,24 +63,24 @@ func (t *DatacenterTask) Start(ctx context.Context) { } // run runs this task in the background. -func (t *DatacenterTask) run(ctx context.Context, index int64) { +func (t *DatacenterTask) run(parentCtx context.Context, index int64) { // synchronize with wait group defer t.WaitGroup.Done() // configure a timeout const defaultTimeout = 15 * time.Second // TODO: change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) defer cancel() // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "Datacenter#%d", index) // TODO: edit + ol := measurexlite.NewOperationLogger(t.Logger, "Datacenter#%d: %s", index, t.Address) // perform the TCP connect tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -100,7 +100,7 @@ func (t *DatacenterTask) run(ctx context.Context, index int64) { ) // create HTTP request - httpReq, err := t.newHTTPRequest(ctx) + httpReq, err := t.newHTTPRequest(opCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) @@ -108,7 +108,7 @@ func (t *DatacenterTask) run(ctx context.Context, index int64) { } // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -148,7 +148,7 @@ func (t *DatacenterTask) newHTTPRequest(ctx context.Context) (*http.Request, err Path: t.URLPath, RawQuery: t.URLRawQuery, } - httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + httpReq, err := http.NewRequestWithContext(ctx, "POST", httpURL.String(), nil) if err != nil { return nil, err } @@ -174,3 +174,9 @@ func (t *DatacenterTask) httpTransaction(ctx context.Context, txp model.HTTPTran _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save return resp, body, err } + +// +// User section +// +// We suggest adding your custom methods and functions here. +// diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index e06ee30670..22b1aa2a29 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -12,10 +12,10 @@ package telegram import ( "context" "errors" + "net" "sync" "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -66,15 +66,30 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, wg := &sync.WaitGroup{} // start background tasks - // TODO: replace this code with code for running your background tasks - - wg.Add(1) - go func(logger model.Logger, idx int64) { - defer wg.Done() - ol := measurexlite.NewOperationLogger(logger, "telegram#%d", idx) - tk.SetFundamentalFailure(errors.New("experiment not implemented")) - ol.Stop(nil) - }(sess.Logger(), idGenerator.Add(1)) + systemDNSTask := &SystemDNSTask{ + IDGenerator: idGenerator, + Logger: sess.Logger(), + TestKeys: tk, + ZeroTime: measurement.MeasurementStartTimeSaved, + WaitGroup: wg, + } + systemDNSTask.Start(ctx) + for _, addr := range dataCenterAddrs { + for _, port := range dataCenterPorts { + dcTask := &DatacenterTask{ + Address: net.JoinHostPort(addr, port), + IDGenerator: idGenerator, + Logger: sess.Logger(), + TestKeys: tk, + ZeroTime: measurement.MeasurementStartTimeSaved, + WaitGroup: wg, + HostHeader: "", + URLPath: "", + URLRawQuery: "", + } + dcTask.Start(ctx) + } + } // wait for background tasks to join wg.Wait() @@ -83,3 +98,16 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // the measurement from being submitted to the OONI collector. return tk.FundamentalFailure() } + +// dataCenterAddrs contains the data center addrs. +var dataCenterAddrs = []string{ + "149.154.175.50", + "149.154.167.51", + "149.154.175.100", + "149.154.167.91", + "149.154.171.5", + "95.161.76.100", +} + +// dataCenterPorts contains the data center ports. +var dataCenterPorts = []string{"80", "443"} diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go index 432b851244..240056a4a0 100644 --- a/internal/experiment/telegram/systemdns.go +++ b/internal/experiment/telegram/systemdns.go @@ -1,11 +1,12 @@ package telegram // -// SystemDNS: Resolves web.telegram.org using the system resolver. +// SystemDNSTask // import ( "context" + "net" "sync" "time" @@ -14,6 +15,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) +// +// Autogenerated section +// +// Suggestion: keep changes in this section minimal to facilitate +// generating the code again next time. +// +// You should insert your own code at the bottom. +// + // Resolves web.telegram.org using the system resolver. // // The zero value of this structure IS NOT valid and you MUST initialize @@ -21,9 +31,6 @@ import ( // // This task implements the system-resolver template. type SystemDNSTask struct { - // Domain is the MANDATORY domain to resolve. - Domain string - // IDGenerator is the MANDATORY atomic int64 to generate task IDs. IDGenerator *atomicx.Int64 @@ -48,13 +55,13 @@ func (t *SystemDNSTask) Start(ctx context.Context) { } // run runs this task in the background. -func (t *SystemDNSTask) run(ctx context.Context, index int64) { +func (t *SystemDNSTask) run(parentCtx context.Context, index int64) { // synchronize with wait group defer t.WaitGroup.Done() - // configure a timeout - const defaultTimeout = 4 * time.Second // TODO: change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: change this default + opCtx, cancel := context.WithTimeout(parentCtx, timeout) defer cancel() // create trace @@ -63,9 +70,9 @@ func (t *SystemDNSTask) run(ctx context.Context, index int64) { // start the operation logger ol := measurexlite.NewOperationLogger(t.Logger, "SystemDNS#%d", index) // TODO: edit - // construct a system resolver. + // runs the lookup reso := trace.NewStdlibResolver(t.Logger) - addrs, err := reso.LookupHost(ctx, t.Domain) + addrs, err := reso.LookupHost(opCtx, webTelegramOrg) _ = trace.DNSLookupsFromRoundTrip() // TODO: save if err != nil { ol.Stop(err) @@ -76,7 +83,51 @@ func (t *SystemDNSTask) run(ctx context.Context, index int64) { ol.Stop(nil) // (typically) fan out a number of child async tasks to use the IP addrs - for range addrs { - // TODO: implement + for _, addr := range addrs { + t.startWebHTTPTask(parentCtx, addr) + t.startWebHTTPSTask(parentCtx, addr) + } +} + +// +// User section +// +// We suggest adding your custom methods and functions here. +// + +// webTelegramOrg is the SNI and host header for telegram web. +const webTelegramOrg = "web.telegram.org" + +// startWebHTTPTask starts a WebHTTPTask for this addr. +func (t *SystemDNSTask) startWebHTTPTask(ctx context.Context, addr string) { + task := &WebHTTPTask{ + Address: net.JoinHostPort(addr, "80"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + HostHeader: webTelegramOrg, + URLPath: "", + URLRawQuery: "", + } + task.Start(ctx) +} + +// startWebHTTPSTask starts a WebHTTPSTask for this addr. +func (t *SystemDNSTask) startWebHTTPSTask(ctx context.Context, addr string) { + task := &WebHTTPSTask{ + Address: net.JoinHostPort(addr, "443"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + ALPN: []string{}, // default is okay + SNI: webTelegramOrg, + HostHeader: webTelegramOrg, + URLPath: "", + URLRawQuery: "", } + task.Start(ctx) } diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go index 4111a6f3fd..aa2b8fcbf4 100644 --- a/internal/experiment/telegram/testkeys.go +++ b/internal/experiment/telegram/testkeys.go @@ -56,7 +56,9 @@ func (tk *TestKeys) FundamentalFailure() error { // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { + // TODO: here you should initialize all the fields return &TestKeys{ - // TODO: here you should initialize all the fields + fundamentalFailure: nil, + mu: &sync.Mutex{}, } } diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index 5cc88398a9..a9b0dd6f3c 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -1,7 +1,7 @@ package telegram // -// WebHTTP: Measures Telegram Web using HTTP. +// WebHTTPTask // import ( @@ -63,24 +63,24 @@ func (t *WebHTTPTask) Start(ctx context.Context) { } // run runs this task in the background. -func (t *WebHTTPTask) run(ctx context.Context, index int64) { +func (t *WebHTTPTask) run(parentCtx context.Context, index int64) { // synchronize with wait group defer t.WaitGroup.Done() // configure a timeout const defaultTimeout = 15 * time.Second // TODO: change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) defer cancel() // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTP#%d", index) // TODO: edit + ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTP#%d: %s", index, t.Address) // perform the TCP connect tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -100,7 +100,7 @@ func (t *WebHTTPTask) run(ctx context.Context, index int64) { ) // create HTTP request - httpReq, err := t.newHTTPRequest(ctx) + httpReq, err := t.newHTTPRequest(opCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) @@ -108,7 +108,7 @@ func (t *WebHTTPTask) run(ctx context.Context, index int64) { } // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -174,3 +174,9 @@ func (t *WebHTTPTask) httpTransaction(ctx context.Context, txp model.HTTPTranspo _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save return resp, body, err } + +// +// User section +// +// We suggest adding your custom methods and functions here. +// diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index 8be57c2d79..9ffac1a68e 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -1,7 +1,7 @@ package telegram // -// WebHTTPS: Measures Telegram Web using HTTPS. +// WebHTTPSTask // import ( @@ -69,24 +69,24 @@ func (t *WebHTTPSTask) Start(ctx context.Context) { } // run runs this task in the background. -func (t *WebHTTPSTask) run(ctx context.Context, index int64) { +func (t *WebHTTPSTask) run(parentCtx context.Context, index int64) { // synchronize with wait group defer t.WaitGroup.Done() // configure a timeout const defaultTimeout = 15 * time.Second // TODO: change this default - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) defer cancel() // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTPS#%d", index) // TODO: edit + ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTPS#%d: %s", index, t.Address) // perform the TCP connect tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(ctx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -111,7 +111,7 @@ func (t *WebHTTPSTask) run(ctx context.Context, index int64) { RootCAs: netxlite.NewDefaultCertPool(), ServerName: tlsSNI, } - tlsConn, _, err := tlsHandshaker.Handshake(ctx, tcpConn, tlsConfig) + tlsConn, _, err := tlsHandshaker.Handshake(opCtx, tcpConn, tlsConfig) _ = <-trace.TLSHandshake // TODO: save if err != nil { ol.Stop(err) @@ -128,7 +128,7 @@ func (t *WebHTTPSTask) run(ctx context.Context, index int64) { ) // create HTTP request - httpReq, err := t.newHTTPRequest(ctx) + httpReq, err := t.newHTTPRequest(opCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) @@ -136,7 +136,7 @@ func (t *WebHTTPSTask) run(ctx context.Context, index int64) { } // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(ctx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -222,3 +222,9 @@ func (t *WebHTTPSTask) httpTransaction(ctx context.Context, txp model.HTTPTransp _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save return resp, body, err } + +// +// User section +// +// We suggest adding your custom methods and functions here. +// From 1478318c498cfe88fe79300f0757445ceafb7e2a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 19:44:08 +0200 Subject: [PATCH 26/83] changes --- internal/experiment/telegram/datacenter.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index 12fdf70720..c806a55f0c 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -20,6 +20,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) +// +// Autogenerated section +// +// Suggestion: keep changes in this section minimal to facilitate +// generating the code again next time. +// +// You should insert your own code at the bottom. +// + // Measures a Telegram data center (DC). // // The zero value of this structure IS NOT valid and you MUST initialize From 7983b7bd838ed10ba40f5d7e08bc4d03b7b43b83 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 19:45:50 +0200 Subject: [PATCH 27/83] changes --- internal/experiment/telegram/webhttp.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index a9b0dd6f3c..904cd4ce4e 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -20,6 +20,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) +// +// Autogenerated section +// +// Suggestion: keep changes in this section minimal to facilitate +// generating the code again next time. +// +// You should insert your own code at the bottom. +// + // Measures Telegram Web using HTTP. // // The zero value of this structure IS NOT valid and you MUST initialize From 9729b80bf5e798b73ca70204b273be70446648e0 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 19:46:38 +0200 Subject: [PATCH 28/83] changes --- internal/experiment/telegram/webhttps.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index 9ffac1a68e..b6693b30f5 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -20,6 +20,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) +// +// Autogenerated section +// +// Suggestion: keep changes in this section minimal to facilitate +// generating the code again next time. +// +// You should insert your own code at the bottom. +// + // Measures Telegram Web using HTTPS. // // The zero value of this structure IS NOT valid and you MUST initialize From 3582e1d20d68f5a7fc650868a678fdc6e61c594e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 19:47:23 +0200 Subject: [PATCH 29/83] x --- internal/experiment/telegram/systemdns.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go index 240056a4a0..2d0a843937 100644 --- a/internal/experiment/telegram/systemdns.go +++ b/internal/experiment/telegram/systemdns.go @@ -68,7 +68,7 @@ func (t *SystemDNSTask) run(parentCtx context.Context, index int64) { trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "SystemDNS#%d", index) // TODO: edit + ol := measurexlite.NewOperationLogger(t.Logger, "SystemDNS#%d: %s", index, webTelegramOrg) // runs the lookup reso := trace.NewStdlibResolver(t.Logger) From ac0a6e3d0de78d9c13022b15dc2b6ec0a1899b50 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 20:54:38 +0200 Subject: [PATCH 30/83] Tweak the code that will be generated --- .../cmd/boilerplate/experiment/config.go.txt | 2 +- .../cmd/boilerplate/experiment/doc.go.txt | 2 +- .../boilerplate/experiment/inputparser.go.txt | 6 +- .../boilerplate/experiment/measurer.go.txt | 40 +++------- .../boilerplate/experiment/registry.go.txt | 3 +- .../cmd/boilerplate/experiment/summary.go.txt | 4 +- .../boilerplate/experiment/testkeys.go.txt | 5 +- internal/cmd/boilerplate/task/endpoint.go.txt | 80 +++++++++---------- .../boilerplate/task/systemresolver.go.txt | 50 +++++------- 9 files changed, 75 insertions(+), 117 deletions(-) diff --git a/internal/cmd/boilerplate/experiment/config.go.txt b/internal/cmd/boilerplate/experiment/config.go.txt index c39944072e..7e987c1d9b 100644 --- a/internal/cmd/boilerplate/experiment/config.go.txt +++ b/internal/cmd/boilerplate/experiment/config.go.txt @@ -1,7 +1,7 @@ package {{ .Package }} // -// Config for {{ .Package }} and associated functions. +// Config // // Config contains {{ .Package }} experiment configuration. diff --git a/internal/cmd/boilerplate/experiment/doc.go.txt b/internal/cmd/boilerplate/experiment/doc.go.txt index e63bc27281..e63175686c 100644 --- a/internal/cmd/boilerplate/experiment/doc.go.txt +++ b/internal/cmd/boilerplate/experiment/doc.go.txt @@ -1,4 +1,4 @@ // Package {{ .Package }} implements the {{ .Name }} experiment. // -// See {{ .SpecURL }}. +// Spec: {{ .SpecURL }}. package {{ .Package }} diff --git a/internal/cmd/boilerplate/experiment/inputparser.go.txt b/internal/cmd/boilerplate/experiment/inputparser.go.txt index 80f6cd7f6b..8619d79cf8 100644 --- a/internal/cmd/boilerplate/experiment/inputparser.go.txt +++ b/internal/cmd/boilerplate/experiment/inputparser.go.txt @@ -1,7 +1,7 @@ package {{ .Package }} // -// Routines for parsing the experiment's input. +// Input parsing // import ( @@ -24,7 +24,7 @@ type InputParser struct { DefaultScheme string } -// Parse parses the experiment input and return the resulting URL. +// Parse parses the experiment input and returns the resulting URL. func (ip *InputParser) Parse(input string) (*url.URL, error) { // put this check at top-level such that we always see the crash if needed runtimex.PanicIfTrue( @@ -44,7 +44,7 @@ func (ip *InputParser) Parse(input string) (*url.URL, error) { return nil, errors.New("cannot parse input") } -// Logic to conditionally allow endpoints if the user is fine with that. +// Conditionally allows endpooints when ip.AllowEndpoints is true. func (ip *InputParser) maybeAllowEndpoints(URL *url.URL, err error) (*url.URL, error) { runtimex.PanicIfNil(err, "expected to be called with a non-nil error") if ip.AllowEndpoints && URL.Scheme != "" && URL.Opaque != "" && URL.User == nil && diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index 18d2662172..6d0b6c56f4 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -1,17 +1,14 @@ package {{ .Package }} // -// Measurer for {{ .Name }}. -// -// The Measurer implements performing a single measurement. Because this -// autogenerated code is optimized to run background tasks, this code's -// main job is to parse inputs and then schedule the "main" task implementing -// the experiment. As such, there is little to change for you here. +// Measurer // import ( "context" + {{ if ne .InputPolicy "InputOptional" -}} "errors" + {{- end }} "sync" "github.com/ooni/probe-cli/v3/internal/atomicx" @@ -19,16 +16,13 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -// Measurer implements the {{ .Name }} experiment. +// Measurer for the {{ .Name }} experiment. type Measurer struct{ - // Config contains the experiment's config. + // Contains the experiment's config. Config *Config } -// NewExperimentMeasurer creates a new model.ExperimentMeasurer for {{ .Name }}. -// -// Code in the internal/registry package binds this function with an ExperimentBuilder -// which is responsible of instantiating a new experiment. +// NewExperimentMeasurer creates a new model.ExperimentMeasurer. func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { return &Measurer{ Config: config, @@ -48,7 +42,7 @@ func (m *Measurer) ExperimentVersion() string { // Run implements model.ExperimentMeasurer. func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { - // Reminder: When this function returns an error, the measurement result + // Reminder: when this function returns an error, the measurement result // WILL NOT be submitted to the OONI backend. You SHOULD only return an error // for fundamental errors (e.g., the input is invalid or missing). @@ -62,7 +56,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // honour {{ .InputPolicy }} input := measurement.Input if input == "" { - input = m.defaultInput() + // TODO: set here the default input value } {{ else }} @@ -95,15 +89,9 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, wg := &sync.WaitGroup{} // start background tasks - // TODO: replace this code with code for running your background tasks + // TODO: write code to start background tasks {{ if ne .InputPolicy "InputNone" }}_ = URL{{ end }} - wg.Add(1) - go func(logger model.Logger, idx int64) { - defer wg.Done() - ol := measurexlite.NewOperationLogger(logger, "{{ .Package }}#%d", idx) - tk.SetFundamentalFailure(errors.New("experiment not implemented")) - ol.Stop(nil) - }(sess.Logger(), idGenerator.Add(1)) + _ = idGenerator // wait for background tasks to join wg.Wait() @@ -112,11 +100,3 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // the measurement from being submitted to the OONI collector. return tk.FundamentalFailure() } - -{{ if eq .InputPolicy "InputOptional" }} -// defaultInput returns the default input value when input is not present -func (m *Measurer) defaultInput() model.MeasurementTarget { - // TODO: implement - panic("not implemented") -} -{{ end }} diff --git a/internal/cmd/boilerplate/experiment/registry.go.txt b/internal/cmd/boilerplate/experiment/registry.go.txt index 1cc64977ab..64472bd083 100644 --- a/internal/cmd/boilerplate/experiment/registry.go.txt +++ b/internal/cmd/boilerplate/experiment/registry.go.txt @@ -1,7 +1,8 @@ package registry // -// Registers the `{{ .Name }}' experiment. +// Registers the `{{ .Name }}' experiment implemented by +// the `./internal/experiment/{{ .Package }}' package. // import ( diff --git a/internal/cmd/boilerplate/experiment/summary.go.txt b/internal/cmd/boilerplate/experiment/summary.go.txt index 7b885a754b..835d063e32 100644 --- a/internal/cmd/boilerplate/experiment/summary.go.txt +++ b/internal/cmd/boilerplate/experiment/summary.go.txt @@ -1,7 +1,7 @@ package {{ .Package }} // -// Experiment summary result returned to ooniprobe. +// Summary // import "github.com/ooni/probe-cli/v3/internal/model" @@ -17,7 +17,7 @@ type SummaryKeys struct { // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + // TODO: fill all the SummaryKeys sk := SummaryKeys{isAnomaly: false} - // TODO: implement return sk, nil } diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt index e9d4356d75..77ae5c3727 100644 --- a/internal/cmd/boilerplate/experiment/testkeys.go.txt +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -11,10 +11,7 @@ import "sync" // TestKeys contains the results produced by {{ .Name }}. type TestKeys struct { - // TODO: add here fields produced by this experiment. They should - // be public such that we can JSON serialize them. - // - // Ideally, try to keep the field names alphabetically sorted. + // TODO: add here public fields produced by this experiment. // // For example: // diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt index 546415854f..4332cede61 100644 --- a/internal/cmd/boilerplate/task/endpoint.go.txt +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -1,7 +1,10 @@ package {{ .Package }} // -// {{ .StructName }}Task +// {{ .StructName }} +// +// This code was generated by `boilerplate' using +// the {{ .Template }} template. // import ( @@ -26,22 +29,11 @@ import ( {{- end }} ) -// -// Autogenerated section -// -// Suggestion: keep changes in this section minimal to facilitate -// generating the code again next time. -// -// You should insert your own code at the bottom. -// - // {{ .Description }} // // The zero value of this structure IS NOT valid and you MUST initialize // all the fields marked as MANDATORY before using this structure. -// -// This task implements the {{ .Template }} template. -type {{ .StructName }}Task struct { +type {{ .StructName }} struct { // Address is the MANDATORY address to connect to. Address string @@ -80,23 +72,18 @@ type {{ .StructName }}Task struct { {{ end }} } -// Start starts this task in a background gorountine. -func (t *{{ .StructName }}Task) Start(ctx context.Context) { +// Start starts this task in a background goroutine. +func (t *{{ .StructName }}) Start(ctx context.Context) { t.WaitGroup.Add(1) index := t.IDGenerator.Add(1) - go t.run(ctx, index) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() } -// run runs this task in the background. -func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { - // synchronize with wait group - defer t.WaitGroup.Done() - - // configure a timeout - const defaultTimeout = 15 * time.Second // TODO: change this default - opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) - defer cancel() - +// Run runs this task in the current goroutine. +func (t *{{ .StructName }}) Run(parentCtx context.Context, index int64) { // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) @@ -105,8 +92,11 @@ func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { {{ if or (eq .Template "tcp") (eq .Template "tls") (eq .Template "https") (eq .Template "http") }} // perform the TCP connect + const tcpTimeout = 10 * time.Second // TODO: consider changing + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -133,8 +123,11 @@ func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { RootCAs: netxlite.NewDefaultCertPool(), ServerName: tlsSNI, } - tlsConn, _, err := tlsHandshaker.Handshake(opCtx, tcpConn, tlsConfig) - _ = <-trace.TLSHandshake // TODO: save + const tlsTimeout = 10 * time.Second // TODO: consider changing + tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) + defer tlsCancel() + tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + _ = <-trace.TLSHandshake // TODO: save if err != nil { ol.Stop(err) return @@ -161,15 +154,18 @@ func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { {{ if or (eq .Template "http") (eq .Template "https") }} // create HTTP request - httpReq, err := t.newHTTPRequest(opCtx) + const httpTimeout = 10 * time.Second // TODO: consider changing + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) return } - // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -186,15 +182,19 @@ func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { {{ if or (eq .Template "tls") (eq .Template "https") }} // alpn returns the user-configured ALPN or a reasonable default -func (t *{{ .StructName }}Task) alpn() []string { +func (t *{{ .StructName }}) alpn() []string { if len(t.ALPN) > 0 { return t.ALPN } + {{ if eq .Template "https" }} return []string{"h2", "http/1.1"} + {{ else }} + return []string{} + {{ end }} } // sni returns the user-configured SNI or a reasonable default -func (t *{{ .StructName }}Task) sni() (string, error) { +func (t *{{ .StructName }}) sni() (string, error) { if t.SNI != "" { return t.SNI, nil } @@ -208,7 +208,7 @@ func (t *{{ .StructName }}Task) sni() (string, error) { {{ if or (eq .Template "http") (eq .Template "https") }} // urlHost computes the host to include into the URL -func (t *{{ .StructName }}Task) urlHost(scheme string) (string, error) { +func (t *{{ .StructName }}) urlHost(scheme string) (string, error) { addr, port, err := net.SplitHostPort(t.Address) if err != nil { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) @@ -227,7 +227,7 @@ func (t *{{ .StructName }}Task) urlHost(scheme string) (string, error) { } // newHTTPRequest creates a new HTTP request. -func (t *{{ .StructName }}Task) newHTTPRequest(ctx context.Context) (*http.Request, error) { +func (t *{{ .StructName }}) newHTTPRequest(ctx context.Context) (*http.Request, error) { const urlScheme = "{{ if eq .Template "http" }}http{{ else }}https{{ end }}" urlHost, err := t.urlHost(urlScheme) if err != nil { @@ -251,7 +251,7 @@ func (t *{{ .StructName }}Task) newHTTPRequest(ctx context.Context) (*http.Reque } // httpTransaction runs the HTTP transaction and saves the results. -func (t *{{ .StructName }}Task) httpTransaction(ctx context.Context, txp model.HTTPTransport, +func (t *{{ .StructName }}) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) @@ -266,9 +266,3 @@ func (t *{{ .StructName }}Task) httpTransaction(ctx context.Context, txp model.H return resp, body, err } {{ end }} - -// -// User section -// -// We suggest adding your custom methods and functions here. -// diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt index 41055074ee..0b484d794e 100644 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ b/internal/cmd/boilerplate/task/systemresolver.go.txt @@ -1,7 +1,10 @@ package {{ .Package }} // -// {{ .StructName }}Task +// {{ .StructName }} +// +// This code was generated by `boilerplate' using +// the {{ .Template }} template. // import ( @@ -14,22 +17,11 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -// -// Autogenerated section -// -// Suggestion: keep changes in this section minimal to facilitate -// generating the code again next time. -// -// You should insert your own code at the bottom. -// - // {{ .Description }} // // The zero value of this structure IS NOT valid and you MUST initialize // all the fields marked as MANDATORY before using this structure. -// -// This task implements the {{ .Template }} template. -type {{ .StructName }}Task struct { +type {{ .StructName }} struct { // Domain is the MANDATORY domain to resolve. Domain string @@ -42,29 +34,29 @@ type {{ .StructName }}Task struct { // TestKeys is MANDATORY and contains the TestKeys. TestKeys *TestKeys - // ZeroTime is the MANDATORY measurement's zero time. + // ZeroTime is the MANDATORY zero time of the measurement. ZeroTime time.Time // WaitGroup is the MANDATORY wait group this task belongs to. WaitGroup *sync.WaitGroup } -// Start starts this task in a background gorountine. -func (t *{{ .StructName }}Task) Start(ctx context.Context) { +// Start starts this task in a background goroutine. +func (t *{{ .StructName }}) Start(ctx context.Context) { t.WaitGroup.Add(1) index := t.IDGenerator.Add(1) - go t.run(ctx, index) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() } -// run runs this task in the background. -func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { - // synchronize with wait group - defer t.WaitGroup.Done() - +// Run runs this task in the current goroutine. +func (t *{{ .StructName }}) Run(parentCtx context.Context, index int64) { // create context with attached a timeout - const timeout = 4 * time.Second // TODO: change this default - opCtx, cancel := context.WithTimeout(parentCtx, timeout) - defer cancel() + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) @@ -74,7 +66,7 @@ func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { // runs the lookup reso := trace.NewStdlibResolver(t.Logger) - addrs, err := reso.LookupHost(opCtx, t.Domain) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) _ = trace.DNSLookupsFromRoundTrip() // TODO: save if err != nil { ol.Stop(err) @@ -89,9 +81,3 @@ func (t *{{ .StructName }}Task) run(parentCtx context.Context, index int64) { // TODO: implement } } - -// -// User section -// -// We suggest adding your custom methods and functions here. -// From 8c3ba3ba30dfe62323c1968dd464ae06c09dc41e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:03:15 +0200 Subject: [PATCH 31/83] x --- internal/experiment/telegram/config.go | 2 +- internal/experiment/telegram/doc.go | 2 +- internal/experiment/telegram/measurer.go | 18 +++++------------- internal/experiment/telegram/summary.go | 4 ++-- internal/experiment/telegram/testkeys.go | 5 +---- internal/registry/telegram.go | 3 ++- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/internal/experiment/telegram/config.go b/internal/experiment/telegram/config.go index 3734b7e2e9..6c29f305d4 100644 --- a/internal/experiment/telegram/config.go +++ b/internal/experiment/telegram/config.go @@ -1,7 +1,7 @@ package telegram // -// Config for telegram and associated functions. +// Config // // Config contains telegram experiment configuration. diff --git a/internal/experiment/telegram/doc.go b/internal/experiment/telegram/doc.go index 09ea0ddb40..e0ddda751d 100644 --- a/internal/experiment/telegram/doc.go +++ b/internal/experiment/telegram/doc.go @@ -1,4 +1,4 @@ // Package telegram implements the telegram experiment. // -// See https://github.com/ooni/spec/blob/master/nettests/ts-020-telegram.md. +// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-020-telegram.md. package telegram diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index 22b1aa2a29..ebcf251dc9 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -1,12 +1,7 @@ package telegram // -// Measurer for telegram. -// -// The Measurer implements performing a single measurement. Because this -// autogenerated code is optimized to run background tasks, this code's -// main job is to parse inputs and then schedule the "main" task implementing -// the experiment. As such, there is little to change for you here. +// Measurer // import ( @@ -19,16 +14,13 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -// Measurer implements the telegram experiment. +// Measurer for the telegram experiment. type Measurer struct { - // Config contains the experiment's config. + // Contains the experiment's config. Config *Config } -// NewExperimentMeasurer creates a new model.ExperimentMeasurer for telegram. -// -// Code in the internal/registry package binds this function with an ExperimentBuilder -// which is responsible of instantiating a new experiment. +// NewExperimentMeasurer creates a new model.ExperimentMeasurer. func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { return &Measurer{ Config: config, @@ -48,7 +40,7 @@ func (m *Measurer) ExperimentVersion() string { // Run implements model.ExperimentMeasurer. func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { - // Reminder: When this function returns an error, the measurement result + // Reminder: when this function returns an error, the measurement result // WILL NOT be submitted to the OONI backend. You SHOULD only return an error // for fundamental errors (e.g., the input is invalid or missing). diff --git a/internal/experiment/telegram/summary.go b/internal/experiment/telegram/summary.go index 108d4ecf52..2391ae2fd5 100644 --- a/internal/experiment/telegram/summary.go +++ b/internal/experiment/telegram/summary.go @@ -1,7 +1,7 @@ package telegram // -// Experiment summary result returned to ooniprobe. +// Summary // import "github.com/ooni/probe-cli/v3/internal/model" @@ -17,7 +17,7 @@ type SummaryKeys struct { // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + // TODO: fill all the SummaryKeys sk := SummaryKeys{isAnomaly: false} - // TODO: implement return sk, nil } diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go index aa2b8fcbf4..ea74af1e61 100644 --- a/internal/experiment/telegram/testkeys.go +++ b/internal/experiment/telegram/testkeys.go @@ -11,10 +11,7 @@ import "sync" // TestKeys contains the results produced by telegram. type TestKeys struct { - // TODO: add here fields produced by this experiment. They should - // be public such that we can JSON serialize them. - // - // Ideally, try to keep the field names alphabetically sorted. + // TODO: add here public fields produced by this experiment. // // For example: // diff --git a/internal/registry/telegram.go b/internal/registry/telegram.go index 2cb03e7e11..f45781c6f6 100644 --- a/internal/registry/telegram.go +++ b/internal/registry/telegram.go @@ -1,7 +1,8 @@ package registry // -// Registers the `telegram' experiment. +// Registers the `telegram' experiment implemented by +// the `./internal/experiment/telegram' package. // import ( From 377d8190cb1f18e3cab94ae3745e76bffa38b166 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:08:11 +0200 Subject: [PATCH 32/83] x --- internal/experiment/telegram/datacenter.go | 65 +++++++++------------- internal/experiment/telegram/measurer.go | 2 +- 2 files changed, 27 insertions(+), 40 deletions(-) diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index c806a55f0c..f72a8d1c65 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -1,7 +1,10 @@ package telegram // -// DatacenterTask +// Datacenter +// +// This code was generated by `boilerplate' using +// the http template. // import ( @@ -20,22 +23,11 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) -// -// Autogenerated section -// -// Suggestion: keep changes in this section minimal to facilitate -// generating the code again next time. -// -// You should insert your own code at the bottom. -// - // Measures a Telegram data center (DC). // // The zero value of this structure IS NOT valid and you MUST initialize // all the fields marked as MANDATORY before using this structure. -// -// This task implements the http template. -type DatacenterTask struct { +type Datacenter struct { // Address is the MANDATORY address to connect to. Address string @@ -64,23 +56,18 @@ type DatacenterTask struct { URLRawQuery string } -// Start starts this task in a background gorountine. -func (t *DatacenterTask) Start(ctx context.Context) { +// Start starts this task in a background goroutine. +func (t *Datacenter) Start(ctx context.Context) { t.WaitGroup.Add(1) index := t.IDGenerator.Add(1) - go t.run(ctx, index) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() } -// run runs this task in the background. -func (t *DatacenterTask) run(parentCtx context.Context, index int64) { - // synchronize with wait group - defer t.WaitGroup.Done() - - // configure a timeout - const defaultTimeout = 15 * time.Second // TODO: change this default - opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) - defer cancel() - +// Run runs this task in the current goroutine. +func (t *Datacenter) Run(parentCtx context.Context, index int64) { // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) @@ -88,8 +75,11 @@ func (t *DatacenterTask) run(parentCtx context.Context, index int64) { ol := measurexlite.NewOperationLogger(t.Logger, "Datacenter#%d: %s", index, t.Address) // perform the TCP connect + const tcpTimeout = 10 * time.Second // TODO: consider changing + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -109,15 +99,18 @@ func (t *DatacenterTask) run(parentCtx context.Context, index int64) { ) // create HTTP request - httpReq, err := t.newHTTPRequest(opCtx) + const httpTimeout = 10 * time.Second // TODO: consider changing + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) return } - // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -132,7 +125,7 @@ func (t *DatacenterTask) run(parentCtx context.Context, index int64) { } // urlHost computes the host to include into the URL -func (t *DatacenterTask) urlHost(scheme string) (string, error) { +func (t *Datacenter) urlHost(scheme string) (string, error) { addr, port, err := net.SplitHostPort(t.Address) if err != nil { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) @@ -145,7 +138,7 @@ func (t *DatacenterTask) urlHost(scheme string) (string, error) { } // newHTTPRequest creates a new HTTP request. -func (t *DatacenterTask) newHTTPRequest(ctx context.Context) (*http.Request, error) { +func (t *Datacenter) newHTTPRequest(ctx context.Context) (*http.Request, error) { const urlScheme = "http" urlHost, err := t.urlHost(urlScheme) if err != nil { @@ -169,7 +162,7 @@ func (t *DatacenterTask) newHTTPRequest(ctx context.Context) (*http.Request, err } // httpTransaction runs the HTTP transaction and saves the results. -func (t *DatacenterTask) httpTransaction(ctx context.Context, txp model.HTTPTransport, +func (t *Datacenter) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) @@ -183,9 +176,3 @@ func (t *DatacenterTask) httpTransaction(ctx context.Context, txp model.HTTPTran _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save return resp, body, err } - -// -// User section -// -// We suggest adding your custom methods and functions here. -// diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index ebcf251dc9..701127d9e4 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -68,7 +68,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, systemDNSTask.Start(ctx) for _, addr := range dataCenterAddrs { for _, port := range dataCenterPorts { - dcTask := &DatacenterTask{ + dcTask := &Datacenter{ Address: net.JoinHostPort(addr, port), IDGenerator: idGenerator, Logger: sess.Logger(), From 3a74d5a47bf614f9b4c67d9520c4aa0d3d7f14a6 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:11:52 +0200 Subject: [PATCH 33/83] x --- internal/experiment/telegram/measurer.go | 2 +- internal/experiment/telegram/systemdns.go | 54 +++++++++-------------- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index 701127d9e4..faa074d193 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -58,7 +58,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, wg := &sync.WaitGroup{} // start background tasks - systemDNSTask := &SystemDNSTask{ + systemDNSTask := &SystemDNS{ IDGenerator: idGenerator, Logger: sess.Logger(), TestKeys: tk, diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go index 2d0a843937..36f635103d 100644 --- a/internal/experiment/telegram/systemdns.go +++ b/internal/experiment/telegram/systemdns.go @@ -1,7 +1,10 @@ package telegram // -// SystemDNSTask +// SystemDNS +// +// This code was generated by `boilerplate' using +// the system-resolver template. // import ( @@ -15,22 +18,11 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -// -// Autogenerated section -// -// Suggestion: keep changes in this section minimal to facilitate -// generating the code again next time. -// -// You should insert your own code at the bottom. -// - // Resolves web.telegram.org using the system resolver. // // The zero value of this structure IS NOT valid and you MUST initialize // all the fields marked as MANDATORY before using this structure. -// -// This task implements the system-resolver template. -type SystemDNSTask struct { +type SystemDNS struct { // IDGenerator is the MANDATORY atomic int64 to generate task IDs. IDGenerator *atomicx.Int64 @@ -40,29 +32,29 @@ type SystemDNSTask struct { // TestKeys is MANDATORY and contains the TestKeys. TestKeys *TestKeys - // ZeroTime is the MANDATORY measurement's zero time. + // ZeroTime is the MANDATORY zero time of the measurement. ZeroTime time.Time // WaitGroup is the MANDATORY wait group this task belongs to. WaitGroup *sync.WaitGroup } -// Start starts this task in a background gorountine. -func (t *SystemDNSTask) Start(ctx context.Context) { +// Start starts this task in a background goroutine. +func (t *SystemDNS) Start(ctx context.Context) { t.WaitGroup.Add(1) index := t.IDGenerator.Add(1) - go t.run(ctx, index) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() } -// run runs this task in the background. -func (t *SystemDNSTask) run(parentCtx context.Context, index int64) { - // synchronize with wait group - defer t.WaitGroup.Done() - +// Run runs this task in the current goroutine. +func (t *SystemDNS) Run(parentCtx context.Context, index int64) { // create context with attached a timeout - const timeout = 4 * time.Second // TODO: change this default - opCtx, cancel := context.WithTimeout(parentCtx, timeout) - defer cancel() + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) @@ -72,7 +64,7 @@ func (t *SystemDNSTask) run(parentCtx context.Context, index int64) { // runs the lookup reso := trace.NewStdlibResolver(t.Logger) - addrs, err := reso.LookupHost(opCtx, webTelegramOrg) + addrs, err := reso.LookupHost(lookupCtx, webTelegramOrg) _ = trace.DNSLookupsFromRoundTrip() // TODO: save if err != nil { ol.Stop(err) @@ -89,17 +81,11 @@ func (t *SystemDNSTask) run(parentCtx context.Context, index int64) { } } -// -// User section -// -// We suggest adding your custom methods and functions here. -// - // webTelegramOrg is the SNI and host header for telegram web. const webTelegramOrg = "web.telegram.org" // startWebHTTPTask starts a WebHTTPTask for this addr. -func (t *SystemDNSTask) startWebHTTPTask(ctx context.Context, addr string) { +func (t *SystemDNS) startWebHTTPTask(ctx context.Context, addr string) { task := &WebHTTPTask{ Address: net.JoinHostPort(addr, "80"), IDGenerator: t.IDGenerator, @@ -115,7 +101,7 @@ func (t *SystemDNSTask) startWebHTTPTask(ctx context.Context, addr string) { } // startWebHTTPSTask starts a WebHTTPSTask for this addr. -func (t *SystemDNSTask) startWebHTTPSTask(ctx context.Context, addr string) { +func (t *SystemDNS) startWebHTTPSTask(ctx context.Context, addr string) { task := &WebHTTPSTask{ Address: net.JoinHostPort(addr, "443"), IDGenerator: t.IDGenerator, From a9114c976c0c49099dac74f4f7304b55c5baa4cf Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:14:15 +0200 Subject: [PATCH 34/83] x --- internal/experiment/telegram/systemdns.go | 2 +- internal/experiment/telegram/webhttp.go | 65 +++++++++-------------- 2 files changed, 27 insertions(+), 40 deletions(-) diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go index 36f635103d..36cdc70864 100644 --- a/internal/experiment/telegram/systemdns.go +++ b/internal/experiment/telegram/systemdns.go @@ -86,7 +86,7 @@ const webTelegramOrg = "web.telegram.org" // startWebHTTPTask starts a WebHTTPTask for this addr. func (t *SystemDNS) startWebHTTPTask(ctx context.Context, addr string) { - task := &WebHTTPTask{ + task := &WebHTTP{ Address: net.JoinHostPort(addr, "80"), IDGenerator: t.IDGenerator, Logger: t.Logger, diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index 904cd4ce4e..971ef063eb 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -1,7 +1,10 @@ package telegram // -// WebHTTPTask +// WebHTTP +// +// This code was generated by `boilerplate' using +// the http template. // import ( @@ -20,22 +23,11 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) -// -// Autogenerated section -// -// Suggestion: keep changes in this section minimal to facilitate -// generating the code again next time. -// -// You should insert your own code at the bottom. -// - // Measures Telegram Web using HTTP. // // The zero value of this structure IS NOT valid and you MUST initialize // all the fields marked as MANDATORY before using this structure. -// -// This task implements the http template. -type WebHTTPTask struct { +type WebHTTP struct { // Address is the MANDATORY address to connect to. Address string @@ -64,23 +56,18 @@ type WebHTTPTask struct { URLRawQuery string } -// Start starts this task in a background gorountine. -func (t *WebHTTPTask) Start(ctx context.Context) { +// Start starts this task in a background goroutine. +func (t *WebHTTP) Start(ctx context.Context) { t.WaitGroup.Add(1) index := t.IDGenerator.Add(1) - go t.run(ctx, index) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() } -// run runs this task in the background. -func (t *WebHTTPTask) run(parentCtx context.Context, index int64) { - // synchronize with wait group - defer t.WaitGroup.Done() - - // configure a timeout - const defaultTimeout = 15 * time.Second // TODO: change this default - opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) - defer cancel() - +// Run runs this task in the current goroutine. +func (t *WebHTTP) Run(parentCtx context.Context, index int64) { // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) @@ -88,8 +75,11 @@ func (t *WebHTTPTask) run(parentCtx context.Context, index int64) { ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTP#%d: %s", index, t.Address) // perform the TCP connect + const tcpTimeout = 10 * time.Second // TODO: consider changing + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -109,15 +99,18 @@ func (t *WebHTTPTask) run(parentCtx context.Context, index int64) { ) // create HTTP request - httpReq, err := t.newHTTPRequest(opCtx) + const httpTimeout = 10 * time.Second // TODO: consider changing + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) return } - // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -132,7 +125,7 @@ func (t *WebHTTPTask) run(parentCtx context.Context, index int64) { } // urlHost computes the host to include into the URL -func (t *WebHTTPTask) urlHost(scheme string) (string, error) { +func (t *WebHTTP) urlHost(scheme string) (string, error) { addr, port, err := net.SplitHostPort(t.Address) if err != nil { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) @@ -145,7 +138,7 @@ func (t *WebHTTPTask) urlHost(scheme string) (string, error) { } // newHTTPRequest creates a new HTTP request. -func (t *WebHTTPTask) newHTTPRequest(ctx context.Context) (*http.Request, error) { +func (t *WebHTTP) newHTTPRequest(ctx context.Context) (*http.Request, error) { const urlScheme = "http" urlHost, err := t.urlHost(urlScheme) if err != nil { @@ -169,7 +162,7 @@ func (t *WebHTTPTask) newHTTPRequest(ctx context.Context) (*http.Request, error) } // httpTransaction runs the HTTP transaction and saves the results. -func (t *WebHTTPTask) httpTransaction(ctx context.Context, txp model.HTTPTransport, +func (t *WebHTTP) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) @@ -183,9 +176,3 @@ func (t *WebHTTPTask) httpTransaction(ctx context.Context, txp model.HTTPTranspo _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save return resp, body, err } - -// -// User section -// -// We suggest adding your custom methods and functions here. -// From 76f4cb7b4725f69723aecf4bc685de67b548a0ad Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:15:44 +0200 Subject: [PATCH 35/83] x --- internal/experiment/telegram/systemdns.go | 2 +- internal/experiment/telegram/webhttps.go | 74 ++++++++++------------- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go index 36cdc70864..c78280f3c5 100644 --- a/internal/experiment/telegram/systemdns.go +++ b/internal/experiment/telegram/systemdns.go @@ -102,7 +102,7 @@ func (t *SystemDNS) startWebHTTPTask(ctx context.Context, addr string) { // startWebHTTPSTask starts a WebHTTPSTask for this addr. func (t *SystemDNS) startWebHTTPSTask(ctx context.Context, addr string) { - task := &WebHTTPSTask{ + task := &WebHTTPS{ Address: net.JoinHostPort(addr, "443"), IDGenerator: t.IDGenerator, Logger: t.Logger, diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index b6693b30f5..b6e154d6bf 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -1,7 +1,10 @@ package telegram // -// WebHTTPSTask +// WebHTTPS +// +// This code was generated by `boilerplate' using +// the https template. // import ( @@ -20,22 +23,11 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) -// -// Autogenerated section -// -// Suggestion: keep changes in this section minimal to facilitate -// generating the code again next time. -// -// You should insert your own code at the bottom. -// - // Measures Telegram Web using HTTPS. // // The zero value of this structure IS NOT valid and you MUST initialize // all the fields marked as MANDATORY before using this structure. -// -// This task implements the https template. -type WebHTTPSTask struct { +type WebHTTPS struct { // Address is the MANDATORY address to connect to. Address string @@ -70,23 +62,18 @@ type WebHTTPSTask struct { URLRawQuery string } -// Start starts this task in a background gorountine. -func (t *WebHTTPSTask) Start(ctx context.Context) { +// Start starts this task in a background goroutine. +func (t *WebHTTPS) Start(ctx context.Context) { t.WaitGroup.Add(1) index := t.IDGenerator.Add(1) - go t.run(ctx, index) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() } -// run runs this task in the background. -func (t *WebHTTPSTask) run(parentCtx context.Context, index int64) { - // synchronize with wait group - defer t.WaitGroup.Done() - - // configure a timeout - const defaultTimeout = 15 * time.Second // TODO: change this default - opCtx, cancel := context.WithTimeout(parentCtx, defaultTimeout) - defer cancel() - +// Run runs this task in the current goroutine. +func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { // create trace trace := measurexlite.NewTrace(index, t.ZeroTime) @@ -94,8 +81,11 @@ func (t *WebHTTPSTask) run(parentCtx context.Context, index int64) { ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTPS#%d: %s", index, t.Address) // perform the TCP connect + const tcpTimeout = 10 * time.Second // TODO: consider changing + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(opCtx, "tcp", t.Address) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) _ = <-trace.TCPConnect // TODO: save if err != nil { ol.Stop(err) @@ -120,7 +110,10 @@ func (t *WebHTTPSTask) run(parentCtx context.Context, index int64) { RootCAs: netxlite.NewDefaultCertPool(), ServerName: tlsSNI, } - tlsConn, _, err := tlsHandshaker.Handshake(opCtx, tcpConn, tlsConfig) + const tlsTimeout = 10 * time.Second // TODO: consider changing + tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) + defer tlsCancel() + tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) _ = <-trace.TLSHandshake // TODO: save if err != nil { ol.Stop(err) @@ -137,15 +130,18 @@ func (t *WebHTTPSTask) run(parentCtx context.Context, index int64) { ) // create HTTP request - httpReq, err := t.newHTTPRequest(opCtx) + const httpTimeout = 10 * time.Second // TODO: consider changing + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) return } - // perform HTTP round trip - httpResp, httpRespBody, err := t.httpTransaction(opCtx, httpTransport, httpReq, trace) + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) if err != nil { ol.Stop(err) return @@ -160,7 +156,7 @@ func (t *WebHTTPSTask) run(parentCtx context.Context, index int64) { } // alpn returns the user-configured ALPN or a reasonable default -func (t *WebHTTPSTask) alpn() []string { +func (t *WebHTTPS) alpn() []string { if len(t.ALPN) > 0 { return t.ALPN } @@ -168,7 +164,7 @@ func (t *WebHTTPSTask) alpn() []string { } // sni returns the user-configured SNI or a reasonable default -func (t *WebHTTPSTask) sni() (string, error) { +func (t *WebHTTPS) sni() (string, error) { if t.SNI != "" { return t.SNI, nil } @@ -180,7 +176,7 @@ func (t *WebHTTPSTask) sni() (string, error) { } // urlHost computes the host to include into the URL -func (t *WebHTTPSTask) urlHost(scheme string) (string, error) { +func (t *WebHTTPS) urlHost(scheme string) (string, error) { addr, port, err := net.SplitHostPort(t.Address) if err != nil { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) @@ -193,7 +189,7 @@ func (t *WebHTTPSTask) urlHost(scheme string) (string, error) { } // newHTTPRequest creates a new HTTP request. -func (t *WebHTTPSTask) newHTTPRequest(ctx context.Context) (*http.Request, error) { +func (t *WebHTTPS) newHTTPRequest(ctx context.Context) (*http.Request, error) { const urlScheme = "https" urlHost, err := t.urlHost(urlScheme) if err != nil { @@ -217,7 +213,7 @@ func (t *WebHTTPSTask) newHTTPRequest(ctx context.Context) (*http.Request, error } // httpTransaction runs the HTTP transaction and saves the results. -func (t *WebHTTPSTask) httpTransaction(ctx context.Context, txp model.HTTPTransport, +func (t *WebHTTPS) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) @@ -231,9 +227,3 @@ func (t *WebHTTPSTask) httpTransaction(ctx context.Context, txp model.HTTPTransp _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save return resp, body, err } - -// -// User section -// -// We suggest adding your custom methods and functions here. -// From a846393a0385420ce1a093d36140f9f5387906ff Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:26:49 +0200 Subject: [PATCH 36/83] feat: add support for transaction_id --- internal/measurexlite/conn.go | 15 +++++++------- internal/measurexlite/dialer.go | 3 ++- internal/measurexlite/dns.go | 1 + internal/measurexlite/http.go | 3 ++- internal/measurexlite/tls.go | 1 + internal/model/archival.go | 35 +++++++++++++++++++-------------- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/internal/measurexlite/conn.go b/internal/measurexlite/conn.go index a688bd4467..54c6cb31b3 100644 --- a/internal/measurexlite/conn.go +++ b/internal/measurexlite/conn.go @@ -73,13 +73,14 @@ func (c *connTrace) Write(b []byte) (int, error) { func NewArchivalNetworkEvent(index int64, started time.Duration, operation string, network string, address string, count int, err error, finished time.Duration) *model.ArchivalNetworkEvent { return &model.ArchivalNetworkEvent{ - Address: address, - Failure: tracex.NewFailure(err), - NumBytes: int64(count), - Operation: operation, - Proto: network, - T: finished.Seconds(), - Tags: []string{}, + Address: address, + Failure: tracex.NewFailure(err), + NumBytes: int64(count), + Operation: operation, + Proto: network, + T: finished.Seconds(), + TransactionID: index, + Tags: []string{}, } } diff --git a/internal/measurexlite/dialer.go b/internal/measurexlite/dialer.go index e7da206f15..7d4bf3c584 100644 --- a/internal/measurexlite/dialer.go +++ b/internal/measurexlite/dialer.go @@ -82,7 +82,8 @@ func NewArchivalTCPConnectResult(index int64, started time.Duration, address str Failure: tracex.NewFailure(err), Success: err == nil, }, - T: finished.Seconds(), + T: finished.Seconds(), + TransactionID: index, } } diff --git a/internal/measurexlite/dns.go b/internal/measurexlite/dns.go index 7a5b0b253e..b75fbf378d 100644 --- a/internal/measurexlite/dns.go +++ b/internal/measurexlite/dns.go @@ -110,6 +110,7 @@ func NewArchivalDNSLookupResultFromRoundTrip(index int64, started time.Duration, ResolverHostname: nil, ResolverAddress: reso.Address(), T: finished.Seconds(), + TransactionID: index, } } diff --git a/internal/measurexlite/http.go b/internal/measurexlite/http.go index e71e610fb4..83bfa80cfc 100644 --- a/internal/measurexlite/http.go +++ b/internal/measurexlite/http.go @@ -45,7 +45,8 @@ func (tx *Trace) NewArchivalHTTPRequestResult( Headers: map[string]model.ArchivalMaybeBinaryData{}, Locations: []string{}, }, - T: tx.TimeSince(tx.ZeroTime).Seconds(), + T: tx.TimeSince(tx.ZeroTime).Seconds(), + TransactionID: tx.Index, } if resp != nil { if body != nil { diff --git a/internal/measurexlite/tls.go b/internal/measurexlite/tls.go index 2d881a79ee..cc67168374 100644 --- a/internal/measurexlite/tls.go +++ b/internal/measurexlite/tls.go @@ -87,6 +87,7 @@ func NewArchivalTLSOrQUICHandshakeResult( T: finished.Seconds(), Tags: []string{}, TLSVersion: netxlite.TLSVersionString(state.Version), + TransactionID: index, } } diff --git a/internal/model/archival.go b/internal/model/archival.go index 42e0e84816..7f96dd5706 100644 --- a/internal/model/archival.go +++ b/internal/model/archival.go @@ -121,6 +121,7 @@ type ArchivalDNSLookupResult struct { ResolverPort *string `json:"resolver_port"` ResolverAddress string `json:"resolver_address"` T float64 `json:"t"` + TransactionID int64 `json:"transaction_id"` } // ArchivalDNSAnswer is a DNS answer. @@ -142,10 +143,11 @@ type ArchivalDNSAnswer struct { // // See https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md. type ArchivalTCPConnectResult struct { - IP string `json:"ip"` - Port int `json:"port"` - Status ArchivalTCPConnectStatus `json:"status"` - T float64 `json:"t"` + IP string `json:"ip"` + Port int `json:"port"` + Status ArchivalTCPConnectStatus `json:"status"` + T float64 `json:"t"` + TransactionID int64 `json:"transaction_id"` } // ArchivalTCPConnectStatus is the status of ArchivalTCPConnectResult. @@ -173,6 +175,7 @@ type ArchivalTLSOrQUICHandshakeResult struct { T float64 `json:"t"` Tags []string `json:"tags"` TLSVersion string `json:"tls_version"` + TransactionID int64 `json:"transaction_id"` } // @@ -183,10 +186,11 @@ type ArchivalTLSOrQUICHandshakeResult struct { // // See https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md. type ArchivalHTTPRequestResult struct { - Failure *string `json:"failure"` - Request ArchivalHTTPRequest `json:"request"` - Response ArchivalHTTPResponse `json:"response"` - T float64 `json:"t"` + Failure *string `json:"failure"` + Request ArchivalHTTPRequest `json:"request"` + Response ArchivalHTTPResponse `json:"response"` + T float64 `json:"t"` + TransactionID int64 `json:"transaction_id"` } // ArchivalHTTPRequest contains an HTTP request. @@ -302,11 +306,12 @@ type ArchivalHTTPTor struct { // // See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md. type ArchivalNetworkEvent struct { - Address string `json:"address,omitempty"` - Failure *string `json:"failure"` - NumBytes int64 `json:"num_bytes,omitempty"` - Operation string `json:"operation"` - Proto string `json:"proto,omitempty"` - T float64 `json:"t"` - Tags []string `json:"tags,omitempty"` + Address string `json:"address,omitempty"` + Failure *string `json:"failure"` + NumBytes int64 `json:"num_bytes,omitempty"` + Operation string `json:"operation"` + Proto string `json:"proto,omitempty"` + T float64 `json:"t"` + TransactionID int64 `json:"transaction_id"` + Tags []string `json:"tags,omitempty"` } From 1c75e038d39c08b4c0061bb108f78a19991a1e04 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:34:21 +0200 Subject: [PATCH 37/83] feat: start saving events --- internal/experiment/telegram/datacenter.go | 4 +-- internal/experiment/telegram/testkeys.go | 36 ++++++++++++++-------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index f72a8d1c65..e788d8514d 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -80,14 +80,14 @@ func (t *Datacenter) Run(parentCtx context.Context, index int64) { defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - _ = <-trace.TCPConnect // TODO: save + t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) if err != nil { ol.Stop(err) return } tcpConn = trace.WrapNetConn(tcpConn) defer func() { - _ = trace.NetworkEvents() // TODO: save + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() }() diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go index ea74af1e61..2c6bddd381 100644 --- a/internal/experiment/telegram/testkeys.go +++ b/internal/experiment/telegram/testkeys.go @@ -7,16 +7,19 @@ package telegram // containing the results produced by OONI experiments. // -import "sync" +import ( + "sync" + + "github.com/ooni/probe-cli/v3/internal/model" +) // TestKeys contains the results produced by telegram. type TestKeys struct { - // TODO: add here public fields produced by this experiment. - // - // For example: - // - // // Blocked indicates that the resource is censored. - // Blocked bool `json:"blocked"` + // NetworkEvents contains network events. + NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + + // TCPConnect contains TCP connect results. + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer @@ -29,12 +32,19 @@ type TestKeys struct { mu *sync.Mutex } -// TODO: implement more thread-safe setters for the real test keys. This allows -// tasks to write directly into the TestKeys. -// -// In some cases, you may also need to write thread-safe getters. For example, -// below we also define a getter for fundamentalFailure because we need to -// read its value inside the autogenerated Runner.Main func. +// AppendNetworkEvents appends to NetworkEvents. +func (tk *TestKeys) AppendNetworkEvents(v ...*model.ArchivalNetworkEvent) { + tk.mu.Lock() + tk.NetworkEvents = append(tk.NetworkEvents, v...) + tk.mu.Unlock() +} + +// AppendTCPConnectResults appends to TCPConnect. +func (tk *TestKeys) AppendTCPConnectResults(v ...*model.ArchivalTCPConnectResult) { + tk.mu.Lock() + tk.TCPConnect = append(tk.TCPConnect, v...) + tk.mu.Unlock() +} // SetFundamentalFailure implements TestKeys. func (tk *TestKeys) SetFundamentalFailure(err error) { From 2e8f975a56f531ee1813bd3293313c34c0a34f84 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:38:15 +0200 Subject: [PATCH 38/83] fix bug in httpTransaction --- internal/cmd/boilerplate/task/endpoint.go.txt | 2 +- internal/experiment/telegram/datacenter.go | 2 +- internal/experiment/telegram/webhttp.go | 2 +- internal/experiment/telegram/webhttps.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt index 4332cede61..0ccbb930f5 100644 --- a/internal/cmd/boilerplate/task/endpoint.go.txt +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -255,8 +255,8 @@ func (t *{{ .StructName }}) httpTransaction(ctx context.Context, txp model.HTTPT req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save if err != nil { + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save return resp, []byte{}, err } defer resp.Body.Close() diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index e788d8514d..714f74f70c 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -166,8 +166,8 @@ func (t *Datacenter) httpTransaction(ctx context.Context, txp model.HTTPTranspor req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save if err != nil { + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save return resp, []byte{}, err } defer resp.Body.Close() diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index 971ef063eb..3a914de6be 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -166,8 +166,8 @@ func (t *WebHTTP) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save if err != nil { + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save return resp, []byte{}, err } defer resp.Body.Close() diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index b6e154d6bf..27e7db8872 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -217,8 +217,8 @@ func (t *WebHTTPS) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 22 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save if err != nil { + _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save return resp, []byte{}, err } defer resp.Body.Close() From 04352ac613bf2b2ebc64d85ad139108475e73c8f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:50:20 +0200 Subject: [PATCH 39/83] save endpoints data --- internal/experiment/telegram/datacenter.go | 13 +++++++------ internal/experiment/telegram/testkeys.go | 20 ++++++++++++++++++++ internal/experiment/telegram/webhttp.go | 17 +++++++++-------- internal/experiment/telegram/webhttps.go | 20 +++++++++++--------- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index 714f74f70c..84a807a4c4 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -9,7 +9,6 @@ package telegram import ( "context" - "io" "net" "net/http" @@ -75,7 +74,7 @@ func (t *Datacenter) Run(parentCtx context.Context, index int64) { ol := measurexlite.NewOperationLogger(t.Logger, "Datacenter#%d: %s", index, t.Address) // perform the TCP connect - const tcpTimeout = 10 * time.Second // TODO: consider changing + const tcpTimeout = 10 * time.Second tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) @@ -99,7 +98,7 @@ func (t *Datacenter) Run(parentCtx context.Context, index int64) { ) // create HTTP request - const httpTimeout = 10 * time.Second // TODO: consider changing + const httpTimeout = 10 * time.Second httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) defer httpCancel() httpReq, err := t.newHTTPRequest(httpCtx) @@ -164,15 +163,17 @@ func (t *Datacenter) newHTTPRequest(ctx context.Context) (*http.Request, error) // httpTransaction runs the HTTP transaction and saves the results. func (t *Datacenter) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 // TODO: you may want to change this default + const maxbody = 1 << 22 resp, err := txp.RoundTrip(req) if err != nil { - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) + t.TestKeys.AppendRequests(ev) return resp, []byte{}, err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) body, err := netxlite.ReadAllContext(ctx, reader) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + t.TestKeys.AppendRequests(ev) return resp, body, err } diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go index 2c6bddd381..104f1bbf00 100644 --- a/internal/experiment/telegram/testkeys.go +++ b/internal/experiment/telegram/testkeys.go @@ -18,9 +18,15 @@ type TestKeys struct { // NetworkEvents contains network events. NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + // Requests contains HTTP results. + Requests []*model.ArchivalHTTPRequestResult `json:"requests"` + // TCPConnect contains TCP connect results. TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` + // TLSHandshakes contains TLS handshakes results. + TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer // such as a failure to parse a URL that was hardcoded in the codebase. When @@ -39,6 +45,13 @@ func (tk *TestKeys) AppendNetworkEvents(v ...*model.ArchivalNetworkEvent) { tk.mu.Unlock() } +// AppendRequests appends to Requests. +func (tk *TestKeys) AppendRequests(v ...*model.ArchivalHTTPRequestResult) { + tk.mu.Lock() + tk.Requests = append(tk.Requests, v...) + tk.mu.Unlock() +} + // AppendTCPConnectResults appends to TCPConnect. func (tk *TestKeys) AppendTCPConnectResults(v ...*model.ArchivalTCPConnectResult) { tk.mu.Lock() @@ -46,6 +59,13 @@ func (tk *TestKeys) AppendTCPConnectResults(v ...*model.ArchivalTCPConnectResult tk.mu.Unlock() } +// AppendTLSHandshakes appends to TLSHandshakes. +func (tk *TestKeys) AppendTLSHandshakes(v ...*model.ArchivalTLSOrQUICHandshakeResult) { + tk.mu.Lock() + tk.TLSHandshakes = append(tk.TLSHandshakes, v...) + tk.mu.Unlock() +} + // SetFundamentalFailure implements TestKeys. func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Lock() diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index 3a914de6be..8854dab022 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -9,7 +9,6 @@ package telegram import ( "context" - "io" "net" "net/http" @@ -75,19 +74,19 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTP#%d: %s", index, t.Address) // perform the TCP connect - const tcpTimeout = 10 * time.Second // TODO: consider changing + const tcpTimeout = 10 * time.Second tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - _ = <-trace.TCPConnect // TODO: save + t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) if err != nil { ol.Stop(err) return } tcpConn = trace.WrapNetConn(tcpConn) defer func() { - _ = trace.NetworkEvents() // TODO: save + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() }() @@ -99,7 +98,7 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { ) // create HTTP request - const httpTimeout = 10 * time.Second // TODO: consider changing + const httpTimeout = 10 * time.Second httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) defer httpCancel() httpReq, err := t.newHTTPRequest(httpCtx) @@ -164,15 +163,17 @@ func (t *WebHTTP) newHTTPRequest(ctx context.Context) (*http.Request, error) { // httpTransaction runs the HTTP transaction and saves the results. func (t *WebHTTP) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 // TODO: you may want to change this default + const maxbody = 1 << 22 resp, err := txp.RoundTrip(req) if err != nil { - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) + t.TestKeys.AppendRequests(ev) return resp, []byte{}, err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) body, err := netxlite.ReadAllContext(ctx, reader) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + t.TestKeys.AppendRequests(ev) return resp, body, err } diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index 27e7db8872..a0c32aa6cb 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -81,19 +81,19 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTPS#%d: %s", index, t.Address) // perform the TCP connect - const tcpTimeout = 10 * time.Second // TODO: consider changing + const tcpTimeout = 10 * time.Second tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - _ = <-trace.TCPConnect // TODO: save + t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) if err != nil { ol.Stop(err) return } tcpConn = trace.WrapNetConn(tcpConn) defer func() { - _ = trace.NetworkEvents() // TODO: save + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() }() @@ -110,11 +110,11 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { RootCAs: netxlite.NewDefaultCertPool(), ServerName: tlsSNI, } - const tlsTimeout = 10 * time.Second // TODO: consider changing + const tlsTimeout = 10 * time.Second tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) defer tlsCancel() tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) - _ = <-trace.TLSHandshake // TODO: save + t.TestKeys.AppendTLSHandshakes(<-trace.TLSHandshake) if err != nil { ol.Stop(err) return @@ -130,7 +130,7 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { ) // create HTTP request - const httpTimeout = 10 * time.Second // TODO: consider changing + const httpTimeout = 10 * time.Second httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) defer httpCancel() httpReq, err := t.newHTTPRequest(httpCtx) @@ -215,15 +215,17 @@ func (t *WebHTTPS) newHTTPRequest(ctx context.Context) (*http.Request, error) { // httpTransaction runs the HTTP transaction and saves the results. func (t *WebHTTPS) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 // TODO: you may want to change this default + const maxbody = 1 << 22 resp, err := txp.RoundTrip(req) if err != nil { - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) + t.TestKeys.AppendRequests(ev) return resp, []byte{}, err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) body, err := netxlite.ReadAllContext(ctx, reader) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + t.TestKeys.AppendRequests(ev) return resp, body, err } From 68642b6f53f58498df817bc8e487e73844d5801f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 12 Aug 2022 21:52:55 +0200 Subject: [PATCH 40/83] save dns queries --- internal/experiment/telegram/systemdns.go | 2 +- internal/experiment/telegram/testkeys.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go index c78280f3c5..9017f90e83 100644 --- a/internal/experiment/telegram/systemdns.go +++ b/internal/experiment/telegram/systemdns.go @@ -65,7 +65,7 @@ func (t *SystemDNS) Run(parentCtx context.Context, index int64) { // runs the lookup reso := trace.NewStdlibResolver(t.Logger) addrs, err := reso.LookupHost(lookupCtx, webTelegramOrg) - _ = trace.DNSLookupsFromRoundTrip() // TODO: save + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) if err != nil { ol.Stop(err) return diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go index 104f1bbf00..d57702fec8 100644 --- a/internal/experiment/telegram/testkeys.go +++ b/internal/experiment/telegram/testkeys.go @@ -18,6 +18,9 @@ type TestKeys struct { // NetworkEvents contains network events. NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + // Queries contains DNS lookup results. + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + // Requests contains HTTP results. Requests []*model.ArchivalHTTPRequestResult `json:"requests"` @@ -45,6 +48,13 @@ func (tk *TestKeys) AppendNetworkEvents(v ...*model.ArchivalNetworkEvent) { tk.mu.Unlock() } +// AppendQueries appends to Queries. +func (tk *TestKeys) AppendQueries(v ...*model.ArchivalDNSLookupResult) { + tk.mu.Lock() + tk.Queries = append(tk.Queries, v...) + tk.mu.Unlock() +} + // AppendRequests appends to Requests. func (tk *TestKeys) AppendRequests(v ...*model.ArchivalHTTPRequestResult) { tk.mu.Lock() From fad09e6909f1334e6eb04946480ec28605e11431 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sat, 13 Aug 2022 10:48:46 +0200 Subject: [PATCH 41/83] feat: implement DCs-related test keys --- .../boilerplate/experiment/measurer.go.txt | 2 +- .../boilerplate/experiment/testkeys.go.txt | 14 +---- internal/experiment/telegram/datacenter.go | 8 +++ internal/experiment/telegram/measurer.go | 2 +- internal/experiment/telegram/testkeys.go | 56 +++++++++++++++---- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index 6d0b6c56f4..b05331eee6 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -98,5 +98,5 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. - return tk.FundamentalFailure() + return tk.fundamentalFailure } diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt index 77ae5c3727..ae9b751a5c 100644 --- a/internal/cmd/boilerplate/experiment/testkeys.go.txt +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -31,26 +31,14 @@ type TestKeys struct { // TODO: implement more thread-safe setters for the real test keys. This allows // tasks to write directly into the TestKeys. -// -// In some cases, you may also need to write thread-safe getters. For example, -// below we also define a getter for fundamentalFailure because we need to -// read its value inside the autogenerated Runner.Main func. -// SetFundamentalFailure implements TestKeys. +// SetFundamentalFailure sets the value of fundamentalFailure. func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Lock() tk.fundamentalFailure = err tk.mu.Unlock() } -// FundamentalFailure implements TestKeys. -func (tk *TestKeys) FundamentalFailure() error { - tk.mu.Lock() - err := tk.fundamentalFailure - tk.mu.Unlock() - return err -} - // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { // TODO: here you should initialize all the fields diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index 84a807a4c4..4d35a184d0 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -90,6 +90,10 @@ func (t *Datacenter) Run(parentCtx context.Context, index int64) { tcpConn.Close() }() + // "If all TCP connections on ports 80 and 443 to Telegram’s access + // point IPs fail we consider Telegram to be blocked." + t.TestKeys.SetTelegramTCPBlocking(false) + // create HTTP transport httpTransport := netxlite.NewHTTPTransport( t.Logger, @@ -115,6 +119,10 @@ func (t *Datacenter) Run(parentCtx context.Context, index int64) { return } + // "If at least an HTTP request returns back a response, we + // consider Telegram [DCs] to not be blocked." + t.TestKeys.SetTelegramHTTPBlocking(false) + // TODO: insert here additional code if needed _ = httpResp _ = httpRespBody diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index faa074d193..b6526fa077 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -88,7 +88,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. - return tk.FundamentalFailure() + return tk.fundamentalFailure } // dataCenterAddrs contains the data center addrs. diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go index d57702fec8..7a63a042bc 100644 --- a/internal/experiment/telegram/testkeys.go +++ b/internal/experiment/telegram/testkeys.go @@ -30,6 +30,18 @@ type TestKeys struct { // TLSHandshakes contains TLS handshakes results. TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + // TelegramTCPBlocking indicates whether we believe DCs + // to be blocked at the TCP/IP layer. From the spec: "If all + // TCP connections on ports 80 and 443 to Telegram’s access + // point IPs fail we consider Telegram to be blocked." + TelegramTCPBlocking bool `json:"telegram_tcp_blocking"` + + // TelegramHTTPBlocking indicates whether we believe DCs + // to be blocked at the TCP/IP layer. From the spec: "If at + // least an HTTP request returns back a response, we + // consider Telegram [DCs] to not be blocked." + TelegramHTTPBlocking bool `json:"telegram_http_blocking"` + // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer // such as a failure to parse a URL that was hardcoded in the codebase. When @@ -76,26 +88,48 @@ func (tk *TestKeys) AppendTLSHandshakes(v ...*model.ArchivalTLSOrQUICHandshakeRe tk.mu.Unlock() } -// SetFundamentalFailure implements TestKeys. -func (tk *TestKeys) SetFundamentalFailure(err error) { +// SetTelegramTCPBlocking sets the value of TelegramTCPBlocking. +func (tk *TestKeys) SetTelegramTCPBlocking(value bool) { tk.mu.Lock() - tk.fundamentalFailure = err + tk.TelegramTCPBlocking = value + tk.mu.Unlock() +} + +// SetTelegramHTTPBlocking sets the value of TelegramHTTPBlocking. +func (tk *TestKeys) SetTelegramHTTPBlocking(value bool) { + tk.mu.Lock() + tk.TelegramHTTPBlocking = value tk.mu.Unlock() } -// FundamentalFailure implements TestKeys. -func (tk *TestKeys) FundamentalFailure() error { +// SetFundamentalFailure sets the value of fundamentalFailure. +func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Lock() - err := tk.fundamentalFailure + tk.fundamentalFailure = err tk.mu.Unlock() - return err } // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { - // TODO: here you should initialize all the fields - return &TestKeys{ - fundamentalFailure: nil, - mu: &sync.Mutex{}, + tk := &TestKeys{ + NetworkEvents: []*model.ArchivalNetworkEvent{}, + Queries: []*model.ArchivalDNSLookupResult{}, + Requests: []*model.ArchivalHTTPRequestResult{}, + TCPConnect: []*model.ArchivalTCPConnectResult{}, + TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, + TelegramTCPBlocking: false, + TelegramHTTPBlocking: false, + fundamentalFailure: nil, + mu: &sync.Mutex{}, } + + // "If all TCP connections on ports 80 and 443 to Telegram’s access + // point IPs fail we consider Telegram to be blocked." + tk.TelegramTCPBlocking = true + + // "If at least an HTTP request returns back a response, we + // consider Telegram [DCs] to not be blocked." + tk.TelegramHTTPBlocking = true + + return tk } From 547dde484efb7aeb89cdac32aa9901c4e1f53153 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sat, 13 Aug 2022 11:08:54 +0200 Subject: [PATCH 42/83] feat: start parsing the results --- internal/experiment/telegram/webhttp.go | 26 +++++++++++++++++++++--- internal/experiment/telegram/webhttps.go | 26 ++++++++++++++++++++---- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index 8854dab022..b217abd309 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -9,7 +9,9 @@ package telegram import ( "context" + "errors" "io" + "log" "net" "net/http" "net/url" @@ -115,9 +117,11 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { return } - // TODO: insert here additional code if needed - _ = httpResp - _ = httpRespBody + // parse HTTP results + if err := t.parseResults(httpResp, httpRespBody); err != nil { + ol.Stop(err) + return + } // completed successfully ol.Stop(nil) @@ -177,3 +181,19 @@ func (t *WebHTTP) httpTransaction(ctx context.Context, txp model.HTTPTransport, t.TestKeys.AppendRequests(ev) return resp, body, err } + +// parseResults parses the results of this sub-measurement. +func (t *WebHTTP) parseResults(resp *http.Response, respBody []byte) error { + if resp.StatusCode != 301 && resp.StatusCode != 308 { + log.Printf("status code: %+v", resp.StatusCode) + return errors.New("http_request_failed") + } + location, err := resp.Location() + if err != nil { + return errors.New("telegram_missing_redirect_error") + } + if location.Scheme != "https" || location.Host != webTelegramOrg { + return errors.New("telegram_invalid_redirect_error") + } + return nil +} diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index a0c32aa6cb..d1c5accd85 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -8,9 +8,12 @@ package telegram // import ( + "bytes" "context" "crypto/tls" + "errors" "io" + "log" "net" "net/http" "net/url" @@ -147,9 +150,11 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { return } - // TODO: insert here additional code if needed - _ = httpResp - _ = httpRespBody + // parse HTTP results + if err := t.parseResults(httpResp, httpRespBody); err != nil { + ol.Stop(err) + return + } // completed successfully ol.Stop(nil) @@ -225,7 +230,20 @@ func (t *WebHTTPS) httpTransaction(ctx context.Context, txp model.HTTPTransport, defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) body, err := netxlite.ReadAllContext(ctx, reader) - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) t.TestKeys.AppendRequests(ev) return resp, body, err } + +// parseResults parses the results of this sub-measurement. +func (t *WebHTTPS) parseResults(resp *http.Response, respBody []byte) error { + if resp.StatusCode != 200 { + log.Printf("status code: %+v", resp.StatusCode) + return errors.New("http_request_failed") + } + title := []byte(`Telegram Web`) + if !bytes.Contains(respBody, title) { + return errors.New("telegram_missing_title_error") + } + return nil +} From de72d5c14b7519dfcc83e06e7a00f8ec6bb13719 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sat, 13 Aug 2022 11:13:10 +0200 Subject: [PATCH 43/83] fix: ensure we send the correct host header --- internal/cmd/boilerplate/task/endpoint.go.txt | 1 + internal/experiment/telegram/webhttp.go | 1 + internal/experiment/telegram/webhttps.go | 1 + 3 files changed, 3 insertions(+) diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt index 0ccbb930f5..22f69422d8 100644 --- a/internal/cmd/boilerplate/task/endpoint.go.txt +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -247,6 +247,7 @@ func (t *{{ .StructName }}) newHTTPRequest(ctx context.Context) (*http.Request, httpReq.Header.Set("Accept", model.HTTPHeaderAccept) httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader return httpReq, nil } diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index b217abd309..8a2484c505 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -161,6 +161,7 @@ func (t *WebHTTP) newHTTPRequest(ctx context.Context) (*http.Request, error) { httpReq.Header.Set("Accept", model.HTTPHeaderAccept) httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader return httpReq, nil } diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index d1c5accd85..d1f29bd0cf 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -214,6 +214,7 @@ func (t *WebHTTPS) newHTTPRequest(ctx context.Context) (*http.Request, error) { httpReq.Header.Set("Accept", model.HTTPHeaderAccept) httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader return httpReq, nil } From 2556ab682f44bfd0fe2501932664c80ec4fa9659 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sat, 13 Aug 2022 12:04:49 +0200 Subject: [PATCH 44/83] feat: implement telegram_web_{status,failure} --- internal/experiment/telegram/measurer.go | 3 ++ internal/experiment/telegram/testkeys.go | 51 ++++++++++++++++++++++++ internal/experiment/telegram/webhttp.go | 4 ++ internal/experiment/telegram/webhttps.go | 6 +++ 4 files changed, 64 insertions(+) diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index b6526fa077..8c9764e525 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -86,6 +86,9 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // wait for background tasks to join wg.Wait() + // perform any deferred computation on the test keys + tk.finalize() + // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. return tk.fundamentalFailure diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go index 7a63a042bc..aa348dd988 100644 --- a/internal/experiment/telegram/testkeys.go +++ b/internal/experiment/telegram/testkeys.go @@ -8,7 +8,9 @@ package telegram // import ( + "errors" "sync" + "syscall" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -42,6 +44,16 @@ type TestKeys struct { // consider Telegram [DCs] to not be blocked." TelegramHTTPBlocking bool `json:"telegram_http_blocking"` + // TelegramWebStatus is either "blocked" or "ok" and indicates + // whether we're able to access the web.telegram.org site. + TelegramWebStatus string `json:"telegram_web_status"` + + // TelegramWebFailure is the failure when accessing web.telegram.org + TelegramWebFailure *string `json:"telegram_web_failure"` + + // webFailures contains the failures occurred when measuring web.telegram.org + webFailures []error + // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer // such as a failure to parse a URL that was hardcoded in the codebase. When @@ -102,6 +114,13 @@ func (tk *TestKeys) SetTelegramHTTPBlocking(value bool) { tk.mu.Unlock() } +// AppendWebFailure appends to the webFailures list. +func (tk *TestKeys) AppendWebFailure(err error) { + tk.mu.Lock() + tk.webFailures = append(tk.webFailures, err) + tk.mu.Unlock() +} + // SetFundamentalFailure sets the value of fundamentalFailure. func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Lock() @@ -119,6 +138,9 @@ func NewTestKeys() *TestKeys { TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, TelegramTCPBlocking: false, TelegramHTTPBlocking: false, + TelegramWebStatus: "", + TelegramWebFailure: nil, + webFailures: []error{}, fundamentalFailure: nil, mu: &sync.Mutex{}, } @@ -131,5 +153,34 @@ func NewTestKeys() *TestKeys { // consider Telegram [DCs] to not be blocked." tk.TelegramHTTPBlocking = true + // We start saying web.telegram.org is blocked and flip to okay + // only when we notice that it's accessible. + tk.TelegramWebStatus = "blocked" + + // We start by saying that the experiment did not actually + // run until completion, and then flip later if needed. + didNotRun := "telegram_did_not_run_error" + tk.TelegramWebFailure = &didNotRun + return tk } + +// finalize performs any delayed computation on the test keys. This function +// must be called from the measurer after all the tasks have completed. +func (tk *TestKeys) finalize() { + var filtered []error + for _, err := range tk.webFailures { + if errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH) { + continue // skip IPv6 errors when there's no working IPv6 support + } + filtered = append(filtered, err) + } + if len(filtered) <= 0 { + tk.TelegramWebStatus = "ok" + tk.TelegramWebFailure = nil + return + } + tk.TelegramWebStatus = "blocked" + first := filtered[0].Error() + tk.TelegramWebFailure = &first +} diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index 8a2484c505..59683b2f39 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -83,6 +83,7 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) if err != nil { + t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } @@ -105,6 +106,7 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { defer httpCancel() httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { + t.TestKeys.AppendWebFailure(err) t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) return @@ -113,12 +115,14 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { // perform HTTP transaction httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) if err != nil { + t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } // parse HTTP results if err := t.parseResults(httpResp, httpRespBody); err != nil { + t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index d1f29bd0cf..5be34be38b 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -91,6 +91,7 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) if err != nil { + t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } @@ -103,6 +104,7 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { // perform TLS handshake tlsSNI, err := t.sni() if err != nil { + t.TestKeys.AppendWebFailure(err) t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) return @@ -119,6 +121,7 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) t.TestKeys.AppendTLSHandshakes(<-trace.TLSHandshake) if err != nil { + t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } @@ -138,6 +141,7 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { defer httpCancel() httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { + t.TestKeys.AppendWebFailure(err) t.TestKeys.SetFundamentalFailure(err) ol.Stop(err) return @@ -146,12 +150,14 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { // perform HTTP transaction httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) if err != nil { + t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } // parse HTTP results if err := t.parseResults(httpResp, httpRespBody); err != nil { + t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } From 0b6452133b2c61fb3bb8aa391225204570715f42 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sat, 13 Aug 2022 12:09:39 +0200 Subject: [PATCH 45/83] feat: add finalize and bail if context is cancelled --- internal/cmd/boilerplate/experiment/measurer.go.txt | 9 +++++++++ internal/cmd/boilerplate/experiment/testkeys.go.txt | 6 ++++++ internal/experiment/telegram/measurer.go | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt index b05331eee6..b3e848ec0f 100644 --- a/internal/cmd/boilerplate/experiment/measurer.go.txt +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -96,6 +96,15 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // wait for background tasks to join wg.Wait() + // If the context passed to us has been cancelled, we cannot + // trust this experiment's results to be okay. + if err := ctx.Err(); err != nil { + return err + } + + // perform any deferred computation on the test keys + tk.finalize() + // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. return tk.fundamentalFailure diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt index ae9b751a5c..639855609a 100644 --- a/internal/cmd/boilerplate/experiment/testkeys.go.txt +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -47,3 +47,9 @@ func NewTestKeys() *TestKeys { mu: &sync.Mutex{}, } } + +// finalize performs any delayed computation on the test keys. This function +// must be called from the measurer after all the tasks have completed. +func (tk *TestKeys) finalize() { + // TODO: implement +} diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index 8c9764e525..816da78cca 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -86,6 +86,12 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // wait for background tasks to join wg.Wait() + // If the context passed to us has been cancelled, we cannot + // trust this experiment's results to be okay. + if err := ctx.Err(); err != nil { + return err + } + // perform any deferred computation on the test keys tk.finalize() From 27df0c7d5ff6f1f11afe31b4da063947dbda0b26 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 17:44:51 +0200 Subject: [PATCH 46/83] define the concept of a multiresolver --- internal/cmd/boilerplate/task.go | 4 + .../cmd/boilerplate/task/multiresolver.go.txt | 198 +++++++++++++++ internal/experiment/telegram/dnsresolvers.go | 240 ++++++++++++++++++ internal/experiment/telegram/measurer.go | 15 +- internal/experiment/telegram/systemdns.go | 119 --------- internal/measurexlite/trace.go | 5 + 6 files changed, 456 insertions(+), 125 deletions(-) create mode 100644 internal/cmd/boilerplate/task/multiresolver.go.txt create mode 100644 internal/experiment/telegram/dnsresolvers.go delete mode 100644 internal/experiment/telegram/systemdns.go diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index c18e7a43e2..1b5c6375a2 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -89,6 +89,9 @@ func getTaskDescription() string { //go:embed "task/endpoint.go.txt" var endpointTemplate string +//go:embed "task/multiresolver.go.txt" +var multiResolverTemplate string + //go:embed "task/systemresolver.go.txt" var systemResolverTemplate string @@ -96,6 +99,7 @@ var systemResolverTemplate string var knownTasks = map[string]string{ "http": endpointTemplate, "https": endpointTemplate, + "multi-resolver": multiResolverTemplate, "system-resolver": systemResolverTemplate, "tcp": endpointTemplate, "tls": endpointTemplate, diff --git a/internal/cmd/boilerplate/task/multiresolver.go.txt b/internal/cmd/boilerplate/task/multiresolver.go.txt new file mode 100644 index 0000000000..1588e7ebd6 --- /dev/null +++ b/internal/cmd/boilerplate/task/multiresolver.go.txt @@ -0,0 +1,198 @@ +package {{ .Package }} + +// +// {{ .StructName }} +// +// This code was generated by `boilerplate' using +// the {{ .Template }} template. +// + +import ( + "context" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// {{ .Description }} +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type {{ .StructName }} struct { + // Domain is the MANDATORY domain to resolve. + Domain string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY zero time of the measurement. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not + // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). + DNSOverHTTPSURL string + + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this + // field is not set we use a default one (e.g., `8.8.8.8:53`). + UDPAddress string +} + +// Start starts this task in a background goroutine. +func (t *{{ .StructName }}) Start(ctx context.Context) { + t.WaitGroup.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx) + }() +} + +// Run runs this task in the current goroutine. +func (t *{{ .StructName }}) Run(parentCtx context.Context) { + // create output channels for the lookup + systemOut := make(chan []string) + udpOut := make(chan []string) + httpsOut := make(chan []string) + + // start asynchronous lookups + go t.lookupHostSystem(parentCtx, systemOut) + go t.lookupHostUDP(parentCtx, udpOut) + go t.lookupHostDNSOverHTTPS(parentCtx, httpsOut) + + // collect resulting IP addresses (which may be nil/empty lists) + systemAddrs := <-systemOut + udpAddrs := <-udpOut + httpsAddrs := <-httpsOut + + // merge the resolved IP addresses + merged := map[string]bool{} + for _, addr := range systemAddrs { + merged[addr] = true + } + for _, addr := range udpAddrs { + merged[addr] = true + } + for _, addr := range httpsAddrs { + merged[addr] = true + } + + // rearrange addresses to have IPv4 first + sorted := []string{} + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && !v6 { + sorted = append(sorted, addr) + } + } + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && v6 { + sorted = append(sorted, addr) + } + } + + // (typically) fan out a number of child async tasks to use the IP addrs + for range sorted { + // TODO: implement + } +} + +// lookupHostSystem performs a DNS lookup using the system resolver. +func (t *{{ .StructName }}) lookupHostSystem(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}+System#%d", index) // TODO: edit + + // runs the lookup + reso := trace.NewStdlibResolver(t.Logger) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + _ = trace.DNSLookupsFromRoundTrip() // TODO: save + ol.Stop(err) + out <- addrs +} + +// lookupHostUDP performs a DNS lookup using an UDP resolver. +func (t *{{ .StructName }}) lookupHostUDP(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}+UDP#%d", index) // TODO: edit + + // runs the lookup + dialer := netxlite.NewDialerWithoutResolver(t.Logger) + reso := trace.NewParallelUDPResolver(t.Logger, dialer, t.udpAddress()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + _ = trace.DNSLookupsFromRoundTrip() // TODO: save + ol.Stop(err) + out <- addrs +} + +// Returns the UDP resolver we should be using by default. +func (t *{{ .StructName }}) udpAddress() string { + if t.UDPAddress != "" { + return t.UDPAddress + } + return "8.8.4.4:53" +} + +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. +func (t *{{ .StructName }}) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}+DNSOverHTTPS#%d", index) // TODO: edit + + // runs the lookup + reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, t.dnsOverHTTPSURL()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + reso.CloseIdleConnections() + _ = trace.DNSLookupsFromRoundTrip() // TODO: save + ol.Stop(err) + out <- addrs +} + +// Returns the DOH resolver URL we should be using by default. +func (t *{{ .StructName }}) dnsOverHTTPSURL() string { + if t.DNSOverHTTPSURL != "" { + return t.DNSOverHTTPSURL + } + return "https://mozilla.cloudflare-dns.com/dns-query" +} diff --git a/internal/experiment/telegram/dnsresolvers.go b/internal/experiment/telegram/dnsresolvers.go new file mode 100644 index 0000000000..4edc739257 --- /dev/null +++ b/internal/experiment/telegram/dnsresolvers.go @@ -0,0 +1,240 @@ +package telegram + +// +// DNSResolvers +// +// This code was generated by `boilerplate' using +// the multi-resolver template. +// + +import ( + "context" + "net" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Lookups web.telegram.org using several resolvers. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type DNSResolvers struct { + // Domain is the MANDATORY domain to resolve. + Domain string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY zero time of the measurement. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not + // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). + DNSOverHTTPSURL string + + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this + // field is not set we use a default one (e.g., `8.8.8.8:53`). + UDPAddress string +} + +// Start starts this task in a background goroutine. +func (t *DNSResolvers) Start(ctx context.Context) { + t.WaitGroup.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx) + }() +} + +// Run runs this task in the current goroutine. +func (t *DNSResolvers) Run(parentCtx context.Context) { + // create output channels for the lookup + systemOut := make(chan []string) + udpOut := make(chan []string) + httpsOut := make(chan []string) + + // start asynchronous lookups + go t.lookupHostSystem(parentCtx, systemOut) + go t.lookupHostUDP(parentCtx, udpOut) + go t.lookupHostDNSOverHTTPS(parentCtx, httpsOut) + + // collect resulting IP addresses (which may be nil/empty lists) + systemAddrs := <-systemOut + udpAddrs := <-udpOut + httpsAddrs := <-httpsOut + + // merge the resolved IP addresses + merged := map[string]bool{} + for _, addr := range systemAddrs { + merged[addr] = true + } + for _, addr := range udpAddrs { + merged[addr] = true + } + for _, addr := range httpsAddrs { + merged[addr] = true + } + + // rearrange addresses to have IPv4 first + sorted := []string{} + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && !v6 { + sorted = append(sorted, addr) + } + } + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && v6 { + sorted = append(sorted, addr) + } + } + + // (typically) fan out a number of child async tasks to use the IP addrs + for _, addr := range sorted { + t.startWebHTTPTask(parentCtx, addr) + t.startWebHTTPSTask(parentCtx, addr) + } +} + +// lookupHostSystem performs a DNS lookup using the system resolver. +func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger( + t.Logger, "DNSResolvers+System#%d: %s", index, t.Domain) + + // runs the lookup + reso := trace.NewStdlibResolver(t.Logger) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// lookupHostUDP performs a DNS lookup using an UDP resolver. +func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger( + t.Logger, "DNSResolvers+UDP#%d: %s", index, t.Domain) + + // runs the lookup + dialer := netxlite.NewDialerWithoutResolver(t.Logger) + reso := trace.NewParallelUDPResolver(t.Logger, dialer, t.udpAddress()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// Returns the UDP resolver we should be using by default. +func (t *DNSResolvers) udpAddress() string { + if t.UDPAddress != "" { + return t.UDPAddress + } + return "8.8.4.4:53" +} + +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. +func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger( + t.Logger, "DNSResolvers+DNSOverHTTPS#%d: %s", index, t.Domain) + + // runs the lookup + reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, t.dnsOverHTTPSURL()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + reso.CloseIdleConnections() + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// Returns the DOH resolver URL we should be using by default. +func (t *DNSResolvers) dnsOverHTTPSURL() string { + if t.DNSOverHTTPSURL != "" { + return t.DNSOverHTTPSURL + } + return "https://mozilla.cloudflare-dns.com/dns-query" +} + +// webTelegramOrg is the SNI and host header for telegram web. +const webTelegramOrg = "web.telegram.org" + +// startWebHTTPTask starts a WebHTTPTask for this addr. +func (t *DNSResolvers) startWebHTTPTask(ctx context.Context, addr string) { + task := &WebHTTP{ + Address: net.JoinHostPort(addr, "80"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + HostHeader: webTelegramOrg, + URLPath: "", + URLRawQuery: "", + } + task.Start(ctx) +} + +// startWebHTTPSTask starts a WebHTTPSTask for this addr. +func (t *DNSResolvers) startWebHTTPSTask(ctx context.Context, addr string) { + task := &WebHTTPS{ + Address: net.JoinHostPort(addr, "443"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + ALPN: []string{}, // default is okay + SNI: webTelegramOrg, + HostHeader: webTelegramOrg, + URLPath: "", + URLRawQuery: "", + } + task.Start(ctx) +} diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go index 816da78cca..f7dc3b6ae4 100644 --- a/internal/experiment/telegram/measurer.go +++ b/internal/experiment/telegram/measurer.go @@ -58,12 +58,15 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, wg := &sync.WaitGroup{} // start background tasks - systemDNSTask := &SystemDNS{ - IDGenerator: idGenerator, - Logger: sess.Logger(), - TestKeys: tk, - ZeroTime: measurement.MeasurementStartTimeSaved, - WaitGroup: wg, + systemDNSTask := &DNSResolvers{ + Domain: webTelegramOrg, + IDGenerator: idGenerator, + Logger: sess.Logger(), + TestKeys: tk, + ZeroTime: measurement.MeasurementStartTimeSaved, + WaitGroup: wg, + DNSOverHTTPSURL: "", + UDPAddress: "", } systemDNSTask.Start(ctx) for _, addr := range dataCenterAddrs { diff --git a/internal/experiment/telegram/systemdns.go b/internal/experiment/telegram/systemdns.go deleted file mode 100644 index 9017f90e83..0000000000 --- a/internal/experiment/telegram/systemdns.go +++ /dev/null @@ -1,119 +0,0 @@ -package telegram - -// -// SystemDNS -// -// This code was generated by `boilerplate' using -// the system-resolver template. -// - -import ( - "context" - "net" - "sync" - "time" - - "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" -) - -// Resolves web.telegram.org using the system resolver. -// -// The zero value of this structure IS NOT valid and you MUST initialize -// all the fields marked as MANDATORY before using this structure. -type SystemDNS struct { - // IDGenerator is the MANDATORY atomic int64 to generate task IDs. - IDGenerator *atomicx.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - - // TestKeys is MANDATORY and contains the TestKeys. - TestKeys *TestKeys - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time - - // WaitGroup is the MANDATORY wait group this task belongs to. - WaitGroup *sync.WaitGroup -} - -// Start starts this task in a background goroutine. -func (t *SystemDNS) Start(ctx context.Context) { - t.WaitGroup.Add(1) - index := t.IDGenerator.Add(1) - go func() { - defer t.WaitGroup.Done() // synchronize with the parent - t.Run(ctx, index) - }() -} - -// Run runs this task in the current goroutine. -func (t *SystemDNS) Run(parentCtx context.Context, index int64) { - // create context with attached a timeout - const timeout = 4 * time.Second // TODO: consider changing - lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) - defer lookpCancel() - - // create trace - trace := measurexlite.NewTrace(index, t.ZeroTime) - - // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "SystemDNS#%d: %s", index, webTelegramOrg) - - // runs the lookup - reso := trace.NewStdlibResolver(t.Logger) - addrs, err := reso.LookupHost(lookupCtx, webTelegramOrg) - t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - if err != nil { - ol.Stop(err) - return - } - - // emit successful log message - ol.Stop(nil) - - // (typically) fan out a number of child async tasks to use the IP addrs - for _, addr := range addrs { - t.startWebHTTPTask(parentCtx, addr) - t.startWebHTTPSTask(parentCtx, addr) - } -} - -// webTelegramOrg is the SNI and host header for telegram web. -const webTelegramOrg = "web.telegram.org" - -// startWebHTTPTask starts a WebHTTPTask for this addr. -func (t *SystemDNS) startWebHTTPTask(ctx context.Context, addr string) { - task := &WebHTTP{ - Address: net.JoinHostPort(addr, "80"), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - HostHeader: webTelegramOrg, - URLPath: "", - URLRawQuery: "", - } - task.Start(ctx) -} - -// startWebHTTPSTask starts a WebHTTPSTask for this addr. -func (t *SystemDNS) startWebHTTPSTask(ctx context.Context, addr string) { - task := &WebHTTPS{ - Address: net.JoinHostPort(addr, "443"), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - ALPN: []string{}, // default is okay - SNI: webTelegramOrg, - HostHeader: webTelegramOrg, - URLPath: "", - URLRawQuery: "", - } - task.Start(ctx) -} diff --git a/internal/measurexlite/trace.go b/internal/measurexlite/trace.go index 946e4e2bf6..31dbfb8129 100644 --- a/internal/measurexlite/trace.go +++ b/internal/measurexlite/trace.go @@ -34,6 +34,11 @@ type Trace struct { // traces, you can use zero to indicate the "default" trace. Index int64 + // TODO(bassosimone): make all these channels private and always + // use ~safe accessor functions to get the data. This protects + // against a case such as DoH where you're not doing actual step + // by step and would like to know what has happened. + // NetworkEvent is MANDATORY and buffers network events. If you create // this channel manually, ensure it has some buffer. NetworkEvent chan *model.ArchivalNetworkEvent From ab6e3c811e8233197e47324dcfaa387256199c8a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 17:48:43 +0200 Subject: [PATCH 47/83] feat: add skeleton of web connectivity experiment --- internal/experiment/webconnectivity/config.go | 8 ++ internal/experiment/webconnectivity/doc.go | 4 + .../experiment/webconnectivity/inputparser.go | 63 +++++++++++++ .../experiment/webconnectivity/measurer.go | 94 +++++++++++++++++++ .../experiment/webconnectivity/summary.go | 23 +++++ .../experiment/webconnectivity/testkeys.go | 55 +++++++++++ internal/registry/webconnectivity.go | 14 +-- 7 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 internal/experiment/webconnectivity/config.go create mode 100644 internal/experiment/webconnectivity/doc.go create mode 100644 internal/experiment/webconnectivity/inputparser.go create mode 100644 internal/experiment/webconnectivity/measurer.go create mode 100644 internal/experiment/webconnectivity/summary.go create mode 100644 internal/experiment/webconnectivity/testkeys.go diff --git a/internal/experiment/webconnectivity/config.go b/internal/experiment/webconnectivity/config.go new file mode 100644 index 0000000000..f5b6bc248d --- /dev/null +++ b/internal/experiment/webconnectivity/config.go @@ -0,0 +1,8 @@ +package webconnectivity + +// +// Config +// + +// Config contains webconnectivity experiment configuration. +type Config struct{} diff --git a/internal/experiment/webconnectivity/doc.go b/internal/experiment/webconnectivity/doc.go new file mode 100644 index 0000000000..bfa80f30fe --- /dev/null +++ b/internal/experiment/webconnectivity/doc.go @@ -0,0 +1,4 @@ +// Package webconnectivity implements the web_connectivity experiment. +// +// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-017-web-connectivity.md. +package webconnectivity diff --git a/internal/experiment/webconnectivity/inputparser.go b/internal/experiment/webconnectivity/inputparser.go new file mode 100644 index 0000000000..cee8775a30 --- /dev/null +++ b/internal/experiment/webconnectivity/inputparser.go @@ -0,0 +1,63 @@ +package webconnectivity + +// +// Input parsing +// + +import ( + "errors" + "net" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// InputParser helps to print the experiment's input. +type InputParser struct { + // List of accepted URL schemes. + AcceptedSchemes []string + + // Whether to allow endpoints in input. + AllowEndpoints bool + + // The default scheme to use if AllowEndpoints == true. + DefaultScheme string +} + +// Parse parses the experiment input and returns the resulting URL. +func (ip *InputParser) Parse(input string) (*url.URL, error) { + // put this check at top-level such that we always see the crash if needed + runtimex.PanicIfTrue( + ip.AllowEndpoints && ip.DefaultScheme == "", + "invalid configuration for InputParser.AllowEndpoints == true", + ) + URL, err := url.Parse(input) + if err != nil { + return ip.maybeAllowEndpoints(URL, err) + } + for _, scheme := range ip.AcceptedSchemes { + if URL.Scheme == scheme { + // TODO: here you may want to perform additional parsing + return URL, nil + } + } + return nil, errors.New("cannot parse input") +} + +// Conditionally allows endpooints when ip.AllowEndpoints is true. +func (ip *InputParser) maybeAllowEndpoints(URL *url.URL, err error) (*url.URL, error) { + runtimex.PanicIfNil(err, "expected to be called with a non-nil error") + if ip.AllowEndpoints && URL.Scheme != "" && URL.Opaque != "" && URL.User == nil && + URL.Host == "" && URL.Path == "" && URL.RawPath == "" && + URL.RawQuery == "" && URL.Fragment == "" && URL.RawFragment == "" { + // See https://go.dev/play/p/Rk5pS_zGY5U + // + // Note that we know that `ip.DefaultScheme != ""` from the above runtime check. + out := &url.URL{ + Scheme: ip.DefaultScheme, + Host: net.JoinHostPort(URL.Scheme, URL.Opaque), + } + return out, nil + } + return nil, err +} diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go new file mode 100644 index 0000000000..84e36a669d --- /dev/null +++ b/internal/experiment/webconnectivity/measurer.go @@ -0,0 +1,94 @@ +package webconnectivity + +// +// Measurer +// + +import ( + "context" + "errors" + "sync" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Measurer for the web_connectivity experiment. +type Measurer struct { + // Contains the experiment's config. + Config *Config +} + +// NewExperimentMeasurer creates a new model.ExperimentMeasurer. +func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { + return &Measurer{ + Config: config, + } +} + +// ExperimentName implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentName() string { + return "web_connectivity" +} + +// ExperimentVersion implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentVersion() string { + return "0.5.0" +} + +// Run implements model.ExperimentMeasurer. +func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { + // Reminder: when this function returns an error, the measurement result + // WILL NOT be submitted to the OONI backend. You SHOULD only return an error + // for fundamental errors (e.g., the input is invalid or missing). + + // honour InputOrQueryBackend + input := measurement.Input + if input == "" { + return errors.New("no input provided") + } + + // convert the input string to a URL + inputParser := &InputParser{ + AcceptedSchemes: []string{ + "http", + "https", + }, + AllowEndpoints: false, + DefaultScheme: "", + } + URL, err := inputParser.Parse(string(measurement.Input)) + if err != nil { + return err + } + + // initialize the experiment's test keys + tk := NewTestKeys() + measurement.TestKeys = tk + + // create variables required to run parallel tasks + idGenerator := &atomicx.Int64{} + wg := &sync.WaitGroup{} + + // start background tasks + // TODO: write code to start background tasks + _ = URL + _ = idGenerator + + // wait for background tasks to join + wg.Wait() + + // If the context passed to us has been cancelled, we cannot + // trust this experiment's results to be okay. + if err := ctx.Err(); err != nil { + return err + } + + // perform any deferred computation on the test keys + tk.finalize() + + // return whether there was a fundamental failure, which would prevent + // the measurement from being submitted to the OONI collector. + return tk.fundamentalFailure +} diff --git a/internal/experiment/webconnectivity/summary.go b/internal/experiment/webconnectivity/summary.go new file mode 100644 index 0000000000..b7de075f29 --- /dev/null +++ b/internal/experiment/webconnectivity/summary.go @@ -0,0 +1,23 @@ +package webconnectivity + +// +// Summary +// + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Summary contains the summary results. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + // TODO: add here additional summary fields. + isAnomaly bool +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + // TODO: fill all the SummaryKeys + sk := SummaryKeys{isAnomaly: false} + return sk, nil +} diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go new file mode 100644 index 0000000000..6e80e0cf11 --- /dev/null +++ b/internal/experiment/webconnectivity/testkeys.go @@ -0,0 +1,55 @@ +package webconnectivity + +// +// TestKeys for web_connectivity. +// +// Note: for historical reasons, we call TestKeys the JSON object +// containing the results produced by OONI experiments. +// + +import "sync" + +// TestKeys contains the results produced by web_connectivity. +type TestKeys struct { + // TODO: add here public fields produced by this experiment. + // + // For example: + // + // // Blocked indicates that the resource is censored. + // Blocked bool `json:"blocked"` + + // fundamentalFailure indicates that some fundamental error occurred + // in a background task. A fundamental error is something like a programmer + // such as a failure to parse a URL that was hardcoded in the codebase. When + // this class of errors happens, you certainly don't want to submit the + // resulting measurement to the OONI collector. + fundamentalFailure error + + // mu provides mutual exclusion for accessing the test keys. + mu *sync.Mutex +} + +// TODO: implement more thread-safe setters for the real test keys. This allows +// tasks to write directly into the TestKeys. + +// SetFundamentalFailure sets the value of fundamentalFailure. +func (tk *TestKeys) SetFundamentalFailure(err error) { + tk.mu.Lock() + tk.fundamentalFailure = err + tk.mu.Unlock() +} + +// NewTestKeys creates a new instance of TestKeys. +func NewTestKeys() *TestKeys { + // TODO: here you should initialize all the fields + return &TestKeys{ + fundamentalFailure: nil, + mu: &sync.Mutex{}, + } +} + +// finalize performs any delayed computation on the test keys. This function +// must be called from the measurer after all the tasks have completed. +func (tk *TestKeys) finalize() { + // TODO: implement +} diff --git a/internal/registry/webconnectivity.go b/internal/registry/webconnectivity.go index 850542033d..8db3ccaedf 100644 --- a/internal/registry/webconnectivity.go +++ b/internal/registry/webconnectivity.go @@ -1,22 +1,24 @@ package registry // -// Registers the `web_connectivity' experiment. +// Registers the `web_connectivity' experiment implemented by +// the `./internal/experiment/webconnectivity' package. // import ( - "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" ) func init() { allexperiments["web_connectivity"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { + build: func(config any) model.ExperimentMeasurer { return webconnectivity.NewExperimentMeasurer( - *config.(*webconnectivity.Config), + config.(*webconnectivity.Config), ) }, - config: &webconnectivity.Config{}, - inputPolicy: model.InputOrQueryBackend, + config: &webconnectivity.Config{}, + interruptible: false, + inputPolicy: model.InputOrQueryBackend, } } From 74a28a5db918e33f318b63c52358dc0e5353d3d5 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 17:58:27 +0200 Subject: [PATCH 48/83] feat: write resolvers step for web connectivity --- .../webconnectivity/dnsresolvers.go | 202 ++++++++++++++++++ .../experiment/webconnectivity/measurer.go | 15 +- .../experiment/webconnectivity/testkeys.go | 22 +- 3 files changed, 227 insertions(+), 12 deletions(-) create mode 100644 internal/experiment/webconnectivity/dnsresolvers.go diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go new file mode 100644 index 0000000000..1c7213f1cd --- /dev/null +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -0,0 +1,202 @@ +package webconnectivity + +// +// DNSResolvers +// +// This code was generated by `boilerplate' using +// the multi-resolver template. +// + +import ( + "context" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Resolves the URL's domain using several resolvers. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type DNSResolvers struct { + // Domain is the MANDATORY domain to resolve. + Domain string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // URL is the MANDATORY URL we're measuring. + URL *url.URL + + // ZeroTime is the MANDATORY zero time of the measurement. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not + // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). + DNSOverHTTPSURL string + + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this + // field is not set we use a default one (e.g., `8.8.8.8:53`). + UDPAddress string +} + +// Start starts this task in a background goroutine. +func (t *DNSResolvers) Start(ctx context.Context) { + t.WaitGroup.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx) + }() +} + +// Run runs this task in the current goroutine. +func (t *DNSResolvers) Run(parentCtx context.Context) { + // create output channels for the lookup + systemOut := make(chan []string) + udpOut := make(chan []string) + httpsOut := make(chan []string) + + // start asynchronous lookups + go t.lookupHostSystem(parentCtx, systemOut) + go t.lookupHostUDP(parentCtx, udpOut) + go t.lookupHostDNSOverHTTPS(parentCtx, httpsOut) + + // collect resulting IP addresses (which may be nil/empty lists) + systemAddrs := <-systemOut + udpAddrs := <-udpOut + httpsAddrs := <-httpsOut + + // merge the resolved IP addresses + merged := map[string]bool{} + for _, addr := range systemAddrs { + merged[addr] = true + } + for _, addr := range udpAddrs { + merged[addr] = true + } + for _, addr := range httpsAddrs { + merged[addr] = true + } + + // rearrange addresses to have IPv4 first + sorted := []string{} + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && !v6 { + sorted = append(sorted, addr) + } + } + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && v6 { + sorted = append(sorted, addr) + } + } + + // (typically) fan out a number of child async tasks to use the IP addrs + for range sorted { + // TODO: implement + } +} + +// lookupHostSystem performs a DNS lookup using the system resolver. +func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "DNSResolvers+System#%d: %s", index, t.Domain) + + // runs the lookup + reso := trace.NewStdlibResolver(t.Logger) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// lookupHostUDP performs a DNS lookup using an UDP resolver. +func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "DNSResolvers+UDP#%d: %s", index, t.Domain) + + // runs the lookup + dialer := netxlite.NewDialerWithoutResolver(t.Logger) + reso := trace.NewParallelUDPResolver(t.Logger, dialer, t.udpAddress()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// Returns the UDP resolver we should be using by default. +func (t *DNSResolvers) udpAddress() string { + if t.UDPAddress != "" { + return t.UDPAddress + } + return "8.8.4.4:53" +} + +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. +func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "DNSResolvers+DNSOverHTTPS#%d: %s", index, t.Domain) + + // runs the lookup + reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, t.dnsOverHTTPSURL()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + reso.CloseIdleConnections() + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// Returns the DOH resolver URL we should be using by default. +func (t *DNSResolvers) dnsOverHTTPSURL() string { + if t.DNSOverHTTPSURL != "" { + return t.DNSOverHTTPSURL + } + return "https://mozilla.cloudflare-dns.com/dns-query" +} diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index 84e36a669d..8ba584c253 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -72,9 +72,18 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, wg := &sync.WaitGroup{} // start background tasks - // TODO: write code to start background tasks - _ = URL - _ = idGenerator + resos := &DNSResolvers{ + Domain: URL.Host, + IDGenerator: idGenerator, + Logger: sess.Logger(), + TestKeys: tk, + URL: URL, + ZeroTime: measurement.MeasurementStartTimeSaved, + WaitGroup: wg, + DNSOverHTTPSURL: "", + UDPAddress: "", + } + resos.Start(ctx) // wait for background tasks to join wg.Wait() diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 6e80e0cf11..97c5ed4646 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -7,16 +7,16 @@ package webconnectivity // containing the results produced by OONI experiments. // -import "sync" +import ( + "sync" + + "github.com/ooni/probe-cli/v3/internal/model" +) // TestKeys contains the results produced by web_connectivity. type TestKeys struct { - // TODO: add here public fields produced by this experiment. - // - // For example: - // - // // Blocked indicates that the resource is censored. - // Blocked bool `json:"blocked"` + // Queries contains DNS queries. + Queries []*model.ArchivalDNSLookupResult `json:"queries"` // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer @@ -29,8 +29,12 @@ type TestKeys struct { mu *sync.Mutex } -// TODO: implement more thread-safe setters for the real test keys. This allows -// tasks to write directly into the TestKeys. +// AppendQueries appends to Queries. +func (tk *TestKeys) AppendQueries(v ...*model.ArchivalDNSLookupResult) { + tk.mu.Lock() + tk.Queries = append(tk.Queries, v...) + tk.mu.Unlock() +} // SetFundamentalFailure sets the value of fundamentalFailure. func (tk *TestKeys) SetFundamentalFailure(err error) { From c1b306f50569dac4ccdb4cbba5e3e6ba7f7de5a5 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 20:19:15 +0200 Subject: [PATCH 49/83] feat: add the connectivity tasks --- .../webconnectivity/dnsresolvers.go | 41 +++++- .../experiment/webconnectivity/measurer.go | 2 +- .../webconnectivity/tcpconnectivity.go | 82 +++++++++++ .../experiment/webconnectivity/testkeys.go | 30 ++++ .../webconnectivity/tlsconnectivity.go | 134 ++++++++++++++++++ 5 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 internal/experiment/webconnectivity/tcpconnectivity.go create mode 100644 internal/experiment/webconnectivity/tlsconnectivity.go diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 1c7213f1cd..1bc46b8a75 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -9,6 +9,7 @@ package webconnectivity import ( "context" + "net" "net/url" "sync" "time" @@ -105,9 +106,13 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { } } - // (typically) fan out a number of child async tasks to use the IP addrs - for range sorted { - // TODO: implement + // TODO(bassosimone): remove bogons + // TODO(bassosimone): what to do if there is an explicit port?! + + // fan out a number of child async tasks to use the IP addrs + for _, addr := range sorted { + t.startTCPTask(parentCtx, addr) + t.startTCPTLSTask(parentCtx, addr) } } @@ -200,3 +205,33 @@ func (t *DNSResolvers) dnsOverHTTPSURL() string { } return "https://mozilla.cloudflare-dns.com/dns-query" } + +// startTCPTask starts a TCP measurement task for the given IP address. +func (t *DNSResolvers) startTCPTask(ctx context.Context, address string) { + task := &TCPConnectivity{ + Address: net.JoinHostPort(address, "80"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + } + task.Start(ctx) +} + +// startTCPTLSTask starts a TCP+TLS measurement task for the given IP address. +func (t *DNSResolvers) startTCPTLSTask(ctx context.Context, address string) { + task := &TLSConnectivity{ + Address: net.JoinHostPort(address, "443"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + ALPN: []string{ + "h2", "http/1.1", + }, + SNI: t.URL.Hostname(), + } + task.Start(ctx) +} diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index 8ba584c253..c10f61116f 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -73,7 +73,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // start background tasks resos := &DNSResolvers{ - Domain: URL.Host, + Domain: URL.Hostname(), IDGenerator: idGenerator, Logger: sess.Logger(), TestKeys: tk, diff --git a/internal/experiment/webconnectivity/tcpconnectivity.go b/internal/experiment/webconnectivity/tcpconnectivity.go new file mode 100644 index 0000000000..50b33ac842 --- /dev/null +++ b/internal/experiment/webconnectivity/tcpconnectivity.go @@ -0,0 +1,82 @@ +package webconnectivity + +// +// TCPConnectivity +// +// This code was generated by `boilerplate' using +// the tcp template. +// + +import ( + "context" + + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Checks for TCP connectivity. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type TCPConnectivity struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup +} + +// Start starts this task in a background goroutine. +func (t *TCPConnectivity) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *TCPConnectivity) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "TCPConnectivity#%d: %s", index, t.Address) + + // perform the TCP connect + const tcpTimeout = 10 * time.Second + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + tcpConn.Close() + }() + + // completed successfully + ol.Stop(nil) +} diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 97c5ed4646..8c35f8aae7 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -15,9 +15,18 @@ import ( // TestKeys contains the results produced by web_connectivity. type TestKeys struct { + // NetworkEvents contains network events. + NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + // Queries contains DNS queries. Queries []*model.ArchivalDNSLookupResult `json:"queries"` + // TCPConnect contains TCP connect results. + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` + + // TLSHandshakes contains TLS handshakes results. + TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer // such as a failure to parse a URL that was hardcoded in the codebase. When @@ -29,6 +38,13 @@ type TestKeys struct { mu *sync.Mutex } +// AppendNetworkEvents appends to NetworkEvents. +func (tk *TestKeys) AppendNetworkEvents(v ...*model.ArchivalNetworkEvent) { + tk.mu.Lock() + tk.NetworkEvents = append(tk.NetworkEvents, v...) + tk.mu.Unlock() +} + // AppendQueries appends to Queries. func (tk *TestKeys) AppendQueries(v ...*model.ArchivalDNSLookupResult) { tk.mu.Lock() @@ -36,6 +52,20 @@ func (tk *TestKeys) AppendQueries(v ...*model.ArchivalDNSLookupResult) { tk.mu.Unlock() } +// AppendTCPConnectResults appends to TCPConnect. +func (tk *TestKeys) AppendTCPConnectResults(v ...*model.ArchivalTCPConnectResult) { + tk.mu.Lock() + tk.TCPConnect = append(tk.TCPConnect, v...) + tk.mu.Unlock() +} + +// AppendTLSHandshakes appends to TLSHandshakes. +func (tk *TestKeys) AppendTLSHandshakes(v ...*model.ArchivalTLSOrQUICHandshakeResult) { + tk.mu.Lock() + tk.TLSHandshakes = append(tk.TLSHandshakes, v...) + tk.mu.Unlock() +} + // SetFundamentalFailure sets the value of fundamentalFailure. func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Lock() diff --git a/internal/experiment/webconnectivity/tlsconnectivity.go b/internal/experiment/webconnectivity/tlsconnectivity.go new file mode 100644 index 0000000000..6216d30a15 --- /dev/null +++ b/internal/experiment/webconnectivity/tlsconnectivity.go @@ -0,0 +1,134 @@ +package webconnectivity + +// +// TLSConnectivity +// +// This code was generated by `boilerplate' using +// the tls template. +// + +import ( + "context" + "crypto/tls" + "net" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Checks for TLS connectivity. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type TLSConnectivity struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // ALPN is the OPTIONAL ALPN to use. + ALPN []string + + // SNI is the OPTIONAL SNI to use. + SNI string +} + +// Start starts this task in a background goroutine. +func (t *TLSConnectivity) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *TLSConnectivity) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "TLSConnectivity#%d: %s", index, t.Address) + + // perform the TCP connect + const tcpTimeout = 10 * time.Second + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + tcpConn.Close() + }() + + // perform TLS handshake + tlsSNI, err := t.sni() + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + tlsHandshaker := trace.NewTLSHandshakerStdlib(t.Logger) + tlsConfig := &tls.Config{ + NextProtos: t.alpn(), + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: tlsSNI, + } + const tlsTimeout = 10 * time.Second + tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) + defer tlsCancel() + tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + t.TestKeys.AppendTLSHandshakes(<-trace.TLSHandshake) + if err != nil { + ol.Stop(err) + return + } + defer tlsConn.Close() + + // completed successfully + ol.Stop(nil) +} + +// alpn returns the user-configured ALPN or a reasonable default +func (t *TLSConnectivity) alpn() []string { + if len(t.ALPN) > 0 { + return t.ALPN + } + return []string{} +} + +// sni returns the user-configured SNI or a reasonable default +func (t *TLSConnectivity) sni() (string, error) { + if t.SNI != "" { + return t.SNI, nil + } + addr, _, err := net.SplitHostPort(t.Address) + if err != nil { + return "", err + } + return addr, nil +} From 88fcd256484d3acbbbec0462a1ff84c4da80bc63 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 20:48:23 +0200 Subject: [PATCH 50/83] fix(endpoint.go.txt): minor fix for clarity --- internal/cmd/boilerplate/task/endpoint.go.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt index 22f69422d8..fa77250d7f 100644 --- a/internal/cmd/boilerplate/task/endpoint.go.txt +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -258,7 +258,7 @@ func (t *{{ .StructName }}) httpTransaction(ctx context.Context, txp model.HTTPT resp, err := txp.RoundTrip(req) if err != nil { _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save - return resp, []byte{}, err + return nil, []byte{}, err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) From ee3cbd8e15cd006258a6524a53563b313ce6be34 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 20:49:10 +0200 Subject: [PATCH 51/83] fix(testkeys.go): start defining HTTP results --- internal/experiment/webconnectivity/testkeys.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 8c35f8aae7..af9d0ee74c 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -21,6 +21,9 @@ type TestKeys struct { // Queries contains DNS queries. Queries []*model.ArchivalDNSLookupResult `json:"queries"` + // Requests contains HTTP results. + Requests []*model.ArchivalHTTPRequestResult `json:"requests"` + // TCPConnect contains TCP connect results. TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` @@ -52,6 +55,13 @@ func (tk *TestKeys) AppendQueries(v ...*model.ArchivalDNSLookupResult) { tk.mu.Unlock() } +// AppendRequests appends to Requests. +func (tk *TestKeys) AppendRequests(v ...*model.ArchivalHTTPRequestResult) { + tk.mu.Lock() + tk.Requests = append(tk.Requests, v...) + tk.mu.Unlock() +} + // AppendTCPConnectResults appends to TCPConnect. func (tk *TestKeys) AppendTCPConnectResults(v ...*model.ArchivalTCPConnectResult) { tk.mu.Lock() From fdaf5fed8de021dd5279e19a0338f0c0a49512f8 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 22:02:55 +0200 Subject: [PATCH 52/83] refactor: create full fledged http/https flows --- .../webconnectivity/cleartextflow.go | 180 ++++++++++++++ .../webconnectivity/dnsresolvers.go | 4 +- .../experiment/webconnectivity/secureflow.go | 232 ++++++++++++++++++ .../webconnectivity/tcpconnectivity.go | 82 ------- .../webconnectivity/tlsconnectivity.go | 134 ---------- 5 files changed, 414 insertions(+), 218 deletions(-) create mode 100644 internal/experiment/webconnectivity/cleartextflow.go create mode 100644 internal/experiment/webconnectivity/secureflow.go delete mode 100644 internal/experiment/webconnectivity/tcpconnectivity.go delete mode 100644 internal/experiment/webconnectivity/tlsconnectivity.go diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go new file mode 100644 index 0000000000..a5988d2b78 --- /dev/null +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -0,0 +1,180 @@ +package webconnectivity + +// +// CleartextFlow +// +// This code was generated by `boilerplate' using +// the http template. +// + +import ( + "context" + "io" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures HTTP endpoints. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type CleartextFlow struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background goroutine. +func (t *CleartextFlow) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "CleartextFlow#%d", index) // TODO: edit + + // perform the TCP connect + const tcpTimeout = 10 * time.Second + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + tcpConn.Close() + }() + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(tcpConn), + netxlite.NewNullTLSDialer(), + ) + + // create HTTP request + const httpTimeout = 10 * time.Second + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + + // completed successfully + ol.Stop(nil) +} + +// urlHost computes the host to include into the URL +func (t *CleartextFlow) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + if port == "80" && scheme == "http" { + return addr, nil + } + return t.Address, nil // there was no need to parse after all 😬 +} + +// newHTTPRequest creates a new HTTP request. +func (t *CleartextFlow) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "http" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *CleartextFlow) httpTransaction(ctx context.Context, txp model.HTTPTransport, + req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 22 // TODO: you may want to change this default + resp, err := txp.RoundTrip(req) + if err != nil { + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) + t.TestKeys.AppendRequests(ev) + return nil, []byte{}, err + } + defer resp.Body.Close() + reader := io.LimitReader(resp.Body, maxbody) + body, err := netxlite.ReadAllContext(ctx, reader) + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + t.TestKeys.AppendRequests(ev) + return resp, body, err +} diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 1bc46b8a75..127ae1f30f 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -208,7 +208,7 @@ func (t *DNSResolvers) dnsOverHTTPSURL() string { // startTCPTask starts a TCP measurement task for the given IP address. func (t *DNSResolvers) startTCPTask(ctx context.Context, address string) { - task := &TCPConnectivity{ + task := &CleartextFlow{ Address: net.JoinHostPort(address, "80"), IDGenerator: t.IDGenerator, Logger: t.Logger, @@ -221,7 +221,7 @@ func (t *DNSResolvers) startTCPTask(ctx context.Context, address string) { // startTCPTLSTask starts a TCP+TLS measurement task for the given IP address. func (t *DNSResolvers) startTCPTLSTask(ctx context.Context, address string) { - task := &TLSConnectivity{ + task := &SecureFlow{ Address: net.JoinHostPort(address, "443"), IDGenerator: t.IDGenerator, Logger: t.Logger, diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go new file mode 100644 index 0000000000..5510803f1a --- /dev/null +++ b/internal/experiment/webconnectivity/secureflow.go @@ -0,0 +1,232 @@ +package webconnectivity + +// +// SecureFlow +// +// This code was generated by `boilerplate' using +// the https template. +// + +import ( + "context" + "crypto/tls" + "io" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures HTTPS endpoints. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type SecureFlow struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // ALPN is the OPTIONAL ALPN to use. + ALPN []string + + // SNI is the OPTIONAL SNI to use. + SNI string + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background goroutine. +func (t *SecureFlow) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *SecureFlow) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "SecureFlow#%d", index) // TODO: edit + + // perform the TCP connect + const tcpTimeout = 10 * time.Second + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + if err != nil { + ol.Stop(err) + return + } + tcpConn = trace.WrapNetConn(tcpConn) + defer func() { + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + tcpConn.Close() + }() + + // perform TLS handshake + tlsSNI, err := t.sni() + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + tlsHandshaker := trace.NewTLSHandshakerStdlib(t.Logger) + tlsConfig := &tls.Config{ + NextProtos: t.alpn(), + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: tlsSNI, + } + const tlsTimeout = 10 * time.Second + tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) + defer tlsCancel() + tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + t.TestKeys.AppendTLSHandshakes(<-trace.TLSHandshake) + if err != nil { + ol.Stop(err) + return + } + defer tlsConn.Close() + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewNullDialer(), + // note: netxlite guarantees that here tlsConn is a netxlite.TLSConn + netxlite.NewSingleUseTLSDialer(tlsConn.(netxlite.TLSConn)), + ) + + // create HTTP request + const httpTimeout = 10 * time.Second + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + + // completed successfully + ol.Stop(nil) +} + +// alpn returns the user-configured ALPN or a reasonable default +func (t *SecureFlow) alpn() []string { + if len(t.ALPN) > 0 { + return t.ALPN + } + return []string{"h2", "http/1.1"} +} + +// sni returns the user-configured SNI or a reasonable default +func (t *SecureFlow) sni() (string, error) { + if t.SNI != "" { + return t.SNI, nil + } + addr, _, err := net.SplitHostPort(t.Address) + if err != nil { + return "", err + } + return addr, nil +} + +// urlHost computes the host to include into the URL +func (t *SecureFlow) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + if port == "443" && scheme == "https" { + return addr, nil + } + return t.Address, nil // there was no need to parse after all 😬 +} + +// newHTTPRequest creates a new HTTP request. +func (t *SecureFlow) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "https" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *SecureFlow) httpTransaction(ctx context.Context, txp model.HTTPTransport, + req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 22 // TODO: you may want to change this default + resp, err := txp.RoundTrip(req) + if err != nil { + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) + t.TestKeys.AppendRequests(ev) + return nil, []byte{}, err + } + defer resp.Body.Close() + reader := io.LimitReader(resp.Body, maxbody) + body, err := netxlite.ReadAllContext(ctx, reader) + ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + t.TestKeys.AppendRequests(ev) + return resp, body, err +} diff --git a/internal/experiment/webconnectivity/tcpconnectivity.go b/internal/experiment/webconnectivity/tcpconnectivity.go deleted file mode 100644 index 50b33ac842..0000000000 --- a/internal/experiment/webconnectivity/tcpconnectivity.go +++ /dev/null @@ -1,82 +0,0 @@ -package webconnectivity - -// -// TCPConnectivity -// -// This code was generated by `boilerplate' using -// the tcp template. -// - -import ( - "context" - - "sync" - "time" - - "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" -) - -// Checks for TCP connectivity. -// -// The zero value of this structure IS NOT valid and you MUST initialize -// all the fields marked as MANDATORY before using this structure. -type TCPConnectivity struct { - // Address is the MANDATORY address to connect to. - Address string - - // IDGenerator is the MANDATORY atomic int64 to generate task IDs. - IDGenerator *atomicx.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - - // TestKeys is MANDATORY and contains the TestKeys. - TestKeys *TestKeys - - // ZeroTime is the MANDATORY measurement's zero time. - ZeroTime time.Time - - // WaitGroup is the MANDATORY wait group this task belongs to. - WaitGroup *sync.WaitGroup -} - -// Start starts this task in a background goroutine. -func (t *TCPConnectivity) Start(ctx context.Context) { - t.WaitGroup.Add(1) - index := t.IDGenerator.Add(1) - go func() { - defer t.WaitGroup.Done() // synchronize with the parent - t.Run(ctx, index) - }() -} - -// Run runs this task in the current goroutine. -func (t *TCPConnectivity) Run(parentCtx context.Context, index int64) { - // create trace - trace := measurexlite.NewTrace(index, t.ZeroTime) - - // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "TCPConnectivity#%d: %s", index, t.Address) - - // perform the TCP connect - const tcpTimeout = 10 * time.Second - tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) - defer tcpCancel() - tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) - if err != nil { - ol.Stop(err) - return - } - tcpConn = trace.WrapNetConn(tcpConn) - defer func() { - t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) - tcpConn.Close() - }() - - // completed successfully - ol.Stop(nil) -} diff --git a/internal/experiment/webconnectivity/tlsconnectivity.go b/internal/experiment/webconnectivity/tlsconnectivity.go deleted file mode 100644 index 6216d30a15..0000000000 --- a/internal/experiment/webconnectivity/tlsconnectivity.go +++ /dev/null @@ -1,134 +0,0 @@ -package webconnectivity - -// -// TLSConnectivity -// -// This code was generated by `boilerplate' using -// the tls template. -// - -import ( - "context" - "crypto/tls" - "net" - "sync" - "time" - - "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// Checks for TLS connectivity. -// -// The zero value of this structure IS NOT valid and you MUST initialize -// all the fields marked as MANDATORY before using this structure. -type TLSConnectivity struct { - // Address is the MANDATORY address to connect to. - Address string - - // IDGenerator is the MANDATORY atomic int64 to generate task IDs. - IDGenerator *atomicx.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - - // TestKeys is MANDATORY and contains the TestKeys. - TestKeys *TestKeys - - // ZeroTime is the MANDATORY measurement's zero time. - ZeroTime time.Time - - // WaitGroup is the MANDATORY wait group this task belongs to. - WaitGroup *sync.WaitGroup - - // ALPN is the OPTIONAL ALPN to use. - ALPN []string - - // SNI is the OPTIONAL SNI to use. - SNI string -} - -// Start starts this task in a background goroutine. -func (t *TLSConnectivity) Start(ctx context.Context) { - t.WaitGroup.Add(1) - index := t.IDGenerator.Add(1) - go func() { - defer t.WaitGroup.Done() // synchronize with the parent - t.Run(ctx, index) - }() -} - -// Run runs this task in the current goroutine. -func (t *TLSConnectivity) Run(parentCtx context.Context, index int64) { - // create trace - trace := measurexlite.NewTrace(index, t.ZeroTime) - - // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "TLSConnectivity#%d: %s", index, t.Address) - - // perform the TCP connect - const tcpTimeout = 10 * time.Second - tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) - defer tcpCancel() - tcpDialer := trace.NewDialerWithoutResolver(t.Logger) - tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) - if err != nil { - ol.Stop(err) - return - } - tcpConn = trace.WrapNetConn(tcpConn) - defer func() { - t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) - tcpConn.Close() - }() - - // perform TLS handshake - tlsSNI, err := t.sni() - if err != nil { - t.TestKeys.SetFundamentalFailure(err) - ol.Stop(err) - return - } - tlsHandshaker := trace.NewTLSHandshakerStdlib(t.Logger) - tlsConfig := &tls.Config{ - NextProtos: t.alpn(), - RootCAs: netxlite.NewDefaultCertPool(), - ServerName: tlsSNI, - } - const tlsTimeout = 10 * time.Second - tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) - defer tlsCancel() - tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) - t.TestKeys.AppendTLSHandshakes(<-trace.TLSHandshake) - if err != nil { - ol.Stop(err) - return - } - defer tlsConn.Close() - - // completed successfully - ol.Stop(nil) -} - -// alpn returns the user-configured ALPN or a reasonable default -func (t *TLSConnectivity) alpn() []string { - if len(t.ALPN) > 0 { - return t.ALPN - } - return []string{} -} - -// sni returns the user-configured SNI or a reasonable default -func (t *TLSConnectivity) sni() (string, error) { - if t.SNI != "" { - return t.SNI, nil - } - addr, _, err := net.SplitHostPort(t.Address) - if err != nil { - return "", err - } - return addr, nil -} From 5d3f8a3f87926eec19a2db53afb6009a8efcae96 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 22:22:37 +0200 Subject: [PATCH 53/83] feat: use semaphores to avoid multiple HTTP fetches --- .../webconnectivity/cleartextflow.go | 12 +++ .../webconnectivity/dnsresolvers.go | 92 +++++++++++++------ .../experiment/webconnectivity/secureflow.go | 12 +++ 3 files changed, 87 insertions(+), 29 deletions(-) diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index a5988d2b78..b0b74fea18 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -36,6 +36,10 @@ type CleartextFlow struct { // Logger is the MANDATORY logger to use. Logger model.Logger + // Sema is the MANDATORY semaphore to allow just a single + // connection to perform the HTTP transaction. + Sema chan any + // TestKeys is MANDATORY and contains the TestKeys. TestKeys *TestKeys @@ -90,6 +94,14 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { tcpConn.Close() }() + // Only allow a single flow to _use_ the connection + select { + case <-t.Sema: + default: + ol.Stop(nil) + return + } + // create HTTP transport httpTransport := netxlite.NewHTTPTransport( t.Logger, diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 127ae1f30f..45cf06b683 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -107,13 +107,10 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { } // TODO(bassosimone): remove bogons - // TODO(bassosimone): what to do if there is an explicit port?! // fan out a number of child async tasks to use the IP addrs - for _, addr := range sorted { - t.startTCPTask(parentCtx, addr) - t.startTCPTLSTask(parentCtx, addr) - } + t.startCleartextFlows(parentCtx, sorted) + t.startSecureFlows(parentCtx, sorted) } // lookupHostSystem performs a DNS lookup using the system resolver. @@ -206,32 +203,69 @@ func (t *DNSResolvers) dnsOverHTTPSURL() string { return "https://mozilla.cloudflare-dns.com/dns-query" } -// startTCPTask starts a TCP measurement task for the given IP address. -func (t *DNSResolvers) startTCPTask(ctx context.Context, address string) { - task := &CleartextFlow{ - Address: net.JoinHostPort(address, "80"), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, +// startCleartextFlows starts a TCP measurement flow for each IP addr. +func (t *DNSResolvers) startCleartextFlows(ctx context.Context, addresses []string) { + if t.URL.Scheme != "http" { + // Do not bother with measuring HTTP when the user + // has asked us to measure an HTTPS URL. + return + } + sema := make(chan any, 1) + sema <- true // allow a single flow to fetch the HTTP body + port := "80" + if urlPort := t.URL.Port(); urlPort != "" { + port = urlPort + } + for _, addr := range addresses { + task := &CleartextFlow{ + Address: net.JoinHostPort(addr, port), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + Sema: sema, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + HostHeader: t.URL.Host, + URLPath: t.URL.Path, + URLRawQuery: t.URL.RawQuery, + } + task.Start(ctx) } - task.Start(ctx) } -// startTCPTLSTask starts a TCP+TLS measurement task for the given IP address. -func (t *DNSResolvers) startTCPTLSTask(ctx context.Context, address string) { - task := &SecureFlow{ - Address: net.JoinHostPort(address, "443"), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - ALPN: []string{ - "h2", "http/1.1", - }, - SNI: t.URL.Hostname(), +// startSecureFlows starts a TCP+TLS measurement flow for each IP addr. +func (t *DNSResolvers) startSecureFlows(ctx context.Context, addresses []string) { + sema := make(chan any, 1) + if t.URL.Scheme == "https" { + // Allows just a single worker to fetch the response body but do that + // only if the test-lists URL uses "https" as the scheme. Otherwise, just + // validate IPs by performing a TLS handshake. + sema <- true + } + port := "443" + if urlPort := t.URL.Port(); urlPort != "" { + if t.URL.Scheme != "https" { + // If the URL is like http://example.com:8080/, we don't know + // which would be the correct port where to use HTTPS. + return + } + port = urlPort + } + for _, addr := range addresses { + task := &SecureFlow{ + Address: net.JoinHostPort(addr, port), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + Sema: sema, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + ALPN: []string{"h2", "http/1.1"}, + SNI: t.URL.Hostname(), + HostHeader: t.URL.Host, + URLPath: t.URL.Path, + URLRawQuery: t.URL.RawQuery, + } + task.Start(ctx) } - task.Start(ctx) } diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index 5510803f1a..86bf07b89c 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -37,6 +37,10 @@ type SecureFlow struct { // Logger is the MANDATORY logger to use. Logger model.Logger + // Sema is the MANDATORY semaphore to allow just a single + // connection to perform the HTTP transaction. + Sema chan any + // TestKeys is MANDATORY and contains the TestKeys. TestKeys *TestKeys @@ -121,6 +125,14 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { } defer tlsConn.Close() + // Only allow a single flow to _use_ the connection + select { + case <-t.Sema: + default: + ol.Stop(nil) + return + } + // create HTTP transport httpTransport := netxlite.NewHTTPTransport( t.Logger, From eb862d6914617b71cf368dfa3ad45926d41cdfb0 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 22:34:02 +0200 Subject: [PATCH 54/83] feat: start preparing the ground for following redirects --- .../webconnectivity/cleartextflow.go | 14 ++++-- .../webconnectivity/dnsresolvers.go | 46 ++++++++++--------- .../experiment/webconnectivity/secureflow.go | 14 ++++-- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index b0b74fea18..77fd07ccc8 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -30,6 +30,10 @@ type CleartextFlow struct { // Address is the MANDATORY address to connect to. Address string + // FollowRedirects is OPTIONAL and instructs this flow + // to follow HTTP redirects (if any). + FollowRedirects bool + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. IDGenerator *atomicx.Int64 @@ -127,12 +131,16 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { return } + // completed successfully + ol.Stop(nil) + + if t.FollowRedirects { + // TODO + } + // TODO: insert here additional code if needed _ = httpResp _ = httpRespBody - - // completed successfully - ol.Stop(nil) } // urlHost computes the host to include into the URL diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 45cf06b683..377f1287ca 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -218,16 +218,17 @@ func (t *DNSResolvers) startCleartextFlows(ctx context.Context, addresses []stri } for _, addr := range addresses { task := &CleartextFlow{ - Address: net.JoinHostPort(addr, port), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - Sema: sema, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - HostHeader: t.URL.Host, - URLPath: t.URL.Path, - URLRawQuery: t.URL.RawQuery, + Address: net.JoinHostPort(addr, port), + FollowRedirects: t.URL.Scheme == "http", + IDGenerator: t.IDGenerator, + Logger: t.Logger, + Sema: sema, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + HostHeader: t.URL.Host, + URLPath: t.URL.Path, + URLRawQuery: t.URL.RawQuery, } task.Start(ctx) } @@ -253,18 +254,19 @@ func (t *DNSResolvers) startSecureFlows(ctx context.Context, addresses []string) } for _, addr := range addresses { task := &SecureFlow{ - Address: net.JoinHostPort(addr, port), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - Sema: sema, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - ALPN: []string{"h2", "http/1.1"}, - SNI: t.URL.Hostname(), - HostHeader: t.URL.Host, - URLPath: t.URL.Path, - URLRawQuery: t.URL.RawQuery, + Address: net.JoinHostPort(addr, port), + FollowRedirects: t.URL.Scheme == "https", + IDGenerator: t.IDGenerator, + Logger: t.Logger, + Sema: sema, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + ALPN: []string{"h2", "http/1.1"}, + SNI: t.URL.Hostname(), + HostHeader: t.URL.Host, + URLPath: t.URL.Path, + URLRawQuery: t.URL.RawQuery, } task.Start(ctx) } diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index 86bf07b89c..10ce42fd26 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -31,6 +31,10 @@ type SecureFlow struct { // Address is the MANDATORY address to connect to. Address string + // FollowRedirects is OPTIONAL and instructs this flow + // to follow HTTP redirects (if any). + FollowRedirects bool + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. IDGenerator *atomicx.Int64 @@ -159,12 +163,16 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { return } + // completed successfully + ol.Stop(nil) + + if t.FollowRedirects { + // TODO + } + // TODO: insert here additional code if needed _ = httpResp _ = httpRespBody - - // completed successfully - ol.Stop(nil) } // alpn returns the user-configured ALPN or a reasonable default From dd1f210bb2c147fbd32af9899b5b3ff9d72ad699 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Sun, 14 Aug 2022 23:06:38 +0200 Subject: [PATCH 55/83] feat: implement following redirects We use a tracing style when following redirects. This reduces the precision of the measurement but still collects useful data. Additionally, we only use the system resolver for following redirects, which is a bummer compared to using several resolvers. The idea here is that we care mostly about the first URL. --- .../webconnectivity/cleartextflow.go | 36 +++- .../experiment/webconnectivity/redirects.go | 188 ++++++++++++++++++ .../experiment/webconnectivity/secureflow.go | 36 +++- .../experiment/webconnectivity/testkeys.go | 8 +- 4 files changed, 253 insertions(+), 15 deletions(-) create mode 100644 internal/experiment/webconnectivity/redirects.go diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 77fd07ccc8..b6633ae228 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -131,16 +131,14 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { return } - // completed successfully - ol.Stop(nil) - - if t.FollowRedirects { - // TODO - } + // if enabled, follow possible redirects + t.maybeFollowRedirects(parentCtx, httpResp) // TODO: insert here additional code if needed - _ = httpResp _ = httpRespBody + + // completed successfully + ol.Stop(nil) } // urlHost computes the host to include into the URL @@ -198,3 +196,27 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, txp model.HTTPTrans t.TestKeys.AppendRequests(ev) return resp, body, err } + +// maybeFollowRedirects follows redirects if configured and needed +func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) { + if t.FollowRedirects { + switch resp.StatusCode { + case 301, 302, 307, 308: + location, err := resp.Location() + if err != nil { + return + } + redirects := &Redirects{ + IDGenerator: t.IDGenerator, + Location: location, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + } + redirects.Start(ctx) + default: + // nothing + } + } +} diff --git a/internal/experiment/webconnectivity/redirects.go b/internal/experiment/webconnectivity/redirects.go new file mode 100644 index 0000000000..61e35e9056 --- /dev/null +++ b/internal/experiment/webconnectivity/redirects.go @@ -0,0 +1,188 @@ +package webconnectivity + +// +// Redirects +// +// This code was generated by `boilerplate' using +// the http template. +// + +import ( + "bytes" + "context" + + "io" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Follows subsequent redirects. We use a step-by-step style for the +// first URL and we use tracing for subsequent redirects. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type Redirects struct { + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Location is the MANDATORY location we were redirected to. + Location *url.URL + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup +} + +// Start starts this task in a background goroutine. +func (t *Redirects) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *Redirects) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "Redirects#%d", index) // TODO: edit + + // create context with a timeout + const tcpTimeout = 10 * time.Second + httpCtx, httpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer httpCancel() + + // create dialers for the transport + dialer := netxlite.NewDialerWithStdlibResolver(t.Logger) + tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(t.Logger)) + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport(t.Logger, dialer, tlsDialer) + + // TODO(bassosimone): we need to configure cookies + // TODO(bassosimone): we need to configure the referer + + // create HTTP client + httpClnt := &http.Client{ + Transport: &redirectsTransport{ + tk: t.TestKeys, + trace: trace, + txp: httpTransport, + }, + CheckRedirect: nil, + Jar: nil, + Timeout: tcpTimeout, + } + defer httpClnt.CloseIdleConnections() + + // create HTTP request + httpReq, err := t.newHTTPRequest(httpCtx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP transaction + httpResp, err := httpClnt.Do(httpReq) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + + // completed successfully + ol.Stop(nil) +} + +// newHTTPRequest creates a new HTTP request. +func (t *Redirects) newHTTPRequest(ctx context.Context) (*http.Request, error) { + httpReq, err := http.NewRequestWithContext(ctx, "GET", t.Location.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.Location.Host) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.Location.Host + return httpReq, nil +} + +// redirectsTransport wraps transport to save minimal information +// about subsequent HTTP redirects +type redirectsTransport struct { + // tk contains the test keys + tk *TestKeys + + // trace is the trace we're using + trace *measurexlite.Trace + + // txp is the transport we're using + txp model.HTTPTransport +} + +// CloseIdleConnections forwards the CloseIdleConnection call +func (t *redirectsTransport) CloseIdleConnections() { + t.txp.CloseIdleConnections() +} + +// RoundTrip implements http.Transport. +func (t *redirectsTransport) RoundTrip(req *http.Request) (*http.Response, error) { + const maxbody = 1 << 22 // TODO: you may want to change this default + resp, err := t.txp.RoundTrip(req) + if err != nil { + ev := t.trace.NewArchivalHTTPRequestResult(t.txp, req, resp, maxbody, []byte{}, err) + t.tk.AppendRequests(ev) + return nil, err + } + defer resp.Body.Close() + reader := io.LimitReader(resp.Body, maxbody) + body, err := netxlite.ReadAllContext(req.Context(), reader) + ev := t.trace.NewArchivalHTTPRequestResult(t.txp, req, resp, maxbody, body, err) + t.tk.AppendRequests(ev) + if err != nil { + return nil, err + } + resp.Body = &redirectsBody{ + r: bytes.NewReader(body), + } + return resp, nil +} + +// redirectsBody is the body used by redirectsTransport +type redirectsBody struct { + // r is the reader + r io.Reader +} + +// Read implements io.Reader +func (rb *redirectsBody) Read(b []byte) (int, error) { + return rb.r.Read(b) +} + +// Close implements io.Closer +func (rb *redirectsBody) Close() error { + return nil +} diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index 10ce42fd26..c6069c4d26 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -163,16 +163,14 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { return } - // completed successfully - ol.Stop(nil) - - if t.FollowRedirects { - // TODO - } + // if enabled, follow possible redirects + t.maybeFollowRedirects(parentCtx, httpResp) // TODO: insert here additional code if needed - _ = httpResp _ = httpRespBody + + // completed successfully + ol.Stop(nil) } // alpn returns the user-configured ALPN or a reasonable default @@ -250,3 +248,27 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, txp model.HTTPTranspor t.TestKeys.AppendRequests(ev) return resp, body, err } + +// maybeFollowRedirects follows redirects if configured and needed +func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) { + if t.FollowRedirects { + switch resp.StatusCode { + case 301, 302, 307, 308: + location, err := resp.Location() + if err != nil { + return + } + redirects := &Redirects{ + IDGenerator: t.IDGenerator, + Location: location, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + } + redirects.Start(ctx) + default: + // nothing + } + } +} diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index af9d0ee74c..78ab76f3df 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -87,6 +87,11 @@ func (tk *TestKeys) SetFundamentalFailure(err error) { func NewTestKeys() *TestKeys { // TODO: here you should initialize all the fields return &TestKeys{ + NetworkEvents: []*model.ArchivalNetworkEvent{}, + Queries: []*model.ArchivalDNSLookupResult{}, + Requests: []*model.ArchivalHTTPRequestResult{}, + TCPConnect: []*model.ArchivalTCPConnectResult{}, + TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, fundamentalFailure: nil, mu: &sync.Mutex{}, } @@ -95,5 +100,6 @@ func NewTestKeys() *TestKeys { // finalize performs any delayed computation on the test keys. This function // must be called from the measurer after all the tasks have completed. func (tk *TestKeys) finalize() { - // TODO: implement + // TODO(bassosimone): set final webconnectivity flags + // TODO(bassosimone): sort requests correctly } From 90f20ee4894dbb2449b1b7db17d65dee6a79e8a8 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 06:35:55 +0200 Subject: [PATCH 56/83] refactor: get webconnectivity measurement flow mostly right --- .../webconnectivity/cleartextflow.go | 84 +++++--- .../experiment/webconnectivity/dnscache.go | 37 ++++ .../webconnectivity/dnsresolvers.go | 53 ++++- .../experiment/webconnectivity/measurer.go | 13 ++ .../experiment/webconnectivity/redirects.go | 188 ------------------ .../experiment/webconnectivity/secureflow.go | 84 +++++--- .../experiment/webconnectivity/testkeys.go | 4 +- 7 files changed, 222 insertions(+), 241 deletions(-) create mode 100644 internal/experiment/webconnectivity/dnscache.go delete mode 100644 internal/experiment/webconnectivity/redirects.go diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index b6633ae228..0b4b9c2045 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -30,9 +30,8 @@ type CleartextFlow struct { // Address is the MANDATORY address to connect to. Address string - // FollowRedirects is OPTIONAL and instructs this flow - // to follow HTTP redirects (if any). - FollowRedirects bool + // DNSCache is the MANDATORY DNS cache. + DNSCache *DNSCache // IDGenerator is the MANDATORY atomic int64 to generate task IDs. IDGenerator *atomicx.Int64 @@ -53,9 +52,27 @@ type CleartextFlow struct { // WaitGroup is the MANDATORY wait group this task belongs to. WaitGroup *sync.WaitGroup + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not + // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). + DNSOverHTTPSURL string + + // FollowRedirects is OPTIONAL and instructs this flow + // to follow HTTP redirects (if any). + FollowRedirects bool + // HostHeader is the OPTIONAL host header to use. HostHeader string + // Referer contains the OPTIONAL referer, used for redirects. + Referer string + + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this + // field is not set we use a default one (e.g., `8.8.8.8:53`). + UDPAddress string + // URLPath is the OPTIONAL URL path. URLPath string @@ -148,10 +165,15 @@ func (t *CleartextFlow) urlHost(scheme string) (string, error) { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) return "", err } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } if port == "80" && scheme == "http" { - return addr, nil + return urlHost, nil } - return t.Address, nil // there was no need to parse after all 😬 + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil } // newHTTPRequest creates a new HTTP request. @@ -174,8 +196,14 @@ func (t *CleartextFlow) newHTTPRequest(ctx context.Context) (*http.Request, erro httpReq.Header.Set("Host", t.HostHeader) httpReq.Header.Set("Accept", model.HTTPHeaderAccept) httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("Referer", t.Referer) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } return httpReq, nil } @@ -190,6 +218,9 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, txp model.HTTPTrans return nil, []byte{}, err } defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } reader := io.LimitReader(resp.Body, maxbody) body, err := netxlite.ReadAllContext(ctx, reader) ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) @@ -199,24 +230,31 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, txp model.HTTPTrans // maybeFollowRedirects follows redirects if configured and needed func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) { - if t.FollowRedirects { - switch resp.StatusCode { - case 301, 302, 307, 308: - location, err := resp.Location() - if err != nil { - return - } - redirects := &Redirects{ - IDGenerator: t.IDGenerator, - Location: location, - Logger: t.Logger, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - } - redirects.Start(ctx) - default: - // nothing + if !t.FollowRedirects { + return + } + switch resp.StatusCode { + case 301, 302, 307, 308: + location, err := resp.Location() + if err != nil { + return } + resolvers := &DNSResolvers{ + CookieJar: t.CookieJar, + DNSCache: t.DNSCache, + Domain: location.Hostname(), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + URL: location, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + DNSOverHTTPSURL: t.DNSOverHTTPSURL, + Referer: resp.Request.URL.String(), + UDPAddress: t.UDPAddress, + } + resolvers.Start(ctx) + default: + // nothing } } diff --git a/internal/experiment/webconnectivity/dnscache.go b/internal/experiment/webconnectivity/dnscache.go new file mode 100644 index 0000000000..c4f5225202 --- /dev/null +++ b/internal/experiment/webconnectivity/dnscache.go @@ -0,0 +1,37 @@ +package webconnectivity + +import "sync" + +// DNSCache wraps a model.Resolver to provide DNS caching. +// +// The zero value is invalid. Please, use NewDNSCache. +type DNSCache struct { + // mu provides mutual exclusion. + mu *sync.Mutex + + // values contains already resolved values. + values map[string][]string +} + +// Get gets values from the cache +func (c *DNSCache) Get(domain string) ([]string, bool) { + c.mu.Lock() + values, found := c.values[domain] + c.mu.Unlock() + return values, found +} + +// Set inserts into the cache +func (c *DNSCache) Set(domain string, values []string) { + c.mu.Lock() + c.values[domain] = values + c.mu.Unlock() +} + +// NewDNSCache creates a new DNSCache instance. +func NewDNSCache() *DNSCache { + return &DNSCache{ + mu: &sync.Mutex{}, + values: map[string][]string{}, + } +} diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 377f1287ca..9d4fffb3d6 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -10,6 +10,7 @@ package webconnectivity import ( "context" "net" + "net/http" "net/url" "sync" "time" @@ -25,6 +26,9 @@ import ( // The zero value of this structure IS NOT valid and you MUST initialize // all the fields marked as MANDATORY before using this structure. type DNSResolvers struct { + // DNSCache is the MANDATORY DNS cache. + DNSCache *DNSCache + // Domain is the MANDATORY domain to resolve. Domain string @@ -46,10 +50,16 @@ type DNSResolvers struct { // WaitGroup is the MANDATORY wait group this task belongs to. WaitGroup *sync.WaitGroup + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). DNSOverHTTPSURL string + // Referer contains the OPTIONAL referer, used for redirects. + Referer string + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this // field is not set we use a default one (e.g., `8.8.8.8:53`). UDPAddress string @@ -64,8 +74,8 @@ func (t *DNSResolvers) Start(ctx context.Context) { }() } -// Run runs this task in the current goroutine. -func (t *DNSResolvers) Run(parentCtx context.Context) { +// run performs a DNS lookup and returns the looked up addrs +func (t *DNSResolvers) run(parentCtx context.Context) []string { // create output channels for the lookup systemOut := make(chan []string) udpOut := make(chan []string) @@ -108,9 +118,30 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { // TODO(bassosimone): remove bogons + return sorted +} + +// Run runs this task in the current goroutine. +func (t *DNSResolvers) Run(parentCtx context.Context) { + var ( + addresses []string + found bool + ) + + // first attempt to use the dns cache + addresses, found = t.DNSCache.Get(t.Domain) + + if !found { + // fall back to performing a real dns lookup + addresses = t.run(parentCtx) + + // insert the addresses we just looked us into the cache + t.DNSCache.Set(t.Domain, addresses) + } + // fan out a number of child async tasks to use the IP addrs - t.startCleartextFlows(parentCtx, sorted) - t.startSecureFlows(parentCtx, sorted) + t.startCleartextFlows(parentCtx, addresses) + t.startSecureFlows(parentCtx, addresses) } // lookupHostSystem performs a DNS lookup using the system resolver. @@ -219,14 +250,19 @@ func (t *DNSResolvers) startCleartextFlows(ctx context.Context, addresses []stri for _, addr := range addresses { task := &CleartextFlow{ Address: net.JoinHostPort(addr, port), - FollowRedirects: t.URL.Scheme == "http", + DNSCache: t.DNSCache, IDGenerator: t.IDGenerator, Logger: t.Logger, Sema: sema, TestKeys: t.TestKeys, ZeroTime: t.ZeroTime, WaitGroup: t.WaitGroup, + CookieJar: t.CookieJar, + DNSOverHTTPSURL: t.DNSOverHTTPSURL, + FollowRedirects: t.URL.Scheme == "http", HostHeader: t.URL.Host, + Referer: t.Referer, + UDPAddress: t.UDPAddress, URLPath: t.URL.Path, URLRawQuery: t.URL.RawQuery, } @@ -255,7 +291,7 @@ func (t *DNSResolvers) startSecureFlows(ctx context.Context, addresses []string) for _, addr := range addresses { task := &SecureFlow{ Address: net.JoinHostPort(addr, port), - FollowRedirects: t.URL.Scheme == "https", + DNSCache: t.DNSCache, IDGenerator: t.IDGenerator, Logger: t.Logger, Sema: sema, @@ -263,8 +299,13 @@ func (t *DNSResolvers) startSecureFlows(ctx context.Context, addresses []string) ZeroTime: t.ZeroTime, WaitGroup: t.WaitGroup, ALPN: []string{"h2", "http/1.1"}, + CookieJar: t.CookieJar, + DNSOverHTTPSURL: t.DNSOverHTTPSURL, + FollowRedirects: t.URL.Scheme == "https", SNI: t.URL.Hostname(), HostHeader: t.URL.Host, + Referer: t.Referer, + UDPAddress: t.UDPAddress, URLPath: t.URL.Path, URLRawQuery: t.URL.RawQuery, } diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index c10f61116f..85422c9c21 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -7,10 +7,12 @@ package webconnectivity import ( "context" "errors" + "net/http/cookiejar" "sync" "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/model" + "golang.org/x/net/publicsuffix" ) // Measurer for the web_connectivity experiment. @@ -71,8 +73,17 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, idGenerator := &atomicx.Int64{} wg := &sync.WaitGroup{} + // create cookiejar + jar, err := cookiejar.New(&cookiejar.Options{ + PublicSuffixList: publicsuffix.List, + }) + if err != nil { + return err + } + // start background tasks resos := &DNSResolvers{ + DNSCache: NewDNSCache(), Domain: URL.Hostname(), IDGenerator: idGenerator, Logger: sess.Logger(), @@ -80,7 +91,9 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, URL: URL, ZeroTime: measurement.MeasurementStartTimeSaved, WaitGroup: wg, + CookieJar: jar, DNSOverHTTPSURL: "", + Referer: "", UDPAddress: "", } resos.Start(ctx) diff --git a/internal/experiment/webconnectivity/redirects.go b/internal/experiment/webconnectivity/redirects.go deleted file mode 100644 index 61e35e9056..0000000000 --- a/internal/experiment/webconnectivity/redirects.go +++ /dev/null @@ -1,188 +0,0 @@ -package webconnectivity - -// -// Redirects -// -// This code was generated by `boilerplate' using -// the http template. -// - -import ( - "bytes" - "context" - - "io" - "net/http" - "net/url" - "sync" - "time" - - "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// Follows subsequent redirects. We use a step-by-step style for the -// first URL and we use tracing for subsequent redirects. -// -// The zero value of this structure IS NOT valid and you MUST initialize -// all the fields marked as MANDATORY before using this structure. -type Redirects struct { - // IDGenerator is the MANDATORY atomic int64 to generate task IDs. - IDGenerator *atomicx.Int64 - - // Location is the MANDATORY location we were redirected to. - Location *url.URL - - // Logger is the MANDATORY logger to use. - Logger model.Logger - - // TestKeys is MANDATORY and contains the TestKeys. - TestKeys *TestKeys - - // ZeroTime is the MANDATORY measurement's zero time. - ZeroTime time.Time - - // WaitGroup is the MANDATORY wait group this task belongs to. - WaitGroup *sync.WaitGroup -} - -// Start starts this task in a background goroutine. -func (t *Redirects) Start(ctx context.Context) { - t.WaitGroup.Add(1) - index := t.IDGenerator.Add(1) - go func() { - defer t.WaitGroup.Done() // synchronize with the parent - t.Run(ctx, index) - }() -} - -// Run runs this task in the current goroutine. -func (t *Redirects) Run(parentCtx context.Context, index int64) { - // create trace - trace := measurexlite.NewTrace(index, t.ZeroTime) - - // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "Redirects#%d", index) // TODO: edit - - // create context with a timeout - const tcpTimeout = 10 * time.Second - httpCtx, httpCancel := context.WithTimeout(parentCtx, tcpTimeout) - defer httpCancel() - - // create dialers for the transport - dialer := netxlite.NewDialerWithStdlibResolver(t.Logger) - tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(t.Logger)) - - // create HTTP transport - httpTransport := netxlite.NewHTTPTransport(t.Logger, dialer, tlsDialer) - - // TODO(bassosimone): we need to configure cookies - // TODO(bassosimone): we need to configure the referer - - // create HTTP client - httpClnt := &http.Client{ - Transport: &redirectsTransport{ - tk: t.TestKeys, - trace: trace, - txp: httpTransport, - }, - CheckRedirect: nil, - Jar: nil, - Timeout: tcpTimeout, - } - defer httpClnt.CloseIdleConnections() - - // create HTTP request - httpReq, err := t.newHTTPRequest(httpCtx) - if err != nil { - t.TestKeys.SetFundamentalFailure(err) - ol.Stop(err) - return - } - - // perform HTTP transaction - httpResp, err := httpClnt.Do(httpReq) - if err != nil { - ol.Stop(err) - return - } - - // TODO: insert here additional code if needed - _ = httpResp - - // completed successfully - ol.Stop(nil) -} - -// newHTTPRequest creates a new HTTP request. -func (t *Redirects) newHTTPRequest(ctx context.Context) (*http.Request, error) { - httpReq, err := http.NewRequestWithContext(ctx, "GET", t.Location.String(), nil) - if err != nil { - return nil, err - } - httpReq.Header.Set("Host", t.Location.Host) - httpReq.Header.Set("Accept", model.HTTPHeaderAccept) - httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) - httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) - httpReq.Host = t.Location.Host - return httpReq, nil -} - -// redirectsTransport wraps transport to save minimal information -// about subsequent HTTP redirects -type redirectsTransport struct { - // tk contains the test keys - tk *TestKeys - - // trace is the trace we're using - trace *measurexlite.Trace - - // txp is the transport we're using - txp model.HTTPTransport -} - -// CloseIdleConnections forwards the CloseIdleConnection call -func (t *redirectsTransport) CloseIdleConnections() { - t.txp.CloseIdleConnections() -} - -// RoundTrip implements http.Transport. -func (t *redirectsTransport) RoundTrip(req *http.Request) (*http.Response, error) { - const maxbody = 1 << 22 // TODO: you may want to change this default - resp, err := t.txp.RoundTrip(req) - if err != nil { - ev := t.trace.NewArchivalHTTPRequestResult(t.txp, req, resp, maxbody, []byte{}, err) - t.tk.AppendRequests(ev) - return nil, err - } - defer resp.Body.Close() - reader := io.LimitReader(resp.Body, maxbody) - body, err := netxlite.ReadAllContext(req.Context(), reader) - ev := t.trace.NewArchivalHTTPRequestResult(t.txp, req, resp, maxbody, body, err) - t.tk.AppendRequests(ev) - if err != nil { - return nil, err - } - resp.Body = &redirectsBody{ - r: bytes.NewReader(body), - } - return resp, nil -} - -// redirectsBody is the body used by redirectsTransport -type redirectsBody struct { - // r is the reader - r io.Reader -} - -// Read implements io.Reader -func (rb *redirectsBody) Read(b []byte) (int, error) { - return rb.r.Read(b) -} - -// Close implements io.Closer -func (rb *redirectsBody) Close() error { - return nil -} diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index c6069c4d26..ea7f0e7a21 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -31,9 +31,8 @@ type SecureFlow struct { // Address is the MANDATORY address to connect to. Address string - // FollowRedirects is OPTIONAL and instructs this flow - // to follow HTTP redirects (if any). - FollowRedirects bool + // DNSCache is the MANDATORY DNS cache. + DNSCache *DNSCache // IDGenerator is the MANDATORY atomic int64 to generate task IDs. IDGenerator *atomicx.Int64 @@ -57,12 +56,30 @@ type SecureFlow struct { // ALPN is the OPTIONAL ALPN to use. ALPN []string + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not + // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). + DNSOverHTTPSURL string + + // FollowRedirects is OPTIONAL and instructs this flow + // to follow HTTP redirects (if any). + FollowRedirects bool + // SNI is the OPTIONAL SNI to use. SNI string // HostHeader is the OPTIONAL host header to use. HostHeader string + // Referer contains the OPTIONAL referer, used for redirects. + Referer string + + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this + // field is not set we use a default one (e.g., `8.8.8.8:53`). + UDPAddress string + // URLPath is the OPTIONAL URL path. URLPath string @@ -200,10 +217,15 @@ func (t *SecureFlow) urlHost(scheme string) (string, error) { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) return "", err } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } if port == "443" && scheme == "https" { - return addr, nil + return urlHost, nil } - return t.Address, nil // there was no need to parse after all 😬 + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil } // newHTTPRequest creates a new HTTP request. @@ -226,8 +248,14 @@ func (t *SecureFlow) newHTTPRequest(ctx context.Context) (*http.Request, error) httpReq.Header.Set("Host", t.HostHeader) httpReq.Header.Set("Accept", model.HTTPHeaderAccept) httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("Referer", t.Referer) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } return httpReq, nil } @@ -242,6 +270,9 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, txp model.HTTPTranspor return nil, []byte{}, err } defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } reader := io.LimitReader(resp.Body, maxbody) body, err := netxlite.ReadAllContext(ctx, reader) ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) @@ -251,24 +282,31 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, txp model.HTTPTranspor // maybeFollowRedirects follows redirects if configured and needed func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) { - if t.FollowRedirects { - switch resp.StatusCode { - case 301, 302, 307, 308: - location, err := resp.Location() - if err != nil { - return - } - redirects := &Redirects{ - IDGenerator: t.IDGenerator, - Location: location, - Logger: t.Logger, - TestKeys: t.TestKeys, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - } - redirects.Start(ctx) - default: - // nothing + if !t.FollowRedirects { + return + } + switch resp.StatusCode { + case 301, 302, 307, 308: + location, err := resp.Location() + if err != nil { + return } + resolvers := &DNSResolvers{ + CookieJar: t.CookieJar, + DNSCache: t.DNSCache, + Domain: location.Hostname(), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + URL: location, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + DNSOverHTTPSURL: t.DNSOverHTTPSURL, + Referer: resp.Request.URL.String(), + UDPAddress: t.UDPAddress, + } + resolvers.Start(ctx) + default: + // nothing } } diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 78ab76f3df..dd24421721 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -58,7 +58,9 @@ func (tk *TestKeys) AppendQueries(v ...*model.ArchivalDNSLookupResult) { // AppendRequests appends to Requests. func (tk *TestKeys) AppendRequests(v ...*model.ArchivalHTTPRequestResult) { tk.mu.Lock() - tk.Requests = append(tk.Requests, v...) + // Implementation note: append at the front since the most recent + // request must be at the beginning of the list. + tk.Requests = append(v, tk.Requests...) tk.mu.Unlock() } From 0b02dd59b20d2c49b63cac5bb536b0677416ccdf Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 07:37:32 +0200 Subject: [PATCH 57/83] feat: issue and save the control request --- .../webconnectivity/cleartextflow.go | 2 + .../experiment/webconnectivity/control.go | 103 ++++++++++++++++++ .../webconnectivity/dnsresolvers.go | 26 +++++ .../experiment/webconnectivity/measurer.go | 20 ++++ .../experiment/webconnectivity/secureflow.go | 2 + .../experiment/webconnectivity/testkeys.go | 24 +++- 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 internal/experiment/webconnectivity/control.go diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 0b4b9c2045..42e3c9882c 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -251,6 +251,8 @@ func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Res WaitGroup: t.WaitGroup, DNSOverHTTPSURL: t.DNSOverHTTPSURL, Referer: resp.Request.URL.String(), + Session: nil, // no need to issue another control request + THAddr: "", // ditto UDPAddress: t.UDPAddress, } resolvers.Start(ctx) diff --git a/internal/experiment/webconnectivity/control.go b/internal/experiment/webconnectivity/control.go new file mode 100644 index 0000000000..b8491d62f8 --- /dev/null +++ b/internal/experiment/webconnectivity/control.go @@ -0,0 +1,103 @@ +package webconnectivity + +import ( + "context" + "net" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/httpx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Control issues a control request and saves the results +// inside of the experiment's TestKeys. +type Control struct { + // Addresses contains the MANDATORY addresses we've looked up. + Addresses []string + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // Session is the MANDATORY session to use. + Session model.ExperimentSession + + // THAddr is the MANDATORY TH's URL. + THAddr string + + // URL is the MANDATORY URL we are measuring. + URL *url.URL + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup +} + +// Start starts this task in a background goroutine. +func (c *Control) Start(ctx context.Context) { + c.WaitGroup.Add(1) + go func() { + defer c.WaitGroup.Done() // synchronize with the parent + c.Run(ctx) + }() +} + +// Run runs this task until completion. +func (c *Control) Run(ctx context.Context) { + // create a subcontext attached to a maximum timeout + const timeout = 30 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // create control request + var endpoints []string + for _, address := range c.Addresses { + if port := c.URL.Port(); port != "" { + endpoints = append(endpoints, net.JoinHostPort(address, port)) + continue + } + endpoints = append(endpoints, net.JoinHostPort(address, "443")) + endpoints = append(endpoints, net.JoinHostPort(address, "80")) + } + creq := &webconnectivity.ControlRequest{ + HTTPRequest: c.URL.String(), + HTTPRequestHeaders: map[string][]string{ + "Accept": {model.HTTPHeaderAccept}, + "Accept-Language": {model.HTTPHeaderAcceptLanguage}, + "User-Agent": {model.HTTPHeaderUserAgent}, + }, + TCPConnect: endpoints, + } + + // create logger for this operation + ol := measurexlite.NewOperationLogger(c.Logger, "control for %s", creq.HTTPRequest) + + // create an API client + clnt := (&httpx.APIClientTemplate{ + BaseURL: c.THAddr, + HTTPClient: c.Session.DefaultHTTPClient(), + Logger: c.Logger, + UserAgent: c.Session.UserAgent(), + }).WithBodyLogging().Build() + + // issue the control request and wait for the response + var cresp webconnectivity.ControlResponse + err := clnt.PostJSON(ctx, "/", creq, &cresp) + if err != nil { + // make sure error is wrapped + err = netxlite.NewTopLevelGenericErrWrapper(err) + c.TestKeys.SetControlFailure(err) + ol.Stop(err) + return + } + + // on success, save the control response + c.TestKeys.SetControl(&cresp) + ol.Stop(nil) +} diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 9d4fffb3d6..89412da3da 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -60,6 +60,15 @@ type DNSResolvers struct { // Referer contains the OPTIONAL referer, used for redirects. Referer string + // Session is the OPTIONAL session. If the session is set, we will use + // it to start the task that issues the control request. This request must + // only be sent during the first iteration. It would be pointless to + // issue such a request for subsequent redirects. + Session model.ExperimentSession + + // THAddr is the OPTIONAL test helper address. + THAddr string + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this // field is not set we use a default one (e.g., `8.8.8.8:53`). UDPAddress string @@ -142,6 +151,7 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { // fan out a number of child async tasks to use the IP addrs t.startCleartextFlows(parentCtx, addresses) t.startSecureFlows(parentCtx, addresses) + t.maybeStartControlFlow(parentCtx, addresses) } // lookupHostSystem performs a DNS lookup using the system resolver. @@ -312,3 +322,19 @@ func (t *DNSResolvers) startSecureFlows(ctx context.Context, addresses []string) task.Start(ctx) } } + +// maybeStartControlFlow starts the control flow, when .Session is set. +func (t *DNSResolvers) maybeStartControlFlow(ctx context.Context, addresses []string) { + if t.Session != nil && t.THAddr != "" { + ctrl := &Control{ + Addresses: addresses, + Logger: t.Logger, + TestKeys: t.TestKeys, + Session: t.Session, + THAddr: t.THAddr, + URL: t.URL, + WaitGroup: t.WaitGroup, + } + ctrl.Start(ctx) + } +} diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index 85422c9c21..3172b5a9fb 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" "golang.org/x/net/publicsuffix" ) @@ -81,6 +82,23 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, return err } + // obtain the test helper's address + testhelpers, _ := sess.GetTestHelpersByName("web-connectivity") + var thAddr string + for _, th := range testhelpers { + if th.Type == "https" { + thAddr = th.Address + measurement.TestHelpers = map[string]any{ + "backend": &th, + } + break + } + } + if thAddr == "" { + sess.Logger().Warnf("continuing without a valid TH address") + tk.SetControlFailure(webconnectivity.ErrNoAvailableTestHelpers) + } + // start background tasks resos := &DNSResolvers{ DNSCache: NewDNSCache(), @@ -94,6 +112,8 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, CookieJar: jar, DNSOverHTTPSURL: "", Referer: "", + Session: sess, + THAddr: thAddr, UDPAddress: "", } resos.Start(ctx) diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index ea7f0e7a21..beb6fd5d1b 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -303,6 +303,8 @@ func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Respon WaitGroup: t.WaitGroup, DNSOverHTTPSURL: t.DNSOverHTTPSURL, Referer: resp.Request.URL.String(), + Session: nil, // no need to issue another control request + THAddr: "", // ditto UDPAddress: t.UDPAddress, } resolvers.Start(ctx) diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index dd24421721..2355738144 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -10,6 +10,7 @@ package webconnectivity import ( "sync" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -30,6 +31,12 @@ type TestKeys struct { // TLSHandshakes contains TLS handshakes results. TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + // Control contains the TH's response. + Control *webconnectivity.ControlResponse `json:"control"` + + // controlFailure is the error associated with accessing the control backend + controlFailure error + // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer // such as a failure to parse a URL that was hardcoded in the codebase. When @@ -78,6 +85,20 @@ func (tk *TestKeys) AppendTLSHandshakes(v ...*model.ArchivalTLSOrQUICHandshakeRe tk.mu.Unlock() } +// SetControl sets the value of Control. +func (tk *TestKeys) SetControl(v *webconnectivity.ControlResponse) { + tk.mu.Lock() + tk.Control = v + tk.mu.Unlock() +} + +// SetControlFailure sets the value of controlFailure. +func (tk *TestKeys) SetControlFailure(err error) { + tk.mu.Lock() + tk.controlFailure = err + tk.mu.Unlock() +} + // SetFundamentalFailure sets the value of fundamentalFailure. func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Lock() @@ -94,6 +115,8 @@ func NewTestKeys() *TestKeys { Requests: []*model.ArchivalHTTPRequestResult{}, TCPConnect: []*model.ArchivalTCPConnectResult{}, TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, + Control: &webconnectivity.ControlResponse{}, + controlFailure: nil, fundamentalFailure: nil, mu: &sync.Mutex{}, } @@ -103,5 +126,4 @@ func NewTestKeys() *TestKeys { // must be called from the measurer after all the tasks have completed. func (tk *TestKeys) finalize() { // TODO(bassosimone): set final webconnectivity flags - // TODO(bassosimone): sort requests correctly } From 705d2ec1775de13b46ea89bcfbd742fd345462ef Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 07:45:56 +0200 Subject: [PATCH 58/83] start to prepare for analysis --- internal/experiment/webconnectivity/analysis.go | 7 +++++++ internal/experiment/webconnectivity/testkeys.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 internal/experiment/webconnectivity/analysis.go diff --git a/internal/experiment/webconnectivity/analysis.go b/internal/experiment/webconnectivity/analysis.go new file mode 100644 index 0000000000..5083bd896b --- /dev/null +++ b/internal/experiment/webconnectivity/analysis.go @@ -0,0 +1,7 @@ +package webconnectivity + +// analysisToplevel is the toplevel function that analyses the results +// of the experiment once all network tasks have completed. +func (tk *TestKeys) analysisToplevel() { + // TODO +} diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 2355738144..da4ee3f749 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -125,5 +125,5 @@ func NewTestKeys() *TestKeys { // finalize performs any delayed computation on the test keys. This function // must be called from the measurer after all the tasks have completed. func (tk *TestKeys) finalize() { - // TODO(bassosimone): set final webconnectivity flags + tk.analysisToplevel() } From 5cb945e5e93ac36dc7e0e0e720ab988c3803fd34 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 09:28:23 +0200 Subject: [PATCH 59/83] feat: implement DNS analysis --- .../experiment/webconnectivity/analysis.go | 7 - .../webconnectivity/analysiscore.go | 18 ++ .../experiment/webconnectivity/analysisdns.go | 298 ++++++++++++++++++ .../experiment/webconnectivity/control.go | 1 + .../experiment/webconnectivity/testkeys.go | 54 +++- 5 files changed, 359 insertions(+), 19 deletions(-) delete mode 100644 internal/experiment/webconnectivity/analysis.go create mode 100644 internal/experiment/webconnectivity/analysiscore.go create mode 100644 internal/experiment/webconnectivity/analysisdns.go diff --git a/internal/experiment/webconnectivity/analysis.go b/internal/experiment/webconnectivity/analysis.go deleted file mode 100644 index 5083bd896b..0000000000 --- a/internal/experiment/webconnectivity/analysis.go +++ /dev/null @@ -1,7 +0,0 @@ -package webconnectivity - -// analysisToplevel is the toplevel function that analyses the results -// of the experiment once all network tasks have completed. -func (tk *TestKeys) analysisToplevel() { - // TODO -} diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go new file mode 100644 index 0000000000..5cb262ede5 --- /dev/null +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -0,0 +1,18 @@ +package webconnectivity + +const ( + // analysisBlockingDNS indicates there's blocking at the DNS level. + analysisBlockingDNS = 1 << iota + + // analysisBlockingTCPIP indicates there's blocking at the TCP/IP level. + analysisBlockingTCPIP + + // analysisBlockingHTTP indicates there's blocking at the HTTP level. + analysisBlockingHTTP +) + +// analysisToplevel is the toplevel function that analyses the results +// of the experiment once all network tasks have completed. +func (tk *TestKeys) analysisToplevel() { + tk.analysisDNSToplevel() +} diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go new file mode 100644 index 0000000000..55a2ae3534 --- /dev/null +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -0,0 +1,298 @@ +package webconnectivity + +import ( + "net" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +const ( + // AnalysisDNSBogon indicates we got any bogon reply + AnalysisDNSBogon = 1 << iota + + // AnalysisDNSUnexpectedFailure indicates the TH could + // resolve a domain while the probe couldn't + AnalysisDNSUnexpectedFailure + + // AnalysisDNSUnexpectedAddrs indicates the TH resolved + // different addresses from the probe + AnalysisDNSUnexpectedAddrs +) + +// analysisDNSToplevel is the toplevel analysis function for DNS results. +func (tk *TestKeys) analysisDNSToplevel() { + tk.analysisDNSExperimentFailure() + tk.analysisDNSBogon() + tk.analysisDNSUnexpectedFailure() + tk.analysisDNSUnexpectedAddrs() + tk.DNSConsistency = "consistent" + if tk.DNSFlags != 0 { + tk.DNSConsistency = "inconsistent" + tk.BlockingFlags |= analysisBlockingDNS + } +} + +// analysisDNSExperimentFailure indicates whether there was any DNS +// experiment failure by inspecting all the queries. +func (tk *TestKeys) analysisDNSExperimentFailure() { + for _, query := range tk.Queries { + if fail := query.Failure; fail != nil { + tk.DNSExperimentFailure = fail + return + } + } +} + +// analysisDNSBogon computes the AnalysisDNSBogon flag. +func (tk *TestKeys) analysisDNSBogon() { + for _, query := range tk.Queries { + for _, answer := range query.Answers { + switch answer.AnswerType { + case "A": + if net.ParseIP(answer.IPv4) != nil && netxlite.IsBogon(answer.IPv4) { + tk.DNSFlags |= AnalysisDNSBogon + return + } + case "AAAA": + if net.ParseIP(answer.IPv6) != nil && netxlite.IsBogon(answer.IPv6) { + tk.DNSFlags |= AnalysisDNSBogon + return + } + default: + // nothing + } + } + } +} + +// analysisDNSUnexpectedFailure computes the AnalysisDNSUnexpectedFailure flags. +func (tk *TestKeys) analysisDNSUnexpectedFailure() { + // make sure we have control before proceeding futher + if tk.Control == nil || tk.controlRequest == nil { + return + } + + // obtain request and response as shortcuts + request := tk.controlRequest + response := tk.Control + + // obtain the domain that the TH has queried for + URL, err := url.Parse(request.HTTPRequest) + if err != nil { + return // this looks like a bug + } + domain := URL.Hostname() + + // we obviously don't care if the domain was an IP adddress + if net.ParseIP(domain) != nil { + return + } + + // we mostly care of whether the control's DNS got back + // any IP address because this is a sign that we had + // unexpected DNS issues locally. + hasAddrs := len(response.DNS.Addrs) > 0 + if !hasAddrs { + return + } + + // therefore, any local query _for the same domain_ queried + // by the probe that contains an error is suspicious + for _, query := range tk.Queries { + if domain != query.Hostname { + continue + } + hasAddrs := false + Loop: + for _, answer := range query.Answers { + switch answer.AnswerType { + case "A", "AAA": + hasAddrs = true + break Loop + } + } + if hasAddrs { + // if the lookup returned any IP address, we are + // not dealing with unexpected failures + continue + } + if query.Failure == nil { + // we expect to see a failure if we don't see + // answers, so this seems a bug + continue + } + tk.DNSFlags |= AnalysisDNSUnexpectedFailure + return + } +} + +// analysisDNSUnexpectedAddrs computes the AnalysisDNSUnexpectedAddrs flags. +func (tk *TestKeys) analysisDNSUnexpectedAddrs() { + // if the list of addresses for which we could not perform a TLS handshake is + // empty, there's no need to compare with the TH, since we can use the results + // of the TLS handshake alone to say that all addresses were correct. + addrsWithoutTLSHandshake := tk.findAddrsWithoutTLSHandshake() + if len(addrsWithoutTLSHandshake) <= 0 { + return + } + + // make sure we have control before proceeding futher + if tk.Control == nil || tk.controlRequest == nil { + return + } + + // obtain request and response as shortcuts + request := tk.controlRequest + response := tk.Control + + // obtain the domain that the TH has queried for + URL, err := url.Parse(request.HTTPRequest) + if err != nil { + return // this looks like a bug + } + domain := URL.Hostname() + + // we obviously don't care if the domain was an IP adddress + if net.ParseIP(domain) != nil { + return + } + + // we mostly care of whether the control's DNS got back + // any IP address because this is a sign that we had + // unexpected DNS issues locally. + thAddrs := response.DNS.Addrs + if len(thAddrs) <= 0 { + return + } + + // gather all the IP addresses queried by the probe + // for the same domain for which the TH queried. + var probeAddrs []string + for _, query := range tk.Queries { + if domain != query.Hostname { + continue + } + for _, answer := range query.Answers { + switch answer.AnswerType { + case "A": + probeAddrs = append(probeAddrs, answer.IPv4) + case "AAAA": + probeAddrs = append(probeAddrs, answer.IPv6) + } + } + } + + // if the probe has not collected any addr for the same domain, it's + // definitely suspicious and counts as a difference + if len(probeAddrs) <= 0 { + tk.DNSFlags |= AnalysisDNSUnexpectedAddrs + return + } + + // if there are no different addresses between the probe and the TH then + // our job here is done and we can just stop searching + differentAddrs := tk.analysisDNSDiffAddrs(probeAddrs, thAddrs) + if len(differentAddrs) <= 0 { + return + } + + // if the different addrs have the same ASN of addrs resolved by + // the TH, then we say everything is still fine. + differentASNS := tk.analysisDNSDiffASN(differentAddrs, thAddrs) + if len(differentASNS) <= 0 { + return + } + + // otherwise, conclude we have unexpected probe addrs + tk.DNSFlags |= AnalysisDNSUnexpectedAddrs +} + +// analysisDNSDiffAddrs returns all the IP addresses that are +// resolved by the probe but not by the test helper. +func (tk *TestKeys) analysisDNSDiffAddrs(probeAddrs, thAddrs []string) (diff []string) { + const ( + inProbe = 1 << iota + inTH + ) + mapping := make(map[string]int) + for _, addr := range probeAddrs { + mapping[addr] |= inProbe + } + for _, addr := range thAddrs { + mapping[addr] = inTH + } + for addr, where := range mapping { + if where&inTH == 0 { + diff = append(diff, addr) + } + } + return +} + +// analysisDNSDiffASN returns whether there are IP addresses in the probe's +// list with different ASNs from the ones in the TH's list. +func (tk *TestKeys) analysisDNSDiffASN(probeAddrs, thAddrs []string) (asns []uint) { + const ( + inProbe = 1 << iota + inTH + ) + mapping := make(map[uint]int) + for _, addr := range probeAddrs { + asn, _, _ := geolocate.LookupASN(addr) + mapping[asn] |= inProbe // including the zero ASN that means unknown + } + for _, addr := range thAddrs { + asn, _, _ := geolocate.LookupASN(addr) + mapping[asn] |= inTH // including the zero ASN that means unknown + } + for asn, where := range mapping { + if where&inTH == 0 { + asns = append(asns, asn) + } + } + return +} + +// findAddrsWithoutTLSHandshake computes the list of probe discovered addresses +// for which we couldn't successfully perform a TLS handshake. +func (tk *TestKeys) findAddrsWithoutTLSHandshake() (output []string) { + const ( + resolved = 1 << iota + handshakeOK + ) + mapping := make(map[string]int) + + // gather all the addrs resolved by the probe + for _, query := range tk.Queries { + for _, answer := range query.Answers { + switch answer.AnswerType { + case "A": + mapping[answer.IPv4] |= resolved + case "AAAA": + mapping[answer.IPv6] |= resolved + } + } + } + + // gather all the addrs with successful handshake + for _, thx := range tk.TLSHandshakes { + addr, _, err := net.SplitHostPort(thx.Address) + if err != nil { + continue // looks like a bug + } + if thx.Failure != nil { + continue // this handshake failed + } + mapping[addr] |= handshakeOK + } + + // compute the list of addresses without the handshakeOK flag + for addr, flags := range mapping { + if flags&handshakeOK == 0 { + output = append(output, addr) + } + } + return +} diff --git a/internal/experiment/webconnectivity/control.go b/internal/experiment/webconnectivity/control.go index b8491d62f8..0dfc24fb6b 100644 --- a/internal/experiment/webconnectivity/control.go +++ b/internal/experiment/webconnectivity/control.go @@ -74,6 +74,7 @@ func (c *Control) Run(ctx context.Context) { }, TCPConnect: endpoints, } + c.TestKeys.SetControlRequest(creq) // create logger for this operation ol := measurexlite.NewOperationLogger(c.Logger, "control for %s", creq.HTTPRequest) diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index da4ee3f749..dffa11940a 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -12,6 +12,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tracex" ) // TestKeys contains the results produced by web_connectivity. @@ -34,8 +35,25 @@ type TestKeys struct { // Control contains the TH's response. Control *webconnectivity.ControlResponse `json:"control"` - // controlFailure is the error associated with accessing the control backend - controlFailure error + // ControlFailure contains the failure of the control experiment. + ControlFailure *string `json:"control_failure"` + + // DNSFlags contains DNS analysis flags. + DNSFlags int64 `json:"x_dns_flags"` + + // DNSExperimentFailure indicates whether there was a failure in any + // of the DNS experiments we performed. + DNSExperimentFailure *string `json:"dns_experiment_failure"` + + // DNSConsistency indicates whether there is consistency between + // the TH's DNS results and the probe's DNS results. + DNSConsistency string `json:"dns_consistency"` + + // BlockingFlags contains blocking flags. + BlockingFlags int64 `json:"x_blocking_flags"` + + // controlRequest is the control request we sent. + controlRequest *webconnectivity.ControlRequest // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer @@ -85,6 +103,13 @@ func (tk *TestKeys) AppendTLSHandshakes(v ...*model.ArchivalTLSOrQUICHandshakeRe tk.mu.Unlock() } +// SetControlRequest sets the value of controlRequest. +func (tk *TestKeys) SetControlRequest(v *webconnectivity.ControlRequest) { + tk.mu.Lock() + tk.controlRequest = v + tk.mu.Unlock() +} + // SetControl sets the value of Control. func (tk *TestKeys) SetControl(v *webconnectivity.ControlResponse) { tk.mu.Lock() @@ -95,7 +120,7 @@ func (tk *TestKeys) SetControl(v *webconnectivity.ControlResponse) { // SetControlFailure sets the value of controlFailure. func (tk *TestKeys) SetControlFailure(err error) { tk.mu.Lock() - tk.controlFailure = err + tk.ControlFailure = tracex.NewFailure(err) tk.mu.Unlock() } @@ -110,15 +135,20 @@ func (tk *TestKeys) SetFundamentalFailure(err error) { func NewTestKeys() *TestKeys { // TODO: here you should initialize all the fields return &TestKeys{ - NetworkEvents: []*model.ArchivalNetworkEvent{}, - Queries: []*model.ArchivalDNSLookupResult{}, - Requests: []*model.ArchivalHTTPRequestResult{}, - TCPConnect: []*model.ArchivalTCPConnectResult{}, - TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, - Control: &webconnectivity.ControlResponse{}, - controlFailure: nil, - fundamentalFailure: nil, - mu: &sync.Mutex{}, + NetworkEvents: []*model.ArchivalNetworkEvent{}, + Queries: []*model.ArchivalDNSLookupResult{}, + Requests: []*model.ArchivalHTTPRequestResult{}, + TCPConnect: []*model.ArchivalTCPConnectResult{}, + TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, + Control: nil, + ControlFailure: nil, + DNSFlags: 0, + DNSExperimentFailure: nil, + DNSConsistency: "", + BlockingFlags: 0, + controlRequest: nil, + fundamentalFailure: nil, + mu: &sync.Mutex{}, } } From c439f0a45260c1c0165d29856fdfc067772068f1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 09:50:40 +0200 Subject: [PATCH 60/83] feat: implement tcp/ip blocking heuristics --- .../webconnectivity/analysiscore.go | 5 ++ .../experiment/webconnectivity/analysisdns.go | 4 ++ .../webconnectivity/analysistcpip.go | 61 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 internal/experiment/webconnectivity/analysistcpip.go diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go index 5cb262ede5..2e47f6652e 100644 --- a/internal/experiment/webconnectivity/analysiscore.go +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -1,5 +1,9 @@ package webconnectivity +// +// Core analysis +// + const ( // analysisBlockingDNS indicates there's blocking at the DNS level. analysisBlockingDNS = 1 << iota @@ -15,4 +19,5 @@ const ( // of the experiment once all network tasks have completed. func (tk *TestKeys) analysisToplevel() { tk.analysisDNSToplevel() + tk.analysisTCPIPToplevel() } diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go index 55a2ae3534..2d896716fa 100644 --- a/internal/experiment/webconnectivity/analysisdns.go +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -1,5 +1,9 @@ package webconnectivity +// +// DNS analysis +// + import ( "net" "net/url" diff --git a/internal/experiment/webconnectivity/analysistcpip.go b/internal/experiment/webconnectivity/analysistcpip.go new file mode 100644 index 0000000000..b065f6b3ec --- /dev/null +++ b/internal/experiment/webconnectivity/analysistcpip.go @@ -0,0 +1,61 @@ +package webconnectivity + +// +// TCP/IP analysis +// + +import ( + "fmt" + "net" + + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// analysisTCPIPToplevel is the toplevel analysis function for TCP/IP results. +func (tk *TestKeys) analysisTCPIPToplevel() { + // if we don't have a control result, do nothing. + if tk.Control == nil || len(tk.Control.TCPConnect) <= 0 { + return + } + var ( + istrue = true + isfalse = false + ) + + // walk the list of probe results and compare with TH results + for _, entry := range tk.TCPConnect { + // skip successful entries + failure := entry.Status.Failure + if failure == nil { + entry.Status.Blocked = &isfalse + continue // did not fail + } + // make sure we exclude the IPv6 failures caused by lack of + // proper IPv6 support by the probe + ipv6, err := netxlite.IsIPv6(entry.IP) + if err != nil { + continue // looks like a bug + } + if ipv6 { + ignore := (*failure == netxlite.FailureNetworkUnreachable || + *failure == netxlite.FailureHostUnreachable) + if ignore { + // this occurs when we don't have IPv6 on the probe + continue + } + } + // obtain the corresponding endpoint + epnt := net.JoinHostPort(entry.IP, fmt.Sprintf("%d", entry.Port)) + ctrl, found := tk.Control.TCPConnect[epnt] + if !found { + continue // only the probe tested this, so hard to say anything + } + if ctrl.Failure != nil { + // if the TH failed as well, don't set any blocking flag + entry.Status.Blocked = &isfalse + continue + } + entry.Status.Blocked = &istrue + tk.BlockingFlags |= analysisBlockingTCPIP + } +} From b4230c6f0e082d682d9d841848b46dd4a4f4fa9b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 10:22:09 +0200 Subject: [PATCH 61/83] feat: write top-level HTTP analysis --- .../webconnectivity/analysiscore.go | 11 ++- .../webconnectivity/analysishttpcore.go | 87 +++++++++++++++++++ .../webconnectivity/analysishttpdiff.go | 14 +++ 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 internal/experiment/webconnectivity/analysishttpcore.go create mode 100644 internal/experiment/webconnectivity/analysishttpdiff.go diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go index 2e47f6652e..fa8a705ad6 100644 --- a/internal/experiment/webconnectivity/analysiscore.go +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -11,8 +11,14 @@ const ( // analysisBlockingTCPIP indicates there's blocking at the TCP/IP level. analysisBlockingTCPIP - // analysisBlockingHTTP indicates there's blocking at the HTTP level. - analysisBlockingHTTP + // analysisBlockingTLSFailure indicates there were TLS issues. + analysisBlockingTLSFailure + + // analysisBlockingHTTPFailure indicates there was an HTTP failure. + analysisBlockingHTTPFailure + + // analysisBlockingHTTPDiff indicates there's an HTTP diff. + analysisBlockingHTTPDiff ) // analysisToplevel is the toplevel function that analyses the results @@ -20,4 +26,5 @@ const ( func (tk *TestKeys) analysisToplevel() { tk.analysisDNSToplevel() tk.analysisTCPIPToplevel() + tk.analysisHTTPToplevel() } diff --git a/internal/experiment/webconnectivity/analysishttpcore.go b/internal/experiment/webconnectivity/analysishttpcore.go new file mode 100644 index 0000000000..5ee12ff80e --- /dev/null +++ b/internal/experiment/webconnectivity/analysishttpcore.go @@ -0,0 +1,87 @@ +package webconnectivity + +import "github.com/ooni/probe-cli/v3/internal/netxlite" + +// analysisHTTPToplevel is the toplevel analysis function for HTTP results. +func (tk *TestKeys) analysisHTTPToplevel() { + // don't perform any analysis if the TH failed + if tk.Control == nil { + return + } + ctrl := tk.Control.HTTPRequest + + // don't perform any analysis if the TH's HTTP measurement failed + if ctrl.Failure != nil { + return + } + + // determine whether we had any TLS handshake issue and, in such a case, + // declare that we had a case of TLS failure. + if tk.hasWellKnownTLSHandshakeIssues() { + tk.BlockingFlags |= analysisBlockingTLSFailure + return + } + + // determine whether we had well known cleartext HTTP round trip issues + // and, in such a case, declare we had an "http-failure". + if tk.hasWellKnownHTTPRoundTripIssues() { + tk.BlockingFlags |= analysisBlockingHTTPFailure + return + } + + // if we don't have any request to check, there's not much more we + // can actually do here, so let's just return. + if len(tk.Requests) <= 0 { + return + } + + // fallback to the original HTTP diff algorithm. The first entry in the + // tk.Requests array is the last entry that was measured. + tk.analysisHTTPDiff(tk.Requests[0], &ctrl) +} + +// hasWellKnownTLSHandshakeIssues returns true in case we observed +// a set of well-known issues during the TLS handshake. +func (tk *TestKeys) hasWellKnownTLSHandshakeIssues() bool { + for _, thx := range tk.TLSHandshakes { + fail := thx.Failure + if fail == nil { + continue // this handshake succeded, so skip it + } + switch *fail { + case netxlite.FailureConnectionReset, + netxlite.FailureGenericTimeoutError, + netxlite.FailureEOFError, + netxlite.FailureSSLInvalidHostname, + netxlite.FailureSSLInvalidCertificate, + netxlite.FailureSSLUnknownAuthority: + return true + default: + // check next handshake + } + } + return false +} + +// hasWellKnownHTTPRoundTripIssues checks whether any HTTP round +// trip failed in a well-known suspicious way +func (tk *TestKeys) hasWellKnownHTTPRoundTripIssues() bool { + for _, rtx := range tk.Requests { + fail := rtx.Failure + if fail == nil { + // This one succeded, so skip it. Note that, in principle, we know + // the fist entry is the last request occurred, but I really do not + // want to embed this bad assumption in one extra place! + continue + } + switch *fail { + case netxlite.FailureConnectionReset, + netxlite.FailureGenericTimeoutError, + netxlite.FailureEOFError: + return true + default: + // check next round trip + } + } + return false +} diff --git a/internal/experiment/webconnectivity/analysishttpdiff.go b/internal/experiment/webconnectivity/analysishttpdiff.go new file mode 100644 index 0000000000..0fe6f9e936 --- /dev/null +++ b/internal/experiment/webconnectivity/analysishttpdiff.go @@ -0,0 +1,14 @@ +package webconnectivity + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// analysisHTTPDiff computes the HTTP diff between the final request-response +// observed by the probe and the TH's result. The caller is responsible of passing +// us a valid probe observation and a valid TH observation. +func (tk *TestKeys) analysisHTTPDiff( + probe *model.ArchivalHTTPRequestResult, th *webconnectivity.ControlHTTPRequestResult) { + // TODO +} From 8705d2f0d455800cae51f589d6901ff597dbf1e3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 11:39:43 +0200 Subject: [PATCH 62/83] feat: add analysis code --- .../webconnectivity/analysiscore.go | 24 ++ .../webconnectivity/analysishttpcore.go | 14 +- .../webconnectivity/analysishttpdiff.go | 217 +++++++++++++++++- .../experiment/webconnectivity/testkeys.go | 36 +++ 4 files changed, 288 insertions(+), 3 deletions(-) diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go index fa8a705ad6..fd79da8d3d 100644 --- a/internal/experiment/webconnectivity/analysiscore.go +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -27,4 +27,28 @@ func (tk *TestKeys) analysisToplevel() { tk.analysisDNSToplevel() tk.analysisTCPIPToplevel() tk.analysisHTTPToplevel() + if (tk.BlockingFlags & analysisBlockingDNS) != 0 { + tk.Blocking = "dns" + return + } + if (tk.BlockingFlags & analysisBlockingTCPIP) != 0 { + tk.Blocking = "tcp_ip" + return + } + if (tk.BlockingFlags & analysisBlockingTLSFailure) != 0 { + tk.Blocking = "http-failure" // backwards compatibility with the spec + return + } + if (tk.BlockingFlags & analysisBlockingHTTPFailure) != 0 { + tk.Blocking = "http-failure" + return + } + if (tk.BlockingFlags & analysisBlockingHTTPDiff) != 0 { + tk.Blocking = "http-diff" + return + } + if tk.Accessible == nil || !*tk.Accessible { + return + } + tk.Blocking = false } diff --git a/internal/experiment/webconnectivity/analysishttpcore.go b/internal/experiment/webconnectivity/analysishttpcore.go index 5ee12ff80e..79df57e6f0 100644 --- a/internal/experiment/webconnectivity/analysishttpcore.go +++ b/internal/experiment/webconnectivity/analysishttpcore.go @@ -1,5 +1,9 @@ package webconnectivity +// +// HTTP core analysis +// + import "github.com/ooni/probe-cli/v3/internal/netxlite" // analysisHTTPToplevel is the toplevel analysis function for HTTP results. @@ -35,9 +39,15 @@ func (tk *TestKeys) analysisHTTPToplevel() { return } - // fallback to the original HTTP diff algorithm. The first entry in the + // if the request has failed in any other way, we don't know. The first entry in the // tk.Requests array is the last entry that was measured. - tk.analysisHTTPDiff(tk.Requests[0], &ctrl) + finalRequest := tk.Requests[0] + if finalRequest.Failure != nil { + return + } + + // fallback to the original HTTP diff algorithm. + tk.analysisHTTPDiff(finalRequest, &ctrl) } // hasWellKnownTLSHandshakeIssues returns true in case we observed diff --git a/internal/experiment/webconnectivity/analysishttpdiff.go b/internal/experiment/webconnectivity/analysishttpdiff.go index 0fe6f9e936..f4517dd44b 100644 --- a/internal/experiment/webconnectivity/analysishttpdiff.go +++ b/internal/experiment/webconnectivity/analysishttpdiff.go @@ -1,6 +1,14 @@ package webconnectivity +// +// HTTP diff analysis +// + import ( + "net/url" + "reflect" + "strings" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -10,5 +18,212 @@ import ( // us a valid probe observation and a valid TH observation. func (tk *TestKeys) analysisHTTPDiff( probe *model.ArchivalHTTPRequestResult, th *webconnectivity.ControlHTTPRequestResult) { - // TODO + + // if we're dealing with an HTTPS request, don't perform any comparison + // under the assumption that we're good if we're using TLS + URL, err := url.Parse(probe.Request.URL) + if err != nil { + return // looks like a bug + } + accessibleTrue := true + if URL.Scheme == "https" { + tk.Accessible = &accessibleTrue + return + } + + // original HTTP diff algorithm adapted for this implementation + tk.httpDiffBodyLengthChecks(probe, th) + tk.httpDiffStatusCodeMatch(probe, th) + tk.httpDiffHeadersMatch(probe, th) + tk.httpDiffTitleMatch(probe, th) + + if tk.StatusCodeMatch != nil && *tk.StatusCodeMatch { + if tk.BodyLengthMatch != nil && *tk.BodyLengthMatch { + tk.Accessible = &accessibleTrue + return + } + if tk.HeadersMatch != nil && *tk.HeadersMatch { + tk.Accessible = &accessibleTrue + return + } + if tk.TitleMatch != nil && *tk.TitleMatch { + tk.Accessible = &accessibleTrue + return + } + } + + tk.BlockingFlags |= analysisBlockingHTTPDiff + accessibleFalse := false + tk.Accessible = &accessibleFalse +} + +// httpDiffBodyLengthChecks compares the bodies lengths. +func (tk *TestKeys) httpDiffBodyLengthChecks( + probe *model.ArchivalHTTPRequestResult, ctrl *webconnectivity.ControlHTTPRequestResult) { + control := ctrl.BodyLength + if control <= 0 { + return // no actual length + } + response := probe.Response + if response.BodyIsTruncated { + return // cannot trust body length in this case + } + measurement := int64(len(response.Body.Value)) + if measurement <= 0 { + return // no actual length + } + const bodyProportionFactor = 0.7 + var proportion float64 + if measurement >= control { + proportion = float64(control) / float64(measurement) + } else { + proportion = float64(measurement) / float64(control) + } + good := proportion > bodyProportionFactor + tk.BodyLengthMatch = &good +} + +// httpDiffStatusCodeMatch compares the status codes. +func (tk *TestKeys) httpDiffStatusCodeMatch( + probe *model.ArchivalHTTPRequestResult, ctrl *webconnectivity.ControlHTTPRequestResult) { + control := ctrl.StatusCode + measurement := probe.Response.Code + if control <= 0 { + return // no real status code + } + if measurement <= 0 { + return // no real status code + } + if control/100 != 2 { + return // avoid comparison if it seems the TH failed + } + good := control == measurement + tk.StatusCodeMatch = &good +} + +// httpDiffHeadersMatch compares the uncommon headers. +func (tk *TestKeys) httpDiffHeadersMatch( + probe *model.ArchivalHTTPRequestResult, ctrl *webconnectivity.ControlHTTPRequestResult) { + control := ctrl.Headers + measurement := probe.Response.Headers + if len(control) <= 0 || len(measurement) <= 0 { + return + } + // Implementation note: using map because we only care about the + // keys being different and we ignore the values. + const ( + inMeasurement = 1 << 0 + inControl = 1 << 1 + inBoth = inMeasurement | inControl + ) + commonHeaders := map[string]bool{ + "date": true, + "content-type": true, + "server": true, + "cache-control": true, + "vary": true, + "set-cookie": true, + "location": true, + "expires": true, + "x-powered-by": true, + "content-encoding": true, + "last-modified": true, + "accept-ranges": true, + "pragma": true, + "x-frame-options": true, + "etag": true, + "x-content-type-options": true, + "age": true, + "via": true, + "p3p": true, + "x-xss-protection": true, + "content-language": true, + "cf-ray": true, + "strict-transport-security": true, + "link": true, + "x-varnish": true, + } + matching := make(map[string]int) + ours := make(map[string]bool) + for key := range measurement { + key = strings.ToLower(key) + if _, ok := commonHeaders[key]; !ok { + matching[key] |= inMeasurement + } + ours[key] = true + } + theirs := make(map[string]bool) + for key := range control { + key = strings.ToLower(key) + if _, ok := commonHeaders[key]; !ok { + matching[key] |= inControl + } + theirs[key] = true + } + // if they are equal we're done + if good := reflect.DeepEqual(ours, theirs); good { + tk.HeadersMatch = &good + return + } + // compute the intersection of uncommon headers + found := false + for _, value := range matching { + if (value & inBoth) == inBoth { + found = true + break + } + } + tk.HeadersMatch = &found +} + +// httpDiffTitleMatch compares the titles. +func (tk *TestKeys) httpDiffTitleMatch( + probe *model.ArchivalHTTPRequestResult, ctrl *webconnectivity.ControlHTTPRequestResult) { + response := probe.Response + if response.Code <= 0 { + return + } + if response.BodyIsTruncated { + return + } + if ctrl.StatusCode <= 0 { + return + } + control := ctrl.Title + measurementBody := response.Body.Value + measurement := webconnectivity.GetTitle(measurementBody) + if control == "" || measurement == "" { + return + } + const ( + inMeasurement = 1 << 0 + inControl = 1 << 1 + inBoth = inMeasurement | inControl + ) + words := make(map[string]int) + // We don't consider to match words that are shorter than 5 + // characters (5 is the average word length for english) + // + // The original implementation considered the word order but + // considering different languages it seems we could have less + // false positives by ignoring the word order. + const minWordLength = 5 + for _, word := range strings.Split(measurement, " ") { + if len(word) >= minWordLength { + words[strings.ToLower(word)] |= inMeasurement + } + } + for _, word := range strings.Split(control, " ") { + if len(word) >= minWordLength { + words[strings.ToLower(word)] |= inControl + } + } + good := true + for _, score := range words { + if (score & inBoth) != inBoth { + good = false + break + } + } + tk.TitleMatch = &good } diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index dffa11940a..25bd8fc771 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -52,6 +52,36 @@ type TestKeys struct { // BlockingFlags contains blocking flags. BlockingFlags int64 `json:"x_blocking_flags"` + // BodyLength match tells us whether the body length matches. + BodyLengthMatch *bool `json:"body_length_match"` + + // HeadersMatch tells us whether the headers match. + HeadersMatch *bool `json:"headers_match"` + + // StatusCodeMatch tells us whether the status code matches. + StatusCodeMatch *bool `json:"status_code_match"` + + // TitleMatch tells us whether the title matches. + TitleMatch *bool `json:"title_match"` + + // Blocking indicates the reason for blocking. This is notoriously a bad + // type because it can be one of the following values: + // + // - "tcp_ip" + // - "dns" + // - "http-diff" + // - "http-failure" + // - false + // - null + // + // In addition to being a bad type, this field has the issue that it + // reduces the reason for blocking to an enum, whereas it's a set of flags, + // hence we introduced the x_blocking_flags field. + Blocking any `json:"blocking"` + + // Accessible indicates whether the resource is accessible. + Accessible *bool `json:"accessible"` + // controlRequest is the control request we sent. controlRequest *webconnectivity.ControlRequest @@ -146,6 +176,12 @@ func NewTestKeys() *TestKeys { DNSExperimentFailure: nil, DNSConsistency: "", BlockingFlags: 0, + BodyLengthMatch: nil, + HeadersMatch: nil, + StatusCodeMatch: nil, + TitleMatch: nil, + Blocking: nil, + Accessible: nil, controlRequest: nil, fundamentalFailure: nil, mu: &sync.Mutex{}, From 0b6c7d5ca61ba46cd31f7ea880cbf22a02da06c8 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 12:03:36 +0200 Subject: [PATCH 63/83] fix: make output easier to read --- .../webconnectivity/cleartextflow.go | 8 +++- .../webconnectivity/dnsresolvers.go | 37 +++++++++++++++---- .../experiment/webconnectivity/secureflow.go | 8 +++- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 42e3c9882c..642a18c037 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -9,6 +9,7 @@ package webconnectivity import ( "context" + "errors" "io" "net" "net/http" @@ -96,7 +97,9 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "CleartextFlow#%d", index) // TODO: edit + ol := measurexlite.NewOperationLogger( + t.Logger, "[#%d] GET http://%s using %s", index, t.HostHeader, t.Address, + ) // perform the TCP connect const tcpTimeout = 10 * time.Second @@ -119,7 +122,7 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { select { case <-t.Sema: default: - ol.Stop(nil) + ol.Stop(errors.New("stop after TCP connect")) // just to emit the right message return } @@ -239,6 +242,7 @@ func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Res if err != nil { return } + t.Logger.Infof("redirect to: %s", location.String()) resolvers := &DNSResolvers{ CookieJar: t.CookieJar, DNSCache: t.DNSCache, diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 89412da3da..1138b6454e 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -9,6 +9,7 @@ package webconnectivity import ( "context" + "fmt" "net" "net/http" "net/url" @@ -168,13 +169,19 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- [] trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "DNSResolvers+System#%d: %s", index, t.Domain) + ol := measurexlite.NewOperationLogger( + t.Logger, "[#%d] lookup %s using system", index, t.Domain, + ) // runs the lookup reso := trace.NewStdlibResolver(t.Logger) addrs, err := reso.LookupHost(lookupCtx, t.Domain) t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - ol.Stop(err) + if err != nil { + ol.Stop(err) + return + } + ol.Stop(fmt.Errorf("%+v", addrs)) // just to emit the correct message out <- addrs } @@ -192,14 +199,21 @@ func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []str trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "DNSResolvers+UDP#%d: %s", index, t.Domain) + udpAddress := t.udpAddress() + ol := measurexlite.NewOperationLogger( + t.Logger, "[#%d] lookup %s using %s", index, t.Domain, udpAddress, + ) // runs the lookup dialer := netxlite.NewDialerWithoutResolver(t.Logger) - reso := trace.NewParallelUDPResolver(t.Logger, dialer, t.udpAddress()) + reso := trace.NewParallelUDPResolver(t.Logger, dialer, udpAddress) addrs, err := reso.LookupHost(lookupCtx, t.Domain) t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - ol.Stop(err) + if err != nil { + ol.Stop(err) + return + } + ol.Stop(fmt.Errorf("%+v", addrs)) // just to emit the correct message out <- addrs } @@ -225,14 +239,21 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "DNSResolvers+DNSOverHTTPS#%d: %s", index, t.Domain) + URL := t.dnsOverHTTPSURL() + ol := measurexlite.NewOperationLogger( + t.Logger, "[#%d] lookup %s using %s", index, t.Domain, URL, + ) // runs the lookup - reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, t.dnsOverHTTPSURL()) + reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, URL) addrs, err := reso.LookupHost(lookupCtx, t.Domain) reso.CloseIdleConnections() t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - ol.Stop(err) + if err != nil { + ol.Stop(err) + return + } + ol.Stop(fmt.Errorf("%+v", addrs)) // just to emit the correct message out <- addrs } diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index beb6fd5d1b..7cb3a7cd7e 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -10,6 +10,7 @@ package webconnectivity import ( "context" "crypto/tls" + "errors" "io" "net" "net/http" @@ -103,7 +104,9 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "SecureFlow#%d", index) // TODO: edit + ol := measurexlite.NewOperationLogger( + t.Logger, "[#%d] GET https://%s using %s", index, t.HostHeader, t.Address, + ) // perform the TCP connect const tcpTimeout = 10 * time.Second @@ -150,7 +153,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { select { case <-t.Sema: default: - ol.Stop(nil) + ol.Stop(errors.New("stop after TLS handshake")) // just to emit the correct message return } @@ -291,6 +294,7 @@ func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Respon if err != nil { return } + t.Logger.Infof("redirect to: %s", location.String()) resolvers := &DNSResolvers{ CookieJar: t.CookieJar, DNSCache: t.DNSCache, From 6709c2b2d43a368a87df15fe42a4b9e15e449b4c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 12:14:18 +0200 Subject: [PATCH 64/83] fix: handle the dns_no_answer case for AAAA --- internal/experiment/webconnectivity/analysisdns.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go index 2d896716fa..27f7ca3136 100644 --- a/internal/experiment/webconnectivity/analysisdns.go +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -43,6 +43,11 @@ func (tk *TestKeys) analysisDNSToplevel() { func (tk *TestKeys) analysisDNSExperimentFailure() { for _, query := range tk.Queries { if fail := query.Failure; fail != nil { + if query.QueryType == "AAAA" && *query.Failure == netxlite.FailureDNSNoAnswer { + // maybe this heuristic could be further improved by checking + // whether the TH did actually see any IPv6 address? + continue + } tk.DNSExperimentFailure = fail return } @@ -127,6 +132,11 @@ func (tk *TestKeys) analysisDNSUnexpectedFailure() { // answers, so this seems a bug continue } + if query.QueryType == "AAAA" && *query.Failure == netxlite.FailureDNSNoAnswer { + // maybe this heuristic could be further improved by checking + // whether the TH did actually see any IPv6 address? + continue + } tk.DNSFlags |= AnalysisDNSUnexpectedFailure return } From 388395f88e23eb7211fd80a7a5c91cdbf7136333 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 12:28:33 +0200 Subject: [PATCH 65/83] fix: remove bug added when trying to beautify output --- .../webconnectivity/dnsresolvers.go | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 1138b6454e..b04485207d 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -9,13 +9,13 @@ package webconnectivity import ( "context" - "fmt" "net" "net/http" "net/url" "sync" "time" + "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" @@ -149,6 +149,8 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { t.DNSCache.Set(t.Domain, addresses) } + log.Infof("using: %+v", addresses) + // fan out a number of child async tasks to use the IP addrs t.startCleartextFlows(parentCtx, addresses) t.startSecureFlows(parentCtx, addresses) @@ -177,12 +179,8 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- [] reso := trace.NewStdlibResolver(t.Logger) addrs, err := reso.LookupHost(lookupCtx, t.Domain) t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - if err != nil { - ol.Stop(err) - return - } - ol.Stop(fmt.Errorf("%+v", addrs)) // just to emit the correct message - out <- addrs + ol.Stop(err) + out <- addrs // must send something -even nil- to the parent } // lookupHostUDP performs a DNS lookup using an UDP resolver. @@ -209,12 +207,8 @@ func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []str reso := trace.NewParallelUDPResolver(t.Logger, dialer, udpAddress) addrs, err := reso.LookupHost(lookupCtx, t.Domain) t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - if err != nil { - ol.Stop(err) - return - } - ol.Stop(fmt.Errorf("%+v", addrs)) // just to emit the correct message - out <- addrs + ol.Stop(err) + out <- addrs // must send something -even nil- to the parent } // Returns the UDP resolver we should be using by default. @@ -249,12 +243,8 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha addrs, err := reso.LookupHost(lookupCtx, t.Domain) reso.CloseIdleConnections() t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - if err != nil { - ol.Stop(err) - return - } - ol.Stop(fmt.Errorf("%+v", addrs)) // just to emit the correct message - out <- addrs + ol.Stop(err) + out <- addrs // must send something -even nil- to the parent } // Returns the DOH resolver URL we should be using by default. From c6e157a0b161839f67c2aef07c9e2f0717b3eb40 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 12:47:54 +0200 Subject: [PATCH 66/83] fix: improve logging and save more DoH data --- .../webconnectivity/analysiscore.go | 6 +++-- .../experiment/webconnectivity/analysisdns.go | 22 +++++++++++++------ .../webconnectivity/dnsresolvers.go | 3 +++ .../experiment/webconnectivity/measurer.go | 2 +- .../experiment/webconnectivity/testkeys.go | 4 ++-- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go index fd79da8d3d..c17d2466e1 100644 --- a/internal/experiment/webconnectivity/analysiscore.go +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -1,5 +1,7 @@ package webconnectivity +import "github.com/ooni/probe-cli/v3/internal/model" + // // Core analysis // @@ -23,8 +25,8 @@ const ( // analysisToplevel is the toplevel function that analyses the results // of the experiment once all network tasks have completed. -func (tk *TestKeys) analysisToplevel() { - tk.analysisDNSToplevel() +func (tk *TestKeys) analysisToplevel(logger model.Logger) { + tk.analysisDNSToplevel(logger) tk.analysisTCPIPToplevel() tk.analysisHTTPToplevel() if (tk.BlockingFlags & analysisBlockingDNS) != 0 { diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go index 27f7ca3136..3a95965dc9 100644 --- a/internal/experiment/webconnectivity/analysisdns.go +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -9,6 +9,7 @@ import ( "net/url" "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -26,13 +27,14 @@ const ( ) // analysisDNSToplevel is the toplevel analysis function for DNS results. -func (tk *TestKeys) analysisDNSToplevel() { +func (tk *TestKeys) analysisDNSToplevel(logger model.Logger) { tk.analysisDNSExperimentFailure() - tk.analysisDNSBogon() - tk.analysisDNSUnexpectedFailure() - tk.analysisDNSUnexpectedAddrs() + tk.analysisDNSBogon(logger) + tk.analysisDNSUnexpectedFailure(logger) + tk.analysisDNSUnexpectedAddrs(logger) tk.DNSConsistency = "consistent" if tk.DNSFlags != 0 { + logger.Warnf("DNSConsistency: inconsistent") tk.DNSConsistency = "inconsistent" tk.BlockingFlags |= analysisBlockingDNS } @@ -55,17 +57,19 @@ func (tk *TestKeys) analysisDNSExperimentFailure() { } // analysisDNSBogon computes the AnalysisDNSBogon flag. -func (tk *TestKeys) analysisDNSBogon() { +func (tk *TestKeys) analysisDNSBogon(logger model.Logger) { for _, query := range tk.Queries { for _, answer := range query.Answers { switch answer.AnswerType { case "A": if net.ParseIP(answer.IPv4) != nil && netxlite.IsBogon(answer.IPv4) { + logger.Warnf("BOGON: %s in #%d", answer.IPv4, query.TransactionID) tk.DNSFlags |= AnalysisDNSBogon return } case "AAAA": if net.ParseIP(answer.IPv6) != nil && netxlite.IsBogon(answer.IPv6) { + logger.Warnf("BOGON: %s in #%d", answer.IPv6, query.TransactionID) tk.DNSFlags |= AnalysisDNSBogon return } @@ -77,7 +81,7 @@ func (tk *TestKeys) analysisDNSBogon() { } // analysisDNSUnexpectedFailure computes the AnalysisDNSUnexpectedFailure flags. -func (tk *TestKeys) analysisDNSUnexpectedFailure() { +func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { // make sure we have control before proceeding futher if tk.Control == nil || tk.controlRequest == nil { return @@ -137,13 +141,14 @@ func (tk *TestKeys) analysisDNSUnexpectedFailure() { // whether the TH did actually see any IPv6 address? continue } + logger.Warnf("DNS: unexpected failure %s in #%d", *query.Failure, query.TransactionID) tk.DNSFlags |= AnalysisDNSUnexpectedFailure return } } // analysisDNSUnexpectedAddrs computes the AnalysisDNSUnexpectedAddrs flags. -func (tk *TestKeys) analysisDNSUnexpectedAddrs() { +func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { // if the list of addresses for which we could not perform a TLS handshake is // empty, there's no need to compare with the TH, since we can use the results // of the TLS handshake alone to say that all addresses were correct. @@ -151,6 +156,7 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs() { if len(addrsWithoutTLSHandshake) <= 0 { return } + logger.Warnf("DNS: addrs without TLS handshake: %+v", addrsWithoutTLSHandshake) // make sure we have control before proceeding futher if tk.Control == nil || tk.controlRequest == nil { @@ -201,6 +207,7 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs() { // if the probe has not collected any addr for the same domain, it's // definitely suspicious and counts as a difference if len(probeAddrs) <= 0 { + logger.Warnf("DNS: no IP address resolved by the probe") tk.DNSFlags |= AnalysisDNSUnexpectedAddrs return } @@ -220,6 +227,7 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs() { } // otherwise, conclude we have unexpected probe addrs + logger.Warnf("DNS: differentAddrs: %+v, differentASNs: %+v", differentAddrs, differentASNS) tk.DNSFlags |= AnalysisDNSUnexpectedAddrs } diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index b04485207d..0709c598a0 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -243,6 +243,9 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha addrs, err := reso.LookupHost(lookupCtx, t.Domain) reso.CloseIdleConnections() t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) + t.TestKeys.AppendTLSHandshakes(trace.TLSHandshakes()...) ol.Stop(err) out <- addrs // must send something -even nil- to the parent } diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index 3172b5a9fb..f84592aced 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -128,7 +128,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, } // perform any deferred computation on the test keys - tk.finalize() + tk.finalize(sess.Logger()) // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 25bd8fc771..f166339c46 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -190,6 +190,6 @@ func NewTestKeys() *TestKeys { // finalize performs any delayed computation on the test keys. This function // must be called from the measurer after all the tasks have completed. -func (tk *TestKeys) finalize() { - tk.analysisToplevel() +func (tk *TestKeys) finalize(logger model.Logger) { + tk.analysisToplevel(logger) } From 844e36bae3ac5e9936abf6244d0780851a42b394 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 12:54:25 +0200 Subject: [PATCH 67/83] fix: only use TLS handshake for interesting addrs --- .../experiment/webconnectivity/analysisdns.go | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go index 3a95965dc9..bfa40d296c 100644 --- a/internal/experiment/webconnectivity/analysisdns.go +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -149,15 +149,6 @@ func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { // analysisDNSUnexpectedAddrs computes the AnalysisDNSUnexpectedAddrs flags. func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { - // if the list of addresses for which we could not perform a TLS handshake is - // empty, there's no need to compare with the TH, since we can use the results - // of the TLS handshake alone to say that all addresses were correct. - addrsWithoutTLSHandshake := tk.findAddrsWithoutTLSHandshake() - if len(addrsWithoutTLSHandshake) <= 0 { - return - } - logger.Warnf("DNS: addrs without TLS handshake: %+v", addrsWithoutTLSHandshake) - // make sure we have control before proceeding futher if tk.Control == nil || tk.controlRequest == nil { return @@ -226,8 +217,16 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { return } + withoutHandshake := tk.findAddrsWithoutTLSHandshake(differentAddrs) + if len(withoutHandshake) <= 0 { + return + } + // otherwise, conclude we have unexpected probe addrs - logger.Warnf("DNS: differentAddrs: %+v, differentASNs: %+v", differentAddrs, differentASNS) + logger.Warnf( + "DNS: differentAddrs: %+v; differentASNs: %+v; withoutHandshake: %+v", + differentAddrs, differentASNS, withoutHandshake, + ) tk.DNSFlags |= AnalysisDNSUnexpectedAddrs } @@ -279,22 +278,34 @@ func (tk *TestKeys) analysisDNSDiffASN(probeAddrs, thAddrs []string) (asns []uin // findAddrsWithoutTLSHandshake computes the list of probe discovered addresses // for which we couldn't successfully perform a TLS handshake. -func (tk *TestKeys) findAddrsWithoutTLSHandshake() (output []string) { +func (tk *TestKeys) findAddrsWithoutTLSHandshake(into []string) (output []string) { const ( resolved = 1 << iota handshakeOK ) mapping := make(map[string]int) + // fill the input map with the addresses we're interested to analyze + for _, addr := range into { + mapping[addr] = 0 + } + // gather all the addrs resolved by the probe for _, query := range tk.Queries { for _, answer := range query.Answers { + var addr string switch answer.AnswerType { case "A": - mapping[answer.IPv4] |= resolved + addr = answer.IPv4 case "AAAA": - mapping[answer.IPv6] |= resolved + addr = answer.IPv6 + default: + continue } + if _, found := mapping[addr]; !found { + continue // we're not interested into this addr + } + mapping[addr] |= resolved } } @@ -307,12 +318,18 @@ func (tk *TestKeys) findAddrsWithoutTLSHandshake() (output []string) { if thx.Failure != nil { continue // this handshake failed } + if _, found := mapping[addr]; !found { + continue // we're not interested into this addr + } mapping[addr] |= handshakeOK } // compute the list of addresses without the handshakeOK flag for addr, flags := range mapping { - if flags&handshakeOK == 0 { + if flags == 0 { + continue // this looks like a bug + } + if (flags & handshakeOK) == 0 { output = append(output, addr) } } From 87f7300112ca3b9eb37a1619edcab4bb71a7af9a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 13:10:34 +0200 Subject: [PATCH 68/83] fix: improve the way in which we log the final status --- .../webconnectivity/analysiscore.go | 33 +++++++++++++++++-- .../webconnectivity/analysishttpcore.go | 25 ++++++++++---- .../webconnectivity/analysishttpdiff.go | 10 +++--- .../webconnectivity/analysistcpip.go | 4 ++- 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go index c17d2466e1..e365d57b02 100644 --- a/internal/experiment/webconnectivity/analysiscore.go +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -27,30 +27,57 @@ const ( // of the experiment once all network tasks have completed. func (tk *TestKeys) analysisToplevel(logger model.Logger) { tk.analysisDNSToplevel(logger) - tk.analysisTCPIPToplevel() - tk.analysisHTTPToplevel() + tk.analysisTCPIPToplevel(logger) + tk.analysisHTTPToplevel(logger) + + accessibleFalse := false + if (tk.BlockingFlags & analysisBlockingDNS) != 0 { + logger.Warnf("BLOCKING: dns => NOT ACCESSIBLE") tk.Blocking = "dns" + tk.Accessible = &accessibleFalse return } + if (tk.BlockingFlags & analysisBlockingTCPIP) != 0 { + logger.Warnf("BLOCKING: tcp_ip => NOT ACCESSIBLE") tk.Blocking = "tcp_ip" + tk.Accessible = &accessibleFalse return } + if (tk.BlockingFlags & analysisBlockingTLSFailure) != 0 { + logger.Warnf("BLOCKING: http-failure (TLS) => NOT ACCESSIBLE") tk.Blocking = "http-failure" // backwards compatibility with the spec + tk.Accessible = &accessibleFalse return } + if (tk.BlockingFlags & analysisBlockingHTTPFailure) != 0 { + logger.Warnf("BLOCKING: http-failure (HTTP) => NOT ACCESSIBLE") tk.Blocking = "http-failure" + tk.Accessible = &accessibleFalse return } + if (tk.BlockingFlags & analysisBlockingHTTPDiff) != 0 { + logger.Warnf("BLOCKING: http-diff => NOT ACCESSIBLE") tk.Blocking = "http-diff" + tk.Accessible = &accessibleFalse return } - if tk.Accessible == nil || !*tk.Accessible { + + if tk.Accessible == nil { + logger.Warnf("ACCESSIBLE: null") return } + + if !*tk.Accessible { + logger.Warnf("ACCESSIBLE: false") // can this happen? + return + } + + logger.Infof("BLOCKING: false") + logger.Infof("ACCESSIBLE: true") tk.Blocking = false } diff --git a/internal/experiment/webconnectivity/analysishttpcore.go b/internal/experiment/webconnectivity/analysishttpcore.go index 79df57e6f0..57bd57d204 100644 --- a/internal/experiment/webconnectivity/analysishttpcore.go +++ b/internal/experiment/webconnectivity/analysishttpcore.go @@ -4,10 +4,13 @@ package webconnectivity // HTTP core analysis // -import "github.com/ooni/probe-cli/v3/internal/netxlite" +import ( + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) // analysisHTTPToplevel is the toplevel analysis function for HTTP results. -func (tk *TestKeys) analysisHTTPToplevel() { +func (tk *TestKeys) analysisHTTPToplevel(logger model.Logger) { // don't perform any analysis if the TH failed if tk.Control == nil { return @@ -21,14 +24,14 @@ func (tk *TestKeys) analysisHTTPToplevel() { // determine whether we had any TLS handshake issue and, in such a case, // declare that we had a case of TLS failure. - if tk.hasWellKnownTLSHandshakeIssues() { + if tk.hasWellKnownTLSHandshakeIssues(logger) { tk.BlockingFlags |= analysisBlockingTLSFailure return } // determine whether we had well known cleartext HTTP round trip issues // and, in such a case, declare we had an "http-failure". - if tk.hasWellKnownHTTPRoundTripIssues() { + if tk.hasWellKnownHTTPRoundTripIssues(logger) { tk.BlockingFlags |= analysisBlockingHTTPFailure return } @@ -47,12 +50,12 @@ func (tk *TestKeys) analysisHTTPToplevel() { } // fallback to the original HTTP diff algorithm. - tk.analysisHTTPDiff(finalRequest, &ctrl) + tk.analysisHTTPDiff(logger, finalRequest, &ctrl) } // hasWellKnownTLSHandshakeIssues returns true in case we observed // a set of well-known issues during the TLS handshake. -func (tk *TestKeys) hasWellKnownTLSHandshakeIssues() bool { +func (tk *TestKeys) hasWellKnownTLSHandshakeIssues(logger model.Logger) bool { for _, thx := range tk.TLSHandshakes { fail := thx.Failure if fail == nil { @@ -65,6 +68,10 @@ func (tk *TestKeys) hasWellKnownTLSHandshakeIssues() bool { netxlite.FailureSSLInvalidHostname, netxlite.FailureSSLInvalidCertificate, netxlite.FailureSSLUnknownAuthority: + logger.Warnf( + "TLS: endpoint %s fails with %s (see #%d)", + thx.Address, *fail, thx.TransactionID, + ) return true default: // check next handshake @@ -75,7 +82,7 @@ func (tk *TestKeys) hasWellKnownTLSHandshakeIssues() bool { // hasWellKnownHTTPRoundTripIssues checks whether any HTTP round // trip failed in a well-known suspicious way -func (tk *TestKeys) hasWellKnownHTTPRoundTripIssues() bool { +func (tk *TestKeys) hasWellKnownHTTPRoundTripIssues(logger model.Logger) bool { for _, rtx := range tk.Requests { fail := rtx.Failure if fail == nil { @@ -88,6 +95,10 @@ func (tk *TestKeys) hasWellKnownHTTPRoundTripIssues() bool { case netxlite.FailureConnectionReset, netxlite.FailureGenericTimeoutError, netxlite.FailureEOFError: + logger.Warnf( + "TLS: endpoint %s fails with %s (see #%d)", + "N/A", *fail, rtx.TransactionID, // TODO(bassosimone): implement + ) return true default: // check next round trip diff --git a/internal/experiment/webconnectivity/analysishttpdiff.go b/internal/experiment/webconnectivity/analysishttpdiff.go index f4517dd44b..8224bc1d9d 100644 --- a/internal/experiment/webconnectivity/analysishttpdiff.go +++ b/internal/experiment/webconnectivity/analysishttpdiff.go @@ -16,7 +16,7 @@ import ( // analysisHTTPDiff computes the HTTP diff between the final request-response // observed by the probe and the TH's result. The caller is responsible of passing // us a valid probe observation and a valid TH observation. -func (tk *TestKeys) analysisHTTPDiff( +func (tk *TestKeys) analysisHTTPDiff(logger model.Logger, probe *model.ArchivalHTTPRequestResult, th *webconnectivity.ControlHTTPRequestResult) { // if we're dealing with an HTTPS request, don't perform any comparison @@ -39,22 +39,20 @@ func (tk *TestKeys) analysisHTTPDiff( if tk.StatusCodeMatch != nil && *tk.StatusCodeMatch { if tk.BodyLengthMatch != nil && *tk.BodyLengthMatch { - tk.Accessible = &accessibleTrue + logger.Infof("HTTP: statusCodeMatch && bodyLengthMatch") return } if tk.HeadersMatch != nil && *tk.HeadersMatch { - tk.Accessible = &accessibleTrue + logger.Infof("HTTP: statusCodeMatch && headersMatch") return } if tk.TitleMatch != nil && *tk.TitleMatch { - tk.Accessible = &accessibleTrue + logger.Infof("HTTP: statusCodeMatch && titleMatch") return } } tk.BlockingFlags |= analysisBlockingHTTPDiff - accessibleFalse := false - tk.Accessible = &accessibleFalse } // httpDiffBodyLengthChecks compares the bodies lengths. diff --git a/internal/experiment/webconnectivity/analysistcpip.go b/internal/experiment/webconnectivity/analysistcpip.go index b065f6b3ec..119263ff23 100644 --- a/internal/experiment/webconnectivity/analysistcpip.go +++ b/internal/experiment/webconnectivity/analysistcpip.go @@ -8,11 +8,12 @@ import ( "fmt" "net" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) // analysisTCPIPToplevel is the toplevel analysis function for TCP/IP results. -func (tk *TestKeys) analysisTCPIPToplevel() { +func (tk *TestKeys) analysisTCPIPToplevel(logger model.Logger) { // if we don't have a control result, do nothing. if tk.Control == nil || len(tk.Control.TCPConnect) <= 0 { return @@ -55,6 +56,7 @@ func (tk *TestKeys) analysisTCPIPToplevel() { entry.Status.Blocked = &isfalse continue } + logger.Warnf("TCP/IP: endpoint %s is blocked (see #%d)", epnt, entry.TransactionID) entry.Status.Blocked = &istrue tk.BlockingFlags |= analysisBlockingTCPIP } From 8e6ad18dc4155d07073ff1ff8d23271144af7bcd Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 14:37:46 +0200 Subject: [PATCH 69/83] fix: move doh specific events into their hier --- .../webconnectivity/dnsresolvers.go | 29 ++++++++++++-- .../experiment/webconnectivity/testkeys.go | 40 ++++++++++++++++++- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 0709c598a0..52bcb20e4a 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -242,14 +242,35 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, URL) addrs, err := reso.LookupHost(lookupCtx, t.Domain) reso.CloseIdleConnections() - t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) - t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) - t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) - t.TestKeys.AppendTLSHandshakes(trace.TLSHandshakes()...) + + // save results making sure we properly splt DoH queries from other queries + doh, other := t.dohSplitQueries(trace.DNSLookupsFromRoundTrip()) + t.TestKeys.Queries = append(t.TestKeys.Queries, doh...) + t.TestKeys.WithTestKeysDoH(func(tkdh *TestKeysDoH) { + tkdh.Queries = append(tkdh.Queries, other...) + tkdh.NetworkEvents = append(tkdh.NetworkEvents, trace.NetworkEvents()...) + tkdh.TCPConnect = append(tkdh.TCPConnect, trace.TCPConnects()...) + tkdh.TLSHandshakes = append(tkdh.TLSHandshakes, trace.TLSHandshakes()...) + }) + ol.Stop(err) out <- addrs // must send something -even nil- to the parent } +// Divides queries generated by DoH in DoH-proper queries and other queries. +func (t *DNSResolvers) dohSplitQueries( + input []*model.ArchivalDNSLookupResult) (doh, other []*model.ArchivalDNSLookupResult) { + for _, query := range input { + switch query.Engine { + case "doh": + doh = append(doh, query) + default: + other = append(other, query) + } + } + return +} + // Returns the DOH resolver URL we should be using by default. func (t *DNSResolvers) dnsOverHTTPSURL() string { if t.DNSOverHTTPSURL != "" { diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index f166339c46..80ad5cd7cd 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -20,6 +20,9 @@ type TestKeys struct { // NetworkEvents contains network events. NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + // DoH contains observations collected by DoH resolvers. + DoH *TestKeysDoH `json:"doh"` + // Queries contains DNS queries. Queries []*model.ArchivalDNSLookupResult `json:"queries"` @@ -96,6 +99,26 @@ type TestKeys struct { mu *sync.Mutex } +// TestKeysDoH contains results collected using DoH. +// +// They are on a separate hierarchy to simplify processing. +type TestKeysDoH struct { + // NetworkEvents contains network events. + NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + + // Queries contains DNS queries. + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + + // Requests contains HTTP results. + Requests []*model.ArchivalHTTPRequestResult `json:"requests"` + + // TCPConnect contains TCP connect results. + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` + + // TLSHandshakes contains TLS handshakes results. + TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` +} + // AppendNetworkEvents appends to NetworkEvents. func (tk *TestKeys) AppendNetworkEvents(v ...*model.ArchivalNetworkEvent) { tk.mu.Lock() @@ -161,11 +184,26 @@ func (tk *TestKeys) SetFundamentalFailure(err error) { tk.mu.Unlock() } +// WithTestKeysDoH calls the given function with the mutex locked passing to +// it as argument the pointer to the DoH field. +func (tk *TestKeys) WithTestKeysDoH(f func(*TestKeysDoH)) { + tk.mu.Lock() + f(tk.DoH) + tk.mu.Unlock() +} + // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { // TODO: here you should initialize all the fields return &TestKeys{ - NetworkEvents: []*model.ArchivalNetworkEvent{}, + NetworkEvents: []*model.ArchivalNetworkEvent{}, + DoH: &TestKeysDoH{ + NetworkEvents: []*model.ArchivalNetworkEvent{}, + Queries: []*model.ArchivalDNSLookupResult{}, + Requests: []*model.ArchivalHTTPRequestResult{}, + TCPConnect: []*model.ArchivalTCPConnectResult{}, + TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, + }, Queries: []*model.ArchivalDNSLookupResult{}, Requests: []*model.ArchivalHTTPRequestResult{}, TCPConnect: []*model.ArchivalTCPConnectResult{}, From 665f0aae29cd18670d31ce85824381a62c55e29c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 14:46:00 +0200 Subject: [PATCH 70/83] feat: also split do53 ancillary queries --- .../webconnectivity/dnsresolvers.go | 26 ++++++++++++++-- .../experiment/webconnectivity/testkeys.go | 30 +++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 52bcb20e4a..42191c58cf 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -206,11 +206,33 @@ func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []str dialer := netxlite.NewDialerWithoutResolver(t.Logger) reso := trace.NewParallelUDPResolver(t.Logger, dialer, udpAddress) addrs, err := reso.LookupHost(lookupCtx, t.Domain) - t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + + // saves the results making sure we split Do53 queries from other queries + do53, other := t.do53SplitQueries(trace.DNSLookupsFromRoundTrip()) + t.TestKeys.AppendQueries(do53...) + t.TestKeys.WithTestKeysDo53(func(tkd *TestKeysDo53) { + tkd.Queries = append(tkd.Queries, other...) + tkd.NetworkEvents = append(tkd.NetworkEvents, trace.NetworkEvents()...) + }) + ol.Stop(err) out <- addrs // must send something -even nil- to the parent } +// Divides queries generated by Do53 in Do53-proper queries and other queries. +func (t *DNSResolvers) do53SplitQueries( + input []*model.ArchivalDNSLookupResult) (do53, other []*model.ArchivalDNSLookupResult) { + for _, query := range input { + switch query.Engine { + case "udp", "tcp": + do53 = append(do53, query) + default: + other = append(other, query) + } + } + return +} + // Returns the UDP resolver we should be using by default. func (t *DNSResolvers) udpAddress() string { if t.UDPAddress != "" { @@ -243,7 +265,7 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha addrs, err := reso.LookupHost(lookupCtx, t.Domain) reso.CloseIdleConnections() - // save results making sure we properly splt DoH queries from other queries + // save results making sure we properly split DoH queries from other queries doh, other := t.dohSplitQueries(trace.DNSLookupsFromRoundTrip()) t.TestKeys.Queries = append(t.TestKeys.Queries, doh...) t.TestKeys.WithTestKeysDoH(func(tkdh *TestKeysDoH) { diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 80ad5cd7cd..552a5463f3 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -20,9 +20,12 @@ type TestKeys struct { // NetworkEvents contains network events. NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` - // DoH contains observations collected by DoH resolvers. + // DoH contains ancillary observations collected by DoH resolvers. DoH *TestKeysDoH `json:"doh"` + // Do53 contains ancillary observations collected by Do53 resolvers. + Do53 *TestKeysDo53 `json:"do53"` + // Queries contains DNS queries. Queries []*model.ArchivalDNSLookupResult `json:"queries"` @@ -99,7 +102,7 @@ type TestKeys struct { mu *sync.Mutex } -// TestKeysDoH contains results collected using DoH. +// TestKeysDoH contains ancillary observations collected using DoH. // // They are on a separate hierarchy to simplify processing. type TestKeysDoH struct { @@ -119,6 +122,17 @@ type TestKeysDoH struct { TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` } +// TestKeysDo53 contains ancillary observations collected using Do53. +// +// They are on a separate hierarchy to simplify processing. +type TestKeysDo53 struct { + // NetworkEvents contains network events. + NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + + // Queries contains DNS queries. + Queries []*model.ArchivalDNSLookupResult `json:"queries"` +} + // AppendNetworkEvents appends to NetworkEvents. func (tk *TestKeys) AppendNetworkEvents(v ...*model.ArchivalNetworkEvent) { tk.mu.Lock() @@ -192,6 +206,14 @@ func (tk *TestKeys) WithTestKeysDoH(f func(*TestKeysDoH)) { tk.mu.Unlock() } +// WithTestKeysDo53 calls the given function with the mutex locked passing to +// it as argument the pointer to the Do53 field. +func (tk *TestKeys) WithTestKeysDo53(f func(*TestKeysDo53)) { + tk.mu.Lock() + f(tk.Do53) + tk.mu.Unlock() +} + // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { // TODO: here you should initialize all the fields @@ -204,6 +226,10 @@ func NewTestKeys() *TestKeys { TCPConnect: []*model.ArchivalTCPConnectResult{}, TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, }, + Do53: &TestKeysDo53{ + NetworkEvents: []*model.ArchivalNetworkEvent{}, + Queries: []*model.ArchivalDNSLookupResult{}, + }, Queries: []*model.ArchivalDNSLookupResult{}, Requests: []*model.ArchivalHTTPRequestResult{}, TCPConnect: []*model.ArchivalTCPConnectResult{}, From d378709e3ed764a4cab0533e2cd63899880e3c78 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 15:27:09 +0200 Subject: [PATCH 71/83] feat: implement detecting DNS interception --- .../webconnectivity/dnsresolvers.go | 33 ++++++++- .../experiment/webconnectivity/dnswhoami.go | 71 +++++++++++++++++++ .../experiment/webconnectivity/testkeys.go | 31 ++++++++ 3 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 internal/experiment/webconnectivity/dnswhoami.go diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 42191c58cf..86270522df 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -90,17 +90,31 @@ func (t *DNSResolvers) run(parentCtx context.Context) []string { systemOut := make(chan []string) udpOut := make(chan []string) httpsOut := make(chan []string) + whoamiSystemV4Out := make(chan []DNSWhoamiInfoEntry) + whoamiUDPv4Out := make(chan []DNSWhoamiInfoEntry) + + udpAddress := t.udpAddress() // start asynchronous lookups go t.lookupHostSystem(parentCtx, systemOut) - go t.lookupHostUDP(parentCtx, udpOut) + go t.lookupHostUDP(parentCtx, udpAddress, udpOut) go t.lookupHostDNSOverHTTPS(parentCtx, httpsOut) + go t.whoamiSystemV4(parentCtx, whoamiSystemV4Out) + go t.whoamiUDPv4(parentCtx, udpAddress, whoamiUDPv4Out) // collect resulting IP addresses (which may be nil/empty lists) systemAddrs := <-systemOut udpAddrs := <-udpOut httpsAddrs := <-httpsOut + // collect whoami results (which also may be nil/empty) + whoamiSystemV4 := <-whoamiSystemV4Out + whoamiUDPv4 := <-whoamiUDPv4Out + t.TestKeys.WithDNSWhoami(func(di *DNSWhoamiInfo) { + di.SystemV4 = whoamiSystemV4 + di.UDPv4[udpAddress] = whoamiUDPv4 + }) + // merge the resolved IP addresses merged := map[string]bool{} for _, addr := range systemAddrs { @@ -157,6 +171,20 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { t.maybeStartControlFlow(parentCtx, addresses) } +// whoamiSystemV4 performs a DNS whoami lookup for the system resolver. +func (t *DNSResolvers) whoamiSystemV4(parentCtx context.Context, out chan<- []DNSWhoamiInfoEntry) { + value, _ := DNSWhoamiSingleton.SystemV4(parentCtx) + t.Logger.Infof("DNS whoami for system resolver: %+v", value) + out <- value +} + +// whoamiUDPv4 performs a DNS whoami lookup for the given UDP resolver. +func (t *DNSResolvers) whoamiUDPv4(parentCtx context.Context, udpAddress string, out chan<- []DNSWhoamiInfoEntry) { + value, _ := DNSWhoamiSingleton.UDPv4(parentCtx, udpAddress) + t.Logger.Infof("DNS whoami for %s/udp resolver: %+v", udpAddress, value) + out <- value +} + // lookupHostSystem performs a DNS lookup using the system resolver. func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout @@ -184,7 +212,7 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- [] } // lookupHostUDP performs a DNS lookup using an UDP resolver. -func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []string) { +func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, udpAddress string, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) @@ -197,7 +225,6 @@ func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []str trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - udpAddress := t.udpAddress() ol := measurexlite.NewOperationLogger( t.Logger, "[#%d] lookup %s using %s", index, t.Domain, udpAddress, ) diff --git a/internal/experiment/webconnectivity/dnswhoami.go b/internal/experiment/webconnectivity/dnswhoami.go new file mode 100644 index 0000000000..456fd5af69 --- /dev/null +++ b/internal/experiment/webconnectivity/dnswhoami.go @@ -0,0 +1,71 @@ +package webconnectivity + +import ( + "context" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// DNSWhoamiService is a service that performs DNS whoami lookups. +type DNSWhoamiService struct { + // mu provides mutual exclusion + mu *sync.Mutex + + // systemv4 contains systemv4 results + systemv4 []DNSWhoamiInfoEntry + + // udpv4 contains udpv4 results + udpv4 map[string][]DNSWhoamiInfoEntry +} + +// SystemV4 returns the results of querying using the system resolver and IPv4. +func (svc *DNSWhoamiService) SystemV4(ctx context.Context) ([]DNSWhoamiInfoEntry, bool) { + svc.mu.Lock() + defer svc.mu.Unlock() + if len(svc.systemv4) <= 0 { + ctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + reso := netxlite.NewStdlibResolver(model.DiscardLogger) + addrs, err := reso.LookupHost(ctx, "whoami.v4.powerdns.org") + if err != nil || len(addrs) < 1 { + return nil, false + } + svc.systemv4 = []DNSWhoamiInfoEntry{{ + Address: addrs[0], + }} + } + return svc.systemv4, len(svc.systemv4) > 0 +} + +// UDPv4 returns the results of querying a given UDP resolver and IPv4. +func (svc *DNSWhoamiService) UDPv4(ctx context.Context, address string) ([]DNSWhoamiInfoEntry, bool) { + svc.mu.Lock() + defer svc.mu.Unlock() + if len(svc.udpv4[address]) <= 0 { + ctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + dialer := netxlite.NewDialerWithStdlibResolver(model.DiscardLogger) + reso := netxlite.NewParallelUDPResolver(model.DiscardLogger, dialer, address) + // TODO(bassosimone): this should actually only send an A query. Sending an AAAA + // query is _way_ unnecessary since we know that only A is going to work. + addrs, err := reso.LookupHost(ctx, "whoami.v4.powerdns.org") + if err != nil || len(addrs) < 1 { + return nil, false + } + svc.udpv4[address] = []DNSWhoamiInfoEntry{{ + Address: addrs[0], + }} + } + value := svc.udpv4[address] + return value, len(value) > 0 +} + +// DNSWhoamiSingleton is the DNSWhoamiService singleton. +var DNSWhoamiSingleton = &DNSWhoamiService{ + mu: &sync.Mutex{}, + systemv4: []DNSWhoamiInfoEntry{}, + udpv4: map[string][]DNSWhoamiInfoEntry{}, +} diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 552a5463f3..4831f3a4c4 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -20,6 +20,10 @@ type TestKeys struct { // NetworkEvents contains network events. NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + // DNSWhoami contains results of using the DNS whoami functionality for the + // possibly cleartext resolvers that we're using. + DNSWoami *DNSWhoamiInfo `json:"dns_whoami"` + // DoH contains ancillary observations collected by DoH resolvers. DoH *TestKeysDoH `json:"doh"` @@ -102,6 +106,21 @@ type TestKeys struct { mu *sync.Mutex } +// DNSWhoamiInfoEntry contains an entry for DNSWhoamiInfo. +type DNSWhoamiInfoEntry struct { + // Address is the IP address + Address string `json:"address"` +} + +// DNSWhoamiInfo contains info about DNS whoami. +type DNSWhoamiInfo struct { + // SystemV4 contains results related to the system resolver using IPv4. + SystemV4 []DNSWhoamiInfoEntry `json:"system_v4"` + + // UDPv4 contains results related to an UDP resolver using IPv4. + UDPv4 map[string][]DNSWhoamiInfoEntry `json:"udp_v4"` +} + // TestKeysDoH contains ancillary observations collected using DoH. // // They are on a separate hierarchy to simplify processing. @@ -214,11 +233,23 @@ func (tk *TestKeys) WithTestKeysDo53(f func(*TestKeysDo53)) { tk.mu.Unlock() } +// WithDNSWhoami calls the given function with the mutex locked passing to +// it as argument the pointer to the DNSWhoami field. +func (tk *TestKeys) WithDNSWhoami(fun func(*DNSWhoamiInfo)) { + tk.mu.Lock() + fun(tk.DNSWoami) + tk.mu.Unlock() +} + // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { // TODO: here you should initialize all the fields return &TestKeys{ NetworkEvents: []*model.ArchivalNetworkEvent{}, + DNSWoami: &DNSWhoamiInfo{ + SystemV4: []DNSWhoamiInfoEntry{}, + UDPv4: map[string][]DNSWhoamiInfoEntry{}, + }, DoH: &TestKeysDoH{ NetworkEvents: []*model.ArchivalNetworkEvent{}, Queries: []*model.ArchivalDNSLookupResult{}, From c62f08c3dc7266f160c6cb8d8c6c5b46f84fc2d5 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 22:39:37 +0200 Subject: [PATCH 72/83] feat: measure new IP addrs discovered by the TH --- .../webconnectivity/cleartextflow.go | 2 +- .../experiment/webconnectivity/control.go | 60 +++++++++++++++++++ .../webconnectivity/dnsresolvers.go | 29 ++++++--- .../experiment/webconnectivity/secureflow.go | 2 +- 4 files changed, 82 insertions(+), 11 deletions(-) diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 642a18c037..78d9f83801 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -42,7 +42,7 @@ type CleartextFlow struct { // Sema is the MANDATORY semaphore to allow just a single // connection to perform the HTTP transaction. - Sema chan any + Sema <-chan any // TestKeys is MANDATORY and contains the TestKeys. TestKeys *TestKeys diff --git a/internal/experiment/webconnectivity/control.go b/internal/experiment/webconnectivity/control.go index 0dfc24fb6b..aa5491ff6c 100644 --- a/internal/experiment/webconnectivity/control.go +++ b/internal/experiment/webconnectivity/control.go @@ -14,12 +14,31 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) +// EndpointMeasurementsStarter is used by Control to start extra +// measurements using new IP addrs discovered by the TH +type EndpointMeasurementsStarter interface { + // startCleartextFlowsWithSema starts a TCP measurement flow for each IP addr. The [sema] + // argument allows to control how many flows are allowed to perform HTTP measurements. Every + // flow will attempt to read from [sema] and won't perform HTTP measurements if a + // nonblocking read fails. Hence, you must create a [sema] channel with buffer equal + // to N and N elements inside it to allow N flows to perform HTTP measuremenets. + startCleartextFlowsWithSema(ctx context.Context, sema <-chan any, addresses []string) + + // startSecureFlowsWithSema starts a TCP+TLS measurement flow for each IP addr. See + // the docs of startCleartextFlowsWithSema for more info on the [sema] arg. + startSecureFlowsWithSema(ctx context.Context, sema <-chan any, addresses []string) +} + // Control issues a control request and saves the results // inside of the experiment's TestKeys. type Control struct { // Addresses contains the MANDATORY addresses we've looked up. Addresses []string + // ExtraMeasurementsStarter is MANDATORY and allows this struct to + // start additional measurements using new TH-discovered addrs. + ExtraMeasurementsStarter EndpointMeasurementsStarter + // Logger is the MANDATORY logger to use. Logger model.Logger @@ -98,7 +117,48 @@ func (c *Control) Run(ctx context.Context) { return } + // if the TH returned us addresses we did not previously were + // aware of, make sure we also measure them + c.maybeStartExtraMeasurements(ctx, cresp.DNS.Addrs) + // on success, save the control response c.TestKeys.SetControl(&cresp) ol.Stop(nil) } + +// This function determines whether we should start new +// background measurements for previously unknown IP addrs. +func (c *Control) maybeStartExtraMeasurements(ctx context.Context, thAddrs []string) { + // determine which addrs are TH only + const ( + inProbe = 1 << iota + inTH + ) + mapping := make(map[string]int) + for _, addr := range c.Addresses { + mapping[addr] |= inProbe + } + for _, addr := range thAddrs { + mapping[addr] |= inTH + } + + // obtain the TH only addresses + var thOnly []string + for addr, flags := range mapping { + if (flags & inProbe) != 0 { + continue // we already measured this + } + thOnly = append(thOnly, addr) + } + + // Start extra measurements for TH only addresses. Because we already + // measured HTTP using IP addrs discovered by the resolvers, we're not + // going to do that again now. I am not sure this is the right policy + // but I think we can just try it and then change if needed... + // + // Also, let's remember that reading from a nil chan blocks forever, so + // we're basically forcing the goroutines to avoid HTTP. + var nohttp chan any = nil + c.ExtraMeasurementsStarter.startCleartextFlowsWithSema(ctx, nohttp, thOnly) + c.ExtraMeasurementsStarter.startSecureFlowsWithSema(ctx, nohttp, thOnly) +} diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 86270522df..8b6bbacf2d 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -330,13 +330,18 @@ func (t *DNSResolvers) dnsOverHTTPSURL() string { // startCleartextFlows starts a TCP measurement flow for each IP addr. func (t *DNSResolvers) startCleartextFlows(ctx context.Context, addresses []string) { + sema := make(chan any, 1) + sema <- true // allow a single flow to fetch the HTTP body + t.startCleartextFlowsWithSema(ctx, sema, addresses) +} + +// startCleartextFlowsWithSema implements EndpointMeasurementsStarter. +func (t *DNSResolvers) startCleartextFlowsWithSema(ctx context.Context, sema <-chan any, addresses []string) { if t.URL.Scheme != "http" { // Do not bother with measuring HTTP when the user // has asked us to measure an HTTPS URL. return } - sema := make(chan any, 1) - sema <- true // allow a single flow to fetch the HTTP body port := "80" if urlPort := t.URL.Port(); urlPort != "" { port = urlPort @@ -373,6 +378,11 @@ func (t *DNSResolvers) startSecureFlows(ctx context.Context, addresses []string) // validate IPs by performing a TLS handshake. sema <- true } + t.startSecureFlowsWithSema(ctx, sema, addresses) +} + +// startSecureFlowsWithSema implements EndpointMeasurementsStarter. +func (t *DNSResolvers) startSecureFlowsWithSema(ctx context.Context, sema <-chan any, addresses []string) { port := "443" if urlPort := t.URL.Port(); urlPort != "" { if t.URL.Scheme != "https" { @@ -411,13 +421,14 @@ func (t *DNSResolvers) startSecureFlows(ctx context.Context, addresses []string) func (t *DNSResolvers) maybeStartControlFlow(ctx context.Context, addresses []string) { if t.Session != nil && t.THAddr != "" { ctrl := &Control{ - Addresses: addresses, - Logger: t.Logger, - TestKeys: t.TestKeys, - Session: t.Session, - THAddr: t.THAddr, - URL: t.URL, - WaitGroup: t.WaitGroup, + Addresses: addresses, + ExtraMeasurementsStarter: t, + Logger: t.Logger, + TestKeys: t.TestKeys, + Session: t.Session, + THAddr: t.THAddr, + URL: t.URL, + WaitGroup: t.WaitGroup, } ctrl.Start(ctx) } diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index 7cb3a7cd7e..acefb7803e 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -43,7 +43,7 @@ type SecureFlow struct { // Sema is the MANDATORY semaphore to allow just a single // connection to perform the HTTP transaction. - Sema chan any + Sema <-chan any // TestKeys is MANDATORY and contains the TestKeys. TestKeys *TestKeys From 35cc9120a271ab91185e17105007a4daf9614125 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 23:18:43 +0200 Subject: [PATCH 73/83] feat: cycle through doh services to measure Also, make it opportunistic rather than measuring each time. So, we'll have some more measurements. --- .../webconnectivity/cleartextflow.go | 31 ++++---- .../webconnectivity/dnsresolvers.go | 76 +++++++++++++++---- .../experiment/webconnectivity/measurer.go | 27 ++++--- .../experiment/webconnectivity/secureflow.go | 31 ++++---- 4 files changed, 100 insertions(+), 65 deletions(-) diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 78d9f83801..25a49f85b4 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -56,10 +56,6 @@ type CleartextFlow struct { // CookieJar contains the OPTIONAL cookie jar, used for redirects. CookieJar http.CookieJar - // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not - // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). - DNSOverHTTPSURL string - // FollowRedirects is OPTIONAL and instructs this flow // to follow HTTP redirects (if any). FollowRedirects bool @@ -244,20 +240,19 @@ func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Res } t.Logger.Infof("redirect to: %s", location.String()) resolvers := &DNSResolvers{ - CookieJar: t.CookieJar, - DNSCache: t.DNSCache, - Domain: location.Hostname(), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - TestKeys: t.TestKeys, - URL: location, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - DNSOverHTTPSURL: t.DNSOverHTTPSURL, - Referer: resp.Request.URL.String(), - Session: nil, // no need to issue another control request - THAddr: "", // ditto - UDPAddress: t.UDPAddress, + CookieJar: t.CookieJar, + DNSCache: t.DNSCache, + Domain: location.Hostname(), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + URL: location, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + Referer: resp.Request.URL.String(), + Session: nil, // no need to issue another control request + THAddr: "", // ditto + UDPAddress: t.UDPAddress, } resolvers.Start(ctx) default: diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 8b6bbacf2d..91c6b2d863 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -9,6 +9,7 @@ package webconnectivity import ( "context" + "math/rand" "net" "net/http" "net/url" @@ -54,10 +55,6 @@ type DNSResolvers struct { // CookieJar contains the OPTIONAL cookie jar, used for redirects. CookieJar http.CookieJar - // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not - // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). - DNSOverHTTPSURL string - // Referer contains the OPTIONAL referer, used for redirects. Referer string @@ -268,8 +265,68 @@ func (t *DNSResolvers) udpAddress() string { return "8.8.4.4:53" } +// OpportunisticDNSOverHTTPS allows to perform opportunistic DNS-over-HTTPS +// measurements as part of Web Connectivity. +type OpportunisticDNSOverHTTPS struct { + // interval is the next interval after which to measure. + interval time.Duration + + // mu provides mutual exclusion + mu *sync.Mutex + + // rnd is the random number generator to use. + rnd *rand.Rand + + // t is when we last run an opportunistic measurement. + t time.Time + + // urls contains the urls of known DoH services. + urls []string +} + +// MaybeNextURL returns the next URL to measure, if any. Our aim is to perform +// some opportunistic DoH measurements as part of Web Connectivity. +func (o *OpportunisticDNSOverHTTPS) MaybeNextURL() (string, bool) { + now := time.Now() + o.mu.Lock() + defer o.mu.Unlock() + if o.t.IsZero() || now.Sub(o.t) > o.interval { + o.rnd.Shuffle(len(o.urls), func(i, j int) { + o.urls[i], o.urls[j] = o.urls[j], o.urls[i] + }) + o.t = now + o.interval = time.Duration(20+o.rnd.Uint32()%20) * time.Second + return o.urls[0], true + } + return "", false +} + +// OpportunisticDNSOverHTTPSSingleton is the singleton used to keep +// track of opportunistic DNS-over-HTTPS measurements. +var OpportunisticDNSOverHTTPSSingleton = &OpportunisticDNSOverHTTPS{ + interval: 0, + mu: &sync.Mutex{}, + rnd: rand.New(rand.NewSource(time.Now().UnixNano())), + t: time.Time{}, + urls: []string{ + "https://mozilla.cloudflare-dns.com/dns-query", + "https://dns.nextdns.io/dns-query", + "https://dns.google/dns-query", + "https://dns.quad9.net/dns-query", + }, +} + // lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { + // obtain an opportunistic DoH URL + URL, good := OpportunisticDNSOverHTTPSSingleton.MaybeNextURL() + if !good { + // no need to perform opportunistic DoH at this time but we still + // need to fake out a lookup to please our caller + out <- []string{} + return + } + // create context with attached a timeout const timeout = 4 * time.Second lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) @@ -282,7 +339,6 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - URL := t.dnsOverHTTPSURL() ol := measurexlite.NewOperationLogger( t.Logger, "[#%d] lookup %s using %s", index, t.Domain, URL, ) @@ -320,14 +376,6 @@ func (t *DNSResolvers) dohSplitQueries( return } -// Returns the DOH resolver URL we should be using by default. -func (t *DNSResolvers) dnsOverHTTPSURL() string { - if t.DNSOverHTTPSURL != "" { - return t.DNSOverHTTPSURL - } - return "https://mozilla.cloudflare-dns.com/dns-query" -} - // startCleartextFlows starts a TCP measurement flow for each IP addr. func (t *DNSResolvers) startCleartextFlows(ctx context.Context, addresses []string) { sema := make(chan any, 1) @@ -357,7 +405,6 @@ func (t *DNSResolvers) startCleartextFlowsWithSema(ctx context.Context, sema <-c ZeroTime: t.ZeroTime, WaitGroup: t.WaitGroup, CookieJar: t.CookieJar, - DNSOverHTTPSURL: t.DNSOverHTTPSURL, FollowRedirects: t.URL.Scheme == "http", HostHeader: t.URL.Host, Referer: t.Referer, @@ -404,7 +451,6 @@ func (t *DNSResolvers) startSecureFlowsWithSema(ctx context.Context, sema <-chan WaitGroup: t.WaitGroup, ALPN: []string{"h2", "http/1.1"}, CookieJar: t.CookieJar, - DNSOverHTTPSURL: t.DNSOverHTTPSURL, FollowRedirects: t.URL.Scheme == "https", SNI: t.URL.Hostname(), HostHeader: t.URL.Host, diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index f84592aced..e2e12a5512 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -101,20 +101,19 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // start background tasks resos := &DNSResolvers{ - DNSCache: NewDNSCache(), - Domain: URL.Hostname(), - IDGenerator: idGenerator, - Logger: sess.Logger(), - TestKeys: tk, - URL: URL, - ZeroTime: measurement.MeasurementStartTimeSaved, - WaitGroup: wg, - CookieJar: jar, - DNSOverHTTPSURL: "", - Referer: "", - Session: sess, - THAddr: thAddr, - UDPAddress: "", + DNSCache: NewDNSCache(), + Domain: URL.Hostname(), + IDGenerator: idGenerator, + Logger: sess.Logger(), + TestKeys: tk, + URL: URL, + ZeroTime: measurement.MeasurementStartTimeSaved, + WaitGroup: wg, + CookieJar: jar, + Referer: "", + Session: sess, + THAddr: thAddr, + UDPAddress: "", } resos.Start(ctx) diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index acefb7803e..dec91e7118 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -60,10 +60,6 @@ type SecureFlow struct { // CookieJar contains the OPTIONAL cookie jar, used for redirects. CookieJar http.CookieJar - // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not - // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). - DNSOverHTTPSURL string - // FollowRedirects is OPTIONAL and instructs this flow // to follow HTTP redirects (if any). FollowRedirects bool @@ -296,20 +292,19 @@ func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Respon } t.Logger.Infof("redirect to: %s", location.String()) resolvers := &DNSResolvers{ - CookieJar: t.CookieJar, - DNSCache: t.DNSCache, - Domain: location.Hostname(), - IDGenerator: t.IDGenerator, - Logger: t.Logger, - TestKeys: t.TestKeys, - URL: location, - ZeroTime: t.ZeroTime, - WaitGroup: t.WaitGroup, - DNSOverHTTPSURL: t.DNSOverHTTPSURL, - Referer: resp.Request.URL.String(), - Session: nil, // no need to issue another control request - THAddr: "", // ditto - UDPAddress: t.UDPAddress, + CookieJar: t.CookieJar, + DNSCache: t.DNSCache, + Domain: location.Hostname(), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + URL: location, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + Referer: resp.Request.URL.String(), + Session: nil, // no need to issue another control request + THAddr: "", // ditto + UDPAddress: t.UDPAddress, } resolvers.Start(ctx) default: From 36c44e7e66012388e67f48e8c79896b54fd283ab Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 23:21:41 +0200 Subject: [PATCH 74/83] mention one more todo --- internal/experiment/webconnectivity/dnswhoami.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/experiment/webconnectivity/dnswhoami.go b/internal/experiment/webconnectivity/dnswhoami.go index 456fd5af69..1d7b9f38e8 100644 --- a/internal/experiment/webconnectivity/dnswhoami.go +++ b/internal/experiment/webconnectivity/dnswhoami.go @@ -9,6 +9,11 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) +// TODO(bassosimone): this code needs refining before we can merge it inside +// master. For one, we already have systemv4 info. Additionally, it would +// be neat to avoid additional AAAA queries. Furthermore, we should also see +// to implement support for IPv6 only clients as well. + // DNSWhoamiService is a service that performs DNS whoami lookups. type DNSWhoamiService struct { // mu provides mutual exclusion From 9214c4b88eee9239fe8c6e0399468ce65c4937f2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 15 Aug 2022 23:25:50 +0200 Subject: [PATCH 75/83] fix: start extra measurements with parent ctx --- internal/experiment/webconnectivity/control.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/experiment/webconnectivity/control.go b/internal/experiment/webconnectivity/control.go index aa5491ff6c..ba66cdc7df 100644 --- a/internal/experiment/webconnectivity/control.go +++ b/internal/experiment/webconnectivity/control.go @@ -68,10 +68,10 @@ func (c *Control) Start(ctx context.Context) { } // Run runs this task until completion. -func (c *Control) Run(ctx context.Context) { +func (c *Control) Run(parentCtx context.Context) { // create a subcontext attached to a maximum timeout const timeout = 30 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) + opCtx, cancel := context.WithTimeout(parentCtx, timeout) defer cancel() // create control request @@ -108,7 +108,7 @@ func (c *Control) Run(ctx context.Context) { // issue the control request and wait for the response var cresp webconnectivity.ControlResponse - err := clnt.PostJSON(ctx, "/", creq, &cresp) + err := clnt.PostJSON(opCtx, "/", creq, &cresp) if err != nil { // make sure error is wrapped err = netxlite.NewTopLevelGenericErrWrapper(err) @@ -119,7 +119,7 @@ func (c *Control) Run(ctx context.Context) { // if the TH returned us addresses we did not previously were // aware of, make sure we also measure them - c.maybeStartExtraMeasurements(ctx, cresp.DNS.Addrs) + c.maybeStartExtraMeasurements(parentCtx, cresp.DNS.Addrs) // on success, save the control response c.TestKeys.SetControl(&cresp) From 4f50590a019df280b31b12daff60765af3a09590 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 16 Aug 2022 07:59:04 +0200 Subject: [PATCH 76/83] changes after code review --- .../boilerplate/experiment/inputparser.go.txt | 2 +- .../webconnectivity/analysiscore.go | 165 ++++++++++++------ .../experiment/webconnectivity/analysisdns.go | 137 +++++++++------ .../webconnectivity/analysishttpcore.go | 52 ++++-- .../webconnectivity/analysishttpdiff.go | 38 +++- .../webconnectivity/analysistcpip.go | 22 ++- .../webconnectivity/cleartextflow.go | 21 ++- .../experiment/webconnectivity/control.go | 41 +++-- .../experiment/webconnectivity/dnscache.go | 2 +- .../webconnectivity/dnsresolvers.go | 44 +++-- .../experiment/webconnectivity/dnswhoami.go | 3 + .../experiment/webconnectivity/inputparser.go | 2 +- .../experiment/webconnectivity/secureflow.go | 21 ++- .../experiment/webconnectivity/summary.go | 2 +- .../experiment/webconnectivity/testkeys.go | 23 +-- 15 files changed, 387 insertions(+), 188 deletions(-) diff --git a/internal/cmd/boilerplate/experiment/inputparser.go.txt b/internal/cmd/boilerplate/experiment/inputparser.go.txt index 8619d79cf8..f37e0b2cc0 100644 --- a/internal/cmd/boilerplate/experiment/inputparser.go.txt +++ b/internal/cmd/boilerplate/experiment/inputparser.go.txt @@ -44,7 +44,7 @@ func (ip *InputParser) Parse(input string) (*url.URL, error) { return nil, errors.New("cannot parse input") } -// Conditionally allows endpooints when ip.AllowEndpoints is true. +// Conditionally allows endpoints when ip.AllowEndpoints is true. func (ip *InputParser) maybeAllowEndpoints(URL *url.URL, err error) (*url.URL, error) { runtimex.PanicIfNil(err, "expected to be called with a non-nil error") if ip.AllowEndpoints && URL.Scheme != "" && URL.Opaque != "" && URL.User == nil && diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go index e365d57b02..023954446f 100644 --- a/internal/experiment/webconnectivity/analysiscore.go +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -6,78 +6,139 @@ import "github.com/ooni/probe-cli/v3/internal/model" // Core analysis // +// These flags determine the context of TestKeys.Blocking. However, while .Blocking +// is an enumeration, these flags allow to describe multiple blocking methods. const ( - // analysisBlockingDNS indicates there's blocking at the DNS level. - analysisBlockingDNS = 1 << iota + // analysisFlagDNSBlocking indicates there's blocking at the DNS level. + analysisFlagDNSBlocking = 1 << iota - // analysisBlockingTCPIP indicates there's blocking at the TCP/IP level. - analysisBlockingTCPIP + // analysisFlagTCPIPBlocking indicates there's blocking at the TCP/IP level. + analysisFlagTCPIPBlocking - // analysisBlockingTLSFailure indicates there were TLS issues. - analysisBlockingTLSFailure + // analysisFlagTLSBlocking indicates there were TLS issues. + analysisFlagTLSBlocking - // analysisBlockingHTTPFailure indicates there was an HTTP failure. - analysisBlockingHTTPFailure + // analysisFlagHTTPBlocking indicates there was an HTTP failure. + analysisFlagHTTPBlocking - // analysisBlockingHTTPDiff indicates there's an HTTP diff. - analysisBlockingHTTPDiff + // analysisFlagHTTPDiff indicates there's an HTTP diff. + analysisFlagHTTPDiff + + // analysisFlagSuccess indicates we did not detect any blocking. + analysisFlagSuccess ) // analysisToplevel is the toplevel function that analyses the results // of the experiment once all network tasks have completed. +// +// The ultimate objective of this function is to set the toplevel flags +// used by the backend to score results. These flags are: +// +// - blocking (and x_blocking_flags) which contain information about +// the detected blocking method (or methods); +// +// - accessible which contains information on whether we think we +// could access the resource somehow. +// +// Originally, Web Connectivity only had a blocking scalar value so +// we could see ourselves in one of the following cases: +// +// +----------+------------+--------------------------+ +// | Blocking | Accessible | Meaning | +// +----------+------------+--------------------------+ +// | null | null | Probe analysis error | +// +----------+------------+--------------------------+ +// | false | true | We detected no blocking | +// +----------+------------+--------------------------+ +// | "..." | false | We detected blocking | +// +----------+------------+--------------------------+ +// +// While it would be possible in this implementation, which has a granular +// definition of blocking (x_blocking_flags), to set accessible to mean +// whether we could access the resource in some conditions, it seems quite +// dangerous to deviate from the original behavior. +// +// Our code will NEVER set .Blocking or .Accessible outside of this function +// and we'll instead rely on XBlockingFlags. This function's job is to call +// other functions that compute the .XBlockingFlags and then to assign the value +// of .Blocking and .Accessible from the .XBlockingFlags value. +// +// Accordingly, this is how we map the value of the .XBlockingFlags to the +// values of .Blocking and .Accessible: +// +// +--------------------------------------+----------------+-------------+ +// | XBlockingFlags | .Blocking | .Accessible | +// +--------------------------------------+----------------+-------------+ +// | (& DNSBlocking) != 0 | "dns" | false | +// +--------------------------------------+----------------+-------------+ +// | (& TCPIPBlocking) != 0 | "tcp_ip" | false | +// +--------------------------------------+----------------+-------------+ +// | (& (TLSBlocking|HTTPBlocking)) != 0 | "http-failure" | false | +// +--------------------------------------+----------------+-------------+ +// | (& HTTPDiff) != 0 | "http-diff" | false | +// +--------------------------------------+----------------+-------------+ +// | == FlagSuccess | false | true | +// +--------------------------------------+----------------+-------------+ +// | otherwise | null | null | +// +--------------------------------------+----------------+-------------+ +// +// It's a very simple rule, that should preserve previous semantics. func (tk *TestKeys) analysisToplevel(logger model.Logger) { + // Since we run after all tasks have completed (or so we assume) we're + // not going to use any form of locking here. + + // these functions compute the value of XBlockingFlags tk.analysisDNSToplevel(logger) tk.analysisTCPIPToplevel(logger) tk.analysisHTTPToplevel(logger) - accessibleFalse := false - - if (tk.BlockingFlags & analysisBlockingDNS) != 0 { - logger.Warnf("BLOCKING: dns => NOT ACCESSIBLE") + // now, let's determine .Accessible and .Blocking + switch { + case (tk.XBlockingFlags & analysisFlagDNSBlocking) != 0: tk.Blocking = "dns" - tk.Accessible = &accessibleFalse - return - } + tk.Accessible = false + logger.Warnf( + "ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + tk.XBlockingFlags, tk.Accessible, tk.Blocking, + ) - if (tk.BlockingFlags & analysisBlockingTCPIP) != 0 { - logger.Warnf("BLOCKING: tcp_ip => NOT ACCESSIBLE") + case (tk.XBlockingFlags & analysisFlagTCPIPBlocking) != 0: tk.Blocking = "tcp_ip" - tk.Accessible = &accessibleFalse - return - } + tk.Accessible = false + logger.Warnf( + "ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + tk.XBlockingFlags, tk.Accessible, tk.Blocking, + ) - if (tk.BlockingFlags & analysisBlockingTLSFailure) != 0 { - logger.Warnf("BLOCKING: http-failure (TLS) => NOT ACCESSIBLE") - tk.Blocking = "http-failure" // backwards compatibility with the spec - tk.Accessible = &accessibleFalse - return - } - - if (tk.BlockingFlags & analysisBlockingHTTPFailure) != 0 { - logger.Warnf("BLOCKING: http-failure (HTTP) => NOT ACCESSIBLE") + case (tk.XBlockingFlags & (analysisFlagTLSBlocking | analysisFlagHTTPBlocking)) != 0: tk.Blocking = "http-failure" - tk.Accessible = &accessibleFalse - return - } + tk.Accessible = false + logger.Warnf("ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + tk.XBlockingFlags, tk.Accessible, tk.Blocking, + ) - if (tk.BlockingFlags & analysisBlockingHTTPDiff) != 0 { - logger.Warnf("BLOCKING: http-diff => NOT ACCESSIBLE") + case (tk.XBlockingFlags & analysisFlagHTTPDiff) != 0: tk.Blocking = "http-diff" - tk.Accessible = &accessibleFalse - return + tk.Accessible = false + logger.Warnf( + "ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + tk.XBlockingFlags, tk.Accessible, tk.Blocking, + ) + + case tk.XBlockingFlags == analysisFlagSuccess: + tk.Blocking = false + tk.Accessible = true + logger.Infof( + "SUCCESS: flags=%d accessible=%+v, blocking=%%+v", + tk.XBlockingFlags, tk.Accessible, tk.Blocking, + ) + + default: + tk.Blocking = nil + tk.Accessible = nil + logger.Warnf( + "UNKNOWN: flags=%d, accessible=%+v, blocking=%+v", + tk.XBlockingFlags, tk.Accessible, tk.Blocking, + ) } - - if tk.Accessible == nil { - logger.Warnf("ACCESSIBLE: null") - return - } - - if !*tk.Accessible { - logger.Warnf("ACCESSIBLE: false") // can this happen? - return - } - - logger.Infof("BLOCKING: false") - logger.Infof("ACCESSIBLE: true") - tk.Blocking = false } diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go index bfa40d296c..cfb33f4f01 100644 --- a/internal/experiment/webconnectivity/analysisdns.go +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -27,21 +27,45 @@ const ( ) // analysisDNSToplevel is the toplevel analysis function for DNS results. +// +// The goals of this function are the following: +// +// 1. Set the legacy .DNSExperimentFailure field to the failure value of the +// first DNS query that failed among the ones we actually tried. Because we +// have multiple queries, unfortunately we are forced to pick one error among +// possibly many to assign to this field. This is why I consider it legacy. +// +// 2. Compute the XDNSFlags value. +// +// From the XDNSFlags value, we determine, in turn DNSConsistency and +// XBlockingFlags according to the following decision table: +// +// +-----------+----------------+---------------------+ +// | XDNSFlags | DNSConsistency | XBlockingFlags | +// +-----------+----------------+---------------------+ +// | 0 | "consistent" | no change | +// +-----------+----------------+---------------------+ +// | nonzero | "inconsistent" | set FlagDNSBlocking | +// +-----------+----------------+---------------------+ +// +// We explain how XDNSFlags is determined in the documentation of +// the functions that this function calls to do its job. func (tk *TestKeys) analysisDNSToplevel(logger model.Logger) { tk.analysisDNSExperimentFailure() tk.analysisDNSBogon(logger) tk.analysisDNSUnexpectedFailure(logger) tk.analysisDNSUnexpectedAddrs(logger) - tk.DNSConsistency = "consistent" - if tk.DNSFlags != 0 { + if tk.XDNSFlags != 0 { logger.Warnf("DNSConsistency: inconsistent") tk.DNSConsistency = "inconsistent" - tk.BlockingFlags |= analysisBlockingDNS + tk.XBlockingFlags |= analysisFlagDNSBlocking + } else { + logger.Warnf("DNSConsistency: consistent") + tk.DNSConsistency = "consistent" } } -// analysisDNSExperimentFailure indicates whether there was any DNS -// experiment failure by inspecting all the queries. +// analysisDNSExperimentFailure sets the legacy DNSExperimentFailure field. func (tk *TestKeys) analysisDNSExperimentFailure() { for _, query := range tk.Queries { if fail := query.Failure; fail != nil { @@ -56,7 +80,8 @@ func (tk *TestKeys) analysisDNSExperimentFailure() { } } -// analysisDNSBogon computes the AnalysisDNSBogon flag. +// analysisDNSBogon computes the AnalysisDNSBogon flag. We set this flag if +// we dectect any bogon in the .Queries field of the TestKeys. func (tk *TestKeys) analysisDNSBogon(logger model.Logger) { for _, query := range tk.Queries { for _, answer := range query.Answers { @@ -64,14 +89,14 @@ func (tk *TestKeys) analysisDNSBogon(logger model.Logger) { case "A": if net.ParseIP(answer.IPv4) != nil && netxlite.IsBogon(answer.IPv4) { logger.Warnf("BOGON: %s in #%d", answer.IPv4, query.TransactionID) - tk.DNSFlags |= AnalysisDNSBogon - return + tk.XDNSFlags |= AnalysisDNSBogon + // continue processing so we print all the bogons we have } case "AAAA": if net.ParseIP(answer.IPv6) != nil && netxlite.IsBogon(answer.IPv6) { logger.Warnf("BOGON: %s in #%d", answer.IPv6, query.TransactionID) - tk.DNSFlags |= AnalysisDNSBogon - return + tk.XDNSFlags |= AnalysisDNSBogon + // continue processing so we print all the bogons we have } default: // nothing @@ -80,19 +105,20 @@ func (tk *TestKeys) analysisDNSBogon(logger model.Logger) { } } -// analysisDNSUnexpectedFailure computes the AnalysisDNSUnexpectedFailure flags. +// analysisDNSUnexpectedFailure computes the AnalysisDNSUnexpectedFailure flags. We say +// a failure is unexpected when the TH could resolve a domain and the probe couldn't. func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { // make sure we have control before proceeding futher if tk.Control == nil || tk.controlRequest == nil { return } - // obtain request and response as shortcuts - request := tk.controlRequest - response := tk.Control + // obtain thRequest and thResponse as shortcuts + thRequest := tk.controlRequest + thResponse := tk.Control // obtain the domain that the TH has queried for - URL, err := url.Parse(request.HTTPRequest) + URL, err := url.Parse(thRequest.HTTPRequest) if err != nil { return // this looks like a bug } @@ -103,19 +129,18 @@ func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { return } - // we mostly care of whether the control's DNS got back - // any IP address because this is a sign that we had - // unexpected DNS issues locally. - hasAddrs := len(response.DNS.Addrs) > 0 + // if the control didn't lookup any IP addresses our job here is done + // because we can't say whether we have unexpected failures + hasAddrs := len(thResponse.DNS.Addrs) > 0 if !hasAddrs { return } - // therefore, any local query _for the same domain_ queried + // with TH-resolved addrs, any local query _for the same domain_ queried // by the probe that contains an error is suspicious for _, query := range tk.Queries { if domain != query.Hostname { - continue + continue // not the domain queried by the test helper } hasAddrs := false Loop: @@ -133,7 +158,7 @@ func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { } if query.Failure == nil { // we expect to see a failure if we don't see - // answers, so this seems a bug + // answers, so this seems a bug? continue } if query.QueryType == "AAAA" && *query.Failure == netxlite.FailureDNSNoAnswer { @@ -142,24 +167,26 @@ func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { continue } logger.Warnf("DNS: unexpected failure %s in #%d", *query.Failure, query.TransactionID) - tk.DNSFlags |= AnalysisDNSUnexpectedFailure - return + tk.XDNSFlags |= AnalysisDNSUnexpectedFailure + // continue processing so we print all the unexpected failures } } -// analysisDNSUnexpectedAddrs computes the AnalysisDNSUnexpectedAddrs flags. +// analysisDNSUnexpectedAddrs computes the AnalysisDNSUnexpectedAddrs flags. This +// algorithm builds upon the original DNSDiff algorithm by introducing an additional +// TLS based heuristic for determining whether an IP address was legit. func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { // make sure we have control before proceeding futher if tk.Control == nil || tk.controlRequest == nil { return } - // obtain request and response as shortcuts - request := tk.controlRequest - response := tk.Control + // obtain thRequest and thResponse as shortcuts + thRequest := tk.controlRequest + thResponse := tk.Control // obtain the domain that the TH has queried for - URL, err := url.Parse(request.HTTPRequest) + URL, err := url.Parse(thRequest.HTTPRequest) if err != nil { return // this looks like a bug } @@ -170,10 +197,9 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { return } - // we mostly care of whether the control's DNS got back - // any IP address because this is a sign that we had - // unexpected DNS issues locally. - thAddrs := response.DNS.Addrs + // if the control didn't resolve any IP address, then we basically + // cannot run this algorithm at all + thAddrs := thResponse.DNS.Addrs if len(thAddrs) <= 0 { return } @@ -183,7 +209,7 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { var probeAddrs []string for _, query := range tk.Queries { if domain != query.Hostname { - continue + continue // not the domain the TH queried for } for _, answer := range query.Answers { switch answer.AnswerType { @@ -199,7 +225,7 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { // definitely suspicious and counts as a difference if len(probeAddrs) <= 0 { logger.Warnf("DNS: no IP address resolved by the probe") - tk.DNSFlags |= AnalysisDNSUnexpectedAddrs + tk.XDNSFlags |= AnalysisDNSUnexpectedAddrs return } @@ -210,24 +236,26 @@ func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { return } - // if the different addrs have the same ASN of addrs resolved by - // the TH, then we say everything is still fine. - differentASNS := tk.analysisDNSDiffASN(differentAddrs, thAddrs) - if len(differentASNS) <= 0 { + // now, let's exclude the differentAddrs for which we successfully + // completed a TLS handshake: those should be good addrs + withoutHandshake := tk.findAddrsWithoutTLSHandshake(domain, differentAddrs) + if len(withoutHandshake) <= 0 { return } - withoutHandshake := tk.findAddrsWithoutTLSHandshake(differentAddrs) - if len(withoutHandshake) <= 0 { + // as a last resort, accept the addresses without an handshake whose + // ASN overlaps with ASNs resolved by the TH + differentASNs := tk.analysisDNSDiffASN(withoutHandshake, thAddrs) + if len(differentASNs) <= 0 { return } // otherwise, conclude we have unexpected probe addrs logger.Warnf( - "DNS: differentAddrs: %+v; differentASNs: %+v; withoutHandshake: %+v", - differentAddrs, differentASNS, withoutHandshake, + "DNSDiff: differentAddrs: %+v; withoutHandshake: %+v; differentASNs: %+v", + differentAddrs, withoutHandshake, differentASNs, ) - tk.DNSFlags |= AnalysisDNSUnexpectedAddrs + tk.XDNSFlags |= AnalysisDNSUnexpectedAddrs } // analysisDNSDiffAddrs returns all the IP addresses that are @@ -245,7 +273,7 @@ func (tk *TestKeys) analysisDNSDiffAddrs(probeAddrs, thAddrs []string) (diff []s mapping[addr] = inTH } for addr, where := range mapping { - if where&inTH == 0 { + if (where & inTH) == 0 { diff = append(diff, addr) } } @@ -269,16 +297,16 @@ func (tk *TestKeys) analysisDNSDiffASN(probeAddrs, thAddrs []string) (asns []uin mapping[asn] |= inTH // including the zero ASN that means unknown } for asn, where := range mapping { - if where&inTH == 0 { + if (where & inTH) == 0 { asns = append(asns, asn) } } return } -// findAddrsWithoutTLSHandshake computes the list of probe discovered addresses -// for which we couldn't successfully perform a TLS handshake. -func (tk *TestKeys) findAddrsWithoutTLSHandshake(into []string) (output []string) { +// findAddrsWithoutTLSHandshake computes the list of probe discovered [addresses] +// for which we couldn't successfully perform a TLS handshake for the given [domain]. +func (tk *TestKeys) findAddrsWithoutTLSHandshake(domain string, addresses []string) (output []string) { const ( resolved = 1 << iota handshakeOK @@ -286,11 +314,11 @@ func (tk *TestKeys) findAddrsWithoutTLSHandshake(into []string) (output []string mapping := make(map[string]int) // fill the input map with the addresses we're interested to analyze - for _, addr := range into { + for _, addr := range addresses { mapping[addr] = 0 } - // gather all the addrs resolved by the probe + // flag the subset of addresses resolved by the probe for _, query := range tk.Queries { for _, answer := range query.Answers { var addr string @@ -309,7 +337,7 @@ func (tk *TestKeys) findAddrsWithoutTLSHandshake(into []string) (output []string } } - // gather all the addrs with successful handshake + // flag the subset of addrs with successful handshake for the right SNI for _, thx := range tk.TLSHandshakes { addr, _, err := net.SplitHostPort(thx.Address) if err != nil { @@ -321,6 +349,9 @@ func (tk *TestKeys) findAddrsWithoutTLSHandshake(into []string) (output []string if _, found := mapping[addr]; !found { continue // we're not interested into this addr } + if thx.ServerName != domain { + continue // the SNI is different, so... + } mapping[addr] |= handshakeOK } @@ -329,7 +360,7 @@ func (tk *TestKeys) findAddrsWithoutTLSHandshake(into []string) (output []string if flags == 0 { continue // this looks like a bug } - if (flags & handshakeOK) == 0 { + if (flags & (resolved | handshakeOK)) == resolved { output = append(output, addr) } } diff --git a/internal/experiment/webconnectivity/analysishttpcore.go b/internal/experiment/webconnectivity/analysishttpcore.go index 57bd57d204..467796e32f 100644 --- a/internal/experiment/webconnectivity/analysishttpcore.go +++ b/internal/experiment/webconnectivity/analysishttpcore.go @@ -10,6 +10,22 @@ import ( ) // analysisHTTPToplevel is the toplevel analysis function for HTTP results. +// +// This function's job is to determine whether there were unexpected TLS +// handshake results (compared to what the TH observed), or unexpected +// failures during HTTP round trips (using the TH as benchmark), or whether +// the obtained body differs from the one obtained by the TH. +// +// This results in possibly setting these XBlockingFlags: +// +// - analysisFlagTLSBlocking +// +// - analysisFlagHTTPBlocking +// +// - analysisFlagHTTPDiff +// +// In websteps fashion, we don't stop at the first failure, rather we +// process all the available data and evaluate all possible errors. func (tk *TestKeys) analysisHTTPToplevel(logger model.Logger) { // don't perform any analysis if the TH failed if tk.Control == nil { @@ -23,17 +39,23 @@ func (tk *TestKeys) analysisHTTPToplevel(logger model.Logger) { } // determine whether we had any TLS handshake issue and, in such a case, - // declare that we had a case of TLS failure. + // declare that we had a case of "http-failure" through TLS. + // + // Note that this would eventually count as an "http-failure" for .Blocking + // because Web Connectivity did not have a concept of TLS based blocking. + // + // This check works ~reliably as long as we ensure to put DoH TLS + // handshakes outside of the main .TLSHandshakes field. if tk.hasWellKnownTLSHandshakeIssues(logger) { - tk.BlockingFlags |= analysisBlockingTLSFailure - return + tk.XBlockingFlags |= analysisFlagTLSBlocking + // continue processing } // determine whether we had well known cleartext HTTP round trip issues // and, in such a case, declare we had an "http-failure". if tk.hasWellKnownHTTPRoundTripIssues(logger) { - tk.BlockingFlags |= analysisBlockingHTTPFailure - return + tk.XBlockingFlags |= analysisFlagHTTPBlocking + // continue processing } // if we don't have any request to check, there's not much more we @@ -42,20 +64,22 @@ func (tk *TestKeys) analysisHTTPToplevel(logger model.Logger) { return } - // if the request has failed in any other way, we don't know. The first entry in the - // tk.Requests array is the last entry that was measured. + // if the request has failed in any other way, we don't know. By convention, the first + // entry in the tk.Requests array is the last entry that was measured. finalRequest := tk.Requests[0] if finalRequest.Failure != nil { return } - // fallback to the original HTTP diff algorithm. + // fallback to the HTTP diff algo. tk.analysisHTTPDiff(logger, finalRequest, &ctrl) } // hasWellKnownTLSHandshakeIssues returns true in case we observed // a set of well-known issues during the TLS handshake. -func (tk *TestKeys) hasWellKnownTLSHandshakeIssues(logger model.Logger) bool { +func (tk *TestKeys) hasWellKnownTLSHandshakeIssues(logger model.Logger) (result bool) { + // TODO(bassosimone): we should return TLS information in the TH + // such that we can perform a TCP-like check for _, thx := range tk.TLSHandshakes { fail := thx.Failure if fail == nil { @@ -72,17 +96,17 @@ func (tk *TestKeys) hasWellKnownTLSHandshakeIssues(logger model.Logger) bool { "TLS: endpoint %s fails with %s (see #%d)", thx.Address, *fail, thx.TransactionID, ) - return true + result = true // flip the result but continue looping so we print them all default: // check next handshake } } - return false + return } // hasWellKnownHTTPRoundTripIssues checks whether any HTTP round // trip failed in a well-known suspicious way -func (tk *TestKeys) hasWellKnownHTTPRoundTripIssues(logger model.Logger) bool { +func (tk *TestKeys) hasWellKnownHTTPRoundTripIssues(logger model.Logger) (result bool) { for _, rtx := range tk.Requests { fail := rtx.Failure if fail == nil { @@ -99,10 +123,10 @@ func (tk *TestKeys) hasWellKnownHTTPRoundTripIssues(logger model.Logger) bool { "TLS: endpoint %s fails with %s (see #%d)", "N/A", *fail, rtx.TransactionID, // TODO(bassosimone): implement ) - return true + result = true // flip the result but continue looping so we print them all default: // check next round trip } } - return false + return } diff --git a/internal/experiment/webconnectivity/analysishttpdiff.go b/internal/experiment/webconnectivity/analysishttpdiff.go index 8224bc1d9d..d293d079cb 100644 --- a/internal/experiment/webconnectivity/analysishttpdiff.go +++ b/internal/experiment/webconnectivity/analysishttpdiff.go @@ -11,13 +11,19 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" ) // analysisHTTPDiff computes the HTTP diff between the final request-response // observed by the probe and the TH's result. The caller is responsible of passing -// us a valid probe observation and a valid TH observation. +// us a valid probe observation and a valid TH observation with nil failure. func (tk *TestKeys) analysisHTTPDiff(logger model.Logger, probe *model.ArchivalHTTPRequestResult, th *webconnectivity.ControlHTTPRequestResult) { + // make sure the caller respected the contract + runtimex.PanicIfTrue( + probe.Failure != nil || th.Failure != nil, + "the caller should have passed us successful HTTP observations", + ) // if we're dealing with an HTTPS request, don't perform any comparison // under the assumption that we're good if we're using TLS @@ -25,9 +31,9 @@ func (tk *TestKeys) analysisHTTPDiff(logger model.Logger, if err != nil { return // looks like a bug } - accessibleTrue := true if URL.Scheme == "https" { - tk.Accessible = &accessibleTrue + logger.Infof("HTTP: HTTPS && no error => #%d is successful", probe.TransactionID) + tk.XBlockingFlags |= analysisFlagSuccess return } @@ -37,22 +43,40 @@ func (tk *TestKeys) analysisHTTPDiff(logger model.Logger, tk.httpDiffHeadersMatch(probe, th) tk.httpDiffTitleMatch(probe, th) + logger.Infof("HTTP: bodyLengthMatch: %v", tk.BodyLengthMatch) + logger.Infof("HTTP: headersMatch: %v", tk.HeadersMatch) + logger.Infof("HTTP: statusCodeMatch: %v", tk.StatusCodeMatch) + logger.Infof("HTTP: titleMatch: %v", tk.TitleMatch) + if tk.StatusCodeMatch != nil && *tk.StatusCodeMatch { if tk.BodyLengthMatch != nil && *tk.BodyLengthMatch { - logger.Infof("HTTP: statusCodeMatch && bodyLengthMatch") + logger.Infof( + "HTTP: statusCodeMatch && bodyLengthMatch => #%d is successful", + probe.TransactionID, + ) + tk.XBlockingFlags |= analysisFlagSuccess return } if tk.HeadersMatch != nil && *tk.HeadersMatch { - logger.Infof("HTTP: statusCodeMatch && headersMatch") + logger.Infof( + "HTTP: statusCodeMatch && headersMatch => #%d is successful", + probe.TransactionID, + ) + tk.XBlockingFlags |= analysisFlagSuccess return } if tk.TitleMatch != nil && *tk.TitleMatch { - logger.Infof("HTTP: statusCodeMatch && titleMatch") + logger.Infof( + "HTTP: statusCodeMatch && titleMatch => #%d is successful", + probe.TransactionID, + ) + tk.XBlockingFlags |= analysisFlagSuccess return } } - tk.BlockingFlags |= analysisBlockingHTTPDiff + tk.XBlockingFlags |= analysisFlagHTTPDiff + logger.Warnf("HTTP: it seems #%d is a case of httpDiff", probe.TransactionID) } // httpDiffBodyLengthChecks compares the bodies lengths. diff --git a/internal/experiment/webconnectivity/analysistcpip.go b/internal/experiment/webconnectivity/analysistcpip.go index 119263ff23..9fb77bc2ca 100644 --- a/internal/experiment/webconnectivity/analysistcpip.go +++ b/internal/experiment/webconnectivity/analysistcpip.go @@ -13,6 +13,16 @@ import ( ) // analysisTCPIPToplevel is the toplevel analysis function for TCP/IP results. +// +// This algorithm has two objectives: +// +// 1. walk the list of TCP connect attempts and mark each of them as +// Status.Blocked = true | false | null depending on what the TH observed +// for the same set of IP addresses (it's ugly to modify a data struct +// in place, but this algorithm is defined by the spec); +// +// 2. assign the analysisFlagTCPIPBlocking flag to XBlockingFlags if +// we see any TCP endpoint for which Status.Blocked is true. func (tk *TestKeys) analysisTCPIPToplevel(logger model.Logger) { // if we don't have a control result, do nothing. if tk.Control == nil || len(tk.Control.TCPConnect) <= 0 { @@ -31,6 +41,7 @@ func (tk *TestKeys) analysisTCPIPToplevel(logger model.Logger) { entry.Status.Blocked = &isfalse continue // did not fail } + // make sure we exclude the IPv6 failures caused by lack of // proper IPv6 support by the probe ipv6, err := netxlite.IsIPv6(entry.IP) @@ -45,19 +56,22 @@ func (tk *TestKeys) analysisTCPIPToplevel(logger model.Logger) { continue } } + // obtain the corresponding endpoint epnt := net.JoinHostPort(entry.IP, fmt.Sprintf("%d", entry.Port)) ctrl, found := tk.Control.TCPConnect[epnt] if !found { - continue // only the probe tested this, so hard to say anything + continue // only the probe tested this, so hard to say anything... } if ctrl.Failure != nil { - // if the TH failed as well, don't set any blocking flag - entry.Status.Blocked = &isfalse + // If the TH failed as well, don't set XBlockingFlags and + // also don't bother with setting .Status.Blocked thus leaving + // it null. Performing precise error mapping should be a job + // for the pipeline rather than for the probe. continue } logger.Warnf("TCP/IP: endpoint %s is blocked (see #%d)", epnt, entry.TransactionID) entry.Status.Blocked = &istrue - tk.BlockingFlags |= analysisBlockingTCPIP + tk.XBlockingFlags |= analysisFlagTCPIPBlocking } } diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 25a49f85b4..96a9a672c7 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -9,7 +9,6 @@ package webconnectivity import ( "context" - "errors" "io" "net" "net/http" @@ -114,11 +113,11 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { tcpConn.Close() }() - // Only allow a single flow to _use_ the connection + // Only allow N flows to _use_ the connection select { case <-t.Sema: default: - ol.Stop(errors.New("stop after TCP connect")) // just to emit the right message + ol.Stop(nil) return } @@ -135,7 +134,13 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { defer httpCancel() httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { - t.TestKeys.SetFundamentalFailure(err) + if t.Referer == "" { + // when the referer is empty, the failing URL comes from our backend + // or from the user, so it's a fundamental failure. After that, we + // are dealing with websites provided URLs, so we should not flag a + // fundamental failure, because we want to see the measurement submitted. + t.TestKeys.SetFundamentalFailure(err) + } ol.Stop(err) return } @@ -209,7 +214,7 @@ func (t *CleartextFlow) newHTTPRequest(ctx context.Context) (*http.Request, erro // httpTransaction runs the HTTP transaction and saves the results. func (t *CleartextFlow) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 // TODO: you may want to change this default + const maxbody = 1 << 19 resp, err := txp.RoundTrip(req) if err != nil { ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) @@ -230,13 +235,13 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, txp model.HTTPTrans // maybeFollowRedirects follows redirects if configured and needed func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) { if !t.FollowRedirects { - return + return // not configured } switch resp.StatusCode { case 301, 302, 307, 308: location, err := resp.Location() if err != nil { - return + return // broken response from server } t.Logger.Infof("redirect to: %s", location.String()) resolvers := &DNSResolvers{ @@ -256,6 +261,6 @@ func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Res } resolvers.Start(ctx) default: - // nothing + // no redirect to follow } } diff --git a/internal/experiment/webconnectivity/control.go b/internal/experiment/webconnectivity/control.go index ba66cdc7df..d296098999 100644 --- a/internal/experiment/webconnectivity/control.go +++ b/internal/experiment/webconnectivity/control.go @@ -15,13 +15,14 @@ import ( ) // EndpointMeasurementsStarter is used by Control to start extra -// measurements using new IP addrs discovered by the TH +// measurements using new IP addrs discovered by the TH. type EndpointMeasurementsStarter interface { // startCleartextFlowsWithSema starts a TCP measurement flow for each IP addr. The [sema] // argument allows to control how many flows are allowed to perform HTTP measurements. Every // flow will attempt to read from [sema] and won't perform HTTP measurements if a // nonblocking read fails. Hence, you must create a [sema] channel with buffer equal - // to N and N elements inside it to allow N flows to perform HTTP measuremenets. + // to N and N elements inside it to allow N flows to perform HTTP measurements. Passing + // a nil [sema] causes no flow to attempt HTTP measurements. startCleartextFlowsWithSema(ctx context.Context, sema <-chan any, addresses []string) // startSecureFlowsWithSema starts a TCP+TLS measurement flow for each IP addr. See @@ -31,6 +32,9 @@ type EndpointMeasurementsStarter interface { // Control issues a control request and saves the results // inside of the experiment's TestKeys. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. type Control struct { // Addresses contains the MANDATORY addresses we've looked up. Addresses []string @@ -77,10 +81,11 @@ func (c *Control) Run(parentCtx context.Context) { // create control request var endpoints []string for _, address := range c.Addresses { - if port := c.URL.Port(); port != "" { + if port := c.URL.Port(); port != "" { // handle the case of a custom port endpoints = append(endpoints, net.JoinHostPort(address, port)) continue } + // otherwise, always attempt to measure both 443 and 80 endpoints endpoints = append(endpoints, net.JoinHostPort(address, "443")) endpoints = append(endpoints, net.JoinHostPort(address, "80")) } @@ -95,16 +100,24 @@ func (c *Control) Run(parentCtx context.Context) { } c.TestKeys.SetControlRequest(creq) + // TODO(bassosimone): the current TH will not perform TLS measurements for + // 443 endpoints. However, we should modify the TH to do that, such that we're + // able to be more confident about TLS measurements results. + // create logger for this operation ol := measurexlite.NewOperationLogger(c.Logger, "control for %s", creq.HTTPRequest) // create an API client clnt := (&httpx.APIClientTemplate{ - BaseURL: c.THAddr, - HTTPClient: c.Session.DefaultHTTPClient(), - Logger: c.Logger, - UserAgent: c.Session.UserAgent(), - }).WithBodyLogging().Build() + Accept: "", + Authorization: "", + BaseURL: c.THAddr, + HTTPClient: c.Session.DefaultHTTPClient(), + Host: "", // use the one inside the URL + LogBody: true, + Logger: c.Logger, + UserAgent: c.Session.UserAgent(), + }).Build() // issue the control request and wait for the response var cresp webconnectivity.ControlResponse @@ -129,7 +142,7 @@ func (c *Control) Run(parentCtx context.Context) { // This function determines whether we should start new // background measurements for previously unknown IP addrs. func (c *Control) maybeStartExtraMeasurements(ctx context.Context, thAddrs []string) { - // determine which addrs are TH only + // classify addeesses by who discovered them const ( inProbe = 1 << iota inTH @@ -142,22 +155,22 @@ func (c *Control) maybeStartExtraMeasurements(ctx context.Context, thAddrs []str mapping[addr] |= inTH } - // obtain the TH only addresses + // obtain the TH-only addresses var thOnly []string for addr, flags := range mapping { if (flags & inProbe) != 0 { - continue // we already measured this + continue // discovered by the probe => already tested } thOnly = append(thOnly, addr) } - // Start extra measurements for TH only addresses. Because we already - // measured HTTP using IP addrs discovered by the resolvers, we're not + // Start extra measurements for TH-only addresses. Because we already + // measured HTTP(S) using IP addrs discovered by the resolvers, we are not // going to do that again now. I am not sure this is the right policy // but I think we can just try it and then change if needed... // // Also, let's remember that reading from a nil chan blocks forever, so - // we're basically forcing the goroutines to avoid HTTP. + // we're basically forcing the goroutines to avoid HTTP(S). var nohttp chan any = nil c.ExtraMeasurementsStarter.startCleartextFlowsWithSema(ctx, nohttp, thOnly) c.ExtraMeasurementsStarter.startSecureFlowsWithSema(ctx, nohttp, thOnly) diff --git a/internal/experiment/webconnectivity/dnscache.go b/internal/experiment/webconnectivity/dnscache.go index c4f5225202..7b62c2683b 100644 --- a/internal/experiment/webconnectivity/dnscache.go +++ b/internal/experiment/webconnectivity/dnscache.go @@ -4,7 +4,7 @@ import "sync" // DNSCache wraps a model.Resolver to provide DNS caching. // -// The zero value is invalid. Please, use NewDNSCache. +// The zero value is invalid; please, use NewDNSCache to construct. type DNSCache struct { // mu provides mutual exclusion. mu *sync.Mutex diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 91c6b2d863..f1388e653c 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -61,7 +61,8 @@ type DNSResolvers struct { // Session is the OPTIONAL session. If the session is set, we will use // it to start the task that issues the control request. This request must // only be sent during the first iteration. It would be pointless to - // issue such a request for subsequent redirects. + // issue such a request for subsequent redirects, because the TH will + // always follow the redirect chain caused by the provided URL. Session model.ExperimentSession // THAddr is the OPTIONAL test helper address. @@ -90,6 +91,10 @@ func (t *DNSResolvers) run(parentCtx context.Context) []string { whoamiSystemV4Out := make(chan []DNSWhoamiInfoEntry) whoamiUDPv4Out := make(chan []DNSWhoamiInfoEntry) + // TODO(bassosimone): add opportunistic support for detecting + // whether DNS queries are answered regardless of dest addr by + // sending a few queries to root DNS servers + udpAddress := t.udpAddress() // start asynchronous lookups @@ -149,7 +154,7 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { found bool ) - // first attempt to use the dns cache + // attempt to use the dns cache addresses, found = t.DNSCache.Get(t.Domain) if !found { @@ -160,7 +165,7 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { t.DNSCache.Set(t.Domain, addresses) } - log.Infof("using: %+v", addresses) + log.Infof("using resolved addrs: %+v", addresses) // fan out a number of child async tasks to use the IP addrs t.startCleartextFlows(parentCtx, addresses) @@ -168,21 +173,24 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { t.maybeStartControlFlow(parentCtx, addresses) } -// whoamiSystemV4 performs a DNS whoami lookup for the system resolver. +// whoamiSystemV4 performs a DNS whoami lookup for the system resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) whoamiSystemV4(parentCtx context.Context, out chan<- []DNSWhoamiInfoEntry) { value, _ := DNSWhoamiSingleton.SystemV4(parentCtx) t.Logger.Infof("DNS whoami for system resolver: %+v", value) out <- value } -// whoamiUDPv4 performs a DNS whoami lookup for the given UDP resolver. +// whoamiUDPv4 performs a DNS whoami lookup for the given UDP resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) whoamiUDPv4(parentCtx context.Context, udpAddress string, out chan<- []DNSWhoamiInfoEntry) { value, _ := DNSWhoamiSingleton.UDPv4(parentCtx, udpAddress) t.Logger.Infof("DNS whoami for %s/udp resolver: %+v", udpAddress, value) out <- value } -// lookupHostSystem performs a DNS lookup using the system resolver. +// lookupHostSystem performs a DNS lookup using the system resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second @@ -208,7 +216,8 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- [] out <- addrs // must send something -even nil- to the parent } -// lookupHostUDP performs a DNS lookup using an UDP resolver. +// lookupHostUDP performs a DNS lookup using an UDP resolver. This function must always +// emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, udpAddress string, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second @@ -257,6 +266,8 @@ func (t *DNSResolvers) do53SplitQueries( return } +// TODO(bassosimone): maybe cycle through a bunch of well known addresses + // Returns the UDP resolver we should be using by default. func (t *DNSResolvers) udpAddress() string { if t.UDPAddress != "" { @@ -285,7 +296,7 @@ type OpportunisticDNSOverHTTPS struct { } // MaybeNextURL returns the next URL to measure, if any. Our aim is to perform -// some opportunistic DoH measurements as part of Web Connectivity. +// periodic, opportunistic DoH measurements as part of Web Connectivity. func (o *OpportunisticDNSOverHTTPS) MaybeNextURL() (string, bool) { now := time.Now() o.mu.Lock() @@ -301,8 +312,14 @@ func (o *OpportunisticDNSOverHTTPS) MaybeNextURL() (string, bool) { return "", false } +// TODO(bassosimone): consider whether factoring out this code +// and storing the state on disk instead of using memory + +// TODO(bassosimone): consider unifying somehow this code and +// the systemresolver code (or maybe just the list of resolvers) + // OpportunisticDNSOverHTTPSSingleton is the singleton used to keep -// track of opportunistic DNS-over-HTTPS measurements. +// track of the opportunistic DNS-over-HTTPS measurements state. var OpportunisticDNSOverHTTPSSingleton = &OpportunisticDNSOverHTTPS{ interval: 0, mu: &sync.Mutex{}, @@ -316,7 +333,8 @@ var OpportunisticDNSOverHTTPSSingleton = &OpportunisticDNSOverHTTPS{ }, } -// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { // obtain an opportunistic DoH URL URL, good := OpportunisticDNSOverHTTPSSingleton.MaybeNextURL() @@ -350,7 +368,7 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha // save results making sure we properly split DoH queries from other queries doh, other := t.dohSplitQueries(trace.DNSLookupsFromRoundTrip()) - t.TestKeys.Queries = append(t.TestKeys.Queries, doh...) + t.TestKeys.AppendQueries(doh...) t.TestKeys.WithTestKeysDoH(func(tkdh *TestKeysDoH) { tkdh.Queries = append(tkdh.Queries, other...) tkdh.NetworkEvents = append(tkdh.NetworkEvents, trace.NetworkEvents()...) @@ -463,12 +481,12 @@ func (t *DNSResolvers) startSecureFlowsWithSema(ctx context.Context, sema <-chan } } -// maybeStartControlFlow starts the control flow, when .Session is set. +// maybeStartControlFlow starts the control flow iff .Session and .THAddr are set. func (t *DNSResolvers) maybeStartControlFlow(ctx context.Context, addresses []string) { if t.Session != nil && t.THAddr != "" { ctrl := &Control{ Addresses: addresses, - ExtraMeasurementsStarter: t, + ExtraMeasurementsStarter: t, // allows starting follow-up measurement flows Logger: t.Logger, TestKeys: t.TestKeys, Session: t.Session, diff --git a/internal/experiment/webconnectivity/dnswhoami.go b/internal/experiment/webconnectivity/dnswhoami.go index 1d7b9f38e8..3a0636cacd 100644 --- a/internal/experiment/webconnectivity/dnswhoami.go +++ b/internal/experiment/webconnectivity/dnswhoami.go @@ -68,6 +68,9 @@ func (svc *DNSWhoamiService) UDPv4(ctx context.Context, address string) ([]DNSWh return value, len(value) > 0 } +// TODO(bassosimone): consider factoring this code and keeping state +// on disk rather than on memory. + // DNSWhoamiSingleton is the DNSWhoamiService singleton. var DNSWhoamiSingleton = &DNSWhoamiService{ mu: &sync.Mutex{}, diff --git a/internal/experiment/webconnectivity/inputparser.go b/internal/experiment/webconnectivity/inputparser.go index cee8775a30..a8b055e225 100644 --- a/internal/experiment/webconnectivity/inputparser.go +++ b/internal/experiment/webconnectivity/inputparser.go @@ -44,7 +44,7 @@ func (ip *InputParser) Parse(input string) (*url.URL, error) { return nil, errors.New("cannot parse input") } -// Conditionally allows endpooints when ip.AllowEndpoints is true. +// Conditionally allows endpoints when ip.AllowEndpoints is true. func (ip *InputParser) maybeAllowEndpoints(URL *url.URL, err error) (*url.URL, error) { runtimex.PanicIfNil(err, "expected to be called with a non-nil error") if ip.AllowEndpoints && URL.Scheme != "" && URL.Opaque != "" && URL.User == nil && diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index dec91e7118..3b9caeb0f9 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -10,7 +10,6 @@ package webconnectivity import ( "context" "crypto/tls" - "errors" "io" "net" "net/http" @@ -145,11 +144,11 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { } defer tlsConn.Close() - // Only allow a single flow to _use_ the connection + // Only allow N flows to _use_ the connection select { case <-t.Sema: default: - ol.Stop(errors.New("stop after TLS handshake")) // just to emit the correct message + ol.Stop(nil) return } @@ -167,7 +166,13 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { defer httpCancel() httpReq, err := t.newHTTPRequest(httpCtx) if err != nil { - t.TestKeys.SetFundamentalFailure(err) + if t.Referer == "" { + // when the referer is empty, the failing URL comes from our backend + // or from the user, so it's a fundamental failure. After that, we + // are dealing with websites provided URLs, so we should not flag a + // fundamental failure, because we want to see the measurement submitted. + t.TestKeys.SetFundamentalFailure(err) + } ol.Stop(err) return } @@ -261,7 +266,7 @@ func (t *SecureFlow) newHTTPRequest(ctx context.Context) (*http.Request, error) // httpTransaction runs the HTTP transaction and saves the results. func (t *SecureFlow) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 // TODO: you may want to change this default + const maxbody = 1 << 19 resp, err := txp.RoundTrip(req) if err != nil { ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) @@ -282,13 +287,13 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, txp model.HTTPTranspor // maybeFollowRedirects follows redirects if configured and needed func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) { if !t.FollowRedirects { - return + return // not configured } switch resp.StatusCode { case 301, 302, 307, 308: location, err := resp.Location() if err != nil { - return + return // broken response from server } t.Logger.Infof("redirect to: %s", location.String()) resolvers := &DNSResolvers{ @@ -308,6 +313,6 @@ func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Respon } resolvers.Start(ctx) default: - // nothing + // no redirect to follow } } diff --git a/internal/experiment/webconnectivity/summary.go b/internal/experiment/webconnectivity/summary.go index b7de075f29..2865c88cff 100644 --- a/internal/experiment/webconnectivity/summary.go +++ b/internal/experiment/webconnectivity/summary.go @@ -17,7 +17,7 @@ type SummaryKeys struct { // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { - // TODO: fill all the SummaryKeys + // TODO(bassosimone): fill all the SummaryKeys sk := SummaryKeys{isAnomaly: false} return sk, nil } diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 4831f3a4c4..bee0f41ef7 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -48,8 +48,8 @@ type TestKeys struct { // ControlFailure contains the failure of the control experiment. ControlFailure *string `json:"control_failure"` - // DNSFlags contains DNS analysis flags. - DNSFlags int64 `json:"x_dns_flags"` + // XDNSFlags contains DNS analysis flags. + XDNSFlags int64 `json:"x_dns_flags"` // DNSExperimentFailure indicates whether there was a failure in any // of the DNS experiments we performed. @@ -59,8 +59,8 @@ type TestKeys struct { // the TH's DNS results and the probe's DNS results. DNSConsistency string `json:"dns_consistency"` - // BlockingFlags contains blocking flags. - BlockingFlags int64 `json:"x_blocking_flags"` + // XBlockingFlags contains blocking flags. + XBlockingFlags int64 `json:"x_blocking_flags"` // BodyLength match tells us whether the body length matches. BodyLengthMatch *bool `json:"body_length_match"` @@ -84,13 +84,14 @@ type TestKeys struct { // - false // - null // - // In addition to being a bad type, this field has the issue that it + // In addition to having a ~bad type, this field has the issue that it // reduces the reason for blocking to an enum, whereas it's a set of flags, // hence we introduced the x_blocking_flags field. Blocking any `json:"blocking"` - // Accessible indicates whether the resource is accessible. - Accessible *bool `json:"accessible"` + // Accessible indicates whether the resource is accessible. Possible + // values for this field are: nil, true, and false. + Accessible any `json:"accessible"` // controlRequest is the control request we sent. controlRequest *webconnectivity.ControlRequest @@ -121,7 +122,8 @@ type DNSWhoamiInfo struct { UDPv4 map[string][]DNSWhoamiInfoEntry `json:"udp_v4"` } -// TestKeysDoH contains ancillary observations collected using DoH. +// TestKeysDoH contains ancillary observations collected using DoH (e.g., the +// DNS lookups, TCP connects, TLS handshakes caused by given DoH lookups). // // They are on a separate hierarchy to simplify processing. type TestKeysDoH struct { @@ -243,7 +245,6 @@ func (tk *TestKeys) WithDNSWhoami(fun func(*DNSWhoamiInfo)) { // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { - // TODO: here you should initialize all the fields return &TestKeys{ NetworkEvents: []*model.ArchivalNetworkEvent{}, DNSWoami: &DNSWhoamiInfo{ @@ -267,10 +268,10 @@ func NewTestKeys() *TestKeys { TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, Control: nil, ControlFailure: nil, - DNSFlags: 0, + XDNSFlags: 0, DNSExperimentFailure: nil, DNSConsistency: "", - BlockingFlags: 0, + XBlockingFlags: 0, BodyLengthMatch: nil, HeadersMatch: nil, StatusCodeMatch: nil, From 429e46ea09296f97c8e3e772a57ee547de505dcc Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 16 Aug 2022 08:07:26 +0200 Subject: [PATCH 77/83] bugfix --- internal/experiment/webconnectivity/analysiscore.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/experiment/webconnectivity/analysiscore.go b/internal/experiment/webconnectivity/analysiscore.go index 023954446f..c9a6737918 100644 --- a/internal/experiment/webconnectivity/analysiscore.go +++ b/internal/experiment/webconnectivity/analysiscore.go @@ -98,7 +98,7 @@ func (tk *TestKeys) analysisToplevel(logger model.Logger) { tk.Blocking = "dns" tk.Accessible = false logger.Warnf( - "ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + "ANOMALY: flags=%d accessible=%+v, blocking=%+v", tk.XBlockingFlags, tk.Accessible, tk.Blocking, ) @@ -106,14 +106,14 @@ func (tk *TestKeys) analysisToplevel(logger model.Logger) { tk.Blocking = "tcp_ip" tk.Accessible = false logger.Warnf( - "ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + "ANOMALY: flags=%d accessible=%+v, blocking=%+v", tk.XBlockingFlags, tk.Accessible, tk.Blocking, ) case (tk.XBlockingFlags & (analysisFlagTLSBlocking | analysisFlagHTTPBlocking)) != 0: tk.Blocking = "http-failure" tk.Accessible = false - logger.Warnf("ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + logger.Warnf("ANOMALY: flags=%d accessible=%+v, blocking=%+v", tk.XBlockingFlags, tk.Accessible, tk.Blocking, ) @@ -121,7 +121,7 @@ func (tk *TestKeys) analysisToplevel(logger model.Logger) { tk.Blocking = "http-diff" tk.Accessible = false logger.Warnf( - "ANOMALY: flags=%d accessible=%+v, blocking=%%+v", + "ANOMALY: flags=%d accessible=%+v, blocking=%+v", tk.XBlockingFlags, tk.Accessible, tk.Blocking, ) @@ -129,7 +129,7 @@ func (tk *TestKeys) analysisToplevel(logger model.Logger) { tk.Blocking = false tk.Accessible = true logger.Infof( - "SUCCESS: flags=%d accessible=%+v, blocking=%%+v", + "SUCCESS: flags=%d accessible=%+v, blocking=%+v", tk.XBlockingFlags, tk.Accessible, tk.Blocking, ) From 307547024d63c46050127e6197e68713d55db5e2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 16 Aug 2022 09:50:35 +0200 Subject: [PATCH 78/83] refactor: make reprocessing from saved measurements ~possible --- .../experiment/webconnectivity/analysisdns.go | 16 ++--- .../webconnectivity/analysishttpcore.go | 67 +++++++++++-------- .../webconnectivity/analysishttpdiff.go | 5 -- .../webconnectivity/analysistcpip.go | 5 ++ .../experiment/webconnectivity/measurer.go | 2 +- .../experiment/webconnectivity/testkeys.go | 14 ++-- 6 files changed, 61 insertions(+), 48 deletions(-) diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go index cfb33f4f01..e065e90f9e 100644 --- a/internal/experiment/webconnectivity/analysisdns.go +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -56,11 +56,11 @@ func (tk *TestKeys) analysisDNSToplevel(logger model.Logger) { tk.analysisDNSUnexpectedFailure(logger) tk.analysisDNSUnexpectedAddrs(logger) if tk.XDNSFlags != 0 { - logger.Warnf("DNSConsistency: inconsistent") + logger.Warn("DNSConsistency: inconsistent") tk.DNSConsistency = "inconsistent" tk.XBlockingFlags |= analysisFlagDNSBlocking } else { - logger.Warnf("DNSConsistency: consistent") + logger.Info("DNSConsistency: consistent") tk.DNSConsistency = "consistent" } } @@ -88,13 +88,13 @@ func (tk *TestKeys) analysisDNSBogon(logger model.Logger) { switch answer.AnswerType { case "A": if net.ParseIP(answer.IPv4) != nil && netxlite.IsBogon(answer.IPv4) { - logger.Warnf("BOGON: %s in #%d", answer.IPv4, query.TransactionID) + logger.Warnf("DNS: BOGON %s in #%d", answer.IPv4, query.TransactionID) tk.XDNSFlags |= AnalysisDNSBogon // continue processing so we print all the bogons we have } case "AAAA": if net.ParseIP(answer.IPv6) != nil && netxlite.IsBogon(answer.IPv6) { - logger.Warnf("BOGON: %s in #%d", answer.IPv6, query.TransactionID) + logger.Warnf("DNS: BOGON %s in #%d", answer.IPv6, query.TransactionID) tk.XDNSFlags |= AnalysisDNSBogon // continue processing so we print all the bogons we have } @@ -109,12 +109,12 @@ func (tk *TestKeys) analysisDNSBogon(logger model.Logger) { // a failure is unexpected when the TH could resolve a domain and the probe couldn't. func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { // make sure we have control before proceeding futher - if tk.Control == nil || tk.controlRequest == nil { + if tk.Control == nil || tk.ControlRequest == nil { return } // obtain thRequest and thResponse as shortcuts - thRequest := tk.controlRequest + thRequest := tk.ControlRequest thResponse := tk.Control // obtain the domain that the TH has queried for @@ -177,12 +177,12 @@ func (tk *TestKeys) analysisDNSUnexpectedFailure(logger model.Logger) { // TLS based heuristic for determining whether an IP address was legit. func (tk *TestKeys) analysisDNSUnexpectedAddrs(logger model.Logger) { // make sure we have control before proceeding futher - if tk.Control == nil || tk.controlRequest == nil { + if tk.Control == nil || tk.ControlRequest == nil { return } // obtain thRequest and thResponse as shortcuts - thRequest := tk.controlRequest + thRequest := tk.ControlRequest thResponse := tk.Control // obtain the domain that the TH has queried for diff --git a/internal/experiment/webconnectivity/analysishttpcore.go b/internal/experiment/webconnectivity/analysishttpcore.go index 467796e32f..441a2db746 100644 --- a/internal/experiment/webconnectivity/analysishttpcore.go +++ b/internal/experiment/webconnectivity/analysishttpcore.go @@ -5,6 +5,8 @@ package webconnectivity // import ( + "net/url" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -27,8 +29,8 @@ import ( // In websteps fashion, we don't stop at the first failure, rather we // process all the available data and evaluate all possible errors. func (tk *TestKeys) analysisHTTPToplevel(logger model.Logger) { - // don't perform any analysis if the TH failed - if tk.Control == nil { + // don't perform any analysis without TH data + if tk.Control == nil || tk.ControlRequest == nil { return } ctrl := tk.Control.HTTPRequest @@ -38,15 +40,19 @@ func (tk *TestKeys) analysisHTTPToplevel(logger model.Logger) { return } + // determine whether the original URL was HTTPS + origURL, err := url.Parse(tk.ControlRequest.HTTPRequest) + if err != nil { + return // this seeems like a bug? + } + isHTTPS := origURL.Scheme == "https" + // determine whether we had any TLS handshake issue and, in such a case, // declare that we had a case of "http-failure" through TLS. // // Note that this would eventually count as an "http-failure" for .Blocking // because Web Connectivity did not have a concept of TLS based blocking. - // - // This check works ~reliably as long as we ensure to put DoH TLS - // handshakes outside of the main .TLSHandshakes field. - if tk.hasWellKnownTLSHandshakeIssues(logger) { + if tk.hasWellKnownTLSHandshakeIssues(isHTTPS, logger) { tk.XBlockingFlags |= analysisFlagTLSBlocking // continue processing } @@ -77,28 +83,35 @@ func (tk *TestKeys) analysisHTTPToplevel(logger model.Logger) { // hasWellKnownTLSHandshakeIssues returns true in case we observed // a set of well-known issues during the TLS handshake. -func (tk *TestKeys) hasWellKnownTLSHandshakeIssues(logger model.Logger) (result bool) { +func (tk *TestKeys) hasWellKnownTLSHandshakeIssues(isHTTPS bool, logger model.Logger) (result bool) { // TODO(bassosimone): we should return TLS information in the TH - // such that we can perform a TCP-like check - for _, thx := range tk.TLSHandshakes { - fail := thx.Failure - if fail == nil { - continue // this handshake succeded, so skip it - } - switch *fail { - case netxlite.FailureConnectionReset, - netxlite.FailureGenericTimeoutError, - netxlite.FailureEOFError, - netxlite.FailureSSLInvalidHostname, - netxlite.FailureSSLInvalidCertificate, - netxlite.FailureSSLUnknownAuthority: - logger.Warnf( - "TLS: endpoint %s fails with %s (see #%d)", - thx.Address, *fail, thx.TransactionID, - ) - result = true // flip the result but continue looping so we print them all - default: - // check next handshake + // such that we can perform a TCP-like check. For now, instead, we + // only perform comparison when the initial URL was HTTPS. Given + // that we unconditionally check for HTTPS even when the URL is HTTP, + // we cannot blindly treat all TLS errors as blocking. A website + // may just not have HTTPS. While in the obvious cases we will see + // certificate errors, in some cases it may actually timeout. + if isHTTPS { + for _, thx := range tk.TLSHandshakes { + fail := thx.Failure + if fail == nil { + continue // this handshake succeded, so skip it + } + switch *fail { + case netxlite.FailureConnectionReset, + netxlite.FailureGenericTimeoutError, + netxlite.FailureEOFError, + netxlite.FailureSSLInvalidHostname, + netxlite.FailureSSLInvalidCertificate, + netxlite.FailureSSLUnknownAuthority: + logger.Warnf( + "TLS: endpoint %s fails with %s (see #%d)", + thx.Address, *fail, thx.TransactionID, + ) + result = true // flip the result but continue looping so we print them all + default: + // check next handshake + } } } return diff --git a/internal/experiment/webconnectivity/analysishttpdiff.go b/internal/experiment/webconnectivity/analysishttpdiff.go index d293d079cb..9e21fa44a5 100644 --- a/internal/experiment/webconnectivity/analysishttpdiff.go +++ b/internal/experiment/webconnectivity/analysishttpdiff.go @@ -43,11 +43,6 @@ func (tk *TestKeys) analysisHTTPDiff(logger model.Logger, tk.httpDiffHeadersMatch(probe, th) tk.httpDiffTitleMatch(probe, th) - logger.Infof("HTTP: bodyLengthMatch: %v", tk.BodyLengthMatch) - logger.Infof("HTTP: headersMatch: %v", tk.HeadersMatch) - logger.Infof("HTTP: statusCodeMatch: %v", tk.StatusCodeMatch) - logger.Infof("HTTP: titleMatch: %v", tk.TitleMatch) - if tk.StatusCodeMatch != nil && *tk.StatusCodeMatch { if tk.BodyLengthMatch != nil && *tk.BodyLengthMatch { logger.Infof( diff --git a/internal/experiment/webconnectivity/analysistcpip.go b/internal/experiment/webconnectivity/analysistcpip.go index 9fb77bc2ca..c1b41b9917 100644 --- a/internal/experiment/webconnectivity/analysistcpip.go +++ b/internal/experiment/webconnectivity/analysistcpip.go @@ -33,6 +33,11 @@ func (tk *TestKeys) analysisTCPIPToplevel(logger model.Logger) { isfalse = false ) + // TODO(bassosimone): the TH should measure also some of the IP addrs it discovered + // and the probe did not discover to improve the analysis. Otherwise, the probe + // is fooled by the TH also failing for countries that return random IP addresses + // that are actually not working. Yet, ooni/data would definitely see this. + // walk the list of probe results and compare with TH results for _, entry := range tk.TCPConnect { // skip successful entries diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index e2e12a5512..86012fa07b 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -127,7 +127,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, } // perform any deferred computation on the test keys - tk.finalize(sess.Logger()) + tk.Finalize(sess.Logger()) // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index bee0f41ef7..1a792d70fe 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -42,6 +42,9 @@ type TestKeys struct { // TLSHandshakes contains TLS handshakes results. TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + // ControlRequest is the control request we sent. + ControlRequest *webconnectivity.ControlRequest `json:"x_control_request"` + // Control contains the TH's response. Control *webconnectivity.ControlResponse `json:"control"` @@ -93,9 +96,6 @@ type TestKeys struct { // values for this field are: nil, true, and false. Accessible any `json:"accessible"` - // controlRequest is the control request we sent. - controlRequest *webconnectivity.ControlRequest - // fundamentalFailure indicates that some fundamental error occurred // in a background task. A fundamental error is something like a programmer // such as a failure to parse a URL that was hardcoded in the codebase. When @@ -194,7 +194,7 @@ func (tk *TestKeys) AppendTLSHandshakes(v ...*model.ArchivalTLSOrQUICHandshakeRe // SetControlRequest sets the value of controlRequest. func (tk *TestKeys) SetControlRequest(v *webconnectivity.ControlRequest) { tk.mu.Lock() - tk.controlRequest = v + tk.ControlRequest = v tk.mu.Unlock() } @@ -278,14 +278,14 @@ func NewTestKeys() *TestKeys { TitleMatch: nil, Blocking: nil, Accessible: nil, - controlRequest: nil, + ControlRequest: nil, fundamentalFailure: nil, mu: &sync.Mutex{}, } } -// finalize performs any delayed computation on the test keys. This function +// Finalize performs any delayed computation on the test keys. This function // must be called from the measurer after all the tasks have completed. -func (tk *TestKeys) finalize(logger model.Logger) { +func (tk *TestKeys) Finalize(logger model.Logger) { tk.analysisToplevel(logger) } From 3567738a3ddde7d76cd1793776c788ae8220cb81 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 16 Aug 2022 12:37:37 +0200 Subject: [PATCH 79/83] Add internal tool to test the scoring rules --- internal/cmd/minipipeline/main.go | 93 +++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 internal/cmd/minipipeline/main.go diff --git a/internal/cmd/minipipeline/main.go b/internal/cmd/minipipeline/main.go new file mode 100644 index 0000000000..af02399e1f --- /dev/null +++ b/internal/cmd/minipipeline/main.go @@ -0,0 +1,93 @@ +// Command minipipeline loads in input Web Connectivity measurements +// and applies the probe's detection heuristics on them again. +// +// By doing that, we can iterate more quickly on improving heuristics. +package main + +import ( + "bufio" + "encoding/json" + "os" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func main() { + for _, arg := range os.Args[1:] { + processfile(arg) + } +} + +func processfile(arg string) { + filep, err := os.Open(arg) + if err != nil { + log.WithError(err).Fatal("os.Open failed") + } + defer filep.Close() + scanner := bufio.NewScanner(filep) + buf := make([]byte, 1<<23) + scanner.Buffer(buf, len(buf)) + index := 1 + for scanner.Scan() { + processmeasurement(scanner.Bytes(), index) + index++ + } + if err := scanner.Err(); err != nil { + log.WithError(err).Fatal("scanner.Err failed") + } +} + +func processmeasurement(mraw []byte, index int) { + var m model.Measurement + if err := json.Unmarshal(mraw, &m); err != nil { + log.WithError(err).Fatal("json.Unmarshal failed") + } + if m.TestName != "web_connectivity" { + return + } + tkraw, err := json.Marshal(m.TestKeys) + if err != nil { + log.WithError(err).Fatal("json.Marshal failed") + } + processtestkeys(tkraw, string(m.Input), index) +} + +func processtestkeys(tkraw []byte, input string, index int) { + var tk webconnectivity.TestKeys + if err := json.Unmarshal(tkraw, &tk); err != nil { + log.WithError(err).Fatal("json.Unmarshal failed") + } + newtk := &webconnectivity.TestKeys{ + NetworkEvents: tk.NetworkEvents, + DNSWoami: tk.DNSWoami, + DoH: tk.DoH, + Do53: tk.Do53, + Queries: tk.Queries, + Requests: tk.Requests, + TCPConnect: tk.TCPConnect, + TLSHandshakes: tk.TLSHandshakes, + ControlRequest: tk.ControlRequest, + Control: tk.Control, + ControlFailure: tk.ControlFailure, + XDNSFlags: 0, + DNSExperimentFailure: nil, + DNSConsistency: "", + XBlockingFlags: 0, + BodyLengthMatch: nil, + HeadersMatch: nil, + StatusCodeMatch: nil, + TitleMatch: nil, + Blocking: nil, + Accessible: nil, + } + reprocesstk(newtk, input, index) +} + +func reprocesstk(tk *webconnectivity.TestKeys, input string, index int) { + log.Infof("\n\n\n") + log.Infof("Input: %s", input) + log.Infof("Idx: %d", index) + tk.Finalize(log.Log) +} From c9be163cbaa39c48372cad137a07e299312d4ae2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 16 Aug 2022 13:26:37 +0200 Subject: [PATCH 80/83] sync templates and implementation --- internal/cmd/boilerplate/task/endpoint.go.txt | 16 ++++++++++------ .../cmd/boilerplate/task/multiresolver.go.txt | 14 ++++++++------ internal/experiment/telegram/datacenter.go | 17 +++++++++++------ internal/experiment/telegram/dnsresolvers.go | 14 ++++++++------ internal/experiment/telegram/webhttp.go | 16 ++++++++++------ internal/experiment/telegram/webhttps.go | 16 ++++++++++------ .../experiment/webconnectivity/cleartextflow.go | 3 +-- .../experiment/webconnectivity/dnsresolvers.go | 9 ++++----- .../experiment/webconnectivity/secureflow.go | 3 +-- 9 files changed, 63 insertions(+), 45 deletions(-) diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt index fa77250d7f..e4d0b6e941 100644 --- a/internal/cmd/boilerplate/task/endpoint.go.txt +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -3,8 +3,7 @@ package {{ .Package }} // // {{ .StructName }} // -// This code was generated by `boilerplate' using -// the {{ .Template }} template. +// Generated by `boilerplate' using the {{ .Template }} template. // import ( @@ -214,16 +213,21 @@ func (t *{{ .StructName }}) urlHost(scheme string) (string, error) { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) return "", err } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } {{ if eq .Template "http" -}} if port == "80" && scheme == "http" { - return addr, nil + return urlHost, nil } {{- else if eq .Template "https" -}} if port == "443" && scheme == "https" { - return addr, nil + return urlHost, nil } {{- end }} - return t.Address, nil // there was no need to parse after all 😬 + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil } // newHTTPRequest creates a new HTTP request. @@ -254,7 +258,7 @@ func (t *{{ .StructName }}) newHTTPRequest(ctx context.Context) (*http.Request, // httpTransaction runs the HTTP transaction and saves the results. func (t *{{ .StructName }}) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 // TODO: you may want to change this default + const maxbody = 1 << 19 // TODO: you may want to change this default resp, err := txp.RoundTrip(req) if err != nil { _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save diff --git a/internal/cmd/boilerplate/task/multiresolver.go.txt b/internal/cmd/boilerplate/task/multiresolver.go.txt index 1588e7ebd6..f225a29ffa 100644 --- a/internal/cmd/boilerplate/task/multiresolver.go.txt +++ b/internal/cmd/boilerplate/task/multiresolver.go.txt @@ -3,8 +3,7 @@ package {{ .Package }} // // {{ .StructName }} // -// This code was generated by `boilerplate' using -// the {{ .Template }} template. +// Generated by `boilerplate' using the {{ .Template }} template. // import ( @@ -101,13 +100,14 @@ func (t *{{ .StructName }}) Run(parentCtx context.Context) { } } - // (typically) fan out a number of child async tasks to use the IP addrs + // fan out a number of child async tasks to use the IP addrs for range sorted { // TODO: implement } } -// lookupHostSystem performs a DNS lookup using the system resolver. +// lookupHostSystem performs a DNS lookup using the system resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. func (t *{{ .StructName }}) lookupHostSystem(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second // TODO: consider changing @@ -131,7 +131,8 @@ func (t *{{ .StructName }}) lookupHostSystem(parentCtx context.Context, out chan out <- addrs } -// lookupHostUDP performs a DNS lookup using an UDP resolver. +// lookupHostUDP performs a DNS lookup using an UDP resolver. This function must always +// emit an ouput on the [out] channel to synchronize with the caller func. func (t *{{ .StructName }}) lookupHostUDP(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second // TODO: consider changing @@ -164,7 +165,8 @@ func (t *{{ .StructName }}) udpAddress() string { return "8.8.4.4:53" } -// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. func (t *{{ .StructName }}) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second // TODO: consider changing diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index 4d35a184d0..ed6c28ba71 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -3,8 +3,7 @@ package telegram // // Datacenter // -// This code was generated by `boilerplate' using -// the http template. +// Generated by `boilerplate' using the http template. // import ( @@ -138,10 +137,15 @@ func (t *Datacenter) urlHost(scheme string) (string, error) { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) return "", err } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } if port == "80" && scheme == "http" { - return addr, nil + return urlHost, nil } - return t.Address, nil // there was no need to parse after all 😬 + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil } // newHTTPRequest creates a new HTTP request. @@ -165,18 +169,19 @@ func (t *Datacenter) newHTTPRequest(ctx context.Context) (*http.Request, error) httpReq.Header.Set("Accept", model.HTTPHeaderAccept) httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader return httpReq, nil } // httpTransaction runs the HTTP transaction and saves the results. func (t *Datacenter) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 + const maxbody = 1 << 19 resp, err := txp.RoundTrip(req) if err != nil { ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) t.TestKeys.AppendRequests(ev) - return resp, []byte{}, err + return nil, []byte{}, err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) diff --git a/internal/experiment/telegram/dnsresolvers.go b/internal/experiment/telegram/dnsresolvers.go index 4edc739257..26f0e78757 100644 --- a/internal/experiment/telegram/dnsresolvers.go +++ b/internal/experiment/telegram/dnsresolvers.go @@ -3,8 +3,7 @@ package telegram // // DNSResolvers // -// This code was generated by `boilerplate' using -// the multi-resolver template. +// Generated by `boilerplate' using the multi-resolver template. // import ( @@ -102,14 +101,15 @@ func (t *DNSResolvers) Run(parentCtx context.Context) { } } - // (typically) fan out a number of child async tasks to use the IP addrs + // fan out a number of child async tasks to use the IP addrs for _, addr := range sorted { t.startWebHTTPTask(parentCtx, addr) t.startWebHTTPSTask(parentCtx, addr) } } -// lookupHostSystem performs a DNS lookup using the system resolver. +// lookupHostSystem performs a DNS lookup using the system resolver. This function must always +// emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second @@ -134,7 +134,8 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- [] out <- addrs } -// lookupHostUDP performs a DNS lookup using an UDP resolver. +// lookupHostUDP performs a DNS lookup using an UDP resolver. This function must always +// emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second @@ -168,7 +169,8 @@ func (t *DNSResolvers) udpAddress() string { return "8.8.4.4:53" } -// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { // create context with attached a timeout const timeout = 4 * time.Second diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index 59683b2f39..d79edcc982 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -3,8 +3,7 @@ package telegram // // WebHTTP // -// This code was generated by `boilerplate' using -// the http template. +// Generated by `boilerplate' using the http template. // import ( @@ -138,10 +137,15 @@ func (t *WebHTTP) urlHost(scheme string) (string, error) { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) return "", err } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } if port == "80" && scheme == "http" { - return addr, nil + return urlHost, nil } - return t.Address, nil // there was no need to parse after all 😬 + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil } // newHTTPRequest creates a new HTTP request. @@ -172,12 +176,12 @@ func (t *WebHTTP) newHTTPRequest(ctx context.Context) (*http.Request, error) { // httpTransaction runs the HTTP transaction and saves the results. func (t *WebHTTP) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 + const maxbody = 1 << 19 resp, err := txp.RoundTrip(req) if err != nil { ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) t.TestKeys.AppendRequests(ev) - return resp, []byte{}, err + return nil, []byte{}, err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index 5be34be38b..66de135a47 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -3,8 +3,7 @@ package telegram // // WebHTTPS // -// This code was generated by `boilerplate' using -// the https template. +// Generated by `boilerplate' using the https template. // import ( @@ -193,10 +192,15 @@ func (t *WebHTTPS) urlHost(scheme string) (string, error) { t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) return "", err } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } if port == "443" && scheme == "https" { - return addr, nil + return urlHost, nil } - return t.Address, nil // there was no need to parse after all 😬 + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil } // newHTTPRequest creates a new HTTP request. @@ -227,12 +231,12 @@ func (t *WebHTTPS) newHTTPRequest(ctx context.Context) (*http.Request, error) { // httpTransaction runs the HTTP transaction and saves the results. func (t *WebHTTPS) httpTransaction(ctx context.Context, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { - const maxbody = 1 << 22 + const maxbody = 1 << 19 resp, err := txp.RoundTrip(req) if err != nil { ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) t.TestKeys.AppendRequests(ev) - return resp, []byte{}, err + return nil, []byte{}, err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, maxbody) diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 96a9a672c7..bdc9d0d0f6 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -3,8 +3,7 @@ package webconnectivity // // CleartextFlow // -// This code was generated by `boilerplate' using -// the http template. +// Generated by `boilerplate' using the http template. // import ( diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index f1388e653c..be28399573 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -3,8 +3,7 @@ package webconnectivity // // DNSResolvers // -// This code was generated by `boilerplate' using -// the multi-resolver template. +// Generated by `boilerplate' using the multi-resolver template. // import ( @@ -213,7 +212,7 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- [] addrs, err := reso.LookupHost(lookupCtx, t.Domain) t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) ol.Stop(err) - out <- addrs // must send something -even nil- to the parent + out <- addrs } // lookupHostUDP performs a DNS lookup using an UDP resolver. This function must always @@ -249,7 +248,7 @@ func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, udpAddress strin }) ol.Stop(err) - out <- addrs // must send something -even nil- to the parent + out <- addrs } // Divides queries generated by Do53 in Do53-proper queries and other queries. @@ -377,7 +376,7 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha }) ol.Stop(err) - out <- addrs // must send something -even nil- to the parent + out <- addrs } // Divides queries generated by DoH in DoH-proper queries and other queries. diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index 3b9caeb0f9..92ea2fa5ae 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -3,8 +3,7 @@ package webconnectivity // // SecureFlow // -// This code was generated by `boilerplate' using -// the https template. +// Generated by `boilerplate' using the https template. // import ( From 70482f97bc08f3a7ea55e35248e709b0b42af794 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 16 Aug 2022 13:27:29 +0200 Subject: [PATCH 81/83] cleanup: remove the standalone system resolver --- internal/cmd/boilerplate/task.go | 14 ++-- .../boilerplate/task/systemresolver.go.txt | 83 ------------------- 2 files changed, 5 insertions(+), 92 deletions(-) delete mode 100644 internal/cmd/boilerplate/task/systemresolver.go.txt diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go index 1b5c6375a2..825478f254 100644 --- a/internal/cmd/boilerplate/task.go +++ b/internal/cmd/boilerplate/task.go @@ -92,17 +92,13 @@ var endpointTemplate string //go:embed "task/multiresolver.go.txt" var multiResolverTemplate string -//go:embed "task/systemresolver.go.txt" -var systemResolverTemplate string - // The list of known tasks var knownTasks = map[string]string{ - "http": endpointTemplate, - "https": endpointTemplate, - "multi-resolver": multiResolverTemplate, - "system-resolver": systemResolverTemplate, - "tcp": endpointTemplate, - "tls": endpointTemplate, + "http": endpointTemplate, + "https": endpointTemplate, + "multi-resolver": multiResolverTemplate, + "tcp": endpointTemplate, + "tls": endpointTemplate, } // Names of known tasks diff --git a/internal/cmd/boilerplate/task/systemresolver.go.txt b/internal/cmd/boilerplate/task/systemresolver.go.txt deleted file mode 100644 index 0b484d794e..0000000000 --- a/internal/cmd/boilerplate/task/systemresolver.go.txt +++ /dev/null @@ -1,83 +0,0 @@ -package {{ .Package }} - -// -// {{ .StructName }} -// -// This code was generated by `boilerplate' using -// the {{ .Template }} template. -// - -import ( - "context" - "sync" - "time" - - "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" -) - -// {{ .Description }} -// -// The zero value of this structure IS NOT valid and you MUST initialize -// all the fields marked as MANDATORY before using this structure. -type {{ .StructName }} struct { - // Domain is the MANDATORY domain to resolve. - Domain string - - // IDGenerator is the MANDATORY atomic int64 to generate task IDs. - IDGenerator *atomicx.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - - // TestKeys is MANDATORY and contains the TestKeys. - TestKeys *TestKeys - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time - - // WaitGroup is the MANDATORY wait group this task belongs to. - WaitGroup *sync.WaitGroup -} - -// Start starts this task in a background goroutine. -func (t *{{ .StructName }}) Start(ctx context.Context) { - t.WaitGroup.Add(1) - index := t.IDGenerator.Add(1) - go func() { - defer t.WaitGroup.Done() // synchronize with the parent - t.Run(ctx, index) - }() -} - -// Run runs this task in the current goroutine. -func (t *{{ .StructName }}) Run(parentCtx context.Context, index int64) { - // create context with attached a timeout - const timeout = 4 * time.Second // TODO: consider changing - lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) - defer lookpCancel() - - // create trace - trace := measurexlite.NewTrace(index, t.ZeroTime) - - // start the operation logger - ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}#%d", index) // TODO: edit - - // runs the lookup - reso := trace.NewStdlibResolver(t.Logger) - addrs, err := reso.LookupHost(lookupCtx, t.Domain) - _ = trace.DNSLookupsFromRoundTrip() // TODO: save - if err != nil { - ol.Stop(err) - return - } - - // emit successful log message - ol.Stop(nil) - - // (typically) fan out a number of child async tasks to use the IP addrs - for range addrs { - // TODO: implement - } -} From 50c6ad9dcbb2a118b3175565684862e068b37301 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 26 Aug 2022 14:29:54 +0200 Subject: [PATCH 82/83] feat: upgrade http flows to support data collection 1. adapt the template to allow for collecting the data we need 2. recognize that support for cookies is optional, so it should probably always be there --- internal/cmd/boilerplate/task/endpoint.go.txt | 68 +++++++++++++++---- internal/experiment/telegram/datacenter.go | 59 ++++++++++++---- internal/experiment/telegram/webhttp.go | 59 ++++++++++++---- internal/experiment/telegram/webhttps.go | 66 +++++++++++++----- .../webconnectivity/cleartextflow.go | 54 ++++++++++----- .../experiment/webconnectivity/secureflow.go | 64 +++++++++++------ 6 files changed, 275 insertions(+), 95 deletions(-) diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt index e4d0b6e941..458233dd6c 100644 --- a/internal/cmd/boilerplate/task/endpoint.go.txt +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -54,15 +54,20 @@ type {{ .StructName }} struct { {{ if or (eq .Template "tls") (eq .Template "https") }} // ALPN is the OPTIONAL ALPN to use. ALPN []string - - // SNI is the OPTIONAL SNI to use. - SNI string {{ end }} + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + {{ if or (eq .Template "http") (eq .Template "https") }} // HostHeader is the OPTIONAL host header to use. HostHeader string + {{ if or (eq .Template "tls") (eq .Template "https") }} + // SNI is the OPTIONAL SNI to use. + SNI string + {{ end }} + // URLPath is the OPTIONAL URL path. URLPath string @@ -101,7 +106,6 @@ func (t *{{ .StructName }}) Run(parentCtx context.Context, index int64) { ol.Stop(err) return } - tcpConn = trace.WrapNetConn(tcpConn) defer func() { _ = trace.NetworkEvents() // TODO: save tcpConn.Close() @@ -125,13 +129,16 @@ func (t *{{ .StructName }}) Run(parentCtx context.Context, index int64) { const tlsTimeout = 10 * time.Second // TODO: consider changing tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) defer tlsCancel() - tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + tlsConn, tlsConnState, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) _ = <-trace.TLSHandshake // TODO: save if err != nil { ol.Stop(err) return } defer tlsConn.Close() + alpn := tlsConnState.NegotiatedProtocol + {{ else }} + alpn := "" // no ALPN because we're not using TLS {{ end }} {{ if eq .Template "http" }} @@ -164,7 +171,15 @@ func (t *{{ .StructName }}) Run(parentCtx context.Context, index int64) { } // perform HTTP transaction - httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) if err != nil { ol.Stop(err) return @@ -252,22 +267,45 @@ func (t *{{ .StructName }}) newHTTPRequest(ctx context.Context) (*http.Request, httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } return httpReq, nil } // httpTransaction runs the HTTP transaction and saves the results. -func (t *{{ .StructName }}) httpTransaction(ctx context.Context, txp model.HTTPTransport, - req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { +func (t *{{ .StructName }}) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 // TODO: you may want to change this default + started := trace.TimeSince(trace.ZeroTime) resp, err := txp.RoundTrip(req) - if err != nil { - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) // TODO: save - return nil, []byte{}, err + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) } - defer resp.Body.Close() - reader := io.LimitReader(resp.Body, maxbody) - body, err := netxlite.ReadAllContext(ctx, reader) - _ = trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) // TODO: save + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) + _ = ev // TODO: save return resp, body, err } {{ end }} diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go index ed6c28ba71..2ecedba4aa 100644 --- a/internal/experiment/telegram/datacenter.go +++ b/internal/experiment/telegram/datacenter.go @@ -44,6 +44,9 @@ type Datacenter struct { // WaitGroup is the MANDATORY wait group this task belongs to. WaitGroup *sync.WaitGroup + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + // HostHeader is the OPTIONAL host header to use. HostHeader string @@ -78,17 +81,18 @@ func (t *Datacenter) Run(parentCtx context.Context, index int64) { defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) if err != nil { ol.Stop(err) return } - tcpConn = trace.WrapNetConn(tcpConn) defer func() { t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() }() + alpn := "" // no ALPN because we're not using TLS + // "If all TCP connections on ports 80 and 443 to Telegram’s access // point IPs fail we consider Telegram to be blocked." t.TestKeys.SetTelegramTCPBlocking(false) @@ -112,7 +116,15 @@ func (t *Datacenter) Run(parentCtx context.Context, index int64) { } // perform HTTP transaction - httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) if err != nil { ol.Stop(err) return @@ -170,23 +182,44 @@ func (t *Datacenter) newHTTPRequest(ctx context.Context) (*http.Request, error) httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } return httpReq, nil } // httpTransaction runs the HTTP transaction and saves the results. -func (t *Datacenter) httpTransaction(ctx context.Context, txp model.HTTPTransport, - req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { +func (t *Datacenter) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) resp, err := txp.RoundTrip(req) - if err != nil { - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) - t.TestKeys.AppendRequests(ev) - return nil, []byte{}, err + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) } - defer resp.Body.Close() - reader := io.LimitReader(resp.Body, maxbody) - body, err := netxlite.ReadAllContext(ctx, reader) - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) t.TestKeys.AppendRequests(ev) return resp, body, err } diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go index d79edcc982..85473e9350 100644 --- a/internal/experiment/telegram/webhttp.go +++ b/internal/experiment/telegram/webhttp.go @@ -46,6 +46,9 @@ type WebHTTP struct { // WaitGroup is the MANDATORY wait group this task belongs to. WaitGroup *sync.WaitGroup + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + // HostHeader is the OPTIONAL host header to use. HostHeader string @@ -80,18 +83,19 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) if err != nil { t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } - tcpConn = trace.WrapNetConn(tcpConn) defer func() { t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() }() + alpn := "" // no ALPN because we're not using TLS + // create HTTP transport httpTransport := netxlite.NewHTTPTransport( t.Logger, @@ -112,7 +116,15 @@ func (t *WebHTTP) Run(parentCtx context.Context, index int64) { } // perform HTTP transaction - httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) if err != nil { t.TestKeys.AppendWebFailure(err) ol.Stop(err) @@ -170,23 +182,44 @@ func (t *WebHTTP) newHTTPRequest(ctx context.Context) (*http.Request, error) { httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } return httpReq, nil } // httpTransaction runs the HTTP transaction and saves the results. -func (t *WebHTTP) httpTransaction(ctx context.Context, txp model.HTTPTransport, - req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { +func (t *WebHTTP) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) resp, err := txp.RoundTrip(req) - if err != nil { - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) - t.TestKeys.AppendRequests(ev) - return nil, []byte{}, err + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) } - defer resp.Body.Close() - reader := io.LimitReader(resp.Body, maxbody) - body, err := netxlite.ReadAllContext(ctx, reader) - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) t.TestKeys.AppendRequests(ev) return resp, body, err } diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go index 66de135a47..f7d1c3edb0 100644 --- a/internal/experiment/telegram/webhttps.go +++ b/internal/experiment/telegram/webhttps.go @@ -51,12 +51,15 @@ type WebHTTPS struct { // ALPN is the OPTIONAL ALPN to use. ALPN []string - // SNI is the OPTIONAL SNI to use. - SNI string + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar // HostHeader is the OPTIONAL host header to use. HostHeader string + // SNI is the OPTIONAL SNI to use. + SNI string + // URLPath is the OPTIONAL URL path. URLPath string @@ -88,13 +91,12 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) if err != nil { t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } - tcpConn = trace.WrapNetConn(tcpConn) defer func() { t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() @@ -117,14 +119,15 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { const tlsTimeout = 10 * time.Second tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) defer tlsCancel() - tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) - t.TestKeys.AppendTLSHandshakes(<-trace.TLSHandshake) + tlsConn, tlsConnState, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + t.TestKeys.AppendTLSHandshakes(trace.TLSHandshakes()...) if err != nil { t.TestKeys.AppendWebFailure(err) ol.Stop(err) return } defer tlsConn.Close() + alpn := tlsConnState.NegotiatedProtocol // create HTTP transport httpTransport := netxlite.NewHTTPTransport( @@ -147,7 +150,15 @@ func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { } // perform HTTP transaction - httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) if err != nil { t.TestKeys.AppendWebFailure(err) ol.Stop(err) @@ -225,23 +236,44 @@ func (t *WebHTTPS) newHTTPRequest(ctx context.Context) (*http.Request, error) { httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } return httpReq, nil } // httpTransaction runs the HTTP transaction and saves the results. -func (t *WebHTTPS) httpTransaction(ctx context.Context, txp model.HTTPTransport, - req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { +func (t *WebHTTPS) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) resp, err := txp.RoundTrip(req) - if err != nil { - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) - t.TestKeys.AppendRequests(ev) - return nil, []byte{}, err + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) } - defer resp.Body.Close() - reader := io.LimitReader(resp.Body, maxbody) - body, err := netxlite.ReadAllContext(ctx, reader) - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) t.TestKeys.AppendRequests(ev) return resp, body, err } diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index bdc9d0d0f6..a1712d8a61 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -101,17 +101,18 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) if err != nil { ol.Stop(err) return } - tcpConn = trace.WrapNetConn(tcpConn) defer func() { t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() }() + alpn := "" // no ALPN because we're not using TLS + // Only allow N flows to _use_ the connection select { case <-t.Sema: @@ -145,7 +146,15 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) { } // perform HTTP transaction - httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) if err != nil { ol.Stop(err) return @@ -211,22 +220,35 @@ func (t *CleartextFlow) newHTTPRequest(ctx context.Context) (*http.Request, erro } // httpTransaction runs the HTTP transaction and saves the results. -func (t *CleartextFlow) httpTransaction(ctx context.Context, txp model.HTTPTransport, - req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { +func (t *CleartextFlow) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) resp, err := txp.RoundTrip(req) - if err != nil { - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) - t.TestKeys.AppendRequests(ev) - return nil, []byte{}, err - } - defer resp.Body.Close() - if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { - t.CookieJar.SetCookies(req.URL, cookies) + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) } - reader := io.LimitReader(resp.Body, maxbody) - body, err := netxlite.ReadAllContext(ctx, reader) - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) t.TestKeys.AppendRequests(ev) return resp, body, err } diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index 92ea2fa5ae..dd837d356e 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -62,15 +62,15 @@ type SecureFlow struct { // to follow HTTP redirects (if any). FollowRedirects bool - // SNI is the OPTIONAL SNI to use. - SNI string - // HostHeader is the OPTIONAL host header to use. HostHeader string // Referer contains the OPTIONAL referer, used for redirects. Referer string + // SNI is the OPTIONAL SNI to use. + SNI string + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this // field is not set we use a default one (e.g., `8.8.8.8:53`). UDPAddress string @@ -108,12 +108,11 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { defer tcpCancel() tcpDialer := trace.NewDialerWithoutResolver(t.Logger) tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) - t.TestKeys.AppendTCPConnectResults(<-trace.TCPConnect) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) if err != nil { ol.Stop(err) return } - tcpConn = trace.WrapNetConn(tcpConn) defer func() { t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) tcpConn.Close() @@ -135,14 +134,16 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { const tlsTimeout = 10 * time.Second tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) defer tlsCancel() - tlsConn, _, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) - t.TestKeys.AppendTLSHandshakes(<-trace.TLSHandshake) + tlsConn, tlsConnState, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + t.TestKeys.AppendTLSHandshakes(trace.TLSHandshakes()...) if err != nil { ol.Stop(err) return } defer tlsConn.Close() + alpn := tlsConnState.NegotiatedProtocol + // Only allow N flows to _use_ the connection select { case <-t.Sema: @@ -177,7 +178,15 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) { } // perform HTTP transaction - httpResp, httpRespBody, err := t.httpTransaction(httpCtx, httpTransport, httpReq, trace) + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) if err != nil { ol.Stop(err) return @@ -263,22 +272,35 @@ func (t *SecureFlow) newHTTPRequest(ctx context.Context) (*http.Request, error) } // httpTransaction runs the HTTP transaction and saves the results. -func (t *SecureFlow) httpTransaction(ctx context.Context, txp model.HTTPTransport, - req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { +func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) resp, err := txp.RoundTrip(req) - if err != nil { - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, []byte{}, err) - t.TestKeys.AppendRequests(ev) - return nil, []byte{}, err - } - defer resp.Body.Close() - if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { - t.CookieJar.SetCookies(req.URL, cookies) + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) } - reader := io.LimitReader(resp.Body, maxbody) - body, err := netxlite.ReadAllContext(ctx, reader) - ev := trace.NewArchivalHTTPRequestResult(txp, req, resp, maxbody, body, err) + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) t.TestKeys.AppendRequests(ev) return resp, body, err } From 738679a009ae523809845910f182946a9a524bb4 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 26 Aug 2022 19:07:13 +0200 Subject: [PATCH 83/83] fix: do not modify webconnectivity v0.4 registry entry --- internal/registry/webconnectivity.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/registry/webconnectivity.go b/internal/registry/webconnectivity.go index 1d18253951..69e27aa2d4 100644 --- a/internal/registry/webconnectivity.go +++ b/internal/registry/webconnectivity.go @@ -1,12 +1,11 @@ package registry // -// Registers the `web_connectivity' experiment implemented by -// the `./internal/experiment/webconnectivity' package. +// Registers the `web_connectivity' experiment. // import ( - "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -14,7 +13,7 @@ func init() { allexperiments["web_connectivity"] = &Factory{ build: func(config any) model.ExperimentMeasurer { return webconnectivity.NewExperimentMeasurer( - config.(*webconnectivity.Config), + config.(webconnectivity.Config), ) }, config: webconnectivity.Config{},