diff --git a/go.mod b/go.mod index c91f2d72dd..4c60c466e3 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( github.com/golang/protobuf v1.5.3-0.20210916003710-5d5e8c018a13 // indirect github.com/grafov/m3u8 v0.11.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/juju/ratelimit v1.0.2 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/cpuid/v2 v2.1.0 // indirect @@ -103,6 +104,8 @@ require ( github.com/refraction-networking/utls v1.0.0 // indirect github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507 // indirect github.com/sirupsen/logrus v1.9.0 // indirect + github.com/spf13/cobra v1.5.0 + github.com/spf13/pflag v1.0.5 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/templexxx/cpu v0.0.9 // indirect github.com/templexxx/xorsimd v0.4.1 // indirect diff --git a/go.sum b/go.sum index 9ac91e1a58..a9e23c581e 100644 --- a/go.sum +++ b/go.sum @@ -155,6 +155,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -395,6 +396,7 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -760,6 +762,7 @@ github.com/rubenv/sql-migrate v1.1.2 h1:9M6oj4e//owVVHYrFISmY9LBRw6gzkCNmD9MV36t github.com/rubenv/sql-migrate v1.1.2/go.mod h1:/7TZymwxN8VWumcIxw1jjHEcR1djpdkMHQPT4FWdnbQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -821,10 +824,13 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= diff --git a/internal/cmd/boilerplate/experiment.go b/internal/cmd/boilerplate/experiment.go new file mode 100644 index 0000000000..845275c7ed --- /dev/null +++ b/internal/cmd/boilerplate/experiment.go @@ -0,0 +1,218 @@ +package main + +// +// Code to generate a new experiment. +// + +import ( + _ "embed" + "path/filepath" + "strings" + "text/template" + + "github.com/AlecAivazis/survey/v2" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// Implements interactively generating a new experiment. +type NewExperimentCommand struct{} + +// Information about the experiment to create. +type ExperimentInfo struct { + // Experiment name + Name string + + // Experiment version + Version string + + // Experiment spec URL + SpecURL string + + // Experiment input policy + InputPolicy string + + // Whether this experiment is interruptible. + Interruptible bool +} + +// Package returns the package name. +func (info *ExperimentInfo) Package() string { + return strings.ReplaceAll(strings.ToLower(info.Name), "_", "") +} + +// Called by the CLI parser +func (c *NewExperimentCommand) Run(*cobra.Command, []string) { + printf("\n") + printf("Welcome! This command will help you to automatically generate code\n") + printf("implementing a new OONI network experiment!\n") + print("\n") + + info := getExperimentInfo() + + printf("\n") + printf("Thank you! Now I'm going to generate boilerplate code for the new experiment!\n") + printf("\n") + + makeExperimentDirectory(info) + generateDocGo(info) + generateMeasurerGo(info) + generateModelsGo(info) + generateRegistryEntryGo(info) + if info.InputPolicy != "InputNone" { + generateInputParserGo(info) + } + + pkg := filepath.Join("internal", "experiment", info.Package(), "/...") + gofmt(pkg) + + printf("\n") + printf("🏁 All done!\n") + printf("\n") +} + +// Obtains the experiment info +func getExperimentInfo() *ExperimentInfo { + return &ExperimentInfo{ + Name: getExperimentName(), + Version: getExperimentVersion(), + SpecURL: getExperimentSpecURL(), + InputPolicy: getExperimentInputPolicy(), + Interruptible: getExperimentInterruptible(), + } +} + +// Obtains the experiment name +func getExperimentName() string { + prompt := &survey.Input{ + Message: "Experiment's name:", + } + var experiment string + err := survey.AskOne(prompt, &experiment) + runtimex.PanicOnError(err, "survey.AskOne failed") + return experiment +} + +// Obtains the experiment version +func getExperimentVersion() string { + prompt := &survey.Input{ + Message: "Experiment's version:", + } + var version string + err := survey.AskOne(prompt, &version) + runtimex.PanicOnError(err, "survey.AskOne failed") + return version +} + +// Obtains the experiment spec URL +func getExperimentSpecURL() string { + prompt := &survey.Input{ + Message: "Experiment's spec URL:", + } + var specURL string + err := survey.AskOne(prompt, &specURL) + runtimex.PanicOnError(err, "survey.AskOne failed") + return specURL +} + +// Obtains the experiment input policy. +func getExperimentInputPolicy() string { + var inputPolicy string + prompt := &survey.Select{ + Message: "Choose an experiment input policy:", + Options: []string{ + "InputOrQueryBackend", + "InputOrStaticDefault", + "InputStrictlyRequired", + "InputNone", + }, + } + err := survey.AskOne(prompt, &inputPolicy) + runtimex.PanicOnError(err, "survey.AskOne failed") + return inputPolicy +} + +// Returns whether we can interrupt experiments midway. +func getExperimentInterruptible() bool { + var interruptible bool + prompt := &survey.Confirm{ + Message: "Should the engine be able to abruptly interrupt a measurement?", + } + err := survey.AskOne(prompt, &interruptible) + runtimex.PanicOnError(err, "survey.AskOne failed") + return interruptible +} + +// Creates a directory for the new experiment. +func makeExperimentDirectory(info *ExperimentInfo) { + fulldir := filepath.Join("internal", "experiment", info.Package()) + mkdirP(fulldir) +} + +//go:embed "experiment/doc.go.txt" +var experimentDocGoTemplate string + +// Generates the doc.go file +func generateDocGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Package(), "doc.go") + tmpl := template.Must(template.New("doc.go").Parse(experimentDocGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/measurer.go.txt" +var experimentMeasurerGoTemplate string + +// Generates the measurer.go file +func generateMeasurerGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Package(), "measurer.go") + tmpl := template.Must(template.New("measurer.go").Parse(experimentMeasurerGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/config.go.txt" +var experimentConfigGoTemplate string + +//go:embed "experiment/summary.go.txt" +var experimentSummaryGoTemplate string + +//go:embed "experiment/testkeys.go.txt" +var experimentTestkeysGoTemplate string + +// Generates the model.go file +func generateModelsGo(info *ExperimentInfo) { + { + fullpath := filepath.Join("internal", "experiment", info.Package(), "config.go") + tmpl := template.Must(template.New("config.go").Parse(experimentConfigGoTemplate)) + writeTemplate(fullpath, tmpl, info) + } + { + fullpath := filepath.Join("internal", "experiment", info.Package(), "summary.go") + tmpl := template.Must(template.New("model.go").Parse(experimentSummaryGoTemplate)) + writeTemplate(fullpath, tmpl, info) + } + { + fullpath := filepath.Join("internal", "experiment", info.Package(), "testkeys.go") + tmpl := template.Must(template.New("model.go").Parse(experimentTestkeysGoTemplate)) + writeTemplate(fullpath, tmpl, info) + } +} + +//go:embed "experiment/registry.go.txt" +var experimentRegistryEntryGoTemplate string + +// Generates the experiment's entry inside ./internal/registry +func generateRegistryEntryGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "registry", info.Package()+".go") + tmpl := template.Must(template.New("registryentry.go").Parse(experimentRegistryEntryGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} + +//go:embed "experiment/inputparser.go.txt" +var experimentInputParserGoTemplate string + +// Generates the experiment's entry inside ./internal/registry +func generateInputParserGo(info *ExperimentInfo) { + fullpath := filepath.Join("internal", "experiment", info.Package(), "inputparser.go") + tmpl := template.Must(template.New("inputparser.go").Parse(experimentInputParserGoTemplate)) + writeTemplate(fullpath, tmpl, info) +} diff --git a/internal/cmd/boilerplate/experiment/config.go.txt b/internal/cmd/boilerplate/experiment/config.go.txt new file mode 100644 index 0000000000..7e987c1d9b --- /dev/null +++ b/internal/cmd/boilerplate/experiment/config.go.txt @@ -0,0 +1,12 @@ +package {{ .Package }} + +// +// Config +// + +// Config contains {{ .Package }} experiment configuration. +type Config struct { + // TODO: add fields here if you need any config +} + +// TODO: implement convenience accessors you may need for Config diff --git a/internal/cmd/boilerplate/experiment/doc.go.txt b/internal/cmd/boilerplate/experiment/doc.go.txt new file mode 100644 index 0000000000..e63175686c --- /dev/null +++ b/internal/cmd/boilerplate/experiment/doc.go.txt @@ -0,0 +1,4 @@ +// Package {{ .Package }} implements the {{ .Name }} experiment. +// +// Spec: {{ .SpecURL }}. +package {{ .Package }} diff --git a/internal/cmd/boilerplate/experiment/inputparser.go.txt b/internal/cmd/boilerplate/experiment/inputparser.go.txt new file mode 100644 index 0000000000..f37e0b2cc0 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/inputparser.go.txt @@ -0,0 +1,63 @@ +package {{ .Package }} + +// +// Input parsing +// + +import ( + "errors" + "net" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// InputParser helps to print the experiment's input. +type InputParser struct { + // List of accepted URL schemes. + AcceptedSchemes []string + + // Whether to allow endpoints in input. + AllowEndpoints bool + + // The default scheme to use if AllowEndpoints == true. + DefaultScheme string +} + +// Parse parses the experiment input and returns the resulting URL. +func (ip *InputParser) Parse(input string) (*url.URL, error) { + // put this check at top-level such that we always see the crash if needed + runtimex.PanicIfTrue( + ip.AllowEndpoints && ip.DefaultScheme == "", + "invalid configuration for InputParser.AllowEndpoints == true", + ) + URL, err := url.Parse(input) + if err != nil { + return ip.maybeAllowEndpoints(URL, err) + } + for _, scheme := range ip.AcceptedSchemes { + if URL.Scheme == scheme { + // TODO: here you may want to perform additional parsing + return URL, nil + } + } + return nil, errors.New("cannot parse input") +} + +// Conditionally allows endpoints when ip.AllowEndpoints is true. +func (ip *InputParser) maybeAllowEndpoints(URL *url.URL, err error) (*url.URL, error) { + runtimex.PanicIfNil(err, "expected to be called with a non-nil error") + if ip.AllowEndpoints && URL.Scheme != "" && URL.Opaque != "" && URL.User == nil && + URL.Host == "" && URL.Path == "" && URL.RawPath == "" && + URL.RawQuery == "" && URL.Fragment == "" && URL.RawFragment == "" { + // See https://go.dev/play/p/Rk5pS_zGY5U + // + // Note that we know that `ip.DefaultScheme != ""` from the above runtime check. + out := &url.URL{ + Scheme: ip.DefaultScheme, + Host: net.JoinHostPort(URL.Scheme, URL.Opaque), + } + return out, nil + } + return nil, err +} diff --git a/internal/cmd/boilerplate/experiment/measurer.go.txt b/internal/cmd/boilerplate/experiment/measurer.go.txt new file mode 100644 index 0000000000..b3e848ec0f --- /dev/null +++ b/internal/cmd/boilerplate/experiment/measurer.go.txt @@ -0,0 +1,111 @@ +package {{ .Package }} + +// +// Measurer +// + +import ( + "context" + {{ if ne .InputPolicy "InputOptional" -}} + "errors" + {{- end }} + "sync" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Measurer for the {{ .Name }} experiment. +type Measurer struct{ + // Contains the experiment's config. + Config *Config +} + +// NewExperimentMeasurer creates a new model.ExperimentMeasurer. +func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { + return &Measurer{ + Config: config, + } +} + +// ExperimentName implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentName() string { + return "{{ .Name }}" +} + +// ExperimentVersion implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentVersion() string { + return "{{ .Version }}" +} + +// Run implements model.ExperimentMeasurer. +func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { + // Reminder: when this function returns an error, the measurement result + // WILL NOT be submitted to the OONI backend. You SHOULD only return an error + // for fundamental errors (e.g., the input is invalid or missing). + + {{ if eq .InputPolicy "InputNone" }} + // honour {{ .InputPolicy }} + if measurement.Input != "" { + return errors.New("this experiment does not take any input") + } + + {{ else if eq .InputPolicy "InputOptional" }} + // honour {{ .InputPolicy }} + input := measurement.Input + if input == "" { + // TODO: set here the default input value + } + + {{ else }} + // honour {{ .InputPolicy }} + input := measurement.Input + if input == "" { + return errors.New("no input provided") + } + {{ end }} + + {{ if ne .InputPolicy "InputNone" }} + // convert the input string to a URL + inputParser := &InputParser{ + AcceptedSchemes: []string{"http", "https"}, // TODO: you may want to change this + AllowEndpoints: false, // TODO: you may want to change this + DefaultScheme: "", // TODO: you may want to change this + } + URL, err := inputParser.Parse(string(measurement.Input)) + if err != nil { + return err + } + {{ end }} + + // initialize the experiment's test keys + tk := NewTestKeys() + measurement.TestKeys = tk + + // create variables required to run parallel tasks + idGenerator := &atomicx.Int64{} + wg := &sync.WaitGroup{} + + // start background tasks + // TODO: write code to start background tasks + {{ if ne .InputPolicy "InputNone" }}_ = URL{{ end }} + _ = idGenerator + + // wait for background tasks to join + wg.Wait() + + // If the context passed to us has been cancelled, we cannot + // trust this experiment's results to be okay. + if err := ctx.Err(); err != nil { + return err + } + + // perform any deferred computation on the test keys + tk.finalize() + + // return whether there was a fundamental failure, which would prevent + // the measurement from being submitted to the OONI collector. + return tk.fundamentalFailure +} diff --git a/internal/cmd/boilerplate/experiment/registry.go.txt b/internal/cmd/boilerplate/experiment/registry.go.txt new file mode 100644 index 0000000000..64472bd083 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/registry.go.txt @@ -0,0 +1,24 @@ +package registry + +// +// Registers the `{{ .Name }}' experiment implemented by +// the `./internal/experiment/{{ .Package }}' package. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/experiment/{{ .Package }}" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + allexperiments["{{ .Name }}"] = &Factory{ + build: func(config any) model.ExperimentMeasurer { + return {{ .Package }}.NewExperimentMeasurer( + config.(*{{ .Package }}.Config), + ) + }, + config: &{{ .Package }}.Config{}, + interruptible: {{ .Interruptible }}, + inputPolicy: model.{{ .InputPolicy }}, + } +} diff --git a/internal/cmd/boilerplate/experiment/summary.go.txt b/internal/cmd/boilerplate/experiment/summary.go.txt new file mode 100644 index 0000000000..835d063e32 --- /dev/null +++ b/internal/cmd/boilerplate/experiment/summary.go.txt @@ -0,0 +1,23 @@ +package {{ .Package }} + +// +// Summary +// + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Summary contains the summary results. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + // TODO: add here additional summary fields. + isAnomaly bool +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + // TODO: fill all the SummaryKeys + sk := SummaryKeys{isAnomaly: false} + return sk, nil +} diff --git a/internal/cmd/boilerplate/experiment/testkeys.go.txt b/internal/cmd/boilerplate/experiment/testkeys.go.txt new file mode 100644 index 0000000000..639855609a --- /dev/null +++ b/internal/cmd/boilerplate/experiment/testkeys.go.txt @@ -0,0 +1,55 @@ +package {{ .Package }} + +// +// TestKeys for {{ .Name }}. +// +// Note: for historical reasons, we call TestKeys the JSON object +// containing the results produced by OONI experiments. +// + +import "sync" + +// TestKeys contains the results produced by {{ .Name }}. +type TestKeys struct { + // TODO: add here public fields produced by this experiment. + // + // For example: + // + // // Blocked indicates that the resource is censored. + // Blocked bool `json:"blocked"` + + // fundamentalFailure indicates that some fundamental error occurred + // in a background task. A fundamental error is something like a programmer + // such as a failure to parse a URL that was hardcoded in the codebase. When + // this class of errors happens, you certainly don't want to submit the + // resulting measurement to the OONI collector. + fundamentalFailure error + + // mu provides mutual exclusion for accessing the test keys. + mu *sync.Mutex +} + +// TODO: implement more thread-safe setters for the real test keys. This allows +// tasks to write directly into the TestKeys. + +// SetFundamentalFailure sets the value of fundamentalFailure. +func (tk *TestKeys) SetFundamentalFailure(err error) { + tk.mu.Lock() + tk.fundamentalFailure = err + tk.mu.Unlock() +} + +// NewTestKeys creates a new instance of TestKeys. +func NewTestKeys() *TestKeys { + // TODO: here you should initialize all the fields + return &TestKeys{ + fundamentalFailure: nil, + mu: &sync.Mutex{}, + } +} + +// finalize performs any delayed computation on the test keys. This function +// must be called from the measurer after all the tasks have completed. +func (tk *TestKeys) finalize() { + // TODO: implement +} diff --git a/internal/cmd/boilerplate/main.go b/internal/cmd/boilerplate/main.go new file mode 100644 index 0000000000..bc8add9b3b --- /dev/null +++ b/internal/cmd/boilerplate/main.go @@ -0,0 +1,37 @@ +// Command boilerplate assists you in generating code for new experiments. +// +// We will generate experiments under the ./internal/experiment folder rather +// than under ./internal/engine/experiment because we are moving away from the +// experiment folder. +package main + +import ( + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +func main() { + root := &cobra.Command{ + Use: "boilerplate", + Short: "Helps to auto-generate code for new experiments", + } + + newExperiment := &cobra.Command{ + Use: "new-experiment", + Args: cobra.NoArgs, + Short: "Interactively generate a new experiment", + Run: (&NewExperimentCommand{}).Run, + } + root.AddCommand(newExperiment) + + newflow := &cobra.Command{ + Use: "new-task", + Args: cobra.NoArgs, + Short: "Interactively generate a new task for an experiment", + Run: (&NewTaskCommand{}).Run, + } + root.AddCommand(newflow) + + err := root.Execute() + runtimex.PanicOnError(err, "root.Execute failed") +} diff --git a/internal/cmd/boilerplate/task.go b/internal/cmd/boilerplate/task.go new file mode 100644 index 0000000000..825478f254 --- /dev/null +++ b/internal/cmd/boilerplate/task.go @@ -0,0 +1,140 @@ +package main + +// +// Code to generate a new experiment flow. +// + +import ( + _ "embed" + "path/filepath" + "sort" + "strings" + "text/template" + + "github.com/AlecAivazis/survey/v2" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// Implements interactively generating a new experiment. +type NewTaskCommand struct{} + +// Information about a task to autogenerate. +type TaskInfo struct { + // The task struct's name. + StructName string + + // Description contains the description. + Description string + + // The task template. + Template string +} + +// Called by the CLI parser +func (c *NewTaskCommand) Run(*cobra.Command, []string) { + printf("\n") + printf("Welcome! This command will help you to automatically generate a task\n") + printf("to include it into an existing OONI experiment!\n") + print("\n") + + experimentName := getExperimentPackageName() + info := getTaskInfo() + + generateTaskGo(experimentName, info) +} + +// Obtains the experiment's package name +func getExperimentPackageName() string { + prompt := &survey.Input{ + Message: "Experiment's package name:", + } + var experiment string + err := survey.AskOne(prompt, &experiment) + runtimex.PanicOnError(err, "survey.AskOne failed") + return experiment +} + +// Obtains information about the task to generate. +func getTaskInfo() *TaskInfo { + return &TaskInfo{ + StructName: getTaskStructName(), + Description: getTaskDescription(), + Template: getTaskTemplate(), + } +} + +// Returns the name of the task struct. +func getTaskStructName() string { + prompt := &survey.Input{ + Message: "Task struct name:", + } + var name string + err := survey.AskOne(prompt, &name) + runtimex.PanicOnError(err, "survey.AskOne failed") + return name +} + +// Returns the task's description +func getTaskDescription() string { + prompt := &survey.Input{ + Message: "Short documentation for this task:", + } + var docs string + err := survey.AskOne(prompt, &docs) + runtimex.PanicOnError(err, "survey.AskOne failed") + return docs +} + +//go:embed "task/endpoint.go.txt" +var endpointTemplate string + +//go:embed "task/multiresolver.go.txt" +var multiResolverTemplate string + +// The list of known tasks +var knownTasks = map[string]string{ + "http": endpointTemplate, + "https": endpointTemplate, + "multi-resolver": multiResolverTemplate, + "tcp": endpointTemplate, + "tls": endpointTemplate, +} + +// Names of known tasks +var knownTaskNames []string + +// Autogenerates the names of the tasks. +func init() { + for name := range knownTasks { + knownTaskNames = append(knownTaskNames, name) + } + sort.Strings(knownTaskNames) +} + +// Returns the task template to use. +func getTaskTemplate() string { + var name string + prompt := &survey.Select{ + Message: "Choose a task you would like to generate:", + Options: knownTaskNames, + } + err := survey.AskOne(prompt, &name) + runtimex.PanicOnError(err, "survey.AskOne failed") + return name +} + +// Generates code for the new task. +func generateTaskGo(experiment string, info *TaskInfo) { + name := strings.ToLower(info.StructName) + ".go" + fullpath := filepath.Join("internal", "experiment", experiment, name) + tmpl := template.Must(template.New("T1").Parse(knownTasks[info.Template])) + mapping := map[string]string{ + "Package": experiment, + "StructName": info.StructName, + "Template": info.Template, + "Description": info.Description, + } + writeTemplate(fullpath, tmpl, mapping) + gofmt(fullpath) +} diff --git a/internal/cmd/boilerplate/task/endpoint.go.txt b/internal/cmd/boilerplate/task/endpoint.go.txt new file mode 100644 index 0000000000..458233dd6c --- /dev/null +++ b/internal/cmd/boilerplate/task/endpoint.go.txt @@ -0,0 +1,311 @@ +package {{ .Package }} + +// +// {{ .StructName }} +// +// Generated by `boilerplate' using the {{ .Template }} template. +// + +import ( + "context" + {{ if or (eq .Template "tls") (eq .Template "https") -}} + "crypto/tls" + {{- end }} + {{ if or (eq .Template "http") (eq .Template "https") -}} + "io" + "net" + "net/http" + "net/url" + {{- end }} + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + {{ if or (eq .Template "tls") (eq .Template "https") (eq .Template "http") -}} + "github.com/ooni/probe-cli/v3/internal/netxlite" + {{- end }} +) + +// {{ .Description }} +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type {{ .StructName }} struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + {{ if or (eq .Template "tls") (eq .Template "https") }} + // ALPN is the OPTIONAL ALPN to use. + ALPN []string + {{ end }} + + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + + {{ if or (eq .Template "http") (eq .Template "https") }} + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + {{ if or (eq .Template "tls") (eq .Template "https") }} + // SNI is the OPTIONAL SNI to use. + SNI string + {{ end }} + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string + {{ end }} +} + +// Start starts this task in a background goroutine. +func (t *{{ .StructName }}) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *{{ .StructName }}) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}#%d", index) // TODO: edit + + {{ if or (eq .Template "tcp") (eq .Template "tls") (eq .Template "https") (eq .Template "http") }} + // perform the TCP connect + const tcpTimeout = 10 * time.Second // TODO: consider changing + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + _ = <-trace.TCPConnect // TODO: save + if err != nil { + ol.Stop(err) + return + } + defer func() { + _ = trace.NetworkEvents() // TODO: save + tcpConn.Close() + }() + {{ end }} + + {{ if or (eq .Template "tls") (eq .Template "https") }} + // perform TLS handshake + tlsSNI, err := t.sni() + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + tlsHandshaker := trace.NewTLSHandshakerStdlib(t.Logger) + tlsConfig := &tls.Config{ + NextProtos: t.alpn(), + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: tlsSNI, + } + const tlsTimeout = 10 * time.Second // TODO: consider changing + tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) + defer tlsCancel() + tlsConn, tlsConnState, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + _ = <-trace.TLSHandshake // TODO: save + if err != nil { + ol.Stop(err) + return + } + defer tlsConn.Close() + alpn := tlsConnState.NegotiatedProtocol + {{ else }} + alpn := "" // no ALPN because we're not using TLS + {{ end }} + + {{ if eq .Template "http" }} + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(tcpConn), + netxlite.NewNullTLSDialer(), + ) + {{ else if eq .Template "https" }} + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewNullDialer(), + // note: netxlite guarantees that here tlsConn is a netxlite.TLSConn + netxlite.NewSingleUseTLSDialer(tlsConn.(netxlite.TLSConn)), + ) + {{ end }} + + {{ if or (eq .Template "http") (eq .Template "https") }} + // create HTTP request + const httpTimeout = 10 * time.Second // TODO: consider changing + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) + if err != nil { + ol.Stop(err) + return + } + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + {{ end }} + + // completed successfully + ol.Stop(nil) +} + +{{ if or (eq .Template "tls") (eq .Template "https") }} +// alpn returns the user-configured ALPN or a reasonable default +func (t *{{ .StructName }}) alpn() []string { + if len(t.ALPN) > 0 { + return t.ALPN + } + {{ if eq .Template "https" }} + return []string{"h2", "http/1.1"} + {{ else }} + return []string{} + {{ end }} +} + +// sni returns the user-configured SNI or a reasonable default +func (t *{{ .StructName }}) sni() (string, error) { + if t.SNI != "" { + return t.SNI, nil + } + addr, _, err := net.SplitHostPort(t.Address) + if err != nil { + return "", err + } + return addr, nil +} +{{ end }} + +{{ if or (eq .Template "http") (eq .Template "https") }} +// urlHost computes the host to include into the URL +func (t *{{ .StructName }}) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } + {{ if eq .Template "http" -}} + if port == "80" && scheme == "http" { + return urlHost, nil + } + {{- else if eq .Template "https" -}} + if port == "443" && scheme == "https" { + return urlHost, nil + } + {{- end }} + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil +} + +// newHTTPRequest creates a new HTTP request. +func (t *{{ .StructName }}) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "{{ if eq .Template "http" }}http{{ else }}https{{ end }}" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *{{ .StructName }}) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 19 // TODO: you may want to change this default + started := trace.TimeSince(trace.ZeroTime) + resp, err := txp.RoundTrip(req) + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) + } + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) + _ = ev // TODO: save + return resp, body, err +} +{{ end }} diff --git a/internal/cmd/boilerplate/task/multiresolver.go.txt b/internal/cmd/boilerplate/task/multiresolver.go.txt new file mode 100644 index 0000000000..f225a29ffa --- /dev/null +++ b/internal/cmd/boilerplate/task/multiresolver.go.txt @@ -0,0 +1,200 @@ +package {{ .Package }} + +// +// {{ .StructName }} +// +// Generated by `boilerplate' using the {{ .Template }} template. +// + +import ( + "context" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// {{ .Description }} +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type {{ .StructName }} struct { + // Domain is the MANDATORY domain to resolve. + Domain string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY zero time of the measurement. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not + // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). + DNSOverHTTPSURL string + + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this + // field is not set we use a default one (e.g., `8.8.8.8:53`). + UDPAddress string +} + +// Start starts this task in a background goroutine. +func (t *{{ .StructName }}) Start(ctx context.Context) { + t.WaitGroup.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx) + }() +} + +// Run runs this task in the current goroutine. +func (t *{{ .StructName }}) Run(parentCtx context.Context) { + // create output channels for the lookup + systemOut := make(chan []string) + udpOut := make(chan []string) + httpsOut := make(chan []string) + + // start asynchronous lookups + go t.lookupHostSystem(parentCtx, systemOut) + go t.lookupHostUDP(parentCtx, udpOut) + go t.lookupHostDNSOverHTTPS(parentCtx, httpsOut) + + // collect resulting IP addresses (which may be nil/empty lists) + systemAddrs := <-systemOut + udpAddrs := <-udpOut + httpsAddrs := <-httpsOut + + // merge the resolved IP addresses + merged := map[string]bool{} + for _, addr := range systemAddrs { + merged[addr] = true + } + for _, addr := range udpAddrs { + merged[addr] = true + } + for _, addr := range httpsAddrs { + merged[addr] = true + } + + // rearrange addresses to have IPv4 first + sorted := []string{} + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && !v6 { + sorted = append(sorted, addr) + } + } + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && v6 { + sorted = append(sorted, addr) + } + } + + // fan out a number of child async tasks to use the IP addrs + for range sorted { + // TODO: implement + } +} + +// lookupHostSystem performs a DNS lookup using the system resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. +func (t *{{ .StructName }}) lookupHostSystem(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}+System#%d", index) // TODO: edit + + // runs the lookup + reso := trace.NewStdlibResolver(t.Logger) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + _ = trace.DNSLookupsFromRoundTrip() // TODO: save + ol.Stop(err) + out <- addrs +} + +// lookupHostUDP performs a DNS lookup using an UDP resolver. This function must always +// emit an ouput on the [out] channel to synchronize with the caller func. +func (t *{{ .StructName }}) lookupHostUDP(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}+UDP#%d", index) // TODO: edit + + // runs the lookup + dialer := netxlite.NewDialerWithoutResolver(t.Logger) + reso := trace.NewParallelUDPResolver(t.Logger, dialer, t.udpAddress()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + _ = trace.DNSLookupsFromRoundTrip() // TODO: save + ol.Stop(err) + out <- addrs +} + +// Returns the UDP resolver we should be using by default. +func (t *{{ .StructName }}) udpAddress() string { + if t.UDPAddress != "" { + return t.UDPAddress + } + return "8.8.4.4:53" +} + +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. +func (t *{{ .StructName }}) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second // TODO: consider changing + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "{{ .StructName }}+DNSOverHTTPS#%d", index) // TODO: edit + + // runs the lookup + reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, t.dnsOverHTTPSURL()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + reso.CloseIdleConnections() + _ = trace.DNSLookupsFromRoundTrip() // TODO: save + ol.Stop(err) + out <- addrs +} + +// Returns the DOH resolver URL we should be using by default. +func (t *{{ .StructName }}) dnsOverHTTPSURL() string { + if t.DNSOverHTTPSURL != "" { + return t.DNSOverHTTPSURL + } + return "https://mozilla.cloudflare-dns.com/dns-query" +} diff --git a/internal/cmd/boilerplate/utils.go b/internal/cmd/boilerplate/utils.go new file mode 100644 index 0000000000..f12db05432 --- /dev/null +++ b/internal/cmd/boilerplate/utils.go @@ -0,0 +1,64 @@ +package main + +// +// Utility functions +// + +import ( + "fmt" + "os" + "path/filepath" + "text/template" + + "github.com/ooni/probe-cli/v3/internal/runtimex" + "golang.org/x/sys/execabs" +) + +// Permissions with which we create new directories +const newDirPermissions = 0755 + +// Helper to write less when printing to stdout +func printf(format string, args ...any) { + fmt.Fprintf(os.Stdout, format, args...) +} + +// Permissions with which we create new files +const newFilePermissions = 0644 + +// Creates a file for writing +func openForWriting(filepath string) *os.File { + filep, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, newFilePermissions) + runtimex.PanicOnError(err, "os.OpenFile failed") + return filep +} + +// Ensures that we close a file without I/O errors +func closeFile(fp *os.File) { + err := fp.Close() + runtimex.PanicOnError(err, "fp.Close failed") +} + +// Generic function for writing a template. +func writeTemplate(fullpath string, tmpl *template.Template, info any) { + printf("🚧 generating %s...\n", fullpath) + filep := openForWriting(fullpath) + defer closeFile(filep) + err := tmpl.Execute(filep, info) + runtimex.PanicOnError(err, "cannot execute a text/template") +} + +// Creates directories recursively +func mkdirP(fulldir string) { + printf("🐚 mkdir -p %s\n", fulldir) + err := os.MkdirAll(fulldir, newDirPermissions) + runtimex.PanicOnError(err, "os.MkdirAll failed") +} + +// Formats a package using go fmt. +func gofmt(packagepath string) { + cmd := execabs.Command("go", "fmt", "."+string(filepath.Separator)+packagepath) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + printf("🐚 %s\n", cmd.String()) + err := cmd.Run() + runtimex.PanicOnError(err, "cmd.Run failed") +} diff --git a/internal/cmd/minipipeline/main.go b/internal/cmd/minipipeline/main.go new file mode 100644 index 0000000000..af02399e1f --- /dev/null +++ b/internal/cmd/minipipeline/main.go @@ -0,0 +1,93 @@ +// Command minipipeline loads in input Web Connectivity measurements +// and applies the probe's detection heuristics on them again. +// +// By doing that, we can iterate more quickly on improving heuristics. +package main + +import ( + "bufio" + "encoding/json" + "os" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func main() { + for _, arg := range os.Args[1:] { + processfile(arg) + } +} + +func processfile(arg string) { + filep, err := os.Open(arg) + if err != nil { + log.WithError(err).Fatal("os.Open failed") + } + defer filep.Close() + scanner := bufio.NewScanner(filep) + buf := make([]byte, 1<<23) + scanner.Buffer(buf, len(buf)) + index := 1 + for scanner.Scan() { + processmeasurement(scanner.Bytes(), index) + index++ + } + if err := scanner.Err(); err != nil { + log.WithError(err).Fatal("scanner.Err failed") + } +} + +func processmeasurement(mraw []byte, index int) { + var m model.Measurement + if err := json.Unmarshal(mraw, &m); err != nil { + log.WithError(err).Fatal("json.Unmarshal failed") + } + if m.TestName != "web_connectivity" { + return + } + tkraw, err := json.Marshal(m.TestKeys) + if err != nil { + log.WithError(err).Fatal("json.Marshal failed") + } + processtestkeys(tkraw, string(m.Input), index) +} + +func processtestkeys(tkraw []byte, input string, index int) { + var tk webconnectivity.TestKeys + if err := json.Unmarshal(tkraw, &tk); err != nil { + log.WithError(err).Fatal("json.Unmarshal failed") + } + newtk := &webconnectivity.TestKeys{ + NetworkEvents: tk.NetworkEvents, + DNSWoami: tk.DNSWoami, + DoH: tk.DoH, + Do53: tk.Do53, + Queries: tk.Queries, + Requests: tk.Requests, + TCPConnect: tk.TCPConnect, + TLSHandshakes: tk.TLSHandshakes, + ControlRequest: tk.ControlRequest, + Control: tk.Control, + ControlFailure: tk.ControlFailure, + XDNSFlags: 0, + DNSExperimentFailure: nil, + DNSConsistency: "", + XBlockingFlags: 0, + BodyLengthMatch: nil, + HeadersMatch: nil, + StatusCodeMatch: nil, + TitleMatch: nil, + Blocking: nil, + Accessible: nil, + } + reprocesstk(newtk, input, index) +} + +func reprocesstk(tk *webconnectivity.TestKeys, input string, index int) { + log.Infof("\n\n\n") + log.Infof("Input: %s", input) + log.Infof("Idx: %d", index) + tk.Finalize(log.Log) +} diff --git a/internal/experiment/telegram/config.go b/internal/experiment/telegram/config.go new file mode 100644 index 0000000000..6c29f305d4 --- /dev/null +++ b/internal/experiment/telegram/config.go @@ -0,0 +1,12 @@ +package telegram + +// +// Config +// + +// Config contains telegram experiment configuration. +type Config struct { + // TODO: add fields here if you need any config +} + +// TODO: implement convenience accessors you may need for Config diff --git a/internal/experiment/telegram/datacenter.go b/internal/experiment/telegram/datacenter.go new file mode 100644 index 0000000000..2ecedba4aa --- /dev/null +++ b/internal/experiment/telegram/datacenter.go @@ -0,0 +1,225 @@ +package telegram + +// +// Datacenter +// +// Generated by `boilerplate' using the http template. +// + +import ( + "context" + "io" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures a Telegram data center (DC). +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type Datacenter struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background goroutine. +func (t *Datacenter) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *Datacenter) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "Datacenter#%d: %s", index, t.Address) + + // perform the TCP connect + const tcpTimeout = 10 * time.Second + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) + if err != nil { + ol.Stop(err) + return + } + defer func() { + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + tcpConn.Close() + }() + + alpn := "" // no ALPN because we're not using TLS + + // "If all TCP connections on ports 80 and 443 to Telegram’s access + // point IPs fail we consider Telegram to be blocked." + t.TestKeys.SetTelegramTCPBlocking(false) + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(tcpConn), + netxlite.NewNullTLSDialer(), + ) + + // create HTTP request + const httpTimeout = 10 * time.Second + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) + if err != nil { + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) + if err != nil { + ol.Stop(err) + return + } + + // "If at least an HTTP request returns back a response, we + // consider Telegram [DCs] to not be blocked." + t.TestKeys.SetTelegramHTTPBlocking(false) + + // TODO: insert here additional code if needed + _ = httpResp + _ = httpRespBody + + // completed successfully + ol.Stop(nil) +} + +// urlHost computes the host to include into the URL +func (t *Datacenter) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } + if port == "80" && scheme == "http" { + return urlHost, nil + } + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil +} + +// newHTTPRequest creates a new HTTP request. +func (t *Datacenter) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "http" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "POST", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *Datacenter) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) + resp, err := txp.RoundTrip(req) + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) + } + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) + t.TestKeys.AppendRequests(ev) + return resp, body, err +} diff --git a/internal/experiment/telegram/dnsresolvers.go b/internal/experiment/telegram/dnsresolvers.go new file mode 100644 index 0000000000..26f0e78757 --- /dev/null +++ b/internal/experiment/telegram/dnsresolvers.go @@ -0,0 +1,242 @@ +package telegram + +// +// DNSResolvers +// +// Generated by `boilerplate' using the multi-resolver template. +// + +import ( + "context" + "net" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Lookups web.telegram.org using several resolvers. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type DNSResolvers struct { + // Domain is the MANDATORY domain to resolve. + Domain string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY zero time of the measurement. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // DNSOverHTTPSURL is the optional DoH URL to use. If this field is not + // set, we use a default one (e.g., `https://mozilla.cloudflare-dns.com/dns-query`). + DNSOverHTTPSURL string + + // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this + // field is not set we use a default one (e.g., `8.8.8.8:53`). + UDPAddress string +} + +// Start starts this task in a background goroutine. +func (t *DNSResolvers) Start(ctx context.Context) { + t.WaitGroup.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx) + }() +} + +// Run runs this task in the current goroutine. +func (t *DNSResolvers) Run(parentCtx context.Context) { + // create output channels for the lookup + systemOut := make(chan []string) + udpOut := make(chan []string) + httpsOut := make(chan []string) + + // start asynchronous lookups + go t.lookupHostSystem(parentCtx, systemOut) + go t.lookupHostUDP(parentCtx, udpOut) + go t.lookupHostDNSOverHTTPS(parentCtx, httpsOut) + + // collect resulting IP addresses (which may be nil/empty lists) + systemAddrs := <-systemOut + udpAddrs := <-udpOut + httpsAddrs := <-httpsOut + + // merge the resolved IP addresses + merged := map[string]bool{} + for _, addr := range systemAddrs { + merged[addr] = true + } + for _, addr := range udpAddrs { + merged[addr] = true + } + for _, addr := range httpsAddrs { + merged[addr] = true + } + + // rearrange addresses to have IPv4 first + sorted := []string{} + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && !v6 { + sorted = append(sorted, addr) + } + } + for addr := range merged { + if v6, err := netxlite.IsIPv6(addr); err == nil && v6 { + sorted = append(sorted, addr) + } + } + + // fan out a number of child async tasks to use the IP addrs + for _, addr := range sorted { + t.startWebHTTPTask(parentCtx, addr) + t.startWebHTTPSTask(parentCtx, addr) + } +} + +// lookupHostSystem performs a DNS lookup using the system resolver. This function must always +// emit an ouput on the [out] channel to synchronize with the caller func. +func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger( + t.Logger, "DNSResolvers+System#%d: %s", index, t.Domain) + + // runs the lookup + reso := trace.NewStdlibResolver(t.Logger) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// lookupHostUDP performs a DNS lookup using an UDP resolver. This function must always +// emit an ouput on the [out] channel to synchronize with the caller func. +func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger( + t.Logger, "DNSResolvers+UDP#%d: %s", index, t.Domain) + + // runs the lookup + dialer := netxlite.NewDialerWithoutResolver(t.Logger) + reso := trace.NewParallelUDPResolver(t.Logger, dialer, t.udpAddress()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// Returns the UDP resolver we should be using by default. +func (t *DNSResolvers) udpAddress() string { + if t.UDPAddress != "" { + return t.UDPAddress + } + return "8.8.4.4:53" +} + +// lookupHostDNSOverHTTPS performs a DNS lookup using a DoH resolver. This function must +// always emit an ouput on the [out] channel to synchronize with the caller func. +func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out chan<- []string) { + // create context with attached a timeout + const timeout = 4 * time.Second + lookupCtx, lookpCancel := context.WithTimeout(parentCtx, timeout) + defer lookpCancel() + + // create trace's index + index := t.IDGenerator.Add(1) + + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger( + t.Logger, "DNSResolvers+DNSOverHTTPS#%d: %s", index, t.Domain) + + // runs the lookup + reso := trace.NewParallelDNSOverHTTPSResolver(t.Logger, t.dnsOverHTTPSURL()) + addrs, err := reso.LookupHost(lookupCtx, t.Domain) + reso.CloseIdleConnections() + t.TestKeys.AppendQueries(trace.DNSLookupsFromRoundTrip()...) + ol.Stop(err) + out <- addrs +} + +// Returns the DOH resolver URL we should be using by default. +func (t *DNSResolvers) dnsOverHTTPSURL() string { + if t.DNSOverHTTPSURL != "" { + return t.DNSOverHTTPSURL + } + return "https://mozilla.cloudflare-dns.com/dns-query" +} + +// webTelegramOrg is the SNI and host header for telegram web. +const webTelegramOrg = "web.telegram.org" + +// startWebHTTPTask starts a WebHTTPTask for this addr. +func (t *DNSResolvers) startWebHTTPTask(ctx context.Context, addr string) { + task := &WebHTTP{ + Address: net.JoinHostPort(addr, "80"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + HostHeader: webTelegramOrg, + URLPath: "", + URLRawQuery: "", + } + task.Start(ctx) +} + +// startWebHTTPSTask starts a WebHTTPSTask for this addr. +func (t *DNSResolvers) startWebHTTPSTask(ctx context.Context, addr string) { + task := &WebHTTPS{ + Address: net.JoinHostPort(addr, "443"), + IDGenerator: t.IDGenerator, + Logger: t.Logger, + TestKeys: t.TestKeys, + ZeroTime: t.ZeroTime, + WaitGroup: t.WaitGroup, + ALPN: []string{}, // default is okay + SNI: webTelegramOrg, + HostHeader: webTelegramOrg, + URLPath: "", + URLRawQuery: "", + } + task.Start(ctx) +} diff --git a/internal/experiment/telegram/doc.go b/internal/experiment/telegram/doc.go new file mode 100644 index 0000000000..e0ddda751d --- /dev/null +++ b/internal/experiment/telegram/doc.go @@ -0,0 +1,4 @@ +// Package telegram implements the telegram experiment. +// +// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-020-telegram.md. +package telegram diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go new file mode 100644 index 0000000000..f7dc3b6ae4 --- /dev/null +++ b/internal/experiment/telegram/measurer.go @@ -0,0 +1,117 @@ +package telegram + +// +// Measurer +// + +import ( + "context" + "errors" + "net" + "sync" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Measurer for the telegram experiment. +type Measurer struct { + // Contains the experiment's config. + Config *Config +} + +// NewExperimentMeasurer creates a new model.ExperimentMeasurer. +func NewExperimentMeasurer(config *Config) model.ExperimentMeasurer { + return &Measurer{ + Config: config, + } +} + +// ExperimentName implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentName() string { + return "telegram" +} + +// ExperimentVersion implements model.ExperimentMeasurer. +func (m *Measurer) ExperimentVersion() string { + return "0.3.0" +} + +// Run implements model.ExperimentMeasurer. +func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { + // Reminder: when this function returns an error, the measurement result + // WILL NOT be submitted to the OONI backend. You SHOULD only return an error + // for fundamental errors (e.g., the input is invalid or missing). + + // honour InputNone + if measurement.Input != "" { + return errors.New("this experiment does not take any input") + } + + // initialize the experiment's test keys + tk := NewTestKeys() + measurement.TestKeys = tk + + // create variables required to run parallel tasks + idGenerator := &atomicx.Int64{} + wg := &sync.WaitGroup{} + + // start background tasks + systemDNSTask := &DNSResolvers{ + Domain: webTelegramOrg, + IDGenerator: idGenerator, + Logger: sess.Logger(), + TestKeys: tk, + ZeroTime: measurement.MeasurementStartTimeSaved, + WaitGroup: wg, + DNSOverHTTPSURL: "", + UDPAddress: "", + } + systemDNSTask.Start(ctx) + for _, addr := range dataCenterAddrs { + for _, port := range dataCenterPorts { + dcTask := &Datacenter{ + Address: net.JoinHostPort(addr, port), + IDGenerator: idGenerator, + Logger: sess.Logger(), + TestKeys: tk, + ZeroTime: measurement.MeasurementStartTimeSaved, + WaitGroup: wg, + HostHeader: "", + URLPath: "", + URLRawQuery: "", + } + dcTask.Start(ctx) + } + } + + // wait for background tasks to join + wg.Wait() + + // If the context passed to us has been cancelled, we cannot + // trust this experiment's results to be okay. + if err := ctx.Err(); err != nil { + return err + } + + // perform any deferred computation on the test keys + tk.finalize() + + // return whether there was a fundamental failure, which would prevent + // the measurement from being submitted to the OONI collector. + return tk.fundamentalFailure +} + +// dataCenterAddrs contains the data center addrs. +var dataCenterAddrs = []string{ + "149.154.175.50", + "149.154.167.51", + "149.154.175.100", + "149.154.167.91", + "149.154.171.5", + "95.161.76.100", +} + +// dataCenterPorts contains the data center ports. +var dataCenterPorts = []string{"80", "443"} diff --git a/internal/experiment/telegram/summary.go b/internal/experiment/telegram/summary.go new file mode 100644 index 0000000000..2391ae2fd5 --- /dev/null +++ b/internal/experiment/telegram/summary.go @@ -0,0 +1,23 @@ +package telegram + +// +// Summary +// + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Summary contains the summary results. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + // TODO: add here additional summary fields. + isAnomaly bool +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (any, error) { + // TODO: fill all the SummaryKeys + sk := SummaryKeys{isAnomaly: false} + return sk, nil +} diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go new file mode 100644 index 0000000000..aa348dd988 --- /dev/null +++ b/internal/experiment/telegram/testkeys.go @@ -0,0 +1,186 @@ +package telegram + +// +// TestKeys for telegram. +// +// Note: for historical reasons, we call TestKeys the JSON object +// containing the results produced by OONI experiments. +// + +import ( + "errors" + "sync" + "syscall" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// TestKeys contains the results produced by telegram. +type TestKeys struct { + // NetworkEvents contains network events. + NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"` + + // Queries contains DNS lookup results. + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + + // Requests contains HTTP results. + Requests []*model.ArchivalHTTPRequestResult `json:"requests"` + + // TCPConnect contains TCP connect results. + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` + + // TLSHandshakes contains TLS handshakes results. + TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + + // TelegramTCPBlocking indicates whether we believe DCs + // to be blocked at the TCP/IP layer. From the spec: "If all + // TCP connections on ports 80 and 443 to Telegram’s access + // point IPs fail we consider Telegram to be blocked." + TelegramTCPBlocking bool `json:"telegram_tcp_blocking"` + + // TelegramHTTPBlocking indicates whether we believe DCs + // to be blocked at the TCP/IP layer. From the spec: "If at + // least an HTTP request returns back a response, we + // consider Telegram [DCs] to not be blocked." + TelegramHTTPBlocking bool `json:"telegram_http_blocking"` + + // TelegramWebStatus is either "blocked" or "ok" and indicates + // whether we're able to access the web.telegram.org site. + TelegramWebStatus string `json:"telegram_web_status"` + + // TelegramWebFailure is the failure when accessing web.telegram.org + TelegramWebFailure *string `json:"telegram_web_failure"` + + // webFailures contains the failures occurred when measuring web.telegram.org + webFailures []error + + // fundamentalFailure indicates that some fundamental error occurred + // in a background task. A fundamental error is something like a programmer + // such as a failure to parse a URL that was hardcoded in the codebase. When + // this class of errors happens, you certainly don't want to submit the + // resulting measurement to the OONI collector. + fundamentalFailure error + + // mu provides mutual exclusion for accessing the test keys. + mu *sync.Mutex +} + +// AppendNetworkEvents appends to NetworkEvents. +func (tk *TestKeys) AppendNetworkEvents(v ...*model.ArchivalNetworkEvent) { + tk.mu.Lock() + tk.NetworkEvents = append(tk.NetworkEvents, v...) + tk.mu.Unlock() +} + +// AppendQueries appends to Queries. +func (tk *TestKeys) AppendQueries(v ...*model.ArchivalDNSLookupResult) { + tk.mu.Lock() + tk.Queries = append(tk.Queries, v...) + tk.mu.Unlock() +} + +// AppendRequests appends to Requests. +func (tk *TestKeys) AppendRequests(v ...*model.ArchivalHTTPRequestResult) { + tk.mu.Lock() + tk.Requests = append(tk.Requests, v...) + tk.mu.Unlock() +} + +// AppendTCPConnectResults appends to TCPConnect. +func (tk *TestKeys) AppendTCPConnectResults(v ...*model.ArchivalTCPConnectResult) { + tk.mu.Lock() + tk.TCPConnect = append(tk.TCPConnect, v...) + tk.mu.Unlock() +} + +// AppendTLSHandshakes appends to TLSHandshakes. +func (tk *TestKeys) AppendTLSHandshakes(v ...*model.ArchivalTLSOrQUICHandshakeResult) { + tk.mu.Lock() + tk.TLSHandshakes = append(tk.TLSHandshakes, v...) + tk.mu.Unlock() +} + +// SetTelegramTCPBlocking sets the value of TelegramTCPBlocking. +func (tk *TestKeys) SetTelegramTCPBlocking(value bool) { + tk.mu.Lock() + tk.TelegramTCPBlocking = value + tk.mu.Unlock() +} + +// SetTelegramHTTPBlocking sets the value of TelegramHTTPBlocking. +func (tk *TestKeys) SetTelegramHTTPBlocking(value bool) { + tk.mu.Lock() + tk.TelegramHTTPBlocking = value + tk.mu.Unlock() +} + +// AppendWebFailure appends to the webFailures list. +func (tk *TestKeys) AppendWebFailure(err error) { + tk.mu.Lock() + tk.webFailures = append(tk.webFailures, err) + tk.mu.Unlock() +} + +// SetFundamentalFailure sets the value of fundamentalFailure. +func (tk *TestKeys) SetFundamentalFailure(err error) { + tk.mu.Lock() + tk.fundamentalFailure = err + tk.mu.Unlock() +} + +// NewTestKeys creates a new instance of TestKeys. +func NewTestKeys() *TestKeys { + tk := &TestKeys{ + NetworkEvents: []*model.ArchivalNetworkEvent{}, + Queries: []*model.ArchivalDNSLookupResult{}, + Requests: []*model.ArchivalHTTPRequestResult{}, + TCPConnect: []*model.ArchivalTCPConnectResult{}, + TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{}, + TelegramTCPBlocking: false, + TelegramHTTPBlocking: false, + TelegramWebStatus: "", + TelegramWebFailure: nil, + webFailures: []error{}, + fundamentalFailure: nil, + mu: &sync.Mutex{}, + } + + // "If all TCP connections on ports 80 and 443 to Telegram’s access + // point IPs fail we consider Telegram to be blocked." + tk.TelegramTCPBlocking = true + + // "If at least an HTTP request returns back a response, we + // consider Telegram [DCs] to not be blocked." + tk.TelegramHTTPBlocking = true + + // We start saying web.telegram.org is blocked and flip to okay + // only when we notice that it's accessible. + tk.TelegramWebStatus = "blocked" + + // We start by saying that the experiment did not actually + // run until completion, and then flip later if needed. + didNotRun := "telegram_did_not_run_error" + tk.TelegramWebFailure = &didNotRun + + return tk +} + +// finalize performs any delayed computation on the test keys. This function +// must be called from the measurer after all the tasks have completed. +func (tk *TestKeys) finalize() { + var filtered []error + for _, err := range tk.webFailures { + if errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH) { + continue // skip IPv6 errors when there's no working IPv6 support + } + filtered = append(filtered, err) + } + if len(filtered) <= 0 { + tk.TelegramWebStatus = "ok" + tk.TelegramWebFailure = nil + return + } + tk.TelegramWebStatus = "blocked" + first := filtered[0].Error() + tk.TelegramWebFailure = &first +} diff --git a/internal/experiment/telegram/webhttp.go b/internal/experiment/telegram/webhttp.go new file mode 100644 index 0000000000..85473e9350 --- /dev/null +++ b/internal/experiment/telegram/webhttp.go @@ -0,0 +1,241 @@ +package telegram + +// +// WebHTTP +// +// Generated by `boilerplate' using the http template. +// + +import ( + "context" + "errors" + "io" + "log" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures Telegram Web using HTTP. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type WebHTTP struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background goroutine. +func (t *WebHTTP) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *WebHTTP) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTP#%d: %s", index, t.Address) + + // perform the TCP connect + const tcpTimeout = 10 * time.Second + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) + if err != nil { + t.TestKeys.AppendWebFailure(err) + ol.Stop(err) + return + } + defer func() { + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + tcpConn.Close() + }() + + alpn := "" // no ALPN because we're not using TLS + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewSingleUseDialer(tcpConn), + netxlite.NewNullTLSDialer(), + ) + + // create HTTP request + const httpTimeout = 10 * time.Second + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) + if err != nil { + t.TestKeys.AppendWebFailure(err) + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) + if err != nil { + t.TestKeys.AppendWebFailure(err) + ol.Stop(err) + return + } + + // parse HTTP results + if err := t.parseResults(httpResp, httpRespBody); err != nil { + t.TestKeys.AppendWebFailure(err) + ol.Stop(err) + return + } + + // completed successfully + ol.Stop(nil) +} + +// urlHost computes the host to include into the URL +func (t *WebHTTP) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } + if port == "80" && scheme == "http" { + return urlHost, nil + } + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil +} + +// newHTTPRequest creates a new HTTP request. +func (t *WebHTTP) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "http" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *WebHTTP) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) + resp, err := txp.RoundTrip(req) + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) + } + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) + t.TestKeys.AppendRequests(ev) + return resp, body, err +} + +// parseResults parses the results of this sub-measurement. +func (t *WebHTTP) parseResults(resp *http.Response, respBody []byte) error { + if resp.StatusCode != 301 && resp.StatusCode != 308 { + log.Printf("status code: %+v", resp.StatusCode) + return errors.New("http_request_failed") + } + location, err := resp.Location() + if err != nil { + return errors.New("telegram_missing_redirect_error") + } + if location.Scheme != "https" || location.Host != webTelegramOrg { + return errors.New("telegram_invalid_redirect_error") + } + return nil +} diff --git a/internal/experiment/telegram/webhttps.go b/internal/experiment/telegram/webhttps.go new file mode 100644 index 0000000000..f7d1c3edb0 --- /dev/null +++ b/internal/experiment/telegram/webhttps.go @@ -0,0 +1,292 @@ +package telegram + +// +// WebHTTPS +// +// Generated by `boilerplate' using the https template. +// + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "io" + "log" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Measures Telegram Web using HTTPS. +// +// The zero value of this structure IS NOT valid and you MUST initialize +// all the fields marked as MANDATORY before using this structure. +type WebHTTPS struct { + // Address is the MANDATORY address to connect to. + Address string + + // IDGenerator is the MANDATORY atomic int64 to generate task IDs. + IDGenerator *atomicx.Int64 + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // TestKeys is MANDATORY and contains the TestKeys. + TestKeys *TestKeys + + // ZeroTime is the MANDATORY measurement's zero time. + ZeroTime time.Time + + // WaitGroup is the MANDATORY wait group this task belongs to. + WaitGroup *sync.WaitGroup + + // ALPN is the OPTIONAL ALPN to use. + ALPN []string + + // CookieJar contains the OPTIONAL cookie jar, used for redirects. + CookieJar http.CookieJar + + // HostHeader is the OPTIONAL host header to use. + HostHeader string + + // SNI is the OPTIONAL SNI to use. + SNI string + + // URLPath is the OPTIONAL URL path. + URLPath string + + // URLRawQuery is the OPTIONAL URL raw query. + URLRawQuery string +} + +// Start starts this task in a background goroutine. +func (t *WebHTTPS) Start(ctx context.Context) { + t.WaitGroup.Add(1) + index := t.IDGenerator.Add(1) + go func() { + defer t.WaitGroup.Done() // synchronize with the parent + t.Run(ctx, index) + }() +} + +// Run runs this task in the current goroutine. +func (t *WebHTTPS) Run(parentCtx context.Context, index int64) { + // create trace + trace := measurexlite.NewTrace(index, t.ZeroTime) + + // start the operation logger + ol := measurexlite.NewOperationLogger(t.Logger, "WebHTTPS#%d: %s", index, t.Address) + + // perform the TCP connect + const tcpTimeout = 10 * time.Second + tcpCtx, tcpCancel := context.WithTimeout(parentCtx, tcpTimeout) + defer tcpCancel() + tcpDialer := trace.NewDialerWithoutResolver(t.Logger) + tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address) + t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...) + if err != nil { + t.TestKeys.AppendWebFailure(err) + ol.Stop(err) + return + } + defer func() { + t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...) + tcpConn.Close() + }() + + // perform TLS handshake + tlsSNI, err := t.sni() + if err != nil { + t.TestKeys.AppendWebFailure(err) + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + tlsHandshaker := trace.NewTLSHandshakerStdlib(t.Logger) + tlsConfig := &tls.Config{ + NextProtos: t.alpn(), + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: tlsSNI, + } + const tlsTimeout = 10 * time.Second + tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) + defer tlsCancel() + tlsConn, tlsConnState, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + t.TestKeys.AppendTLSHandshakes(trace.TLSHandshakes()...) + if err != nil { + t.TestKeys.AppendWebFailure(err) + ol.Stop(err) + return + } + defer tlsConn.Close() + alpn := tlsConnState.NegotiatedProtocol + + // create HTTP transport + httpTransport := netxlite.NewHTTPTransport( + t.Logger, + netxlite.NewNullDialer(), + // note: netxlite guarantees that here tlsConn is a netxlite.TLSConn + netxlite.NewSingleUseTLSDialer(tlsConn.(netxlite.TLSConn)), + ) + + // create HTTP request + const httpTimeout = 10 * time.Second + httpCtx, httpCancel := context.WithTimeout(parentCtx, httpTimeout) + defer httpCancel() + httpReq, err := t.newHTTPRequest(httpCtx) + if err != nil { + t.TestKeys.AppendWebFailure(err) + t.TestKeys.SetFundamentalFailure(err) + ol.Stop(err) + return + } + + // perform HTTP transaction + httpResp, httpRespBody, err := t.httpTransaction( + httpCtx, + "tcp", + t.Address, + alpn, + httpTransport, + httpReq, + trace, + ) + if err != nil { + t.TestKeys.AppendWebFailure(err) + ol.Stop(err) + return + } + + // parse HTTP results + if err := t.parseResults(httpResp, httpRespBody); err != nil { + t.TestKeys.AppendWebFailure(err) + ol.Stop(err) + return + } + + // completed successfully + ol.Stop(nil) +} + +// alpn returns the user-configured ALPN or a reasonable default +func (t *WebHTTPS) alpn() []string { + if len(t.ALPN) > 0 { + return t.ALPN + } + return []string{"h2", "http/1.1"} +} + +// sni returns the user-configured SNI or a reasonable default +func (t *WebHTTPS) sni() (string, error) { + if t.SNI != "" { + return t.SNI, nil + } + addr, _, err := net.SplitHostPort(t.Address) + if err != nil { + return "", err + } + return addr, nil +} + +// urlHost computes the host to include into the URL +func (t *WebHTTPS) urlHost(scheme string) (string, error) { + addr, port, err := net.SplitHostPort(t.Address) + if err != nil { + t.Logger.Warnf("BUG: net.SplitHostPort failed for %s: %s", t.Address, err.Error()) + return "", err + } + urlHost := t.HostHeader + if urlHost == "" { + urlHost = addr + } + if port == "443" && scheme == "https" { + return urlHost, nil + } + urlHost = net.JoinHostPort(urlHost, port) + return urlHost, nil +} + +// newHTTPRequest creates a new HTTP request. +func (t *WebHTTPS) newHTTPRequest(ctx context.Context) (*http.Request, error) { + const urlScheme = "https" + urlHost, err := t.urlHost(urlScheme) + if err != nil { + return nil, err + } + httpURL := &url.URL{ + Scheme: urlScheme, + Host: urlHost, + Path: t.URLPath, + RawQuery: t.URLRawQuery, + } + httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Host", t.HostHeader) + httpReq.Header.Set("Accept", model.HTTPHeaderAccept) + httpReq.Header.Set("Accept-Language", model.HTTPHeaderAcceptLanguage) + httpReq.Header.Set("User-Agent", model.HTTPHeaderUserAgent) + httpReq.Host = t.HostHeader + if t.CookieJar != nil { + for _, cookie := range t.CookieJar.Cookies(httpURL) { + httpReq.AddCookie(cookie) + } + } + return httpReq, nil +} + +// httpTransaction runs the HTTP transaction and saves the results. +func (t *WebHTTPS) httpTransaction(ctx context.Context, network, address, alpn string, + txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { + const maxbody = 1 << 19 + started := trace.TimeSince(trace.ZeroTime) + resp, err := txp.RoundTrip(req) + var body []byte + if err == nil { + defer resp.Body.Close() + if cookies := resp.Cookies(); t.CookieJar != nil && len(cookies) > 0 { + t.CookieJar.SetCookies(req.URL, cookies) + } + reader := io.LimitReader(resp.Body, maxbody) + body, err = netxlite.ReadAllContext(ctx, reader) + } + finished := trace.TimeSince(trace.ZeroTime) + ev := measurexlite.NewArchivalHTTPRequestResult( + trace.Index, + started, + network, + address, + alpn, + txp.Network(), + req, + resp, + maxbody, + body, + err, + finished, + ) + t.TestKeys.AppendRequests(ev) + return resp, body, err +} + +// parseResults parses the results of this sub-measurement. +func (t *WebHTTPS) parseResults(resp *http.Response, respBody []byte) error { + if resp.StatusCode != 200 { + log.Printf("status code: %+v", resp.StatusCode) + return errors.New("http_request_failed") + } + title := []byte(`Telegram Web`) + if !bytes.Contains(respBody, title) { + return errors.New("telegram_missing_title_error") + } + return nil +} diff --git a/internal/registry/telegram.go b/internal/registry/telegram.go index 3c20980aba..c0d4c5e5d8 100644 --- a/internal/registry/telegram.go +++ b/internal/registry/telegram.go @@ -1,11 +1,12 @@ package registry // -// Registers the `telegram' experiment. +// Registers the `telegram' experiment implemented by +// the `./internal/experiment/telegram' package. // import ( - "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/experiment/telegram" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -13,7 +14,7 @@ func init() { allexperiments["telegram"] = &Factory{ build: func(config any) model.ExperimentMeasurer { return telegram.NewExperimentMeasurer( - config.(telegram.Config), + config.(*telegram.Config), ) }, config: telegram.Config{},