From f8da3fce0846a642f5b8c70bc4650a4d27a62fc1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 18 Aug 2022 16:08:01 +0200 Subject: [PATCH 1/2] chore: upgrade ooni/oohttp dependency --- go.mod | 6 +++--- go.sum | 12 ++++++------ internal/model/mocks/tls.go | 8 ++++++++ internal/netxlite/utls.go | 20 +++++++++++++++++++- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index a6be194053..9d2ba4aa54 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/mitchellh/go-wordwrap v1.0.1 github.com/montanaflynn/stats v0.6.6 github.com/ooni/go-libtor v1.1.5 - github.com/ooni/oohttp v0.0.0-20220602055714-3d81a8b41c3a + github.com/ooni/oohttp v0.2.1-0.20220818134039-01e4ffcde985 github.com/ooni/probe-assets v0.10.0 github.com/ooni/psiphon/tunnel-core v0.0.0-20220519122549-9c044eb6bd83 github.com/oschwald/geoip2-golang v1.7.0 @@ -38,8 +38,8 @@ require ( gitlab.com/yawning/obfs4.git v0.0.0-20220204003609-77af0cba934d gitlab.com/yawning/utls.git v0.0.12-1 golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 - golang.org/x/net v0.0.0-20220531201128-c960675eff93 - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + golang.org/x/net v0.0.0-20220812174116-3211cb980234 + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 ) require ( diff --git a/go.sum b/go.sum index bd7170ccd2..eecff2aa07 100644 --- a/go.sum +++ b/go.sum @@ -614,8 +614,8 @@ github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/ooni/go-libtor v1.1.5 h1:YbwXR9aLuL37EwL7rksPCQQhcHwoxU+M/v+jwZR+n5Y= github.com/ooni/go-libtor v1.1.5/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI= -github.com/ooni/oohttp v0.0.0-20220602055714-3d81a8b41c3a h1:2BH9AplJ7meOhtzfMiTvsClYQ9+qjk3Z2mw8cRUPYqE= -github.com/ooni/oohttp v0.0.0-20220602055714-3d81a8b41c3a/go.mod h1:p2VVLbs+BXBIgTHITV9Vw8Rv6G1u66JUWP/8KCgDGNo= +github.com/ooni/oohttp v0.2.1-0.20220818134039-01e4ffcde985 h1:PmScZvhuRmnyJVvhmOYIApdEoyMA4WLtEJKhkjQ3qQI= +github.com/ooni/oohttp v0.2.1-0.20220818134039-01e4ffcde985/go.mod h1:fgNDPYw+nsgEKCDBpT/4R06bgnrCRtvgNmAWOCmm4JE= github.com/ooni/probe-assets v0.10.0 h1:FSk5nNPEFfRhglJDjtFHlZCHE+0eERcW5R+Ujs425Tk= github.com/ooni/probe-assets v0.10.0/go.mod h1:N0PyNM3aadlYDDCFXAPzs54HC54+MZA/4/xnCtd9EAo= github.com/ooni/psiphon/tunnel-core v0.0.0-20220519122549-9c044eb6bd83 h1:xflU9CdKoHLMhVpt/beum7xw5erAR20wKawZSNWoJAA= @@ -1075,8 +1075,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220531201128-c960675eff93 h1:MYimHLfoXEpOhqd/zgoA/uoXzHB86AEky4LAx5ij9xA= -golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1182,8 +1182,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= diff --git a/internal/model/mocks/tls.go b/internal/model/mocks/tls.go index 347cd34009..c90f69eafe 100644 --- a/internal/model/mocks/tls.go +++ b/internal/model/mocks/tls.go @@ -28,6 +28,9 @@ type TLSConn struct { // MockHandshakeContext allows to mock the HandshakeContext method. MockHandshakeContext func(ctx context.Context) error + + // MockNetConn allows to mock the NetConn method + MockNetConn func() net.Conn } // ConnectionState calls MockConnectionState. @@ -40,6 +43,11 @@ func (c *TLSConn) HandshakeContext(ctx context.Context) error { return c.MockHandshakeContext(ctx) } +// NetConn calls MockNetConn. +func (c *TLSConn) NetConn() net.Conn { + return c.MockNetConn() +} + // TLSDialer allows to mock netxlite.TLSDialer. type TLSDialer struct { // MockCloseIdleConnections allows to mock the CloseIdleConnections method. diff --git a/internal/netxlite/utls.go b/internal/netxlite/utls.go index c789c3e62b..2e20c08c6d 100644 --- a/internal/netxlite/utls.go +++ b/internal/netxlite/utls.go @@ -36,7 +36,13 @@ func NewTLSHandshakerUTLS(logger model.DebugLogger, id *utls.ClientHelloID) mode // utlsConn implements TLSConn and uses a utls UConn as its underlying connection type utlsConn struct { + // The underlying uTLS conn *utls.UConn + + // The underlying net.Conn used to implement .NetConn + cc net.Conn + + // A hook to write better unit tests testableHandshake func() error } @@ -85,7 +91,12 @@ func newConnUTLSWithHelloID(conn net.Conn, config *tls.Config, cid *utls.ClientH ServerName: config.ServerName, } tlsConn := utls.UClient(conn, uConfig, *cid) - return &utlsConn{UConn: tlsConn}, nil + uconn := &utlsConn{ + UConn: tlsConn, + cc: conn, + testableHandshake: nil, + } + return uconn, nil } // ErrUTLSHandshakePanic indicates that there was panic handshaking @@ -93,6 +104,7 @@ func newConnUTLSWithHelloID(conn net.Conn, config *tls.Config, cid *utls.ClientH // See https://github.com/ooni/probe/issues/1770 for more information. var ErrUTLSHandshakePanic = errors.New("utls: handshake panic") +// HandshakeContext implements oohttp.TLSConn func (c *utlsConn) HandshakeContext(ctx context.Context) (err error) { errch := make(chan error, 1) go func() { @@ -119,6 +131,7 @@ func (c *utlsConn) handshakefn() func() error { return c.UConn.Handshake } +// ConnectionState implements oohttp.TLSConn func (c *utlsConn) ConnectionState() tls.ConnectionState { uState := c.Conn.ConnectionState() return tls.ConnectionState{ @@ -136,3 +149,8 @@ func (c *utlsConn) ConnectionState() tls.ConnectionState { TLSUnique: uState.TLSUnique, } } + +// NetConn implements oohttp.TLSConn +func (c *utlsConn) NetConn() net.Conn { + return c.cc +} From 82ae51b5af5fef2267875eefa4b704f26e31fa70 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 18 Aug 2022 16:08:45 +0200 Subject: [PATCH 2/2] feat(netxlite/http): gather ?ALPN, {src,dst}Addr See https://github.com/ooni/probe/issues/2220 --- internal/netxlite/http.go | 70 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/internal/netxlite/http.go b/internal/netxlite/http.go index 3eaefb3439..9aa464c796 100644 --- a/internal/netxlite/http.go +++ b/internal/netxlite/http.go @@ -7,12 +7,14 @@ package netxlite import ( "context" "errors" + "log" "net" "net/http" "net/url" "time" oohttp "github.com/ooni/oohttp" + oohttptrace "github.com/ooni/oohttp/httptrace" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -231,11 +233,75 @@ func (txp *httpTransportStdlib) Network() string { // This is a low level factory. Consider not using it directly. func WrapHTTPTransport(logger model.DebugLogger, txp model.HTTPTransport) model.HTTPTransport { return &httpTransportLogger{ - HTTPTransport: &httpTransportErrWrapper{txp}, - Logger: logger, + HTTPTransport: &httpTransportErrWrapper{ + &httpTransportTracer{txp}, + }, + Logger: logger, } } +// httpTransportTracer provides support for implementing HTTP tracing. +type httpTransportTracer struct { + // child is the underlying transport + child model.HTTPTransport +} + +var _ model.HTTPTransport = &httpTransportTracer{} + +// CloseIdleConnections implements model.HTTPTransport +func (tt *httpTransportTracer) CloseIdleConnections() { + tt.child.CloseIdleConnections() +} + +// Network implements model.HTTPTransport +func (tt *httpTransportTracer) Network() string { + return tt.child.Network() +} + +// httpConnRequestBinding contains the binding between a request +// and the connection being used to perform such a request. +type httpConnRequestBinding struct { + // The ALPN value (if any) + alpn string + + // The connection's local address + localAddress string + + // The connection's remote address + remoteAddress string +} + +// RoundTrip implements model.HTTPTransport +func (tt *httpTransportTracer) RoundTrip(req *http.Request) (*http.Response, error) { + bindingch := make(chan *httpConnRequestBinding, 2) + trace := &oohttptrace.ClientTrace{ + GotConn: func(gci oohttptrace.GotConnInfo) { + binding := &httpConnRequestBinding{ + alpn: "", + localAddress: gci.Conn.LocalAddr().String(), + remoteAddress: gci.Conn.RemoteAddr().String(), + } + if tconn, good := gci.Conn.(TLSConn); good { + binding.alpn = tconn.ConnectionState().NegotiatedProtocol + } + bindingch <- binding + }, + } + ctx := req.Context() + ctx = oohttptrace.WithClientTraceWithoutComposition(ctx, trace) + req = req.WithContext(ctx) + resp, err := tt.child.RoundTrip(req) + select { + case binding := <-bindingch: + // TODO(bassosimone): replace this logging message with proper routing + // of the information related to this HTTP round trip + log.Printf("*** CONNECTION BINDING: %+v %+v", binding, req.URL) + default: + // nothing + } + return resp, err +} + // httpDialerWithReadTimeout enforces a read timeout for all HTTP // connections. See https://github.com/ooni/probe/issues/1609. type httpDialerWithReadTimeout struct {