pax_global_header00006660000000000000000000000064147246675340014534gustar00rootroot0000000000000052 comment=f9c934dbc54cf968d3807155830ad948f1050ce7 golang-ptutil-0.0~git20240710.6c4d8ed/000077500000000000000000000000001472466753400170435ustar00rootroot00000000000000golang-ptutil-0.0~git20240710.6c4d8ed/.gitlab-ci.yml000066400000000000000000000001521472466753400214750ustar00rootroot00000000000000test: image: golang script: - test -z "$(go fmt ./...)" - go vet ./... - go test -v ./... golang-ptutil-0.0~git20240710.6c4d8ed/LICENSE000066400000000000000000000034361472466753400200560ustar00rootroot00000000000000 This file contains the license for "ptutil" a free software project which provides a utils for pluggable transport. ================================================================================ Copyright (c) 2016, Serene Han, Arlo Breault Copyright (c) 2019-2024, The Tor Project, Inc Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the names of the copyright owners nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ golang-ptutil-0.0~git20240710.6c4d8ed/README.md000066400000000000000000000012441472466753400203230ustar00rootroot00000000000000PTutil ====== A collection of utilities for Pluggable Transports. SafeLog ------- A safer logging wrapper around the standard logging package. It removes private parts from the logs, like IP addresses. ```go log.SetOutput(&safelog.LogScrubber{Output: os.Stderr}) ``` SafeProm -------- Implements some additional prometheus metrics that we need for privacy preserving counts of users and proxies. ```go usersTotal = safeprom.NewRoundedCounterVec( prometheus.CounterOpts{ Name: "rounded_users_total", Help: "The number of users, rounded up to a multiple of 8", }, []string{"status"}, ) usersTotal.WithLabelValues("active").Inc() ``` golang-ptutil-0.0~git20240710.6c4d8ed/go.mod000066400000000000000000000007471472466753400201610ustar00rootroot00000000000000module gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil go 1.21 toolchain go1.22.3 require ( github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.6.1 google.golang.org/protobuf v1.34.1 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/prometheus/common v0.54.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect golang.org/x/sys v0.21.0 // indirect ) golang-ptutil-0.0~git20240710.6c4d8ed/go.sum000066400000000000000000000033101472466753400201730ustar00rootroot00000000000000github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= golang-ptutil-0.0~git20240710.6c4d8ed/safelog/000077500000000000000000000000001472466753400204635ustar00rootroot00000000000000golang-ptutil-0.0~git20240710.6c4d8ed/safelog/log.go000066400000000000000000000045651472466753400216050ustar00rootroot00000000000000//Package for a safer logging wrapper around the standard logging package // import "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil/safelog" package safelog import ( "bytes" "io" "regexp" "sync" ) const ipv4Address = `\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}` // %3A and %3a are for matching : in URL-encoded IPv6 addresses const colon = `(:|%3a|%3A)` const ipv6Address = `([0-9a-fA-F]{0,4}` + colon + `){5,7}([0-9a-fA-F]{0,4})?` const ipv6Compressed = `([0-9a-fA-F]{0,4}` + colon + `){0,5}([0-9a-fA-F]{0,4})?(` + colon + `){2}([0-9a-fA-F]{0,4}` + colon + `){0,5}([0-9a-fA-F]{0,4})?` const ipv6Full = `(` + ipv6Address + `(` + ipv4Address + `))` + `|(` + ipv6Compressed + `(` + ipv4Address + `))` + `|(` + ipv6Address + `)` + `|(` + ipv6Compressed + `)` const optionalPort = `(:\d{1,5})?` const addressPattern = `((` + ipv4Address + `)|(\[(` + ipv6Full + `)\])|(` + ipv6Full + `))` + optionalPort const fullAddrPattern = `(?:^|\s|[^\w:])(` + addressPattern + `)(?:\s|(:\s)|[^\w:]|$)` var scrubberPatterns = []*regexp.Regexp{ regexp.MustCompile(fullAddrPattern), } var addressRegexp = regexp.MustCompile(addressPattern) // An io.Writer that can be used as the output for a logger that first // sanitizes logs and then writes to the provided io.Writer type LogScrubber struct { Output io.Writer buffer []byte lock sync.Mutex } func (ls *LogScrubber) Lock() { (*ls).lock.Lock() } func (ls *LogScrubber) Unlock() { (*ls).lock.Unlock() } func Scrub(b []byte) []byte { scrubbedBytes := b for _, pattern := range scrubberPatterns { // this is a workaround since go does not yet support look ahead or look // behind for regular expressions. var newBytes []byte index := 0 for { loc := pattern.FindSubmatchIndex(scrubbedBytes[index:]) if loc == nil { break } newBytes = append(newBytes, scrubbedBytes[index:index+loc[2]]...) newBytes = append(newBytes, []byte("[scrubbed]")...) index = index + loc[3] } scrubbedBytes = append(newBytes, scrubbedBytes[index:]...) } return scrubbedBytes } func (ls *LogScrubber) Write(b []byte) (n int, err error) { ls.Lock() defer ls.Unlock() n = len(b) ls.buffer = append(ls.buffer, b...) for { i := bytes.LastIndexByte(ls.buffer, '\n') if i == -1 { return } fullLines := ls.buffer[:i+1] _, err = ls.Output.Write(Scrub(fullLines)) if err != nil { return } ls.buffer = ls.buffer[i+1:] } } golang-ptutil-0.0~git20240710.6c4d8ed/safelog/log_test.go000066400000000000000000000136161472466753400226410ustar00rootroot00000000000000package safelog import ( "bytes" "log" "testing" ) // Check to make sure that addresses split across calls to write are still scrubbed func TestLogScrubberSplit(t *testing.T) { input := []byte("test\nhttp2: panic serving [2620:101:f000:780:9097:75b1:519f:dbb8]:58344: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack\n") expected := "test\nhttp2: panic serving [scrubbed]: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack\n" var buff bytes.Buffer scrubber := &LogScrubber{Output: &buff} n, err := scrubber.Write(input[:12]) //test\nhttp2: if n != 12 { t.Errorf("wrong number of bytes %d", n) } if err != nil { t.Errorf("%q", err) } if buff.String() != "test\n" { t.Errorf("Got %q, expected %q", buff.String(), "test\n") } n, err = scrubber.Write(input[12:30]) //panic serving [2620:101:f if n != 18 { t.Errorf("wrong number of bytes %d", n) } if err != nil { t.Errorf("%q", err) } if buff.String() != "test\n" { t.Errorf("Got %q, expected %q", buff.String(), "test\n") } n, err = scrubber.Write(input[30:]) //000:780:9097:75b1:519f:dbb8]:58344: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack\n if n != (len(input) - 30) { t.Errorf("wrong number of bytes %d", n) } if err != nil { t.Errorf("%q", err) } if buff.String() != expected { t.Errorf("Got %q, expected %q", buff.String(), expected) } } // Test the log scrubber on known problematic log messages func TestLogScrubberMessages(t *testing.T) { for _, test := range []struct { input, expected string }{ { "http: TLS handshake error from 129.97.208.23:38310: ", "http: TLS handshake error from [scrubbed]: \n", }, { "http2: panic serving [2620:101:f000:780:9097:75b1:519f:dbb8]:58344: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack", "http2: panic serving [scrubbed]: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack\n", }, { //Make sure it doesn't scrub fingerprint "a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74", "a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74\n", }, { //try with enclosing parens "(1:2:3:4:c:d:e:f) {1:2:3:4:c:d:e:f}", "([scrubbed]) {[scrubbed]}\n", }, { //Make sure it doesn't scrub timestamps "2019/05/08 15:37:31 starting", "2019/05/08 15:37:31 starting\n", }, { //Make sure ipv6 addresses where : are encoded as %3A or %3a are scrubbed "error dialing relay: wss://snowflake.torproject.net/?client_ip=6201%3ac8%3A3004%3A%3A1234", "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed]\n", }, { // make sure url encoded IPv6 IPs get scrubbed (%3a) "http2: panic serving [fd00%3a111%3af000%3a777%3a9999%3abbbb%3affff%3adddd]:58344: xxx", "http2: panic serving [scrubbed]: xxx\n", }, { // make sure url encoded IPv6 IPs get scrubbed (%3A) "http2: panic serving [fd00%3a111%3af000%3a777%3a9999%3abbbb%3affff%3adddd]:58344: xxx", "http2: panic serving [scrubbed]: xxx\n", }, { // make sure url encoded IPv6 IPs get scrubbed, different URL (%3A) "error dialing relay: wss://snowflake.torproject.net/?client_ip=fd00%3A8888%3Abbbb%3Acccc%3Adddd%3Aeeee%3A2222%3A123 = dial tcp xxx", "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed] = dial tcp xxx\n", }, { // make sure url encoded IPv6 IPs get scrubbed (%3A), compressed "http2: panic serving [1%3A2%3A3%3A%3Ad%3Ae%3Af]:55: xxx", "http2: panic serving [scrubbed]: xxx\n", }, { // make sure url encoded IPv6 IPs get scrubbed (%3A), compressed "error dialing relay: wss://snowflake.torproject.net/?client_ip=1%3A2%3A3%3A%3Ad%3Ae%3Af = dial tcp xxx", "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed] = dial tcp xxx\n", }, { // multiple space-separated IP addresses "Allowed stations: [10.0.1.1 10.0.1.2 10.0.1.3 10.0.1.4]\n", "Allowed stations: [[scrubbed] [scrubbed] [scrubbed] [scrubbed]]\n", }, } { var buff bytes.Buffer log.SetFlags(0) //remove all extra log output for test comparisons log.SetOutput(&LogScrubber{Output: &buff}) log.Print(test.input) if buff.String() != test.expected { t.Errorf("%q: got %q, expected %q", test.input, buff.String(), test.expected) } } } func TestLogScrubberGoodFormats(t *testing.T) { for _, addr := range []string{ // IPv4 "1.2.3.4", "255.255.255.255", // IPv4 with port "1.2.3.4:55", "255.255.255.255:65535", // IPv6 "1:2:3:4:c:d:e:f", "1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF", // IPv6 with brackets "[1:2:3:4:c:d:e:f]", "[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]", // IPv6 with brackets and port "[1:2:3:4:c:d:e:f]:55", "[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]:65535", // compressed IPv6 "::f", "::d:e:f", "1:2:3::", "1:2:3::d:e:f", "1:2:3:d:e:f::", "::1:2:3:d:e:f", "1111:2222:3333::DDDD:EEEE:FFFF", // compressed IPv6 with brackets "[::d:e:f]", "[1:2:3::]", "[1:2:3::d:e:f]", "[1111:2222:3333::DDDD:EEEE:FFFF]", "[1:2:3:4:5:6::8]", "[1::7:8]", // compressed IPv6 with brackets and port "[1::]:58344", "[::d:e:f]:55", "[1:2:3::]:55", "[1:2:3::d:e:f]:55", "[1111:2222:3333::DDDD:EEEE:FFFF]:65535", // IPv4-compatible and IPv4-mapped "::255.255.255.255", "::ffff:255.255.255.255", "[::255.255.255.255]", "[::ffff:255.255.255.255]", "[::255.255.255.255]:65535", "[::ffff:255.255.255.255]:65535", "[::ffff:0:255.255.255.255]", "[2001:db8:3:4::192.0.2.33]", } { var buff bytes.Buffer log.SetFlags(0) //remove all extra log output for test comparisons log.SetOutput(&LogScrubber{Output: &buff}) log.Print(addr) if buff.String() != "[scrubbed]\n" { t.Errorf("%q: Got %q, expected %q", addr, buff.String(), "[scrubbed]\n") } } } golang-ptutil-0.0~git20240710.6c4d8ed/safeprom/000077500000000000000000000000001472466753400206575ustar00rootroot00000000000000golang-ptutil-0.0~git20240710.6c4d8ed/safeprom/prometheus.go000066400000000000000000000053371472466753400234110ustar00rootroot00000000000000/* Implements some additional prometheus metrics that we need for privacy preserving counts of users and proxies */ package safeprom import ( "sync/atomic" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "google.golang.org/protobuf/proto" ) // New Prometheus counter type that produces rounded counts of metrics // for privacy preserving reasons type Counter interface { prometheus.Metric Inc() } type counter struct { total uint64 //reflects the true count value uint64 //reflects the rounded count desc *prometheus.Desc labelPairs []*dto.LabelPair } // Implements the Counter interface func (c *counter) Inc() { atomic.AddUint64(&c.total, 1) if c.total > c.value { atomic.AddUint64(&c.value, 8) } } // Implements the prometheus.Metric interface func (c *counter) Desc() *prometheus.Desc { return c.desc } // Implements the prometheus.Metric interface func (c *counter) Write(m *dto.Metric) error { m.Label = c.labelPairs m.Counter = &dto.Counter{Value: proto.Float64(float64(c.value))} return nil } // New prometheus vector type that will track Counter metrics // accross multiple labels type CounterVec struct { *prometheus.MetricVec } // NewCounterVec will create a CounterVec but will not register it, users must register it themselves. // The behaviour of this function is similar to github.com/prometheus/client_golang/prometheus func NewCounterVec(opts prometheus.CounterOpts, labelNames []string) *CounterVec { desc := prometheus.NewDesc( prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, labelNames, opts.ConstLabels, ) c := &CounterVec{ MetricVec: prometheus.NewMetricVec(desc, func(lvs ...string) prometheus.Metric { if len(lvs) != len(labelNames) { panic("inconsistent cardinality") } return &counter{desc: desc, labelPairs: prometheus.MakeLabelPairs(desc, lvs)} }), } return c } // NewCounterVecRegistered will create a CounterVec and register it. // The behaviour of this function is similar to github.com/prometheus/client_golang/prometheus/promauto func NewCounterVecRegistered(opts prometheus.CounterOpts, labelNames []string) *CounterVec { c := NewCounterVec(opts, labelNames) prometheus.DefaultRegisterer.MustRegister(c) return c } // Helper function to return the underlying Counter metric from MetricVec func (v *CounterVec) With(labels prometheus.Labels) Counter { metric, err := v.GetMetricWith(labels) if err != nil { panic(err) } return metric.(Counter) } // Helper function to return the underlying Counter metric from MetricVec func (v *CounterVec) WithLabelValues(lvs ...string) Counter { metric, err := v.GetMetricWithLabelValues(lvs...) if err != nil { panic(err) } return metric.(Counter) } golang-ptutil-0.0~git20240710.6c4d8ed/safeprom/prometheus_test.go000066400000000000000000000033421472466753400244420ustar00rootroot00000000000000package safeprom import ( "bufio" "net/http" "net/http/httptest" "strconv" "strings" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) func TestIncVecWithLabelValues(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() metricName := "label_values" usersTotal := NewCounterVecRegistered( prometheus.CounterOpts{ Name: metricName, }, []string{"status"}, ) for i := 0; i < 8; i++ { usersTotal.WithLabelValues("active").Inc() expectedValue(t, ts.URL, metricName, 8) } usersTotal.WithLabelValues("active").Inc() expectedValue(t, ts.URL, metricName, 16) } func TestIncVecWith(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() metricName := "with" usersTotal := NewCounterVecRegistered( prometheus.CounterOpts{ Name: metricName, }, []string{"status"}, ) for i := 0; i < 8; i++ { usersTotal.With(prometheus.Labels{"status": "active"}).Inc() expectedValue(t, ts.URL, metricName, 8) } usersTotal.With(prometheus.Labels{"status": "active"}).Inc() expectedValue(t, ts.URL, metricName, 16) } func expectedValue(t *testing.T, url string, metric string, expected int) { res, err := http.Get(url) if err != nil { t.Fatal(err) } defer res.Body.Close() value := -1 scanner := bufio.NewScanner(res.Body) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, metric) { continue } parts := strings.Split(line, " ") value, err = strconv.Atoi(parts[len(parts)-1]) if err != nil { t.Fatal(err) } } if value == -1 { t.Fatal("Metric", metric, "was not present") } if value != expected { t.Error("Metric value is not", expected, ":", value) } }