package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"testing"

	"code.justin.tv/rhys/nursery/internal/_vendor/trace"
)

func TestTrimVendor(t *testing.T) {
	testcase := func(input, want string) func(t *testing.T) {
		return func(t *testing.T) {
			have := trimVendor(input)
			if have != want {
				t.Errorf("trimVendor(%q); %q != %q", input, have, want)
			}
		}
	}

	t.Run("", testcase("a", "a"))
	t.Run("", testcase("a/b", "a/b"))
	t.Run("", testcase("a/vendor/b", "b"))
	t.Run("", testcase("a/vendor/b/vendor/c", "c"))
	t.Run("", testcase("vendor/b", "b"))
	t.Run("", testcase("vendor/b/vendor/c", "c"))
}

func TestParseEventString(t *testing.T) {
	str := `
36184678 GoStart p=6 g=3017828 off=130971 g=3017828 seq=0
36215995 GoSysCall p=6 g=3017828 off=130978
  47ef09 syscall.write /usr/local/go/src/syscall/zsyscall_linux_amd64.go:914
  4d2d98 syscall.Write /usr/local/go/src/syscall/syscall_unix.go:214
  4d2d23 internal/poll.(*FD).Write /usr/local/go/src/internal/poll/fd_unix.go:268
  5adade net.(*netFD).Write /usr/local/go/src/net/fd_unix.go:220
  5c084d net.(*conn).Write /usr/local/go/src/net/net.go:196
  61a87f crypto/tls.(*Conn).write /usr/local/go/src/crypto/tls/conn.go:895
  61ac70 crypto/tls.(*Conn).writeRecordLocked /usr/local/go/src/crypto/tls/conn.go:944
  61b963 crypto/tls.(*Conn).Write /usr/local/go/src/crypto/tls/conn.go:1111
  6f7071 net/http.persistConnWriter.Write /usr/local/go/src/net/http/transport.go:1663
  5672f4 bufio.(*Writer).Flush /usr/local/go/src/bufio/bufio.go:591
  6fa0a7 net/http.(*persistConn).writeLoop /usr/local/go/src/net/http/transport.go:2293
36221521 GoStart p=5 g=258 off=68060 g=258 seq=143
36224379 HeapAlloc p=5 g=258 off=68067 mem=209330856
36226235 ProcStop p=2 g=0 off=581320
36229201 ProcStart p=2 g=0 off=581323 thread=22
36229435 GoUnblock p=1000002 g=0 off=581327 g=15469 seq=4
36229883 GoStart p=2 g=15469 off=581334 g=15469 seq=0
36230481 GoSysCall p=2 g=15469 off=581338
  47ed19 syscall.read /usr/local/go/src/syscall/zsyscall_linux_amd64.go:686
  4d2112 syscall.Read /usr/local/go/src/syscall/syscall_unix.go:189
  4d20d9 internal/poll.(*FD).Read /usr/local/go/src/internal/poll/fd_unix.go:165
  5ad64e net.(*netFD).Read /usr/local/go/src/net/fd_unix.go:202
  5c060d net.(*conn).Read /usr/local/go/src/net/net.go:184
  6d8023 net/http.(*connReader).Read /usr/local/go/src/net/http/server.go:786
  565ece bufio.(*Reader).Read /usr/local/go/src/bufio/bufio.go:226
  4cdf02 io.(*LimitedReader).Read /usr/local/go/src/io/io.go:451
  6ecdfe net/http.(*body).readLocked /usr/local/go/src/net/http/transfer.go:847
  6ecd21 net/http.(*body).Read /usr/local/go/src/net/http/transfer.go:839
  113bcfe code.justin.tv/common/chitin.(*body).Read /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:363
  4f0eb0 bytes.(*Buffer).ReadFrom /usr/local/go/src/bytes/buffer.go:204
  5886f2 io/ioutil.readAll /usr/local/go/src/io/ioutil/ioutil.go:36
  f64092 io/ioutil.ReadAll /usr/local/go/src/io/ioutil/ioutil.go:45
  f64048 code.justin.tv/revenue/offer-tenant-schema/rpc.(*offersTenantServer).serveIsEligibleProtobuf /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/revenue/offer-tenant-schema/rpc/offers-tenant.twirp.go:326
  f631c8 code.justin.tv/revenue/offer-tenant-schema/rpc.(*offersTenantServer).serveIsEligible /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/revenue/offer-tenant-schema/rpc/offers-tenant.twirp.go:254
  f62e08 code.justin.tv/revenue/offer-tenant-schema/rpc.(*offersTenantServer).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/revenue/offer-tenant-schema/rpc/offers-tenant.twirp.go:228
  1718359 goji%2eio.dispatch.ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/dispatch.go:17
  17b25cd code.justin.tv/revenue/subscriptions/internal/api/server.handlePanic.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/server/server.go:98
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  171885c goji%2eio.(*Mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/mux.go:74
  113ac2a code.justin.tv/common/chitin.(*mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:248
  6e1f72 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2807
  6dd9eb net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1895
36233083 GoBlockSelect p=3 g=3021749 off=23653
  448f02 runtime.selectgo /usr/local/go/src/runtime/select.go:316
  6faaf5 net/http.(*persistConn).roundTrip /usr/local/go/src/net/http/transport.go:2498
  6ef8b8 net/http.(*Transport).roundTrip /usr/local/go/src/net/http/transport.go:565
  6d60d4 net/http.(*Transport).RoundTrip /usr/local/go/src/net/http/roundtrip.go:17
  11de7f0 code.justin.tv/foundation/twitchclient.(*twitchRoundTripper).RoundTrip /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/foundation/twitchclient/httpclient.go:109
  6991fd net/http.send /usr/local/go/src/net/http/client.go:252
  698c19 net/http.(*Client).send /usr/local/go/src/net/http/client.go:176
  69a8e9 net/http.(*Client).do /usr/local/go/src/net/http/client.go:699
  11dc4dd net/http.(*Client).Do /usr/local/go/src/net/http/client.go:567
  11dc4c4 code.justin.tv/foundation/twitchclient.(*clientImpl).Do /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/foundation/twitchclient/client.go:68
  163a604 code.justin.tv/web/users-service/client/usersclient_internal.(*clientImpl).GetUserByIDAndParams /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/web/users-service/client/usersclient_internal/usersservice.go:97
  16d57e8 code.justin.tv/revenue/subscriptions/internal/app/gifts.(*giftsImpl).checkUserSuspended /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/gifts/eligibility.go:408
  16d55c6 code.justin.tv/revenue/subscriptions/internal/app/gifts.(*giftsImpl).checkSenderSuspended /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/gifts/eligibility.go:378
  16e7c15 code.justin.tv/revenue/subscriptions/internal/app/gifts.(*giftsImpl).eligibleForGiftWithChecks.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/gifts/eligibility.go:264
  11e9458 golang.org/x/sync/errgroup.(*Group).Go.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/golang.org/x/sync/errgroup/errgroup.go:57
`[1:]

	for _, s := range splitEvents(str) {
		t.Run("", func(t *testing.T) {
			ev, err := parseEventString(s)
			if err != nil {
				t.Fatalf("parseEventString; err = %v", err)
			}
			have := eventString(ev, true)
			if want := s; have != want {
				t.Errorf("eventString(parseEventString) does not round-trip:\n%s\n!=\n%s", have, want)
			}
		})
	}
}

func TestRegionSyncMutex(t *testing.T) {
	testcase := func(filename string, t1, t2 int64) func(t *testing.T) {
		return func(t *testing.T) {
			buf, err := ioutil.ReadFile(filename)
			if err != nil {
				t.Fatalf("ReadFile(%q); err = %v", filename, err)
			}

			var evs []*trace.Event
			for _, s := range splitEvents(string(buf)) {
				ev, err := parseEventString(s)
				if err != nil {
					t.Fatalf("parseEventString; err = %v", err)
				}
				evs = append(evs, ev)
			}

			var regions []*region
			rt := &mutexTracker{
				verbose: false,

				flush: func(evs []*trace.Event) {
					regions = append(regions, &region{Kind: "internal/mutex", Events: evs})
				},
			}
			state := rt.idle
			for _, ev := range evs {
				if state != nil {
					state = state(ev)
				}
			}

			if l := len(regions); l != 1 {
				t.Fatalf("len(regions); %d != 1", l)
			}
			r := regions[0]
			if have, want := r.Kind, "internal/mutex"; have != want {
				t.Errorf("region.Kind; %q != %q", have, want)
			}
			// The region starts with a GoBlockSync event.
			if have, want := r.Events[0].Ts, t1; have != want {
				t.Errorf("region start timestamp; %d != %d", have, want)
			}
			// The region ends with the subsequent GoStart event.
			if have, want := r.Events[len(r.Events)-1].Ts, t2; have != want {
				t.Errorf("region end timestamp; %d != %d", have, want)
			}

			if t.Failed() {
				for _, ev := range r.Events {
					t.Logf("%s", eventString(ev, false))
				}
			}
		}
	}

	t.Run("", testcase("./testdata/mutex-1.txt", 779274831, 880504078))
}

func TestRegionNydusHTTPServer(t *testing.T) {
	testcase := func(filename string, t1, t2 int64) func(t *testing.T) {
		return func(t *testing.T) {
			buf, err := ioutil.ReadFile(filename)
			if err != nil {
				t.Fatalf("ReadFile(%q); err = %v", filename, err)
			}

			var evs []*trace.Event
			for _, s := range splitEvents(string(buf)) {
				ev, err := parseEventString(s)
				if err != nil {
					t.Fatalf("parseEventString; err = %v", err)
				}
				evs = append(evs, ev)
			}

			var regions []*region
			rt := &track{
				verbose: false,

				flushInbound: func(evs []*trace.Event) {
					regions = append(regions, &region{Kind: "server/http", Events: evs})
				},
			}
			state := rt.inboundRequestPending
			for _, ev := range evs {
				if state != nil {
					state = state(ev)
				}
			}

			if l := len(regions); l != 1 {
				t.Fatalf("len(regions); %d != 1", l)
			}
			r := regions[0]
			if have, want := r.Kind, "server/http"; have != want {
				t.Errorf("region.Kind; %q != %q", have, want)
			}
			// The region starts immediately after a GoSysCall event for the
			// "read" syscall as called by net/http.ReadRequest.
			if have, want := r.Events[0].Ts, t1; have != want {
				t.Errorf("region start timestamp; %d != %d", have, want)
			}
			// The region ends when the goroutine exits, or when it has another
			// GoSysCall event under net/http.ReadRequest.
			if have, want := r.Events[len(r.Events)-1].Ts, t2; have != want {
				t.Errorf("region end timestamp; %d != %d", have, want)
			}

			if t.Failed() {
				for _, ev := range r.Events {
					t.Logf("%s", eventString(ev, false))
				}
			}
		}
	}

	t.Run("", testcase("./testdata/nydushttpserver-1.txt", 561724209, 561880282))
}

func TestRegionGithubGoredisRedisV7(t *testing.T) {
	testcase := func(filename string, t1, t2 int64) func(t *testing.T) {
		return func(t *testing.T) {
			buf, err := ioutil.ReadFile(filename)
			if err != nil {
				t.Fatalf("ReadFile(%q); err = %v", filename, err)
			}

			var evs []*trace.Event
			for _, s := range splitEvents(string(buf)) {
				ev, err := parseEventString(s)
				if err != nil {
					t.Fatalf("parseEventString; err = %v", err)
				}
				evs = append(evs, ev)
			}

			var regions []*region
			rt := &redisTracker{
				verbose: false,

				flush: func(evs []*trace.Event) {
					regions = append(regions, &region{Kind: "client/redis", Events: evs})
				},
			}
			state := rt.idle
			for _, ev := range evs {
				if state != nil {
					state = state(ev)
				}
			}

			if l := len(regions); l != 1 {
				t.Fatalf("len(regions); %d != 1", l)
			}
			r := regions[0]
			if have, want := r.Kind, "client/redis"; have != want {
				t.Errorf("region.Kind; %q != %q", have, want)
			}
			// The region starts when the driver does a write syscall.
			if have, want := r.Events[0].Ts, t1; have != want {
				t.Errorf("region start timestamp; %d != %d", have, want)
			}
			// The region ends as soon as the goroutine has any meaningful event other
			// than a Redis read. The end timestamp corresponds to the read syscall, so
			// it won't count processing time in driver (or application) code.
			if have, want := r.Events[len(r.Events)-1].Ts, t2; have != want {
				t.Errorf("region end timestamp; %d != %d", have, want)
			}

			if t.Failed() {
				for _, ev := range r.Events {
					t.Logf("%s", eventString(ev, false))
				}
			}
		}
	}

	t.Run("", testcase("./testdata/redisclient-1.txt", 134836145, 135065754))

	t.Run("", testcase("./testdata/redisclient-2.txt", 816909690, 818233491))
}

func TestRegionGithubLibPq(t *testing.T) {
	str := `
894626614 GoStart p=8 g=3026725 off=858072 g=3026725 seq=0
  115a120 code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:407
894632139 HeapAlloc p=8 g=3026725 off=858079 mem=264225432
894708086 GoSysCall p=8 g=3026725 off=858086
  47ef09 syscall.write /usr/local/go/src/syscall/zsyscall_linux_amd64.go:914
  4d2d98 syscall.Write /usr/local/go/src/syscall/syscall_unix.go:214
  4d2d23 internal/poll.(*FD).Write /usr/local/go/src/internal/poll/fd_unix.go:268
  5adade net.(*netFD).Write /usr/local/go/src/net/fd_unix.go:220
  5c084d net.(*conn).Write /usr/local/go/src/net/net.go:196
  10f6bcb code.justin.tv/common/chitin/internal/trace.(*batchWriter).Write /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/internal/trace/send.go:403
  10f70d6 code.justin.tv/common/chitin/internal/trace.(*perPWriter).Write.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/internal/trace/send.go:338
  10f10e7 code.justin.tv/common/chitin/internal/perp.(*Pool).Do /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/internal/perp/perp.go:73
  10f67b7 code.justin.tv/common/chitin/internal/trace.(*perPWriter).Write /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/internal/trace/send.go:335
  10f6165 code.justin.tv/common/chitin/internal/trace.SendEvent /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/internal/trace/send.go:290
  1153190 code.justin.tv/common/chitin/dbtrace.BasicRequestHeadPrepared /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/dbtrace/event.go:36
  1159d4a code.justin.tv/chat/db.traceStart /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/util.go:177
  1159fb4 code.justin.tv/chat/db.(*dbImpl).Query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:392
  115a15b code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:408
894727093 GoCreate p=8 g=3026725 off=858091 g=3026726 stack=289
  11a83cb github.com/lib/pq.(*conn).watchCancel /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn_go18.go:93
  11a7bb5 github.com/lib/pq.(*conn).QueryContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn_go18.go:19
  109fd06 database/sql.ctxDriverQuery /usr/local/go/src/database/sql/ctxutil.go:48
  10afbba database/sql.(*DB).queryDC.func1 /usr/local/go/src/database/sql/sql.go:1592
  10af008 database/sql.withLock /usr/local/go/src/database/sql/sql.go:3184
  10a792c database/sql.(*DB).queryDC /usr/local/go/src/database/sql/sql.go:1587
  10a72b5 database/sql.(*DB).query /usr/local/go/src/database/sql/sql.go:1570
  10a6f30 database/sql.(*DB).QueryContext /usr/local/go/src/database/sql/sql.go:1547
  11532f6 code.justin.tv/chat/db.(*sqlDBImpl).Query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:91
  115a040 code.justin.tv/chat/db.(*dbImpl).Query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:394
  115a15b code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:408
894736139 GoSysCall p=8 g=3026725 off=858103
  47ef09 syscall.write /usr/local/go/src/syscall/zsyscall_linux_amd64.go:914
  4d2d98 syscall.Write /usr/local/go/src/syscall/syscall_unix.go:214
  4d2d23 internal/poll.(*FD).Write /usr/local/go/src/internal/poll/fd_unix.go:268
  5adade net.(*netFD).Write /usr/local/go/src/net/fd_unix.go:220
  5c084d net.(*conn).Write /usr/local/go/src/net/net.go:196
  119ec29 github.com/lib/pq.(*conn).send /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:890
  11a47f0 github.com/lib/pq.(*conn).sendBinaryModeQuery /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:1608
  119e46b github.com/lib/pq.(*conn).query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:836
  11a7c06 github.com/lib/pq.(*conn).QueryContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn_go18.go:20
  109fd06 database/sql.ctxDriverQuery /usr/local/go/src/database/sql/ctxutil.go:48
  10afbba database/sql.(*DB).queryDC.func1 /usr/local/go/src/database/sql/sql.go:1592
  10af008 database/sql.withLock /usr/local/go/src/database/sql/sql.go:3184
  10a792c database/sql.(*DB).queryDC /usr/local/go/src/database/sql/sql.go:1587
  10a72b5 database/sql.(*DB).query /usr/local/go/src/database/sql/sql.go:1570
  10a6f30 database/sql.(*DB).QueryContext /usr/local/go/src/database/sql/sql.go:1547
  11532f6 code.justin.tv/chat/db.(*sqlDBImpl).Query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:91
  115a040 code.justin.tv/chat/db.(*dbImpl).Query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:394
  115a15b code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:408
894751541 GoSysCall p=8 g=3026725 off=858108
  47ed19 syscall.read /usr/local/go/src/syscall/zsyscall_linux_amd64.go:686
  4d2112 syscall.Read /usr/local/go/src/syscall/syscall_unix.go:189
  4d20d9 internal/poll.(*FD).Read /usr/local/go/src/internal/poll/fd_unix.go:165
  5ad64e net.(*netFD).Read /usr/local/go/src/net/fd_unix.go:202
  5c060d net.(*conn).Read /usr/local/go/src/net/net.go:184
  565ece bufio.(*Reader).Read /usr/local/go/src/bufio/bufio.go:226
  4cd656 io.ReadAtLeast /usr/local/go/src/io/io.go:310
  119ef72 io.ReadFull /usr/local/go/src/io/io.go:329
  119ef31 github.com/lib/pq.(*conn).recvMessage /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:936
  119f1f8 github.com/lib/pq.(*conn).recv1Buf /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:985
  11a51e8 github.com/lib/pq.(*conn).recv1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:1006
  11a51b1 github.com/lib/pq.(*conn).readParseResponse /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:1658
  119e47c github.com/lib/pq.(*conn).query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:838
  11a7c06 github.com/lib/pq.(*conn).QueryContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn_go18.go:20
  109fd06 database/sql.ctxDriverQuery /usr/local/go/src/database/sql/ctxutil.go:48
  10afbba database/sql.(*DB).queryDC.func1 /usr/local/go/src/database/sql/sql.go:1592
  10af008 database/sql.withLock /usr/local/go/src/database/sql/sql.go:3184
  10a792c database/sql.(*DB).queryDC /usr/local/go/src/database/sql/sql.go:1587
  10a72b5 database/sql.(*DB).query /usr/local/go/src/database/sql/sql.go:1570
  10a6f30 database/sql.(*DB).QueryContext /usr/local/go/src/database/sql/sql.go:1547
  11532f6 code.justin.tv/chat/db.(*sqlDBImpl).Query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:91
  115a040 code.justin.tv/chat/db.(*dbImpl).Query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:394
  115a15b code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:408
894756491 GoBlockNet p=8 g=3026725 off=858113
  4d215c internal/poll.(*FD).Read /usr/local/go/src/internal/poll/fd_unix.go:169
  5ad64e net.(*netFD).Read /usr/local/go/src/net/fd_unix.go:202
  5c060d net.(*conn).Read /usr/local/go/src/net/net.go:184
  565ece bufio.(*Reader).Read /usr/local/go/src/bufio/bufio.go:226
  4cd656 io.ReadAtLeast /usr/local/go/src/io/io.go:310
  119ef72 io.ReadFull /usr/local/go/src/io/io.go:329
  119ef31 github.com/lib/pq.(*conn).recvMessage /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:936
  119f1f8 github.com/lib/pq.(*conn).recv1Buf /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:985
  11a51e8 github.com/lib/pq.(*conn).recv1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:1006
  11a51b1 github.com/lib/pq.(*conn).readParseResponse /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:1658
  119e47c github.com/lib/pq.(*conn).query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:838
  11a7c06 github.com/lib/pq.(*conn).QueryContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn_go18.go:20
  109fd06 database/sql.ctxDriverQuery /usr/local/go/src/database/sql/ctxutil.go:48
  10afbba database/sql.(*DB).queryDC.func1 /usr/local/go/src/database/sql/sql.go:1592
  10af008 database/sql.withLock /usr/local/go/src/database/sql/sql.go:3184
  10a792c database/sql.(*DB).queryDC /usr/local/go/src/database/sql/sql.go:1587
  10a72b5 database/sql.(*DB).query /usr/local/go/src/database/sql/sql.go:1570
  10a6f30 database/sql.(*DB).QueryContext /usr/local/go/src/database/sql/sql.go:1547
  11532f6 code.justin.tv/chat/db.(*sqlDBImpl).Query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:91
  115a040 code.justin.tv/chat/db.(*dbImpl).Query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:394
  115a15b code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:408
895334833 GoStart p=3 g=3026725 off=657683 g=3026725 seq=0
895335665 GoSysCall p=3 g=3026725 off=657689
  47ed19 syscall.read /usr/local/go/src/syscall/zsyscall_linux_amd64.go:686
  4d2112 syscall.Read /usr/local/go/src/syscall/syscall_unix.go:189
  4d20d9 internal/poll.(*FD).Read /usr/local/go/src/internal/poll/fd_unix.go:165
  5ad64e net.(*netFD).Read /usr/local/go/src/net/fd_unix.go:202
  5c060d net.(*conn).Read /usr/local/go/src/net/net.go:184
  565ece bufio.(*Reader).Read /usr/local/go/src/bufio/bufio.go:226
  4cd656 io.ReadAtLeast /usr/local/go/src/io/io.go:310
  119ef72 io.ReadFull /usr/local/go/src/io/io.go:329
  119ef31 github.com/lib/pq.(*conn).recvMessage /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:936
  119f1f8 github.com/lib/pq.(*conn).recv1Buf /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:985
  11a51e8 github.com/lib/pq.(*conn).recv1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:1006
  11a51b1 github.com/lib/pq.(*conn).readParseResponse /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:1658
  119e47c github.com/lib/pq.(*conn).query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn.go:838
  11a7c06 github.com/lib/pq.(*conn).QueryContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/lib/pq/conn_go18.go:20
  109fd06 database/sql.ctxDriverQuery /usr/local/go/src/database/sql/ctxutil.go:48
  10afbba database/sql.(*DB).queryDC.func1 /usr/local/go/src/database/sql/sql.go:1592
  10af008 database/sql.withLock /usr/local/go/src/database/sql/sql.go:3184
  10a792c database/sql.(*DB).queryDC /usr/local/go/src/database/sql/sql.go:1587
  10a72b5 database/sql.(*DB).query /usr/local/go/src/database/sql/sql.go:1570
  10a6f30 database/sql.(*DB).QueryContext /usr/local/go/src/database/sql/sql.go:1547
  11532f6 code.justin.tv/chat/db.(*sqlDBImpl).Query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:91
  115a040 code.justin.tv/chat/db.(*dbImpl).Query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:394
  115a15b code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:408
895358769 GoCreate p=3 g=3026725 off=657693 g=3026010 stack=295
  10acdc0 database/sql.(*Rows).initContextClose /usr/local/go/src/database/sql/sql.go:2717
  10a79f6 database/sql.(*DB).queryDC /usr/local/go/src/database/sql/sql.go:1606
  10a72b5 database/sql.(*DB).query /usr/local/go/src/database/sql/sql.go:1570
  10a6f30 database/sql.(*DB).QueryContext /usr/local/go/src/database/sql/sql.go:1547
  11532f6 code.justin.tv/chat/db.(*sqlDBImpl).Query /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:91
  115a040 code.justin.tv/chat/db.(*dbImpl).Query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:394
  115a15b code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:408
895374876 GoUnblock p=3 g=3026725 off=657705 g=1752 seq=18
  448b37 runtime.selectgo /usr/local/go/src/runtime/select.go:481
  115a26f code.justin.tv/chat/db.(*dbImpl).query.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/db/db.go:421
895376305 GoEnd p=3 g=3026725 off=657714
`[1:]

	var evs []*trace.Event
	for _, s := range splitEvents(str) {
		ev, err := parseEventString(s)
		if err != nil {
			t.Fatalf("parseEventString; err = %v", err)
		}
		evs = append(evs, ev)
	}

	var regions []*region
	rt := &libpqTracker{
		verbose: false,

		flush: func(evs []*trace.Event) {
			regions = append(regions, &region{Kind: "client/postgres", Events: evs})
		},
	}
	state := rt.idle
	for _, ev := range evs {
		if state != nil {
			state = state(ev)
		}
	}

	if l := len(regions); l != 1 {
		t.Fatalf("len(regions); %d != 1", l)
	}
	r := regions[0]
	if have, want := r.Kind, "client/postgres"; have != want {
		t.Errorf("region.Kind; %q != %q", have, want)
	}
	// The region starts when the driver does a write syscall.
	if have, want := r.Events[0].Ts, int64(894736139); have != want {
		t.Errorf("region start timestamp; %d != %d", have, want)
	}
	// The region ends as soon as the goroutine has any meaningful event other
	// than a Postgres read. The end timestamp corresponds to the read syscall,
	// so it won't count processing time in driver (or application) code.
	if have, want := r.Events[len(r.Events)-1].Ts, int64(895335665); have != want {
		t.Errorf("region end timestamp; %d != %d", have, want)
	}

	if t.Failed() {
		for _, ev := range r.Events {
			t.Logf("%s", eventString(ev, false))
		}
	}
}

func TestRegionHTTPClient(t *testing.T) {
	testcase := func(filename string, t1, t2 int64) func(t *testing.T) {
		return func(t *testing.T) {
			buf, err := ioutil.ReadFile(filename)
			if err != nil {
				t.Fatalf("ReadFile(%q); err = %v", filename, err)
			}

			var evs []*trace.Event
			for _, s := range splitEvents(string(buf)) {
				ev, err := parseEventString(s)
				if err != nil {
					t.Fatalf("parseEventString; err = %v", err)
				}
				evs = append(evs, ev)
			}

			var regions []*region
			rt := &httpClientTracker{
				verbose: false,

				flush: func(evs []*trace.Event) {
					regions = append(regions, &region{Kind: "client/http", Events: evs})
				},
			}
			state := rt.idle
			for _, ev := range evs {
				if state != nil {
					state = state(ev)
				}
			}

			if l := len(regions); l != 1 {
				t.Fatalf("len(regions); %d != 1", l)
			}
			r := regions[0]
			if have, want := r.Kind, "client/http"; have != want {
				t.Errorf("region.Kind; %q != %q", have, want)
			}
			// The region starts when a RoundTrip call queues to dial a connection, or when TODO.
			if have, want := r.Events[0].Ts, t1; have != want {
				t.Errorf("region start timestamp; %d != %d", have, want)
			}
			// The region ends as soon as the goroutine has any meaningful event other
			// than waiting for helper goroutines to do work. The end timestamp
			// corresponds to when the goroutine unblocks after waiting on the helpers,
			// so it won't count processing time in driver (or application) code. It
			// also does not count the time for the application to read the response
			// body.
			if have, want := r.Events[len(r.Events)-1].Ts, t2; have != want {
				t.Errorf("region end timestamp; %d != %d", have, want)
			}

			if t.Failed() {
				for _, ev := range r.Events {
					t.Logf("%s", eventString(ev, false))
				}
			}
		}
	}

	// dial new http/1.1 connection
	t.Run("", testcase("./testdata/httpclient-1.txt", 905030236, 919604241))
	// http/1.1 connection available from pool
	t.Run("", testcase("./testdata/httpclient-2.txt", 786653993, 889754433))
}

func TestRegionHTTPServer(t *testing.T) {
	testcase := func(filename string, ts ...int64) func(t *testing.T) {
		return func(t *testing.T) {
			buf, err := ioutil.ReadFile(filename)
			if err != nil {
				t.Fatalf("ReadFile(%q); err = %v", filename, err)
			}

			var evs []*trace.Event
			for _, s := range splitEvents(string(buf)) {
				ev, err := parseEventString(s)
				if err != nil {
					t.Fatalf("parseEventString; err = %v", err)
				}
				evs = append(evs, ev)
			}

			var regions []*region
			rt := &track{
				verbose: false,

				flushInbound: func(evs []*trace.Event) {
					regions = append(regions, &region{Kind: "server/http", Events: evs})
				},
			}
			state := rt.inboundRequestPending
			for _, ev := range evs {
				if state != nil {
					state = state(ev)
				}
			}

			if have, want := len(regions), len(ts)/2; have != want {
				t.Errorf("len(regions); %d != %d", have, want)
			}
			for i, r := range regions {
				if (i+1)*2 > len(ts) {
					break
				}
				t1 := ts[i*2]
				t2 := ts[i*2+1]

				t.Run("", func(t *testing.T) {
					if have, want := r.Kind, "server/http"; have != want {
						t.Errorf("region.Kind; %q != %q", have, want)
					}
					// The region starts when a RoundTrip call queues to dial a connection, or when TODO.
					if have, want := r.Events[0].Ts, t1; have != want {
						t.Errorf("region start timestamp; %d != %d", have, want)
					}
					// The region ends as soon as the goroutine has any meaningful event other
					// than waiting for helper goroutines to do work. The end timestamp
					// corresponds to when the goroutine unblocks after waiting on the helpers,
					// so it won't count processing time in driver (or application) code. It
					// also does not count the time for the application to read the response
					// body.
					if have, want := r.Events[len(r.Events)-1].Ts, t2; have != want {
						t.Errorf("region end timestamp; %d != %d", have, want)
					}

					if t.Failed() {
						for _, ev := range r.Events {
							t.Logf("%s", eventString(ev, false))
						}
					}
				})
			}
		}
	}

	// TODO: check the start time on this... should it be the GoSysCall at
	// 182359267 that returns the last of the headers, or should it be the first
	// non-header-read event, GoCreate at 182397112?
	t.Run("", testcase("./testdata/httpserver-1.txt", 182397112, 185654925))

	// go test | grep pending | grep active | awk '{print $4}' | paste -d, - - | sed -e 's/$/,/'

	t.Run("", testcase("./testdata/httpserver-2.txt",
		529613392, 530681403,
		534320337, 534985596,
		566464642, 567076012,
		576832046, 579665732,
		584726106, 585313797,
		586867760, 587335046,
		587437830, 588784113,
		601553032, 602240264,
		607924575, 609678751,
		623712034, 624619255,
		625313165, 626970659,
		628736910, 633199502,
		634835151, 640225146,
		640595088, 641263333,
		641633659, 642338277,
		644133926, 645439121,
		647869308, 648556305,
		651802002, 652364797,
		653674365, 656898152,
		658596563, 659107859,
		659526611, 666350079,
		667098325, 670844864,
		673699692, 675077740,
		680370306, 681905496,
		682048514, 684841688,
		686755459, 687871150,
		690583215, 691320986,
		691939012, 694413807,
		695577242, 696160282,
		697228507, 697769349,
		710987464, 711993054,
		728118689, 729580342,
		731733921, 736371405,
		737947256, 740057806,
		741489422, 744426894,
		746619236, 748190330,
		749237263, 750829818,
		752270053, 752731109,
		755544635, 756798971,
		761045031, 762689319,
		763794663, 765707539,
		780309675, 781059605,
		782070187, 783984961,
		785150017, 785933313,
		786542487, 787880065,
		788293634, 789684268,
		797017902, 797762030,
		798668932, 802011268,
		806607877, 807976112,
		809594779, 811671174,
		821974024, 824356659,
		825174579, 825764083,
		826842974, 828171380,
		831608671, 832223114,
		833833952, 836424459,
		838015840, 839543478,
		842731809, 843358839,
		847382072, 848581624,
		881414035, 882204116,
		885506794, 886091007,
		887510143, 889038784,
		890639275, 894649281,
		897974145, 898862849,
		912842265, 915498031,
		916182575, 917430853,
		918546693, 920179675,
		920488838, 921056432,
		922044315, 923737948,
		928080796, 929804381,
		931620210, 934361715,
		935660574, 937701876,
		938356233, 942661428,
		943052767, 943888479,
		950008075, 950693003,
		957157751, 957702263,
		965480697, 966078201,
		967978831, 969777828,
		970831226, 971406821,
		974856400, 977867707,
		979333713, 981388519,
	))

	// When the server sets a keepalive idle time limit, net/http will set the
	// read time and then peek at the connection for new data. That can result
	// in an entire request being available in the buffer and no event for the
	// process blocking on the network in readRequest.
	t.Run("users-service-loadtest", testcase("./testdata/httpserver-3.txt",
		185419294, 187451677,
		196223341, 198518891,
		200131605, 202382163,
		203344723, 205621180,
		211166627, 213216332,
		214021452, 217705204,
		218755379, 220797789,
		227266585, 229237335,
		242001530, 244193635,
		246404002, 248347361,
		248626144, 250581983,
		250959306, 252962654,
		253322931, 255072690,
		255569223, 257517168,
		262824130, 265172139,
		266759807, 269325992,
		269823805, 271716156,
		275829135, 277879758,
		288584305, 291041947,
		291412805, 293356462,
		294895299, 297669249,
		327033540, 329050904,
		329949335, 331921750,
		333144640, 335000894,
		335498387, 337410834,
		337779666, 340264592,
		341408592, 343249572,
		358178906, 360648835,
		361887064, 365159318,
		370619624, 373360614,
		378696056, 381141473,
		383237898, 387313714,
		393177903, 395371864,
		427840643, 437550610,
		441835770, 443866489,
		444562552, 446667703,
	))
}

func TestRegionDAX(t *testing.T) {
	str := `
622908512 GoUnblock p=9 g=8102 off=889006 g=258 seq=2968
  4080f3 runtime.selectnbsend /usr/local/go/src/runtime/chan.go:625
  abe012 code.justin.tv/amzn/TwitchTelemetry.(*BufferedAggregator).ObserveSample /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/amzn/TwitchTelemetry/buffered_aggregator.go:64
  ac18a6 code.justin.tv/amzn/TwitchTelemetry.(*SampleReporter).ReportDurationSample /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/amzn/TwitchTelemetry/sample_reporter.go:34
  afe3dc code.justin.tv/commerce/splatter.(*TwitchTelemetryStatter).TimingDuration /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/commerce/splatter/twitch_telemetry_statter.go:99
  12a55e8 code.justin.tv/chat/rediczar/runmetrics/statsdrunmetrics.(*StatTracker).timingDuration /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/rediczar/runmetrics/statsdrunmetrics/stattracker.go:42
  12a54ae code.justin.tv/chat/rediczar/runmetrics/statsdrunmetrics.(*StatTracker).Success /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/rediczar/runmetrics/statsdrunmetrics/stattracker.go:29
  1059ae7 code.justin.tv/chat/rediczar.trackCommand /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/rediczar/metrics.go:24
  1044cda code.justin.tv/chat/rediczar.(*PrefixedCommands).Get /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/chat/rediczar/client_commands.gen.go:1800
  16cc367 code.justin.tv/revenue/subscriptions/internal/app/badges/cache.(*cache).GetCachedBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/cache/badge_cache.go:54
  16f3b2a code.justin.tv/revenue/subscriptions/internal/app/badges.(*badge).getFounderBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/get_subscriber_badge.go:224
  16f1e7b code.justin.tv/revenue/subscriptions/internal/app/badges.(*badge).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/get_subscriber_badge.go:113
  17678ee code.justin.tv/revenue/subscriptions/internal/api/subs.(*BadgesAPI).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/badges_api.go:249
  d0e8bd code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14118
  179615a code.justin.tv/revenue/subscriptions/internal/api/subs.ReqMetaInterceptor.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:198
  d0ea2e code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14120
  d0ec03 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func2 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14136
  c5bc17 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14137
  c5aa38 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14003
  c35bfc code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:10573
  17956d6 code.justin.tv/revenue/subscriptions/internal/api/subs.WithTwitchRepository.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:111
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  1718359 goji%2eio.dispatch.ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/dispatch.go:17
  17b25cd code.justin.tv/revenue/subscriptions/internal/api/server.handlePanic.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/server/server.go:98
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  171885c goji%2eio.(*Mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/mux.go:74
  113ac2a code.justin.tv/common/chitin.(*mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:248
  6e1f72 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2807
  6dd9eb net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1895
622930784 GoSysCall p=9 g=8102 off=889016
  47ef09 syscall.write /usr/local/go/src/syscall/zsyscall_linux_amd64.go:914
  4d2d98 syscall.Write /usr/local/go/src/syscall/syscall_unix.go:214
  4d2d23 internal/poll.(*FD).Write /usr/local/go/src/internal/poll/fd_unix.go:268
  5adade net.(*netFD).Write /usr/local/go/src/net/fd_unix.go:220
  5c084d net.(*conn).Write /usr/local/go/src/net/net.go:196
  5672f4 bufio.(*Writer).Flush /usr/local/go/src/bufio/bufio.go:591
  1469525 github.com/aws/aws-dax-go/dax/internal/cbor.(*Writer).Flush /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/cbor/cbor.go:71
  1469530 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:668
  1468d46 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithRetries /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:630
  1464ae1 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:287
  146c991 github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:209
  144098e github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).retry /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:331
  143f43b github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:212
  147539a github.com/aws/aws-dax-go/dax.(*Dax).GetItemWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/api.go:117
  147a1b5 code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb.(*tenureBenefitsDBImpl).GetTenureBenefit /build/go/src/code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb/tenurebenefitsdb.go:117
  16f1f66 code.justin.tv/revenue/subscriptions/internal/app/badges.(*badge).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/get_subscriber_badge.go:119
  17678ee code.justin.tv/revenue/subscriptions/internal/api/subs.(*BadgesAPI).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/badges_api.go:249
  d0e8bd code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14118
  179615a code.justin.tv/revenue/subscriptions/internal/api/subs.ReqMetaInterceptor.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:198
  d0ea2e code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14120
  d0ec03 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func2 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14136
  c5bc17 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14137
  c5aa38 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14003
  c35bfc code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:10573
  17956d6 code.justin.tv/revenue/subscriptions/internal/api/subs.WithTwitchRepository.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:111
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  1718359 goji%2eio.dispatch.ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/dispatch.go:17
  17b25cd code.justin.tv/revenue/subscriptions/internal/api/server.handlePanic.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/server/server.go:98
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  171885c goji%2eio.(*Mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/mux.go:74
  113ac2a code.justin.tv/common/chitin.(*mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:248
  6e1f72 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2807
  6dd9eb net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1895
622962101 GoSysCall p=9 g=8102 off=889021
  47ed19 syscall.read /usr/local/go/src/syscall/zsyscall_linux_amd64.go:686
  4d2112 syscall.Read /usr/local/go/src/syscall/syscall_unix.go:189
  4d20d9 internal/poll.(*FD).Read /usr/local/go/src/internal/poll/fd_unix.go:165
  5ad64e net.(*netFD).Read /usr/local/go/src/net/fd_unix.go:202
  5c060d net.(*conn).Read /usr/local/go/src/net/net.go:184
  565812 bufio.(*Reader).fill /usr/local/go/src/bufio/bufio.go:100
  5660b8 bufio.(*Reader).ReadByte /usr/local/go/src/bufio/bufio.go:252
  131e832 github.com/aws/aws-dax-go/dax/internal/cbor.(*Reader).readTypeHeader /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/cbor/cbor.go:409
  131e3ae github.com/aws/aws-dax-go/dax/internal/cbor.(*Reader).ReadArrayLength /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/cbor/cbor.go:349
  1443dc2 github.com/aws/aws-dax-go/dax/internal/client.decodeError /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/error.go:96
  146955f github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:674
  1468d46 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithRetries /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:630
  1464ae1 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:287
  146c991 github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:209
  144098e github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).retry /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:331
  143f43b github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:212
  147539a github.com/aws/aws-dax-go/dax.(*Dax).GetItemWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/api.go:117
  147a1b5 code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb.(*tenureBenefitsDBImpl).GetTenureBenefit /build/go/src/code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb/tenurebenefitsdb.go:117
  16f1f66 code.justin.tv/revenue/subscriptions/internal/app/badges.(*badge).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/get_subscriber_badge.go:119
  17678ee code.justin.tv/revenue/subscriptions/internal/api/subs.(*BadgesAPI).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/badges_api.go:249
  d0e8bd code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14118
  179615a code.justin.tv/revenue/subscriptions/internal/api/subs.ReqMetaInterceptor.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:198
  d0ea2e code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14120
  d0ec03 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func2 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14136
  c5bc17 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14137
  c5aa38 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14003
  c35bfc code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:10573
  17956d6 code.justin.tv/revenue/subscriptions/internal/api/subs.WithTwitchRepository.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:111
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  1718359 goji%2eio.dispatch.ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/dispatch.go:17
  17b25cd code.justin.tv/revenue/subscriptions/internal/api/server.handlePanic.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/server/server.go:98
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  171885c goji%2eio.(*Mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/mux.go:74
  113ac2a code.justin.tv/common/chitin.(*mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:248
  6e1f72 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2807
  6dd9eb net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1895
622977184 GoBlockNet p=9 g=8102 off=889026
  4d215c internal/poll.(*FD).Read /usr/local/go/src/internal/poll/fd_unix.go:169
  5ad64e net.(*netFD).Read /usr/local/go/src/net/fd_unix.go:202
  5c060d net.(*conn).Read /usr/local/go/src/net/net.go:184
  565812 bufio.(*Reader).fill /usr/local/go/src/bufio/bufio.go:100
  5660b8 bufio.(*Reader).ReadByte /usr/local/go/src/bufio/bufio.go:252
  131e832 github.com/aws/aws-dax-go/dax/internal/cbor.(*Reader).readTypeHeader /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/cbor/cbor.go:409
  131e3ae github.com/aws/aws-dax-go/dax/internal/cbor.(*Reader).ReadArrayLength /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/cbor/cbor.go:349
  1443dc2 github.com/aws/aws-dax-go/dax/internal/client.decodeError /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/error.go:96
  146955f github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:674
  1468d46 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithRetries /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:630
  1464ae1 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:287
  146c991 github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:209
  144098e github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).retry /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:331
  143f43b github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:212
  147539a github.com/aws/aws-dax-go/dax.(*Dax).GetItemWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/api.go:117
  147a1b5 code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb.(*tenureBenefitsDBImpl).GetTenureBenefit /build/go/src/code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb/tenurebenefitsdb.go:117
  16f1f66 code.justin.tv/revenue/subscriptions/internal/app/badges.(*badge).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/get_subscriber_badge.go:119
  17678ee code.justin.tv/revenue/subscriptions/internal/api/subs.(*BadgesAPI).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/badges_api.go:249
  d0e8bd code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14118
  179615a code.justin.tv/revenue/subscriptions/internal/api/subs.ReqMetaInterceptor.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:198
  d0ea2e code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14120
  d0ec03 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func2 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14136
  c5bc17 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14137
  c5aa38 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14003
  c35bfc code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:10573
  17956d6 code.justin.tv/revenue/subscriptions/internal/api/subs.WithTwitchRepository.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:111
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  1718359 goji%2eio.dispatch.ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/dispatch.go:17
  17b25cd code.justin.tv/revenue/subscriptions/internal/api/server.handlePanic.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/server/server.go:98
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  171885c goji%2eio.(*Mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/mux.go:74
  113ac2a code.justin.tv/common/chitin.(*mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:248
  6e1f72 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2807
  6dd9eb net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1895
623237406 GoStart p=9 g=8102 off=889043 g=8102 seq=0
623238216 GoSysCall p=9 g=8102 off=889047
  47ed19 syscall.read /usr/local/go/src/syscall/zsyscall_linux_amd64.go:686
  4d2112 syscall.Read /usr/local/go/src/syscall/syscall_unix.go:189
  4d20d9 internal/poll.(*FD).Read /usr/local/go/src/internal/poll/fd_unix.go:165
  5ad64e net.(*netFD).Read /usr/local/go/src/net/fd_unix.go:202
  5c060d net.(*conn).Read /usr/local/go/src/net/net.go:184
  565812 bufio.(*Reader).fill /usr/local/go/src/bufio/bufio.go:100
  5660b8 bufio.(*Reader).ReadByte /usr/local/go/src/bufio/bufio.go:252
  131e832 github.com/aws/aws-dax-go/dax/internal/cbor.(*Reader).readTypeHeader /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/cbor/cbor.go:409
  131e3ae github.com/aws/aws-dax-go/dax/internal/cbor.(*Reader).ReadArrayLength /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/cbor/cbor.go:349
  1443dc2 github.com/aws/aws-dax-go/dax/internal/client.decodeError /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/error.go:96
  146955f github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:674
  1468d46 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).executeWithRetries /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:630
  1464ae1 github.com/aws/aws-dax-go/dax/internal/client.(*SingleDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/single.go:287
  146c991 github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions.func1 /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:209
  144098e github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).retry /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:331
  143f43b github.com/aws/aws-dax-go/dax/internal/client.(*ClusterDaxClient).GetItemWithOptions /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/internal/client/cluster.go:212
  147539a github.com/aws/aws-dax-go/dax.(*Dax).GetItemWithContext /build/go/src/code.justin.tv/revenue/subscriptions/vendor/github.com/aws/aws-dax-go/dax/api.go:117
  147a1b5 code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb.(*tenureBenefitsDBImpl).GetTenureBenefit /build/go/src/code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb/tenurebenefitsdb.go:117
  16f1f66 code.justin.tv/revenue/subscriptions/internal/app/badges.(*badge).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/get_subscriber_badge.go:119
  17678ee code.justin.tv/revenue/subscriptions/internal/api/subs.(*BadgesAPI).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/badges_api.go:249
  d0e8bd code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14118
  179615a code.justin.tv/revenue/subscriptions/internal/api/subs.ReqMetaInterceptor.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:198
  d0ea2e code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14120
  d0ec03 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func2 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14136
  c5bc17 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14137
  c5aa38 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14003
  c35bfc code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:10573
  17956d6 code.justin.tv/revenue/subscriptions/internal/api/subs.WithTwitchRepository.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:111
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  1718359 goji%2eio.dispatch.ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/dispatch.go:17
  17b25cd code.justin.tv/revenue/subscriptions/internal/api/server.handlePanic.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/server/server.go:98
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  171885c goji%2eio.(*Mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/mux.go:74
  113ac2a code.justin.tv/common/chitin.(*mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:248
  6e1f72 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2807
  6dd9eb net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1895
623276360 HeapAlloc p=9 g=8102 off=889051 mem=246913136
623299528 HeapAlloc p=9 g=8102 off=889058 mem=246921232
623300488 GoUnblock p=9 g=8102 off=889065 g=258 seq=2970
  4080f3 runtime.selectnbsend /usr/local/go/src/runtime/chan.go:625
  abe012 code.justin.tv/amzn/TwitchTelemetry.(*BufferedAggregator).ObserveSample /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/amzn/TwitchTelemetry/buffered_aggregator.go:64
  ac18a6 code.justin.tv/amzn/TwitchTelemetry.(*SampleReporter).ReportDurationSample /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/amzn/TwitchTelemetry/sample_reporter.go:34
  afe3dc code.justin.tv/commerce/splatter.(*TwitchTelemetryStatter).TimingDuration /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/commerce/splatter/twitch_telemetry_statter.go:99
  147d8d8 code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb.(*tenureBenefitsDBImpl).startTimer.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb/tenurebenefitsdb.go:354
  147a225 code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb.(*tenureBenefitsDBImpl).GetTenureBenefit /build/go/src/code.justin.tv/revenue/subscriptions/internal/clients/datastore/tenurebenefitsdb/tenurebenefitsdb.go:126
  16f1f66 code.justin.tv/revenue/subscriptions/internal/app/badges.(*badge).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/app/badges/get_subscriber_badge.go:119
  17678ee code.justin.tv/revenue/subscriptions/internal/api/subs.(*BadgesAPI).GetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/badges_api.go:249
  d0e8bd code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14118
  179615a code.justin.tv/revenue/subscriptions/internal/api/subs.ReqMetaInterceptor.func1.1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:198
  d0ea2e code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func1 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14120
  d0ec03 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf.func2 /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14136
  c5bc17 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadgeProtobuf /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14137
  c5aa38 code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).serveGetSubscriberBadge /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:14003
  c35bfc code.justin.tv/revenue/subscriptions/twirp.(*subscriptionsServer).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/twirp/subscriptions.twirp.go:10573
  17956d6 code.justin.tv/revenue/subscriptions/internal/api/subs.WithTwitchRepository.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/subs/subs_api.go:111
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  1718359 goji%2eio.dispatch.ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/dispatch.go:17
  17b25cd code.justin.tv/revenue/subscriptions/internal/api/server.handlePanic.func1 /build/go/src/code.justin.tv/revenue/subscriptions/internal/api/server/server.go:98
  6debe3 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2012
  171885c goji%2eio.(*Mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/goji.io/mux.go:74
  113ac2a code.justin.tv/common/chitin.(*mux).ServeHTTP /build/go/src/code.justin.tv/revenue/subscriptions/vendor/code.justin.tv/common/chitin/http.go:248
  6e1f72 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2807
  6dd9eb net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1895
`[1:]

	var evs []*trace.Event
	for _, s := range splitEvents(str) {
		ev, err := parseEventString(s)
		if err != nil {
			t.Fatalf("parseEventString; err = %v", err)
		}
		evs = append(evs, ev)
	}

	var regions []*region
	rt := &daxTracker{
		verbose: false,

		flush: func(evs []*trace.Event) {
			regions = append(regions, &region{Kind: "client/dax", Events: evs})
		},
	}
	state := rt.idle
	for _, ev := range evs {
		if state != nil {
			state = state(ev)
		}
	}

	if l := len(regions); l != 1 {
		t.Fatalf("len(regions); %d != 1", l)
	}
	r := regions[0]
	if have, want := r.Kind, "client/dax"; have != want {
		t.Errorf("region.Kind; %q != %q", have, want)
	}
	// The region starts when the driver does a write syscall.
	if have, want := r.Events[0].Ts, int64(622930784); have != want {
		t.Errorf("region start timestamp; %d != %d", have, want)
	}
	// The region ends as soon as the goroutine has any meaningful event other
	// than a DAX read. The end timestamp corresponds to the read syscall, so it
	// won't count processing time in driver (or application) code.
	if have, want := r.Events[len(r.Events)-1].Ts, int64(623238216); have != want {
		t.Errorf("region end timestamp; %d != %d", have, want)
	}

	if t.Failed() {
		for _, ev := range r.Events {
			t.Logf("%s", eventString(ev, false))
		}
	}
}

func splitEvents(s string) []string {
	sc := bufio.NewScanner(strings.NewReader(s))
	var buf strings.Builder
	var evs []string
	flush := func() {
		ev := buf.String()
		buf.Reset()
		if ev != "" {
			evs = append(evs, ev)
		}
	}
	for sc.Scan() {
		if !strings.HasPrefix(sc.Text(), "  ") {
			flush()
		}
		fmt.Fprintf(&buf, "%s\n", sc.Bytes())
	}
	flush()
	return evs
}

func parseEventString(s string) (*trace.Event, error) {
	s = strings.TrimSuffix(s, "\n")
	lines := strings.Split(s, "\n")
	title, stack := lines[0], lines[1:]
	ev, err := parseEventTitle(title)
	if err != nil {
		return nil, err
	}
	for _, line := range stack {
		frame, err := parseEventFrame(line)
		if err != nil {
			return ev, err
		}
		ev.Stk = append(ev.Stk, frame)
	}
	return ev, nil
}

func parseEventTitle(s string) (*trace.Event, error) {
	ev := &trace.Event{}
	parts := strings.SplitN(s, " ", 6)
	if len(parts) < 5 {
		return nil, fmt.Errorf("need at least five parts for event")
	}

	for i, desc := range trace.EventDescriptions {
		if parts[1] == desc.Name {
			ev.Type = uint8(i)
			break
		}
	}

	var err error

	trimPrefix := func(s, prefix string) string {
		if err != nil {
			return ""
		}
		v := strings.TrimPrefix(s, prefix)
		if len(s) != len(prefix)+len(v) {
			err = fmt.Errorf("missing %q prefix: %q", prefix, s)
		}
		return v
	}

	getInt := func(s string, base int, bitSize int) int64 {
		var i int64
		if err == nil {
			i, err = strconv.ParseInt(s, base, bitSize)
		}
		return i
	}

	getUint := func(s string, base int, bitSize int) uint64 {
		var i uint64
		if err == nil {
			i, err = strconv.ParseUint(s, base, bitSize)
		}
		return i
	}

	ev.Ts = getInt(parts[0], 10, 64)
	ev.P = int(getInt(trimPrefix(parts[2], "p="), 10, 0))
	ev.G = getUint(trimPrefix(parts[3], "g="), 10, 64)
	ev.Off = int(getInt(trimPrefix(parts[4], "off="), 10, 0))

	desc := trace.EventDescriptions[ev.Type]
	args := strings.Join(parts[5:], " ")
	for i, k := range desc.Args {
		parts := strings.SplitN(args, " ", 2)
		arg := parts[0]
		args = strings.Join(parts[1:], " ")
		if i <= len(ev.Args) {
			ev.Args[i] = getUint(trimPrefix(arg, k+"="), 10, 64)
		}
	}

	// TODO: string args (for EvUserLog and EvUserTaskCreate)

	return ev, err
}

func parseEventFrame(s string) (*trace.Frame, error) {
	frame := &trace.Frame{}
	parts := strings.SplitN(s, " ", 5)
	if len(parts) != 5 {
		return nil, fmt.Errorf("need five parts per frame")
	}

	if parts[0] != "" || parts[1] != "" {
		return nil, fmt.Errorf("need to start with two spaces")
	}

	pc, err := strconv.ParseInt(parts[2], 16, 64)
	if err != nil {
		return nil, fmt.Errorf("expect hex pc: %w", err)
	}
	frame.PC = uint64(pc)

	frame.Fn = parts[3]

	i := strings.LastIndex(parts[4], ":")
	if i < 0 {
		return nil, fmt.Errorf("no colon in file:line")
	}
	line, err := strconv.ParseInt(parts[4][i+len(":"):], 10, 0)
	if err != nil {
		return nil, fmt.Errorf("line number: %w", err)
	}
	frame.File = parts[4][:i]
	frame.Line = int(line)

	return frame, nil
}
