Bug #20208
Updated by Brett Smith about 1 year ago
I've seen the following tests fail repeatedly. sdk/go/keepclient failing: https://ci.arvados.org/job/developer-run-tests-remainder/3718/console <pre>---------------------------------------------------------------------- FAIL: keepclient_test.go:1087: ServerRequiredSuite.TestPutGetHead keepclient_test.go:1099: c.Check(err, Equals, BlockNotFound) ... obtained *keepclient.ErrNotFound = &keepclient.ErrNotFound{multipleResponseError:keepclient.multipleResponseError{error:(*errors.errorString)(0xc0014aafd0), isTemp:false}} ("HEAD a8383a37e70cc4c384694c32c3face5f failed: [http://localhost:33893/a8383a37e70cc4c384694c32c3face5f: HTTP 404 \"\" http://localhost:36625/a8383a37e70cc4c384694c32c3face5f: HTTP 405 \"\"]") ... expected *keepclient.ErrNotFound = &keepclient.ErrNotFound{multipleResponseError:keepclient.multipleResponseError{error:(*errors.errorString)(0xc00020c5d0), isTemp:false}} ("Block not found") ... Difference: ... multipleResponseError.error.s: "HEAD a8383a37e70cc4c384694c32c3face5f failed: [http://localhost:33893/a8383a37e70cc4c384694c32c3face5f: HTTP 404 \"\" http://localhost:36625/a8383a37e70cc4c384694c32c3face5f: HTTP 405 \"\"]" != "Block not found" Sent SIGTERM to 10052 (/tmp/workspace/developer-run-tests-remainder/tmp/keep0.pid) 2023/03/06 20:27:59 TestUploadToStubKeepServer OOPS: 47 passed, 1 FAILED --- FAIL: Test (9.55s) </pre> services/keep-balance failing: https://ci.arvados.org/job/developer-run-tests-remainder/3720/console <pre>2023/03/06 22:35:36 zzzzz-bi6l4-000000000000000 (keep0.zzzzz.arvadosapi.com:25107, disk): mount zzzzz-ivpuk-000000000000000 reports replication=0, using replication=1 2023/03/06 22:35:36 zzzzz-bi6l4-000000000000001 (keep1.zzzzz.arvadosapi.com:25107, disk): mount zzzzz-ivpuk-100000000000000 reports replication=0, using replication=1 2023/03/06 22:35:36 zzzzz-bi6l4-000000000000002 (keep2.zzzzz.arvadosapi.com:25107, disk): mount zzzzz-ivpuk-200000000000000 reports replication=0, using replication=1 coverage: 72.2% of statements panic: test timed out after 10m0s goroutine 824 [running]: testing.(*M).startAlarm.func1() /usr/src/go/src/testing/testing.go:2029 +0x8e created by time.goFunc /usr/src/go/src/time/sleep.go:176 +0x32 goroutine 1 [chan receive, 9 minutes]: testing.(*T).Run(0xc000102b60, {0xbf35f9?, 0x519e05?}, 0xc42990) /usr/src/go/src/testing/testing.go:1487 +0x37a testing.runTests.func1(0xc000102b60?) /usr/src/go/src/testing/testing.go:1839 +0x6e testing.tRunner(0xc000102b60, 0xc0000d7ca0) /usr/src/go/src/testing/testing.go:1439 +0x102 testing.runTests(0xc00021d400?, {0x1188ce0, 0x1, 0x1}, {0x7f7b684ec5b8?, 0x40?, 0x119cd20?}) /usr/src/go/src/testing/testing.go:1837 +0x457 testing.(*M).Run(0xc00021d400) /usr/src/go/src/testing/testing.go:1719 +0x5d9 main.main() _testmain.go:109 +0x255 goroutine 18 [chan receive, 9 minutes]: gopkg.in/check%2ev1.(*suiteRunner).runTest(...) /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/check.go:813 gopkg.in/check%2ev1.(*suiteRunner).run(0xc000127e80) /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/check.go:618 +0x1be gopkg.in/check%2ev1.Run({0xbb8ce0?, 0xc00010e180?}, 0x0?) /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/run.go:92 +0x2a gopkg.in/check%2ev1.RunAll(0x22?) /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/run.go:84 +0x85 gopkg.in/check%2ev1.TestingT(0xc000102d00) /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/run.go:72 +0x292 git.arvados.org/arvados.git/services/keep-balance.Test(0x408279?) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance_test.go:22 +0x19 testing.tRunner(0xc000102d00, 0xc42990) /usr/src/go/src/testing/testing.go:1439 +0x102 created by testing.(*T).Run /usr/src/go/src/testing/testing.go:1486 +0x35f goroutine 19 [select, 9 minutes]: gopkg.in/check%2ev1.(*resultTracker)._loopRoutine(0xc00016c240) /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/check.go:464 +0x8d created by gopkg.in/check%2ev1.(*resultTracker).start /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/check.go:444 +0x56 goroutine 23 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc0003ba680, {0xd17420, 0xc0003c2a40}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 150 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc00011c750, {0xd17420, 0xc000300180}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 481 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc000383040, {0xd17420, 0xc000358dc0}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 478 [chan receive, 9 minutes]: git.arvados.org/arvados.git/services/keep-balance.(*runSuite).TestRunForever(0xc00010e180, 0x0?) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance_run_test.go:670 +0x6bb reflect.Value.call({0xbb8ce0?, 0xc00010e180?, 0x2a13?}, {0xbf350d, 0x4}, {0xc0001a5f08, 0x1, 0x1?}) /usr/src/go/src/reflect/value.go:556 +0x845 reflect.Value.Call({0xbb8ce0?, 0xc00010e180?, 0xc00038acf0?}, {0xc000663708, 0x1, 0x1}) /usr/src/go/src/reflect/value.go:339 +0xbf gopkg.in/check%2ev1.(*suiteRunner).forkTest.func1(0xc0001be0f0) /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/check.go:775 +0x612 gopkg.in/check%2ev1.(*suiteRunner).forkCall.func1() /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/check.go:669 +0xa2 created by gopkg.in/check%2ev1.(*suiteRunner).forkCall /home/jenkins/tmp/GOPATH/pkg/mod/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/check.go:666 +0x285 goroutine 562 [IO wait, 9 minutes]: internal/poll.runtime_pollWait(0x7f7b38f05708, 0x72) /usr/src/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc000299180?, 0x0?, 0x0) /usr/src/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/src/go/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Accept(0xc000299180) /usr/src/go/src/internal/poll/fd_unix.go:614 +0x22c net.(*netFD).accept(0xc000299180) /usr/src/go/src/net/fd_unix.go:172 +0x35 net.(*TCPListener).accept(0xc000523da0) /usr/src/go/src/net/tcpsock_posix.go:139 +0x28 net.(*TCPListener).Accept(0xc000523da0) /usr/src/go/src/net/tcpsock.go:288 +0x3d net/http.(*Server).Serve(0xc000234380, {0xd16998, 0xc000523da0}) /usr/src/go/src/net/http/server.go:3039 +0x385 net/http/httptest.(*Server).goServe.func1() /usr/src/go/src/net/http/httptest/server.go:308 +0x6a created by net/http/httptest.(*Server).goServe /usr/src/go/src/net/http/httptest/server.go:306 +0x6a goroutine 565 [syscall, 9 minutes]: os/signal.signal_recv() /usr/src/go/src/runtime/sigqueue.go:151 +0x2f os/signal.loop() /usr/src/go/src/os/signal/signal_unix.go:23 +0x19 created by os/signal.Notify.func1.1 /usr/src/go/src/os/signal/signal.go:151 +0x2a goroutine 237 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc0005ee1a0, {0xd17420, 0xc00064d440}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 316 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc00011ca90, {0xd17420, 0xc00012e700}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 302 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc0003e2d00, {0xd17420, 0xc0001e0a40}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 459 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc0003bb1e0, {0xd17420, 0xc0003c2b00}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 352 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc00009f790, {0xd17420, 0xc0003585c0}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 321 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc000382340, {0xd17420, 0xc0001e1640}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 303 [IO wait, 9 minutes]: internal/poll.runtime_pollWait(0x7f7b38f06158, 0x72) /usr/src/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc000185300?, 0x0?, 0x0) /usr/src/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/src/go/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Accept(0xc000185300) /usr/src/go/src/internal/poll/fd_unix.go:614 +0x22c net.(*netFD).accept(0xc000185300) /usr/src/go/src/net/fd_unix.go:172 +0x35 net.(*TCPListener).accept(0xc0002d8468) /usr/src/go/src/net/tcpsock_posix.go:139 +0x28 net.(*TCPListener).Accept(0xc0002d8468) /usr/src/go/src/net/tcpsock.go:288 +0x3d net/http.(*Server).Serve(0xc000234000, {0xd16998, 0xc0002d8468}) /usr/src/go/src/net/http/server.go:3039 +0x385 net/http/httptest.(*Server).goServe.func1() /usr/src/go/src/net/http/httptest/server.go:308 +0x6a created by net/http/httptest.(*Server).goServe /usr/src/go/src/net/http/httptest/server.go:306 +0x6a goroutine 305 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc0005ee820, {0xd17420, 0xc0003c2400}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 338 [IO wait, 9 minutes]: internal/poll.runtime_pollWait(0x7f7b38f05ac8, 0x72) /usr/src/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0001f9980?, 0x0?, 0x0) /usr/src/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/src/go/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Accept(0xc0001f9980) /usr/src/go/src/internal/poll/fd_unix.go:614 +0x22c net.(*netFD).accept(0xc0001f9980) /usr/src/go/src/net/fd_unix.go:172 +0x35 net.(*TCPListener).accept(0xc00057bd88) /usr/src/go/src/net/tcpsock_posix.go:139 +0x28 net.(*TCPListener).Accept(0xc00057bd88) /usr/src/go/src/net/tcpsock.go:288 +0x3d net/http.(*Server).Serve(0xc0002341c0, {0xd16998, 0xc00057bd88}) /usr/src/go/src/net/http/server.go:3039 +0x385 net/http/httptest.(*Server).goServe.func1() /usr/src/go/src/net/http/httptest/server.go:308 +0x6a created by net/http/httptest.(*Server).goServe /usr/src/go/src/net/http/httptest/server.go:306 +0x6a goroutine 343 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc00011d380, {0xd17420, 0xc000309500}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 344 [IO wait, 9 minutes]: internal/poll.runtime_pollWait(0x7f7b38f058e8, 0x72) /usr/src/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0002ae800?, 0x0?, 0x0) /usr/src/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/src/go/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Accept(0xc0002ae800) /usr/src/go/src/internal/poll/fd_unix.go:614 +0x22c net.(*netFD).accept(0xc0002ae800) /usr/src/go/src/net/fd_unix.go:172 +0x35 net.(*TCPListener).accept(0xc00057bdd0) /usr/src/go/src/net/tcpsock_posix.go:139 +0x28 net.(*TCPListener).Accept(0xc00057bdd0) /usr/src/go/src/net/tcpsock.go:288 +0x3d net/http.(*Server).Serve(0xc0002340e0, {0xd16998, 0xc00057bdd0}) /usr/src/go/src/net/http/server.go:3039 +0x385 net/http/httptest.(*Server).goServe.func1() /usr/src/go/src/net/http/httptest/server.go:308 +0x6a created by net/http/httptest.(*Server).goServe /usr/src/go/src/net/http/httptest/server.go:306 +0x6a goroutine 438 [select, 9 minutes]: database/sql.(*DB).connectionOpener(0xc00011cea0, {0xd17420, 0xc00064ca80}) /usr/src/go/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/src/go/src/database/sql/sql.go:794 +0x18d goroutine 563 [chan receive, 9 minutes]: git.arvados.org/arvados.git/services/keep-balance.(*Balancer).commitAsync(0xc0004cc200, 0xbfb124?, {0xbfa48b, 0xf}, 0xc00004d7a0) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1193 +0x25a git.arvados.org/arvados.git/services/keep-balance.(*Balancer).CommitTrash(0xc0004cc200?, {0xd17490?, 0xc0000887e0}, 0xc000383860) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1172 +0xf1 git.arvados.org/arvados.git/services/keep-balance.(*Balancer).Run(0xc0004cc200, {0xd17420, 0xc000032500}, 0xc000383860, 0xc000242800, {0x0, 0x1, 0x1, 0x0, {0x0, ...}, ...}) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:182 +0xb79 git.arvados.org/arvados.git/services/keep-balance.(*Server).runOnce(0xc00021c1e0, {0xd17420, 0xc000032500}) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/server.go:93 +0x205 git.arvados.org/arvados.git/services/keep-balance.(*Server).runForever(0xc00021c1e0, {0xd17420, 0xc000032500}) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/server.go:124 +0x390 git.arvados.org/arvados.git/services/keep-balance.(*runSuite).TestRunForever.func1() /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance_run_test.go:658 +0x2e created by git.arvados.org/arvados.git/services/keep-balance.(*runSuite).TestRunForever /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance_run_test.go:657 +0x631 goroutine 845 [select, 9 minutes]: io.(*pipe).write(0xc0005523c0, {0xc00028c000, 0x5, 0xd35}) /usr/src/go/src/io/pipe.go:86 +0x212 io.(*PipeWriter).Write(0xc000383ee0?, {0xc00028c000?, 0x11ce940?, 0xc000663790?}) /usr/src/go/src/io/pipe.go:165 +0x25 encoding/json.(*Encoder).Encode(0xc000663780, {0xae92c0, 0x11ce940}) /usr/src/go/src/encoding/json/stream.go:231 +0x1f6 git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put.func1() /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:63 +0x93 created by git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:61 +0x232 goroutine 782 [chan receive, 9 minutes]: git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put(0xc00004d340?, {0xd17490, 0xc0000887e0}, 0xc0002c0a00?, {0xbf41ea, 0x5}, {0xae92c0?, 0xc00056a000}) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:77 +0x42a git.arvados.org/arvados.git/services/keep-balance.(*KeepService).CommitTrash(...) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:46 git.arvados.org/arvados.git/services/keep-balance.(*Balancer).CommitTrash.func1(0xc00021c320) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1174 +0x8f git.arvados.org/arvados.git/services/keep-balance.(*Balancer).commitAsync.func1(0xc00021c320) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1185 +0x122 created by git.arvados.org/arvados.git/services/keep-balance.(*Balancer).commitAsync /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1181 +0xac goroutine 843 [chan receive, 9 minutes]: git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put(0xc00004d320?, {0xd17490, 0xc0000887e0}, 0xc0002c6200?, {0xbf41ea, 0x5}, {0xae92c0?, 0xc000364948}) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:77 +0x42a git.arvados.org/arvados.git/services/keep-balance.(*KeepService).CommitTrash(...) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:46 git.arvados.org/arvados.git/services/keep-balance.(*Balancer).CommitTrash.func1(0xc00021d540) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1174 +0x8f git.arvados.org/arvados.git/services/keep-balance.(*Balancer).commitAsync.func1(0xc00021d540) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1185 +0x122 created by git.arvados.org/arvados.git/services/keep-balance.(*Balancer).commitAsync /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1181 +0xac goroutine 851 [select, 9 minutes]: io.(*pipe).write(0xc0005741e0, {0xc0000cc000, 0x7c, 0x594}) /usr/src/go/src/io/pipe.go:86 +0x212 io.(*PipeWriter).Write(0xc000540f28?, {0xc0000cc000?, 0xc00056a000?, 0xa77e73?}) /usr/src/go/src/io/pipe.go:165 +0x25 encoding/json.(*Encoder).Encode(0xc0002bbf80, {0xae92c0, 0xc00056a000}) /usr/src/go/src/encoding/json/stream.go:231 +0x1f6 git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put.func1() /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:63 +0x93 created by git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:61 +0x232 goroutine 780 [chan receive, 9 minutes]: git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put(0x0?, {0xd17490, 0xc0000887e0}, 0xc0002c7c00?, {0xbf41ea, 0x5}, {0xae92c0?, 0x11ce940}) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:77 +0x42a git.arvados.org/arvados.git/services/keep-balance.(*KeepService).CommitTrash(...) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:46 git.arvados.org/arvados.git/services/keep-balance.(*Balancer).CommitTrash.func1(0xc00021d680) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1174 +0x8f git.arvados.org/arvados.git/services/keep-balance.(*Balancer).commitAsync.func1(0xc00021d680) /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1185 +0x122 created by git.arvados.org/arvados.git/services/keep-balance.(*Balancer).commitAsync /tmp/workspace/developer-run-tests-remainder/services/keep-balance/balance.go:1181 +0xac goroutine 844 [select, 9 minutes]: io.(*pipe).write(0xc000552ba0, {0xc0002646c0, 0x7c, 0x10f}) /usr/src/go/src/io/pipe.go:86 +0x212 io.(*PipeWriter).Write(0x44dc9a?, {0xc0002646c0?, 0xc000364948?, 0x477bc0?}) /usr/src/go/src/io/pipe.go:165 +0x25 encoding/json.(*Encoder).Encode(0xc0001a1f80, {0xae92c0, 0xc000364948}) /usr/src/go/src/encoding/json/stream.go:231 +0x1f6 git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put.func1() /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:63 +0x93 created by git.arvados.org/arvados.git/services/keep-balance.(*KeepService).put /tmp/workspace/developer-run-tests-remainder/services/keep-balance/keep_service.go:61 +0x232 FAIL git.arvados.org/arvados.git/services/keep-balance 600.025s FAIL ======= services/keep-balance tests -- FAILED ======= test services/keep-balance -- 602s</pre>