Arvados

I am following the install guide for a manual production install of Arvados.

When I log into workbench and upload a file it fails and I get the following error in the Keepproxy log:

• Uploading to keep directly using arv-put works fine.

{"RequestID":"req-jo3uga8uno3k1ja9mjzh","level":"info","msg":"request","remoteAddr":"10.211.55.111:39086","reqBytes":8281848,"reqForwardedFor":"","reqHost":"keep.paarv.local:25107","reqMethod":"POST","reqPath":"","reqQuery":"","time":"2019-01-18T04:00:10.962217416Z"} {"RequestID":"req-bjs4nxov884e10apr1mw","level":"info","msg":"request","remoteAddr":"10.211.55.111:39088","reqBytes":8281848,"reqForwardedFor":"","reqHost":"keep.paarv.local:25107","reqMethod":"POST","reqPath":"","reqQuery":"","time":"2019-01-18T04:00:10.968886902Z"} {"level":"info","msg":"10.211.55.111:39088 POST / 500 8281848 2 0 - \u003cnil\u003e","time":"2019-01-18T04:00:10.969009593Z"} {"RequestID":"req-bjs4nxov884e10apr1mw","level":"info","msg":"response","remoteAddr":"10.211.55.111:39088","reqBytes":8281848,"reqForwardedFor":"","reqHost":"keep.paarv.local:25107","reqMethod":"POST","reqPath":"","reqQuery":"","respBytes":0,"respStatus":"OK","respStatusCode":200,"time":"2019-01-18T04:00:10.969049239Z","timeToStatus":-9223372036.854776,"timeTotal":0.000160,"timeWriteBody":9223372036.854776}
2019/01/18 04:00:10 http: panic serving 10.211.55.111:39088: runtime error: invalid memory address or nil pointer dereference
goroutine 286 [running]:
net/http.(*conn).serve.func1(0xc420180320)
/usr/local/go/src/net/http/server.go:1726 +0xd0
panic(0x7a7180, 0xa5ba30)
/usr/local/go/src/runtime/panic.go:502 +0x229
main.(*proxyHandler).Put.func1(0xc420020600, 0xc420177930, 0xc4203b47e0, 0xc420180820, 0xc420177920, 0xc4201779c0, 0xc4201779d0, 0x86de20, 0xc4203ba240)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:484 +0x303
main.(*proxyHandler).Put(0xc42036d380, 0x86de20, 0xc4203ba240, 0xc420020600)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:546 +0x7a3
main.(*proxyHandler).Put-fm(0x86de20, 0xc4203ba240, 0xc420020600)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:330 +0x48
net/http.HandlerFunc.ServeHTTP(0xc420394310, 0x86de20, 0xc4203ba240, 0xc420020600)
/usr/local/go/src/net/http/server.go:1947 +0x44
git.curoverse.com/arvados.git/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc4201de3c0, 0x86de20, 0xc4203ba240, 0xc420020600)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/vendor/github.com/gorilla/mux/mux.go:150 +0xed
git.curoverse.com/arvados.git/sdk/go/httpserver.LogRequests.func1(0x86eb20, 0xc420068000, 0xc420020400)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/sdk/go/httpserver/logger.go:50 +0x877
net/http.HandlerFunc.ServeHTTP(0xc42036df20, 0x86eb20, 0xc420068000, 0xc420020200)
/usr/local/go/src/net/http/server.go:1947 +0x44
git.curoverse.com/arvados.git/sdk/go/httpserver.AddRequestIDs.func1(0x86eb20, 0xc420068000, 0xc420020200)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/sdk/go/httpserver/id_generator.go:57 +0xfe
net/http.HandlerFunc.ServeHTTP(0xc420378dc0, 0x86eb20, 0xc420068000, 0xc420020200)
/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc420170750, 0x86eb20, 0xc420068000, 0xc420020200)
/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc420180320, 0x86efa0, 0xc420066040)
/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2795 +0x27b {"level":"info","msg":"10.211.55.111:39086 POST / 500 8281848 2 0 - \u003cnil\u003e","time":"2019-01-18T04:00:10.970136834Z"} {"RequestID":"req-jo3uga8uno3k1ja9mjzh","level":"info","msg":"response","remoteAddr":"10.211.55.111:39086","reqBytes":8281848,"reqForwardedFor":"","reqHost":"keep.paarv.local:25107","reqMethod":"POST","reqPath":"","reqQuery":"","respBytes":0,"respStatus":"OK","respStatusCode":200,"time":"2019-01-18T04:00:10.970176692Z","timeToStatus":-9223372036.854776,"timeTotal":0.007957,"timeWriteBody":9223372036.854776}
2019/01/18 04:00:10 http: panic serving 10.211.55.111:39086: runtime error: invalid memory address or nil pointer dereference
goroutine 285 [running]:
net/http.(*conn).serve.func1(0xc420180500)
/usr/local/go/src/net/http/server.go:1726 +0xd0
panic(0x7a7180, 0xa5ba30)
/usr/local/go/src/runtime/panic.go:502 +0x229
main.(*proxyHandler).Put.func1(0xc420021f00, 0xc420179930, 0xc4203b4e28, 0xc4201805a0, 0xc420179920, 0xc4201799c0, 0xc4201799d0, 0x86de20, 0xc4203bb5c0)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:484 +0x303
main.(*proxyHandler).Put(0xc42036d380, 0x86de20, 0xc4203bb5c0, 0xc420021f00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:546 +0x7a3
main.(*proxyHandler).Put-fm(0x86de20, 0xc4203bb5c0, 0xc420021f00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:330 +0x48
net/http.HandlerFunc.ServeHTTP(0xc420394310, 0x86de20, 0xc4203bb5c0, 0xc420021f00)
/usr/local/go/src/net/http/server.go:1947 +0x44
git.curoverse.com/arvados.git/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc4201de3c0, 0x86de20, 0xc4203bb5c0, 0xc420021f00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/vendor/github.com/gorilla/mux/mux.go:150 +0xed
git.curoverse.com/arvados.git/sdk/go/httpserver.LogRequests.func1(0x86eb20, 0xc420068700, 0xc420021c00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/sdk/go/httpserver/logger.go:50 +0x877
net/http.HandlerFunc.ServeHTTP(0xc42036df20, 0x86eb20, 0xc420068700, 0xc420021b00)
/usr/local/go/src/net/http/server.go:1947 +0x44
git.curoverse.com/arvados.git/sdk/go/httpserver.AddRequestIDs.func1(0x86eb20, 0xc420068700, 0xc420021b00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/sdk/go/httpserver/id_generator.go:57 +0xfe
net/http.HandlerFunc.ServeHTTP(0xc420378dc0, 0x86eb20, 0xc420068700, 0xc420021b00)
/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc420170750, 0x86eb20, 0xc420068700, 0xc420021b00)
/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc420180500, 0x86efa0, 0xc420066dc0)
/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2795 +0x27b {"RequestID":"req-1w8w2mbzm27babkjnvzu","level":"info","msg":"request","remoteAddr":"10.211.55.111:39090","reqBytes":8281848,"reqForwardedFor":"","reqHost":"keep.paarv.local:25107","reqMethod":"POST","reqPath":"","reqQuery":"","time":"2019-01-18T04:00:10.975552253Z"} {"level":"info","msg":"10.211.55.111:39090 POST / 500 8281848 2 0 - \u003cnil\u003e","time":"2019-01-18T04:00:10.977102187Z"} {"RequestID":"req-1w8w2mbzm27babkjnvzu","level":"info","msg":"response","remoteAddr":"10.211.55.111:39090","reqBytes":8281848,"reqForwardedFor":"","reqHost":"keep.paarv.local:25107","reqMethod":"POST","reqPath":"","reqQuery":"","respBytes":0,"respStatus":"OK","respStatusCode":200,"time":"2019-01-18T04:00:10.977162032Z","timeToStatus":-9223372036.854776,"timeTotal":0.001604,"timeWriteBody":9223372036.854776}
2019/01/18 04:00:10 http: panic serving 10.211.55.111:39090: runtime error: invalid memory address or nil pointer dereference
goroutine 289 [running]:
net/http.(*conn).serve.func1(0xc420180aa0)
/usr/local/go/src/net/http/server.go:1726 +0xd0
panic(0x7a7180, 0xa5ba30)
/usr/local/go/src/runtime/panic.go:502 +0x229
main.(*proxyHandler).Put.func1(0xc420020a00, 0xc420179930, 0xc4203b4aa8, 0xc420180be0, 0xc420179920, 0xc4201799c0, 0xc4201799d0, 0x86de20, 0xc4203baa20)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:484 +0x303
main.(*proxyHandler).Put(0xc42036d380, 0x86de20, 0xc4203baa20, 0xc420020a00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:546 +0x7a3
main.(*proxyHandler).Put-fm(0x86de20, 0xc4203baa20, 0xc420020a00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/services/keepproxy/keepproxy.go:330 +0x48
net/http.HandlerFunc.ServeHTTP(0xc420394310, 0x86de20, 0xc4203baa20, 0xc420020a00)
/usr/local/go/src/net/http/server.go:1947 +0x44
git.curoverse.com/arvados.git/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc4201de3c0, 0x86de20, 0xc4203baa20, 0xc420020a00)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/vendor/github.com/gorilla/mux/mux.go:150 +0xed
git.curoverse.com/arvados.git/sdk/go/httpserver.LogRequests.func1(0x86eb20, 0xc4200680e0, 0xc420020800)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/sdk/go/httpserver/logger.go:50 +0x877
net/http.HandlerFunc.ServeHTTP(0xc42036df20, 0x86eb20, 0xc4200680e0, 0xc420020700)
/usr/local/go/src/net/http/server.go:1947 +0x44
git.curoverse.com/arvados.git/sdk/go/httpserver.AddRequestIDs.func1(0x86eb20, 0xc4200680e0, 0xc420020700)
/tmp/tmp.xqAdEvnpDj/src/git.curoverse.com/arvados.git/sdk/go/httpserver/id_generator.go:57 +0xfe
net/http.HandlerFunc.ServeHTTP(0xc420378dc0, 0x86eb20, 0xc4200680e0, 0xc420020700)
/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc420170750, 0x86eb20, 0xc4200680e0, 0xc420020700)
/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc420180aa0, 0x86efa0, 0xc420066280)
/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2795 +0x27b

"runtime error: invalid memory address or nil pointer dereference"

Searching this issue seems to show that this issue was a bug in Keepproxy and was then solved again.

I have tried compiling Keepproxy from master, and using older versions of Keepproxy as well as keep (back to 1.14) but it is still throwing this error. I think this may be a regression as this bug was previously fixed.

Keepproxy.yml => {
"Client": {
"APIHost": "paarv.local",
"AuthToken": "mytoken",
"Insecure": true
},
"Listen": ":25107",
"DisableGet": false,
"DisablePut": false,
"DefaultReplicas": 0,
"Timeout": "15s",
"PIDFile": "",
"Debug": true,
"ManagementToken": ""
}
}

How can I resolve this error?