|
1
|
Sep 05 21:06:44 arv-demo arvados-server[18694]: {"PID":18694,"level":"info","msg":"starting","task":"dispatch-cloud","time":"2022-09-05T21:06:44.584131956Z"}
|
|
2
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: {"PID":18694,"command":"[/var/lib/arvados/bin/arvados-server dispatch-cloud -config /var/lib/arvados/wwwtmp/config.yml]","dir":"/var/lib/arvados/tmp","level":"info","msg":"executing","time":"2022-09-05T21:07:00.365163407Z"}
|
|
3
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceSHA256","time":"2022-09-05T21:07:00.563376798Z"}
|
|
4
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceTimestamp","time":"2022-09-05T21:07:00.563543156Z"}
|
|
5
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: {"PID":18694,"level":"info","msg":"waiting","task":"dispatch-cloud","time":"2022-09-05T21:07:00.595675664Z"}
|
|
6
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: {"PID":18694,"level":"info","msg":"ready","task":"dispatch-cloud","time":"2022-09-05T21:07:00.595692231Z"}
|
|
7
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"error","msg":"error updating queue: request failed: https://arva3.di-pentima.com.ar:4440/arvados/v1/api_client_authorizations/current: 502 Bad Gateway","time":"2022-09-05T21:07:00.659025703Z"}
|
|
8
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"info","msg":"waiting 1s before retry","time":"2022-09-05T21:07:00.659068889Z"}
|
|
9
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","N":0,"PID":18918,"level":"info","msg":"loaded initial instance list","time":"2022-09-05T21:07:00.757135488Z"}
|
|
10
|
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Listen":"[::]:9006","PID":18918,"Service":"arvados-dispatch-cloud","URL":"http://0.0.0.0:9006/","Version":"2.1.0-2866-g10440ac12 (go1.17.7)","level":"info","msg":"listening","time":"2022-09-05T21:07:00.757288097Z"}
|
|
11
|
Sep 05 21:07:01 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-46ng47b63nwr18rjcai2","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:01.317922327Z"}
|
|
12
|
Sep 05 21:07:01 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-46ng47b63nwr18rjcai2","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:01.317995442Z","timeToStatus":0.000047,"timeTotal":0.000050,"timeWriteBody":0.000003}
|
|
13
|
Sep 05 21:07:02 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-vqo6lzdxbq7b1i2w3yq9","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:02.315921673Z"}
|
|
14
|
Sep 05 21:07:02 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-vqo6lzdxbq7b1i2w3yq9","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:02.316004985Z","timeToStatus":0.000072,"timeTotal":0.000076,"timeWriteBody":0.000003}
|
|
15
|
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-5hp1k53wysc1l7mkywlz","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:06.320717785Z"}
|
|
16
|
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-5hp1k53wysc1l7mkywlz","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:06.320786291Z","timeToStatus":0.000057,"timeTotal":0.000063,"timeWriteBody":0.000006}
|
|
17
|
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"info","msg":"FixStaleLocks starting.","time":"2022-09-05T21:07:06.876306400Z"}
|
|
18
|
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"info","msg":"FixStaleLocks finished (44.205µs), starting scheduling.","time":"2022-09-05T21:07:06.876351186Z"}
|
|
19
|
Sep 05 21:07:07 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-20uccx8khav1qrzbloem","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:07.326096156Z"}
|
|
20
|
Sep 05 21:07:07 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-20uccx8khav1qrzbloem","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:07.326243837Z","timeToStatus":0.000136,"timeTotal":0.000140,"timeWriteBody":0.000004}
|
|
21
|
Sep 05 21:07:08 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-1pm4pjns58e6b183ce6g","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:08.315649572Z"}
|
|
22
|
Sep 05 21:07:08 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-1pm4pjns58e6b183ce6g","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:08.315734964Z","timeToStatus":0.000078,"timeTotal":0.000081,"timeWriteBody":0.000004}
|
|
23
|
Sep 05 21:08:16 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","InstanceType":"localhost","PID":18918,"Priority":562948291009222799,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:08:16.951020182Z"}
|
|
24
|
Sep 05 21:08:17 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:08:17.031358511Z"}
|
|
25
|
Sep 05 21:08:17 arv-demo arvados-server[18694]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","IdleBehavior":"run","Instance":"localhost","InstanceType":"localhost","PID":18918,"State":"booting","level":"info","msg":"instance appeared in cloud","time":"2022-09-05T21:08:17.119222371Z"}
|
|
26
|
Sep 05 21:15:26 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","PID":18918,"Priority":0,"State":"Locked","level":"info","msg":"requeueing locked container because priority=0","time":"2022-09-05T21:15:26.937755675Z"}
|
|
27
|
Sep 05 21:15:27 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","PID":18918,"Priority":0,"State":"Queued","level":"info","msg":"container on hold -- dropping from queue","time":"2022-09-05T21:15:27.013736444Z"}
|
|
28
|
Sep 05 21:15:27 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","PID":18918,"State":"Queued","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:15:27.013789633Z"}
|
|
29
|
Sep 05 21:15:46 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"Priority":562948291008766280,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:15:46.943927315Z"}
|
|
30
|
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":603671548445,"Instance":"localhost","InstanceType":"localhost","PID":18918,"Since":"2022-09-05T21:08:17.119435471Z","State":"booting","level":"warning","msg":"new instance unresponsive, shutting down","time":"2022-09-05T21:18:20.798336720Z"}
|
|
31
|
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":603671548445,"Instance":"localhost","InstanceType":"localhost","PID":18918,"ProbeStart":"2022-09-05T21:18:20.790983922Z","level":"info","msg":"boot failed","stderr":"","time":"2022-09-05T21:18:20.798447965Z"}
|
|
32
|
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:18:20.798513056Z"}
|
|
33
|
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:18:20.928607177Z"}
|
|
34
|
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:18:20.978498952Z"}
|
|
35
|
Sep 05 21:19:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":18918,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:19:00.765191191Z"}
|
|
36
|
Sep 05 21:19:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:19:21.063489911Z"}
|
|
37
|
Sep 05 21:19:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:19:21.084653683Z"}
|
|
38
|
Sep 05 21:19:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:19:21.084750987Z"}
|
|
39
|
Sep 05 21:20:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":18918,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:20:00.766247298Z"}
|
|
40
|
Sep 05 21:20:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:20:21.149823831Z"}
|
|
41
|
Sep 05 21:20:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:20:21.192124593Z"}
|
|
42
|
Sep 05 21:20:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:20:21.192245708Z"}
|
|
43
|
Sep 05 21:21:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":18918,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:21:00.766925995Z"}
|
|
44
|
Sep 05 21:21:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:21:21.259514320Z"}
|
|
45
|
Sep 05 21:21:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:21:21.308168867Z"}
|
|
46
|
Sep 05 21:21:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:21:21.308354095Z"}
|
|
47
|
Sep 05 21:21:46 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","PID":18918,"Priority":0,"State":"Queued","level":"info","msg":"container on hold -- dropping from queue","time":"2022-09-05T21:21:46.957838156Z"}
|
|
48
|
Sep 05 21:21:46 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","PID":18918,"State":"Queued","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:21:46.957877222Z"}
|
|
49
|
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"level":"info","msg":"starting","task":"dispatch-cloud","time":"2022-09-05T21:21:58.811163032Z"}
|
|
50
|
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"command":"[/var/lib/arvados/bin/arvados-server dispatch-cloud -config /var/lib/arvados/wwwtmp/config.yml]","dir":"/var/lib/arvados/tmp","level":"info","msg":"executing","time":"2022-09-05T21:21:58.890516378Z"}
|
|
51
|
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"level":"info","msg":"waiting","task":"dispatch-cloud","time":"2022-09-05T21:21:58.952617406Z"}
|
|
52
|
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"level":"info","msg":"ready","task":"dispatch-cloud","time":"2022-09-05T21:21:58.952662731Z"}
|
|
53
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceTimestamp","time":"2022-09-05T21:21:59.071738355Z"}
|
|
54
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceSHA256","time":"2022-09-05T21:21:59.071993162Z"}
|
|
55
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"error","msg":"error updating queue: request failed: https://arva3.di-pentima.com.ar:4440/arvados/v1/api_client_authorizations/current: 500 Internal Server Error: Get \"//railsapi.internal/arvados/v1/api_client_authorizations/current\": dial tcp 0.0.0.0:9001: connect: connection refused","time":"2022-09-05T21:21:59.163531971Z"}
|
|
56
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"info","msg":"waiting 1s before retry","time":"2022-09-05T21:21:59.163588666Z"}
|
|
57
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","N":0,"PID":22503,"level":"info","msg":"loaded initial instance list","time":"2022-09-05T21:21:59.229689776Z"}
|
|
58
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","Listen":"[::]:9006","PID":22503,"Service":"arvados-dispatch-cloud","URL":"http://0.0.0.0:9006/","Version":"2.1.0-2866-g10440ac12 (go1.17.7)","level":"info","msg":"listening","time":"2022-09-05T21:21:59.231077905Z"}
|
|
59
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-jwgqm4myr8pnqx1aek5a","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:21:59.548510077Z"}
|
|
60
|
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-jwgqm4myr8pnqx1aek5a","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:21:59.548622979Z","timeToStatus":0.000103,"timeTotal":0.000105,"timeWriteBody":0.000002}
|
|
61
|
Sep 05 21:22:00 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1ptvtvsf62jxm1mb884q","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:00.541776580Z"}
|
|
62
|
Sep 05 21:22:00 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1ptvtvsf62jxm1mb884q","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:00.541851545Z","timeToStatus":0.000066,"timeTotal":0.000069,"timeWriteBody":0.000003}
|
|
63
|
Sep 05 21:22:04 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1l4mvzi3pyj7p6o76f8x","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:04.549349097Z"}
|
|
64
|
Sep 05 21:22:04 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1l4mvzi3pyj7p6o76f8x","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:04.549680702Z","timeToStatus":0.000086,"timeTotal":0.000089,"timeWriteBody":0.000003}
|
|
65
|
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"info","msg":"FixStaleLocks starting.","time":"2022-09-05T21:22:05.114120250Z"}
|
|
66
|
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"info","msg":"FixStaleLocks finished (65.77µs), starting scheduling.","time":"2022-09-05T21:22:05.114183077Z"}
|
|
67
|
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1sanbqph43nbw7njaboq","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:05.542217118Z"}
|
|
68
|
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1sanbqph43nbw7njaboq","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:05.542270689Z","timeToStatus":0.000046,"timeTotal":0.000048,"timeWriteBody":0.000002}
|
|
69
|
Sep 05 21:22:06 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1jp2mdrbs7wwc1xta91v","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:06.543357858Z"}
|
|
70
|
Sep 05 21:22:06 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1jp2mdrbs7wwc1xta91v","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:06.543417908Z","timeToStatus":0.000057,"timeTotal":0.000059,"timeWriteBody":0.000002}
|
|
71
|
Sep 05 21:25:49 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"starting","task":"dispatch-cloud","time":"2022-09-05T21:25:49.074531616Z"}
|
|
72
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"command":"[/var/lib/arvados/bin/arvados-server dispatch-cloud -config /var/lib/arvados/wwwtmp/config.yml]","dir":"/var/lib/arvados/tmp","level":"info","msg":"executing","time":"2022-09-05T21:25:58.565875498Z"}
|
|
73
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceSHA256","time":"2022-09-05T21:25:58.698955888Z"}
|
|
74
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceTimestamp","time":"2022-09-05T21:25:58.699610849Z"}
|
|
75
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"error","msg":"error updating queue: request failed: https://arva3.di-pentima.com.ar:4440/arvados/v1/api_client_authorizations/current: 502 Bad Gateway","time":"2022-09-05T21:25:58.768320924Z"}
|
|
76
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"info","msg":"waiting 1s before retry","time":"2022-09-05T21:25:58.768363871Z"}
|
|
77
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"waiting","task":"dispatch-cloud","time":"2022-09-05T21:25:58.775825031Z"}
|
|
78
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"ready","task":"dispatch-cloud","time":"2022-09-05T21:25:58.775980480Z"}
|
|
79
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"waiting","targets":"keepproxy+http://0.0.0.0:9007/_health/ping arvados-api-server+http://0.0.0.0:9001/_health/ping arvados-dispatch-cloud+http://0.0.0.0:9006/_health/ping","time":"2022-09-05T21:25:58.844972005Z"}
|
|
80
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","N":0,"PID":23463,"level":"info","msg":"loaded initial instance list","time":"2022-09-05T21:25:58.867033756Z"}
|
|
81
|
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Listen":"[::]:9006","PID":23463,"Service":"arvados-dispatch-cloud","URL":"http://0.0.0.0:9006/","Version":"2.1.0-2866-g10440ac12 (go1.17.7)","level":"info","msg":"listening","time":"2022-09-05T21:25:58.868291411Z"}
|
|
82
|
Sep 05 21:25:59 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-teg395swgg421xh5wox3","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:25:59.811110787Z"}
|
|
83
|
Sep 05 21:25:59 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-teg395swgg421xh5wox3","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:25:59.811194259Z","timeToStatus":0.000075,"timeTotal":0.000077,"timeWriteBody":0.000002}
|
|
84
|
Sep 05 21:26:03 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-1xrw7ponex0pl1jaussq","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:26:03.813513266Z"}
|
|
85
|
Sep 05 21:26:03 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-1xrw7ponex0pl1jaussq","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:26:03.813604953Z","timeToStatus":0.000065,"timeTotal":0.000069,"timeWriteBody":0.000003}
|
|
86
|
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"info","msg":"FixStaleLocks starting.","time":"2022-09-05T21:26:04.701494864Z"}
|
|
87
|
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"info","msg":"FixStaleLocks finished (40.255µs), starting scheduling.","time":"2022-09-05T21:26:04.701532766Z"}
|
|
88
|
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-12vzub46fxf8v1xb0frg","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:26:04.809979602Z"}
|
|
89
|
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-12vzub46fxf8v1xb0frg","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:26:04.810084360Z","timeToStatus":0.000096,"timeTotal":0.000099,"timeWriteBody":0.000004}
|
|
90
|
Sep 05 21:26:05 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-yet3fxp6uuk91vujq2l9","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:26:05.810003244Z"}
|
|
91
|
Sep 05 21:26:05 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-yet3fxp6uuk91vujq2l9","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:26:05.810089943Z","timeToStatus":0.000060,"timeTotal":0.000063,"timeWriteBody":0.000003}
|
|
92
|
Sep 05 21:27:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"Priority":562948291008091902,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:27:04.765740214Z"}
|
|
93
|
Sep 05 21:27:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:27:04.835338227Z"}
|
|
94
|
Sep 05 21:27:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","IdleBehavior":"run","Instance":"localhost","InstanceType":"localhost","PID":23463,"State":"booting","level":"info","msg":"instance appeared in cloud","time":"2022-09-05T21:27:04.858262134Z"}
|
|
95
|
Sep 05 21:37:08 arv-demo arvados-server[23258]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":604112469667,"Instance":"localhost","InstanceType":"localhost","PID":23463,"Since":"2022-09-05T21:27:04.858354147Z","State":"booting","level":"warning","msg":"new instance unresponsive, shutting down","time":"2022-09-05T21:37:08.977753431Z"}
|
|
96
|
Sep 05 21:37:08 arv-demo arvados-server[23258]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":604112469667,"Instance":"localhost","InstanceType":"localhost","PID":23463,"ProbeStart":"2022-09-05T21:37:08.970823821Z","level":"info","msg":"boot failed","stderr":"","time":"2022-09-05T21:37:08.977850147Z"}
|
|
97
|
Sep 05 21:37:08 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:37:08.977937998Z"}
|
|
98
|
Sep 05 21:37:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:37:09.040512748Z"}
|
|
99
|
Sep 05 21:37:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:37:09.113587800Z"}
|
|
100
|
Sep 05 21:37:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:37:58.873608854Z"}
|
|
101
|
Sep 05 21:38:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:38:09.186182290Z"}
|
|
102
|
Sep 05 21:38:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:38:09.220550715Z"}
|
|
103
|
Sep 05 21:38:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:38:09.220672443Z"}
|
|
104
|
Sep 05 21:38:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:38:58.874888645Z"}
|
|
105
|
Sep 05 21:39:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:39:09.292512013Z"}
|
|
106
|
Sep 05 21:39:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:39:09.317574018Z"}
|
|
107
|
Sep 05 21:39:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:39:09.317708137Z"}
|
|
108
|
Sep 05 21:39:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:39:58.876026877Z"}
|
|
109
|
Sep 05 21:40:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:40:09.396941284Z"}
|
|
110
|
Sep 05 21:40:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:40:09.429551715Z"}
|
|
111
|
Sep 05 21:40:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:40:09.429703846Z"}
|
|
112
|
Sep 05 21:40:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:40:58.876880506Z"}
|
|
113
|
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:41:09.514528381Z"}
|
|
114
|
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:41:09.583702273Z"}
|
|
115
|
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:41:09.612859836Z"}
|
|
116
|
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","PID":23463,"State":"Cancelled","level":"info","msg":"container finished -- dropping from queue","time":"2022-09-05T21:41:09.852550279Z"}
|
|
117
|
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","PID":23463,"State":"Cancelled","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:41:09.852598345Z"}
|
|
118
|
Sep 05 21:41:14 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"Priority":562948291008091902,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:41:14.788757552Z"}
|
|
119
|
Sep 05 21:41:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:41:58.877483495Z"}
|
|
120
|
Sep 05 21:42:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:42:09.701242569Z"}
|
|
121
|
Sep 05 21:42:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:42:09.743228835Z"}
|
|
122
|
Sep 05 21:42:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:42:09.743342744Z"}
|
|
123
|
Sep 05 21:42:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:42:58.878066903Z"}
|
|
124
|
Sep 05 21:43:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:43:09.829092644Z"}
|
|
125
|
Sep 05 21:43:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:43:09.890649505Z"}
|
|
126
|
Sep 05 21:43:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:43:09.913470801Z"}
|
|
127
|
Sep 05 21:43:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:43:58.879069982Z"}
|
|
128
|
Sep 05 21:44:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:44:09.982321967Z"}
|
|
129
|
Sep 05 21:44:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:44:10.011331099Z"}
|
|
130
|
Sep 05 21:44:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:44:10.013689029Z"}
|
|
131
|
Sep 05 21:44:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:44:58.880127720Z"}
|
|
132
|
Sep 05 21:45:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:45:10.098063737Z"}
|
|
133
|
Sep 05 21:45:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:45:10.155721661Z"}
|
|
134
|
Sep 05 21:45:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:45:10.155828278Z"}
|
|
135
|
Sep 05 21:45:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:45:58.880752526Z"}
|
|
136
|
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:46:10.243968458Z"}
|
|
137
|
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:46:10.305993217Z"}
|
|
138
|
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:46:10.313527865Z"}
|
|
139
|
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","PID":23463,"State":"Cancelled","level":"info","msg":"container finished -- dropping from queue","time":"2022-09-05T21:46:10.537127534Z"}
|
|
140
|
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","PID":23463,"State":"Cancelled","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:46:10.537182983Z"}
|
|
141
|
Sep 05 21:46:14 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"Priority":562948291008091902,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:46:14.801358192Z"}
|
|
142
|
Sep 05 21:46:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:46:58.881494427Z"}
|
|
143
|
Sep 05 21:47:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:47:10.391469080Z"}
|
|
144
|
Sep 05 21:47:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:47:10.452296709Z"}
|
|
145
|
Sep 05 21:47:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:47:10.452463191Z"}
|
|
146
|
Sep 05 21:47:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:47:58.882116779Z"}
|
|
147
|
Sep 05 21:48:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:48:10.523595630Z"}
|
|
148
|
Sep 05 21:48:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:48:10.564103461Z"}
|
|
149
|
Sep 05 21:48:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:48:10.612886689Z"}
|
|
150
|
Sep 05 21:48:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:48:58.883219014Z"}
|
|
151
|
Sep 05 21:49:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:49:10.683777674Z"}
|
|
152
|
Sep 05 21:49:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:49:10.719801134Z"}
|
|
153
|
Sep 05 21:49:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:49:10.719924690Z"}
|
|
154
|
Sep 05 21:49:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:49:58.884665391Z"}
|
|
155
|
Sep 05 21:50:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:50:10.794297782Z"}
|
|
156
|
Sep 05 21:50:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:50:10.830300543Z"}
|
|
157
|
Sep 05 21:50:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:50:10.830430799Z"}
|
|
158
|
Sep 05 21:50:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:50:58.885736286Z"}
|
|
159
|
Sep 05 21:51:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:51:10.906573704Z"}
|
|
160
|
Sep 05 21:51:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:51:10.934605924Z"}
|
|
161
|
Sep 05 21:51:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:51:10.934717651Z"}
|
|
162
|
Sep 05 21:51:11 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","PID":23463,"State":"Cancelled","level":"info","msg":"container finished -- dropping from queue","time":"2022-09-05T21:51:11.080144717Z"}
|
|
163
|
Sep 05 21:51:11 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","PID":23463,"State":"Cancelled","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:51:11.080231263Z"}
|
|
164
|
Sep 05 21:51:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:51:58.886024352Z"}
|
|
165
|
Sep 05 21:52:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"WorkerState":"shutdown","level":"info","msg":"instance disappeared in cloud","time":"2022-09-05T21:52:58.887131300Z"}
|