|
1
|
Apr 07 18:51:12 arvados-dispatch-cloud[100746]: {"ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","InstanceTypes":"t3small","PID":100746,"Priority":562948240905439964,"State":"Queued","level":"info","msg":"adding container to queue","time":"2024-04-07T18:51:12.174643188Z"}
|
|
2
|
Apr 07 18:51:12 arvados-dispatch-cloud[100746]: {"ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","InstanceType":"t3small","PID":100746,"level":"info","msg":"creating new instance","time":"2024-04-07T18:51:12.225076799Z"}
|
|
3
|
Apr 07 18:51:13 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","IdleBehavior":"run","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"State":"booting","level":"info","msg":"instance appeared in cloud","time":"2024-04-07T18:51:13.570415050Z"}
|
|
4
|
Apr 07 18:52:44 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","Command":"systemctl is-system-running","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"level":"info","msg":"boot probe succeeded","stderr":"","stdout":"running\n","time":"2024-04-07T18:52:44.517691886Z"}
|
|
5
|
Apr 07 18:52:44 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"cmd":"sudo sh -c 'set -e; dstdir=\"/tmp/arvados-crunch-run/\"; dstfile=\"/tmp/arvados-crunch-run/crunch-run~7c0856a5783c632b27d3ecd83e3442a3\"; mkdir -p \"$dstdir\"; touch \"$dstfile\"; chmod 0755 \"$dstdir\" \"$dstfile\"; cat \u003e\"$dstfile\"'","hash":"7c0856a5783c632b27d3ecd83e3442a3","level":"info","msg":"installing runner binary on worker","path":"/tmp/arvados-crunch-run/crunch-run~7c0856a5783c632b27d3ecd83e3442a3","time":"2024-04-07T18:52:44.678856390Z"}
|
|
6
|
Apr 07 18:52:45 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"ProbeStart":"2024-04-07T18:52:44.50905918Z","level":"info","msg":"instance booted; will try probeRunning","time":"2024-04-07T18:52:45.019848433Z"}
|
|
7
|
Apr 07 18:52:45 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"ProbeStart":"2024-04-07T18:52:44.50905918Z","RunningContainers":0,"State":"idle","level":"info","msg":"probes succeeded, instance is in service","time":"2024-04-07T18:52:45.050440842Z"}
|
|
8
|
Apr 07 18:52:45 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"level":"info","msg":"crunch-run process started","time":"2024-04-07T18:52:45.122491232Z"}
|
|
9
|
Apr 07 19:01:14 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Reason":"priority=0","level":"info","msg":"killing crunch-run process","time":"2024-04-07T19:01:14.831853359Z"}
|
|
10
|
Apr 07 19:01:14 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:14.831953732Z"}
|
|
11
|
Apr 07 19:01:15 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:15.910772764Z"}
|
|
12
|
Apr 07 19:01:19 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:19.832552383Z"}
|
|
13
|
Apr 07 19:01:20 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:20.867850527Z"}
|
|
14
|
Apr 07 19:01:24 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:24.832184857Z"}
|
|
15
|
Apr 07 19:01:25 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:25.869517180Z"}
|
|
16
|
Apr 07 19:01:29 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:29.832123213Z"}
|
|
17
|
Apr 07 19:01:30 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:30.913364995Z"}
|
|
18
|
Apr 07 19:01:34 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:34.832237710Z"}
|
|
19
|
Apr 07 19:01:35 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:35.872298630Z"}
|
|
20
|
Apr 07 19:01:39 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:39.832894519Z"}
|
|
21
|
Apr 07 19:01:40 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:40.870450195Z"}
|
|
22
|
Apr 07 19:01:44 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:44.832991493Z"}
|
|
23
|
Apr 07 19:01:45 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:45.865468178Z"}
|
|
24
|
Apr 07 19:01:49 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:49.832794215Z"}
|
|
25
|
Apr 07 19:01:50 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:50.871029467Z"}
|
|
26
|
Apr 07 19:01:54 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:54.832009163Z"}
|
|
27
|
Apr 07 19:01:55 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:01:55.871477867Z"}
|
|
28
|
Apr 07 19:01:59 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:01:59.832240511Z"}
|
|
29
|
Apr 07 19:02:00 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:00.879173214Z"}
|
|
30
|
Apr 07 19:02:04 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:04.832946440Z"}
|
|
31
|
Apr 07 19:02:05 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:05.871279926Z"}
|
|
32
|
Apr 07 19:02:09 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:09.832898122Z"}
|
|
33
|
Apr 07 19:02:10 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:10.882219283Z"}
|
|
34
|
Apr 07 19:02:14 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:14.832728232Z"}
|
|
35
|
Apr 07 19:02:15 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:15.916779503Z"}
|
|
36
|
Apr 07 19:02:19 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:19.832839914Z"}
|
|
37
|
Apr 07 19:02:20 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:20.880556595Z"}
|
|
38
|
Apr 07 19:02:24 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:24.832186644Z"}
|
|
39
|
Apr 07 19:02:25 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:25.871478118Z"}
|
|
40
|
Apr 07 19:02:29 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:29.832202615Z"}
|
|
41
|
Apr 07 19:02:30 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:30.873354662Z"}
|
|
42
|
Apr 07 19:02:34 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:34.832251069Z"}
|
|
43
|
Apr 07 19:02:35 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:35.871028019Z"}
|
|
44
|
Apr 07 19:02:39 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:39.832554567Z"}
|
|
45
|
Apr 07 19:02:40 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:40.872405158Z"}
|
|
46
|
Apr 07 19:02:44 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:44.833025184Z"}
|
|
47
|
Apr 07 19:02:45 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:45.875575027Z"}
|
|
48
|
Apr 07 19:02:49 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:49.832530807Z"}
|
|
49
|
Apr 07 19:02:50 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:50.870825429Z"}
|
|
50
|
Apr 07 19:02:54 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:54.832957263Z"}
|
|
51
|
Apr 07 19:02:55 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:02:55.877747309Z"}
|
|
52
|
Apr 07 19:02:59 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:02:59.832539818Z"}
|
|
53
|
Apr 07 19:03:00 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:03:00.903079022Z"}
|
|
54
|
Apr 07 19:03:04 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:03:04.832702526Z"}
|
|
55
|
Apr 07 19:03:05 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:03:05.897148523Z"}
|
|
56
|
Apr 07 19:03:09 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"level":"info","msg":"sending signal","time":"2024-04-07T19:03:09.832026102Z"}
|
|
57
|
Apr 07 19:03:10 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"Signal":15,"error":"Process exited with status 1","level":"info","msg":"kill attempt unsuccessful","stderr":"z2a07-dz642-fcdoi6pbsnakqkc: pid 1375: sent signal 15 (terminated) but process is still alive\n","stdout":"","time":"2024-04-07T19:03:10.868837434Z"}
|
|
58
|
Apr 07 19:03:14 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"level":"warning","msg":"unkillable container, draining worker","time":"2024-04-07T19:03:14.832246059Z"}
|
|
59
|
Apr 07 19:03:14 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","IdleBehavior":"drain","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"level":"info","msg":"set idle behavior","time":"2024-04-07T19:03:14.832327881Z"}
|
|
60
|
Apr 07 19:03:14 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","IdleBehavior":"drain","IdleDuration":0.645286,"Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"State":"running","level":"info","msg":"shutdown worker","time":"2024-04-07T19:03:14.832371622Z"}
|
|
61
|
Apr 07 19:03:30 arvados-dispatch-cloud[100746]: {"ClusterID":"z2a07","Instance":"i-0eeffadca1a877551","PID":100746,"WorkerState":"shutdown","level":"info","msg":"instance disappeared in cloud","time":"2024-04-07T19:03:30.612856785Z"}
|
|
62
|
Apr 07 19:03:30 arvados-dispatch-cloud[100746]: {"Address":"10.1.1.190","ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","Instance":"i-0eeffadca1a877551","InstanceType":"t3small","PID":100746,"level":"info","msg":"crunch-run process abandoned","time":"2024-04-07T19:03:30.616242379Z"}
|
|
63
|
Apr 07 19:03:30 arvados-dispatch-cloud[100746]: {"ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","PID":100746,"Priority":0,"State":"Locked","level":"info","msg":"requeueing locked container because priority=0","time":"2024-04-07T19:03:30.616397263Z"}
|
|
64
|
Apr 07 19:03:30 arvados-dispatch-cloud[100746]: {"ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","PID":100746,"Priority":0,"State":"Queued","level":"info","msg":"container on hold -- dropping from queue","time":"2024-04-07T19:03:30.660808031Z"}
|
|
65
|
Apr 07 19:03:30 arvados-dispatch-cloud[100746]: {"ClusterID":"z2a07","ContainerUUID":"z2a07-dz642-fcdoi6pbsnakqkc","PID":100746,"State":"Queued","level":"info","msg":"dropping container from queue","time":"2024-04-07T19:03:30.660863012Z"}
|