Arvados

We seem to regularly get containers that get stuck "forever"

An example of slurm job output is:

2018/07/11 08:28:03 crunch-run 1.1.4.20180426193406 started
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T08:28:03.296077299Z crunch-run 1.1.4.20180426193406 started
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T08:28:03.296095210Z Executing container 'eglyx-dz642-4f6gf681uq3dch1'
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T08:28:03.296113417Z Executing on host 'arvados-compute-node-eglyx-053'
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T08:28:03.405392931Z Fetching Docker image from collection 'bd833c802183670d11bc4cbe06f40edd+468'
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T08:28:03.432383071Z Using Docker image id 'sha256:5ee1237c127786973c80d32640cd68e081ff3834a293b0f4d3f5ec0eefb58176'
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T08:28:03.433879333Z Docker image is available
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T08:28:03.434043301Z Running [arv-mount --foreground --allow-other --read-write --crunchstat-interval=10 --file-cache 1073741824 --mount-tmp tmp0 --mount-by-pdh by_id /tmp/crunch-run.eglyx-dz642-4f6gf681uq3dch1.712862961/keep863762844]
slurmstepd: error: *** JOB 1132746 ON arvados-compute-node-eglyx-053 CANCELLED AT 2018-07-11T09:08:19 ***
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T09:08:19.153316911Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:42:24.939121470Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:43:54.014791351Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:44:51.500177244Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:45:43.890095860Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:46:17.222535967Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:46:49.637860320Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:47:27.564609242Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:48:26.901312529Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:48:34.183325068Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:49:06.724882298Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:49:54.701456002Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:50:27.737378267Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:51:35.339572045Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:52:09.483574629Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:53:05.549591237Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:53:29.728284418Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:53:58.925214421Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:54:27.903706853Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:55:06.235889582Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:55:43.955694684Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T11:56:34.579516838Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:10:08.346997502Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:10:51.315560809Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:11:45.037012225Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:12:18.939801538Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:12:47.869753873Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:19:18.656704531Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:20:49.392934526Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:21:49.217415086Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:22:36.043822805Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:23:09.998391793Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:23:44.007051633Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:24:27.482233161Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:25:24.616579136Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:26:28.200938339Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:27:18.599679217Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:28:08.473988653Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:28:43.100730926Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:29:15.584159376Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:30:03.734036119Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:30:51.502861931Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:31:46.567809038Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:32:37.352102250Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:39:40.652544151Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:39:43.926379938Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:41:15.750686813Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:42:11.793027212Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:43:00.920826915Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:43:36.484032705Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:44:07.874078055Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:50:50.224211713Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:52:19.463807508Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:53:24.309297093Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T12:54:26.946381439Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:00:46.099240482Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:00:49.869695246Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:02:20.293866568Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:03:18.387622092Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:04:00.277182503Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:04:42.799266326Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:05:13.132826948Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:06:16.092834042Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:07:24.962125097Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:08:27.377479040Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:14:47.095049137Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:16:04.565321212Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:17:06.640365762Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:17:40.676025943Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:18:10.557260129Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:19:02.566212837Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:20:25.089357197Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:21:19.754482096Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:22:08.349841794Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:22:46.669757437Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:23:29.046631507Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:24:24.436758136Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:25:13.087400465Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:26:02.126270399Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:26:50.614941567Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:27:44.539695825Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:28:32.050822884Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:29:30.022675822Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:30:09.453728647Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:30:54.284866480Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:36:34.103328416Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:39:13.668872468Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:39:56.258838471Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:40:23.358608202Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:40:54.844554552Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:41:46.560857647Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:49:40.806246188Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:50:36.251824003Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:58:06.408017507Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T13:59:16.573997398Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:00:12.099434795Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:00:45.949909099Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:07:20.967630051Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:08:31.631906929Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:09:32.674380521Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:10:06.090435119Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:11:00.247309870Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:11:42.653577360Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:12:46.538573406Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:13:49.722333559Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:15:20.824222038Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:15:39.528282902Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:16:11.432571149Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:24:01.153175190Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:24:35.483547337Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:25:25.576987533Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:25:56.957508292Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:26:29.393457824Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:33:17.816954300Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:34:49.303733443Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:36:29.714811445Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:36:44.477106735Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:37:13.792348979Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:38:33.966177202Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:38:46.539417458Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:40:00.105074954Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:40:58.364604206Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:47:35.210883348Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:48:20.446923428Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:49:10.652229620Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:50:30.458091510Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:50:44.357387740Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:51:29.031605392Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T14:59:16.509619989Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:00:20.557070644Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:01:16.892881031Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:01:44.922986752Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:03:11.590758061Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:03:22.302375459Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:04:32.785610733Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:05:34.864901637Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:06:29.253419656Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:07:08.026829081Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:07:52.681900453Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:08:48.176816431Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:10:15.519448069Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:10:30.780272577Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:11:13.399005744Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:12:14.143842743Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:13:06.027263444Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:13:51.024183716Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:14:29.335006098Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:15:31.844946240Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:16:17.560152361Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:17:09.449492484Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:17:57.395190368Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:18:59.197958397Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:19:39.091408034Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:27:32.885828701Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:28:40.819067890Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:29:33.794598786Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:30:02.574315171Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:30:57.876371980Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:32:04.035840931Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:32:58.761835692Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:39:45.342343971Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:41:37.793425126Z caught signal: terminated
eglyx-dz642-4f6gf681uq3dch1 2018-07-11T15:44:46.455097806Z caught signal: terminated

It is now over 8 hours since the last logged message and the slurm job is still active with idle crunch-run and arv-mount tasks:

root@arvados-compute-node-eglyx-053:~# pstree -ap 29810
slurmstepd,29810
  |-crunch-run,29814 -cgroup-parent-subsystem=memory eglyx-dz642-4f6gf681uq3dch1
  |   |-arv-mount,29838 /usr/bin/arv-mount --foreground --allow-other --read-write --crunchstat-interval=10 --file-cache ...
  |   |   |-{arv-mount},29844
  |   |   |-{arv-mount},29846
  |   |   |-{arv-mount},29847
  |   |   |-{arv-mount},29849
  |   |   |-{arv-mount},29850
  |   |   |-{arv-mount},29851
  |   |   |-{arv-mount},29852
  |   |   |-{arv-mount},29853
  |   |   |-{arv-mount},29854
  |   |   |-{arv-mount},29855
  |   |   |-{arv-mount},29856
  |   |   |-{arv-mount},29857
  |   |   |-{arv-mount},29858
  |   |   |-{arv-mount},29860
  |   |   |-{arv-mount},29861
  |   |   |-{arv-mount},29862
  |   |   |-{arv-mount},29863
  |   |   |-{arv-mount},29864
  |   |   |-{arv-mount},29865
  |   |   |-{arv-mount},29866
  |   |   |-{arv-mount},29867
  |   |   |-{arv-mount},29868
  |   |   |-{arv-mount},29869
  |   |   |-{arv-mount},29870
  |   |   |-{arv-mount},29871
  |   |   |-{arv-mount},29872
  |   |   |-{arv-mount},29874
  |   |   |-{arv-mount},29875
  |   |   |-{arv-mount},29876
  |   |   |-{arv-mount},29877
  |   |   |-{arv-mount},29878
  |   |   |-{arv-mount},29879
  |   |   `-{arv-mount},29880
  |   |-{crunch-run},29821
  |   |-{crunch-run},29822
  |   |-{crunch-run},29823
  |   |-{crunch-run},29827
  |   |-{crunch-run},29835
  |   |-{crunch-run},29836
  |   |-{crunch-run},29837
  |   |-{crunch-run},29882
  |   `-{crunch-run},30044
  |-{slurmstepd},29811
  |-{slurmstepd},29812
  `-{slurmstepd},29813

root@arvados-compute-node-eglyx-053:~# scontrol show job 1132746
JobId=1132746 JobName=eglyx-dz642-4f6gf681uq3dch1
   UserId=root(0) GroupId=root(0)
   Priority=4294799883 Nice=2988 Account=(null) QOS=(null)
   JobState=RUNNING Reason=None Dependency=(null)
   Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0
   RunTime=14:38:02 TimeLimit=UNLIMITED TimeMin=N/A
   SubmitTime=2018-07-11T08:28:03 EligibleTime=2018-07-11T08:28:03
   StartTime=2018-07-11T08:28:03 EndTime=Unknown
   PreemptTime=None SuspendTime=None SecsPreSuspend=0
   Partition=compute AllocNode:Sid=arvados-master-eglyx:26326
   ReqNodeList=(null) ExcNodeList=(null)
   NodeList=arvados-compute-node-eglyx-053
   BatchHost=arvados-compute-node-eglyx-053
   NumNodes=1 NumCPUs=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:*
   TRES=cpu=1,mem=9536,node=1
   Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=*
   MinCPUsNode=1 MinMemoryNode=9536M MinTmpDiskNode=1000M
   Features=(null) Gres=(null) Reservation=(null)
   Shared=1 Contiguous=0 Licenses=(null) Network=(null)
   Command=(null)
   WorkDir=/
   StdErr=//slurm-1132746.out
   StdIn=/dev/null
   StdOut=//slurm-1132746.out
   Power= SICP=0