Version 10 - History - Pipeline template template - Arvados

Pipeline template template » History » Version 10

Bryan Cosca, 05/21/2015 07:48 PM

-Bryan Cosca
+h1. Pipeline template template
 h2. Run-Command Template
-Bryan Cosca
+<pre>
-Bryan Cosca
+  "NAME":{
-Bryan Cosca
+   "script_version":"29009a1c1f8a9653042c5853832881aca4141cf2",
-Bryan Cosca
+   "repository":"arvados",
    "script":"run-command",
    "script_parameters":{
     "input":{
-Bryan Cosca
+     "output_of":"OUTPUT_OF_PREVIOUS_JOB_NAME"
-Bryan Cosca
+    },
     "reference":{
      "dataclass":"Collection",
      "default":"3514b8e5da0e8d109946bc809b20a78a+5698",
      "link_name":"human_g1k_v37 reference data",
-Bryan Cosca
+     "title":"NAME Input Reference genome (FASTA)"
-Bryan Cosca
+    },
     "command":[
      "java",
      "-Xmx60g",
      "-jar",
      "$(dir $(gatk3))/GenomeAnalysisTK.jar",
      "-T",
      "PrintReads",
      "-R",
      "$(glob $(dir $(reference))/*.fasta)",
+     {
       "foreach":"iterator",
       "command":[
        "-I",
        "$(iterator)"
+      ]
      },
      "-BQSR",
      "$(bqsr_table)",
      "-nct",
      "16",
      "-o",
      "$(outputname)"
     ],
     "outputname":{
      "value":{
       "list":"iterator",
       "index":"0",
       "command":"$(basename $(iterator)).bqsrCal.bam"
+     }
     },
     "bqsr_table":{
      "value":{
       "list":"iterator",
       "index":"0",
       "command":"$(dir $(bqsr))/$(basename $(basename $(iterator))).recal_data.table"
+     }
     },
     "input_dir":"$(dir $(input))",
     "task.foreach":[
      "iterator"
     ],
     "iterator":{
      "value":{
       "group":"input_dir",
       "regex":"(.*)\\.realigned.bam"
 Bryan Cosca
-Bryan Cosca
+    },
     "gatk3":{
      "dataclass":"Collection",
      "default":"2e98fdc8e90f4c48a0714b711767c9ce+76",
-Bryan Cosca
+     "link_name":"Genome Analysis Toolkit 3.2-2",
-Bryan Cosca
+     "title":"NAME Input Version of GATK3 jar"
-Bryan Cosca
+    },
     "dbsnp":{
      "dataclass":"Collection",
      "default":"8ac324bfa3dfff1ff81ed34b433869b1+6712",
      "title":"Single Nucleotide Polymorphism database",
-Bryan Cosca
+     "description":"NAME Input DBsnp"
 Bryan Cosca
    },
    "runtime_constraints":{
     "max_tasks_per_node":1,
     "min_nodes":1,
     "docker_image":"bcosc/arv-base-java",
-Bryan Cosca
+    "arvados_sdk_version":"29009a1c1f8a9653042c5853832881aca4141cf2"
-Bryan Cosca
+   },
    "output_name":false
 Bryan Cosca
-Bryan Cosca
+</pre>
 Bryan Cosca
-Bryan Cosca
+h2. Crunch Script Template Template
 <pre>
-Bryan Cosca
+ "NAME":{
    "script_version":"GIT_BRANCH_NAME",
    "repository":"GIT_REPO_NAME",
    "script":"GIT_SCRIPT_NAME",
-Bryan Cosca
+   "script_parameters":{
     "input":{
-Bryan Cosca
+     "output_of":"PREVIOUS_JOB_NAME"
-Bryan Cosca
+    },
     "samtools":{
      "required":true,
      "dataclass":"Collection",
      "default":"0b5dd5ad3fd555dbb9ef81a027b69dec+18147",
-Bryan Cosca
+     "title":"NAME Input Samtools Collection"
-Bryan Cosca
+    },
     "gatk_jar":{
      "required":true,
      "dataclass":"Collection",
-Bryan Cosca
+     "default":"2e98fdc8e90f4c48a0714b711767c9ce+76",
-Bryan Cosca
+     "title":"NAME Input GATK Collection"
-Bryan Cosca
+    },
     "reference":{
      "required":true,
      "dataclass":"Collection",
      "default":"3514b8e5da0e8d109946bc809b20a78a+5698",
-Bryan Cosca
+     "title":"NAME Input Reference Collection"
     },
     "picard":{
      "required":true,
      "dataclass":"Collection",
      "default":"0eaa58017c3689414a9e644a2297df5c+165",
      "title":"NAME Input Picard Collection"
     },
     "bedtools_bin":{
      "required":true,
      "dataclass":"Collection",
      "default":"b2f86c26e05e7a0686e7f39a86d406bf+34584",
      "title":"NAME Input Bedtools Collection"
     },
     "bed_path":{
      "required":true,
      "dataclass":"Collection",
      "default":"a0e2c5057d64bfb9ce2cdefd46f71b27+516",
      "title":"NAME Input Bed Collection"
 Bryan Cosca
    },
    "runtime_constraints":{
     "max_tasks_per_node":1,
     "min_nodes":1,
     "docker_image":"bcosc/arv-base-java",
     "arvados_sdk_version":"749b87143ebb0bdcbe2d49deee9c66f6de9f86dd"
    },
    "output_name":false
   },
 </pre>
-Bryan Cosca
+h2. Crunch Script Template
 <pre>
 #!/usr/bin/env python
 import arvados
 import subprocess
 import os
 import sys
 import re
 from arvados.collection import Collection as coll
 import arvados_tools
 import shutil
 arvados_tools.spawn_new_task_per_file('input','.*realigned.bqsrCal.bam$',if_sequence=0, and_end_task=True)
 this_job = arvados.current_job()
 this_task = arvados.current_task()
 tmpdir = arvados.current_task().tmpdir
 input_1 = this_task['parameters']['input_1']
 input_1_path = os.path.join(arvados.get_job_param_mount("input"),input_1)
 tmp_input_1_path = os.path.join(tmpdir,input_1) # If we need to copy to tmpdir
 shutil.copyfile(input_1_path,tmp_input_1_path) # If we need to copy to tmpdir
 </pre>
 <pre>
 samtools_path = arvados_tools.get_file_path('samtools','^samtools$')
 gatk_path = arvados_tools.get_file_path('gatk_jar','^GenomeAnalysisTK.jar$')
 reference_path = arvados_tools.get_file_path('reference','.*f(ast)?a(.gz)?$')
 dbsnp_path = arvados_tools.get_file_path('dbsnp','^dbsnp.*vcf$')
 </pre>
 <pre>
 gatk_inserttool_args = []
 gatk_inserttool_output_name
 gatk_inserttool_output_path
 gatk_inserttool_pipe = subprocess.check_output()
 </pre>
 <pre>
 samtools_inserttool_args = []
 </pre>
 h2. Script Parameter Template
-Bryan Cosca
+h2. Random stuff
 Bryan Cosca
-Bryan Cosca
+h2. Latest arvados_sdk_version:
 Bryan Cosca
 https://arvados.org/projects/arvados/repository/revisions/master/show/sdk/python
 Bryan Cosca
 h2. Random tools I use
 os.path.join(arvados.get_job_param_mount("param"),name)

Project

General

Profile

Arvados

Pipeline template template » History » Version 10