Arvados

#!/usr/bin/env python

from __future__ import print_function

import os

import sys

import arvados

implementation = arvados.getjobparam('implementation')

if implementation == 'shutil':

    print("Using shutil copy implementation", file=sys.stderr)

    import shutil

    def mkdir(path):

        if not os.path.isdir(path):

            os.mkdir(path)

    copyfile = shutil.copyfile

else:

    print("Using builtin copy implementation", file=sys.stderr)

    import errno

    def mkdir(path):

        try:

            os.mkdir(out_dir)

        except OSError as error:

            if error.errno != errno.EEXIST:

                raise

    def copyfile(in_path, out_path):

        with open(in_path) as in_file, open(out_path, 'w') as out_file:

            for file_data in iter(lambda: in_file.read(1024), ''):

                out_file.write(file_data)

task = arvados.current_task()

out_root = task.tmpdir

os.chdir(os.environ['CRUNCH_SRC'])

for subdir, _, files in os.walk('.'):

    print("Traversing {!r}".format(subdir), file=sys.stderr)

    out_dir = os.path.join(out_root, subdir)

    mkdir(out_dir)

    for basename in files:

        print("Writing {!r}".format(basename), file=sys.stderr)

        copyfile(os.path.join(subdir, basename), os.path.join(out_dir, basename))

task.set_output('')