<commit_before>import numpy as np
import os
import unittest
from nose.tools import assert_equals, assert_true

from test_utils import PySparkTestCaseWithOutputDir
from thunder import ThunderContext
from thunder.utils.common import pil_to_array

_have_image = False
try:
    from PIL import Image
    _have_image = True
except ImportError:
    # PIL not available; skip tests that require it
    Image = None

class TestContextLoading(PySparkTestCaseWithOutputDir):
    def setUp(self):
        super(TestContextLoading, self).setUp()
        self.tsc = ThunderContext(self.sc)

    @staticmethod
    def _findTestResourcesDir(resourcesdirname="resources"):
        testdirpath = os.path.dirname(os.path.realpath(__file__))
        testresourcesdirpath = os.path.join(testdirpath, resourcesdirname)
        if not os.path.isdir(testresourcesdirpath):
            raise IOError("Test resources directory "+testresourcesdirpath+" not found")
        return testresourcesdirpath

    def test_loadStacksAsSeriesNoShuffle(self):
        rangeary = np.arange(64*128, dtype=np.dtype('int16'))
        rangeary.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary.stack")
        rangeary.tofile(filepath)

        range_series_noshuffle = self.tsc.loadImagesAsSeries(filepath, dims=(128, 64))
        range_series_noshuffle_ary = range_series_noshuffle.pack()

        assert_equals((128, 64), range_series_noshuffle.dims.count)
        assert_equals((64, 128), range_series_noshuffle_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_noshuffle_ary))

    def test_load3dStackAsSeriesNoShuffle(self):
        rangeary = np.arange(32*64*4, dtype=np.dtype('int16'))
        rangeary.shape = (4, 64, 32)
        filepath = os.path.join(self.outputdir, "rangeary.stack")
        rangeary.tofile(filepath)

        range_series_noshuffle = self.tsc.loadImagesAsSeries(filepath, dims=(32, 64, 4))
        range_series_noshuffle_ary = range_series_noshuffle.pack()

        assert_equals((32, 64, 4), range_series_noshuffle.dims.count)
        assert_equals((4, 64, 32), range_series_noshuffle_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_noshuffle_ary))

    def test_loadStacksAsSeriesWithShuffle(self):
        rangeary = np.arange(64*128, dtype=np.dtype('int16'))
        rangeary.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary.stack")
        rangeary.tofile(filepath)
        range_series_withshuffle = self.tsc.loadImagesAsSeries(filepath, dims=(128, 64), shuffle=True)
        range_series_withshuffle_ary = range_series_withshuffle.pack()

        assert_equals((128, 64), range_series_withshuffle.dims.count)
        assert_equals((64, 128), range_series_withshuffle_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_withshuffle_ary))

    def test_loadMultipleStacksAsSeriesNoShuffle(self):
        rangeary = np.arange(64*128, dtype=np.dtype('int16'))
        rangeary.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary01.stack")
        rangeary.tofile(filepath)
        rangeary2 = np.arange(64*128, 2*64*128, dtype=np.dtype('int16'))
        rangeary2.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary02.stack")
        rangeary2.tofile(filepath)

        range_series_noshuffle = self.tsc.loadImagesAsSeries(self.outputdir, dims=(128, 64))
        range_series_noshuffle_ary = range_series_noshuffle.pack()

        assert_equals((128, 64), range_series_noshuffle.dims.count)
        assert_equals((2, 64, 128), range_series_noshuffle_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_noshuffle_ary[0]))
        assert_true(np.array_equal(rangeary2, range_series_noshuffle_ary[1]))

    def test_loadMultipleStacksAsSeriesWithShuffle(self):
        rangeary = np.arange(64*128, dtype=np.dtype('int16'))
        rangeary.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary01.stack")
        rangeary.tofile(filepath)
        rangeary2 = np.arange(64*128, 2*64*128, dtype=np.dtype('int16'))
        rangeary2.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary02.stack")
        rangeary2.tofile(filepath)

        range_series_shuffle = self.tsc.loadImagesAsSeries(self.outputdir, dims=(128, 64), shuffle=True)
        range_series_shuffle_ary = range_series_shuffle.pack()

        assert_equals((128, 64), range_series_shuffle.dims.count)
        assert_equals((2, 64, 128), range_series_shuffle_ary.shape)

        assert_true(np.array_equal(rangeary, range_series_shuffle_ary[0]))
        assert_true(np.array_equal(rangeary2, range_series_shuffle_ary[1]))

    def __run_loadTifAsSeries(self, shuffle):
        tmpary = np.arange(60*120, dtype=np.dtype('uint16'))
        rangeary = np.mod(tmpary, 255).astype('uint8').reshape((60, 120))
        pilimg = Image.fromarray(rangeary)
        filepath = os.path.join(self.outputdir, "rangetif01.tif")
        pilimg.save(filepath)
        del pilimg, tmpary

        range_series = self.tsc.loadImagesAsSeries(self.outputdir, inputformat="tif-stack", shuffle=shuffle)
        range_series_ary = range_series.pack()

        assert_equals((1, 120, 60), range_series.dims.count)
        assert_equals((60, 120), range_series_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_ary))

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTifAsSeriesNoShuffle(self):
        self.__run_loadTifAsSeries(False)

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTifAsSeriesWithShuffle(self):
        self.__run_loadTifAsSeries(True)

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTestTifAsSeriesNoShuffle(self):
        testresourcesdir = TestContextLoading._findTestResourcesDir()
        imagepath = os.path.join(testresourcesdir, "multilayer_tif", "dotdotdot_lzw.tif")

        testimg_pil = Image.open(imagepath)
        testimg_arys = list()
        testimg_arys.append(pil_to_array(testimg_pil))
        testimg_pil.seek(1)
        testimg_arys.append(pil_to_array(testimg_pil))
        testimg_pil.seek(2)
        testimg_arys.append(pil_to_array(testimg_pil))

        range_series_noshuffle = self.tsc.loadImagesAsSeries(imagepath, inputformat="tif-stack")
        range_series_noshuffle_ary = range_series_noshuffle.pack()

        assert_equals((75, 70, 3), range_series_noshuffle.dims.count)
        #assert_equals((70, 75, 3), range_series_noshuffle_ary.shape)
        assert_equals((3, 70, 75), range_series_noshuffle_ary.shape)
        # assert_true(np.array_equal(testimg_arys[0], range_series_noshuffle_ary[:, :, 0]))
        # assert_true(np.array_equal(testimg_arys[1], range_series_noshuffle_ary[:, :, 1]))
        # assert_true(np.array_equal(testimg_arys[2], range_series_noshuffle_ary[:, :, 2]))
        assert_true(np.array_equal(testimg_arys[0], range_series_noshuffle_ary[0]))
        assert_true(np.array_equal(testimg_arys[1], range_series_noshuffle_ary[1]))
        assert_true(np.array_equal(testimg_arys[2], range_series_noshuffle_ary[2]))

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTestTifAsSeriesWithShuffle(self):
        testresourcesdir = TestContextLoading._findTestResourcesDir()
        imagepath = os.path.join(testresourcesdir, "multilayer_tif", "dotdotdot_lzw.tif")

        testimg_pil = Image.open(imagepath)
        testimg_arys = list()
        testimg_arys.append(pil_to_array(testimg_pil))
        testimg_pil.seek(1)
        testimg_arys.append(pil_to_array(testimg_pil))
        testimg_pil.seek(2)
        testimg_arys.append(pil_to_array(testimg_pil))

        range_series_shuffle = self.tsc.loadImagesAsSeries(imagepath, inputformat="tif-stack", shuffle=True)
        range_series_shuffle_ary = range_series_shuffle.pack()

        assert_equals((3, 75, 70), range_series_shuffle.dims.count)
        assert_equals((70, 75, 3), range_series_shuffle_ary.shape)
        assert_true(np.array_equal(testimg_arys[0], range_series_shuffle_ary[:, :, 0]))
        assert_true(np.array_equal(testimg_arys[1], range_series_shuffle_ary[:, :, 1]))
        assert_true(np.array_equal(testimg_arys[2], range_series_shuffle_ary[:, :, 2]))

    def __run_loadMultipleTifsAsSeries(self, shuffle):
        tmpary = np.arange(60*120, dtype=np.dtype('uint16'))
        rangeary = np.mod(tmpary, 255).astype('uint8').reshape((60, 120))
        pilimg = Image.fromarray(rangeary)
        filepath = os.path.join(self.outputdir, "rangetif01.tif")
        pilimg.save(filepath)

        tmpary = np.arange(60*120, 2*60*120, dtype=np.dtype('uint16'))
        rangeary2 = np.mod(tmpary, 255).astype('uint8').reshape((60, 120))
        pilimg = Image.fromarray(rangeary2)
        filepath = os.path.join(self.outputdir, "rangetif02.tif")
        pilimg.save(filepath)

        del pilimg, tmpary

        range_series = self.tsc.loadImagesAsSeries(self.outputdir, inputformat="tif-stack", shuffle=shuffle)
        range_series_ary = range_series.pack()

        assert_equals((1, 120, 60), range_series.dims.count)
        assert_equals((2, 60, 120), range_series_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_ary[0]))
        assert_true(np.array_equal(rangeary2, range_series_ary[1]))

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadMultipleTifsAsSeriesNoShuffle(self):
        self.__run_loadMultipleTifsAsSeries(False)

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadMultipleTifsAsSeriesWithShuffle(self):
        self.__run_loadMultipleTifsAsSeries(True)<commit_msg>consolidate tests in test_context<commit_after>import numpy as np
import os
import unittest
from nose.tools import assert_equals, assert_true

from test_utils import PySparkTestCaseWithOutputDir
from thunder import ThunderContext
from thunder.utils.common import pil_to_array

_have_image = False
try:
    from PIL import Image
    _have_image = True
except ImportError:
    # PIL not available; skip tests that require it
    Image = None

class TestContextLoading(PySparkTestCaseWithOutputDir):
    def setUp(self):
        super(TestContextLoading, self).setUp()
        self.tsc = ThunderContext(self.sc)

    @staticmethod
    def _findTestResourcesDir(resourcesdirname="resources"):
        testdirpath = os.path.dirname(os.path.realpath(__file__))
        testresourcesdirpath = os.path.join(testdirpath, resourcesdirname)
        if not os.path.isdir(testresourcesdirpath):
            raise IOError("Test resources directory "+testresourcesdirpath+" not found")
        return testresourcesdirpath

    def __run_loadStacksAsSeries(self, shuffle):
        rangeary = np.arange(64*128, dtype=np.dtype('int16'))
        rangeary.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary.stack")
        rangeary.tofile(filepath)

        range_series = self.tsc.loadImagesAsSeries(filepath, dims=(128, 64), shuffle=shuffle)
        range_series_ary = range_series.pack()

        assert_equals((128, 64), range_series.dims.count)
        assert_equals((64, 128), range_series_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_ary))

    def test_loadStacksAsSeriesNoShuffle(self):
        self.__run_loadStacksAsSeries(False)

    def test_loadStacksAsSeriesWithShuffle(self):
        self.__run_loadStacksAsSeries(True)

    def __run_load3dStackAsSeries(self, shuffle):
        rangeary = np.arange(32*64*4, dtype=np.dtype('int16'))
        rangeary.shape = (4, 64, 32)
        filepath = os.path.join(self.outputdir, "rangeary.stack")
        rangeary.tofile(filepath)

        range_series_noshuffle = self.tsc.loadImagesAsSeries(filepath, dims=(32, 64, 4), shuffle=shuffle)
        range_series_noshuffle_ary = range_series_noshuffle.pack()

        assert_equals((32, 64, 4), range_series_noshuffle.dims.count)
        assert_equals((4, 64, 32), range_series_noshuffle_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_noshuffle_ary))

    def test_load3dStackAsSeriesNoShuffle(self):
        self.__run_load3dStackAsSeries(False)

    def test_load3dStackAsSeriesWithShuffle(self):
        self.__run_load3dStackAsSeries(True)

    def __run_loadMultipleStacksAsSeries(self, shuffle):
        rangeary = np.arange(64*128, dtype=np.dtype('int16'))
        rangeary.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary01.stack")
        rangeary.tofile(filepath)
        rangeary2 = np.arange(64*128, 2*64*128, dtype=np.dtype('int16'))
        rangeary2.shape = (64, 128)
        filepath = os.path.join(self.outputdir, "rangeary02.stack")
        rangeary2.tofile(filepath)

        range_series = self.tsc.loadImagesAsSeries(self.outputdir, dims=(128, 64), shuffle=shuffle)
        range_series_ary = range_series.pack()

        assert_equals((128, 64), range_series.dims.count)
        assert_equals((2, 64, 128), range_series_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_ary[0]))
        assert_true(np.array_equal(rangeary2, range_series_ary[1]))

    def test_loadMultipleStacksAsSeriesNoShuffle(self):
        self.__run_loadMultipleStacksAsSeries(False)

    def test_loadMultipleStacksAsSeriesWithShuffle(self):
        self.__run_loadMultipleStacksAsSeries(True)

    def __run_loadTifAsSeries(self, shuffle):
        tmpary = np.arange(60*120, dtype=np.dtype('uint16'))
        rangeary = np.mod(tmpary, 255).astype('uint8').reshape((60, 120))
        pilimg = Image.fromarray(rangeary)
        filepath = os.path.join(self.outputdir, "rangetif01.tif")
        pilimg.save(filepath)
        del pilimg, tmpary

        range_series = self.tsc.loadImagesAsSeries(self.outputdir, inputformat="tif-stack", shuffle=shuffle)
        range_series_ary = range_series.pack()

        assert_equals((1, 120, 60), range_series.dims.count)
        assert_equals((60, 120), range_series_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_ary))

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTifAsSeriesNoShuffle(self):
        self.__run_loadTifAsSeries(False)

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTifAsSeriesWithShuffle(self):
        self.__run_loadTifAsSeries(True)

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTestTifAsSeriesNoShuffle(self):
        testresourcesdir = TestContextLoading._findTestResourcesDir()
        imagepath = os.path.join(testresourcesdir, "multilayer_tif", "dotdotdot_lzw.tif")

        testimg_pil = Image.open(imagepath)
        testimg_arys = list()
        testimg_arys.append(pil_to_array(testimg_pil))  # original shape 70, 75
        testimg_pil.seek(1)
        testimg_arys.append(pil_to_array(testimg_pil))
        testimg_pil.seek(2)
        testimg_arys.append(pil_to_array(testimg_pil))

        range_series_noshuffle = self.tsc.loadImagesAsSeries(imagepath, inputformat="tif-stack")
        range_series_noshuffle_ary = range_series_noshuffle.pack()

        assert_equals((75, 70, 3), range_series_noshuffle.dims.count)
        #assert_equals((70, 75, 3), range_series_noshuffle_ary.shape)
        assert_equals((3, 70, 75), range_series_noshuffle_ary.shape)
        # assert_true(np.array_equal(testimg_arys[0], range_series_noshuffle_ary[:, :, 0]))
        # assert_true(np.array_equal(testimg_arys[1], range_series_noshuffle_ary[:, :, 1]))
        # assert_true(np.array_equal(testimg_arys[2], range_series_noshuffle_ary[:, :, 2]))
        assert_true(np.array_equal(testimg_arys[0], range_series_noshuffle_ary[0]))
        assert_true(np.array_equal(testimg_arys[1], range_series_noshuffle_ary[1]))
        assert_true(np.array_equal(testimg_arys[2], range_series_noshuffle_ary[2]))

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadTestTifAsSeriesWithShuffle(self):
        testresourcesdir = TestContextLoading._findTestResourcesDir()
        imagepath = os.path.join(testresourcesdir, "multilayer_tif", "dotdotdot_lzw.tif")

        testimg_pil = Image.open(imagepath)
        testimg_arys = list()
        testimg_arys.append(pil_to_array(testimg_pil))
        testimg_pil.seek(1)
        testimg_arys.append(pil_to_array(testimg_pil))
        testimg_pil.seek(2)
        testimg_arys.append(pil_to_array(testimg_pil))

        range_series_shuffle = self.tsc.loadImagesAsSeries(imagepath, inputformat="tif-stack", shuffle=True)
        range_series_shuffle_ary = range_series_shuffle.pack()

        assert_equals((3, 75, 70), range_series_shuffle.dims.count)
        assert_equals((70, 75, 3), range_series_shuffle_ary.shape)
        assert_true(np.array_equal(testimg_arys[0], range_series_shuffle_ary[:, :, 0]))
        assert_true(np.array_equal(testimg_arys[1], range_series_shuffle_ary[:, :, 1]))
        assert_true(np.array_equal(testimg_arys[2], range_series_shuffle_ary[:, :, 2]))

    def __run_loadMultipleTifsAsSeries(self, shuffle):
        tmpary = np.arange(60*120, dtype=np.dtype('uint16'))
        rangeary = np.mod(tmpary, 255).astype('uint8').reshape((60, 120))
        pilimg = Image.fromarray(rangeary)
        filepath = os.path.join(self.outputdir, "rangetif01.tif")
        pilimg.save(filepath)

        tmpary = np.arange(60*120, 2*60*120, dtype=np.dtype('uint16'))
        rangeary2 = np.mod(tmpary, 255).astype('uint8').reshape((60, 120))
        pilimg = Image.fromarray(rangeary2)
        filepath = os.path.join(self.outputdir, "rangetif02.tif")
        pilimg.save(filepath)

        del pilimg, tmpary

        range_series = self.tsc.loadImagesAsSeries(self.outputdir, inputformat="tif-stack", shuffle=shuffle)
        range_series_ary = range_series.pack()

        assert_equals((1, 120, 60), range_series.dims.count)
        assert_equals((2, 60, 120), range_series_ary.shape)
        assert_true(np.array_equal(rangeary, range_series_ary[0]))
        assert_true(np.array_equal(rangeary2, range_series_ary[1]))

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadMultipleTifsAsSeriesNoShuffle(self):
        self.__run_loadMultipleTifsAsSeries(False)

    @unittest.skipIf(not _have_image, "PIL/pillow not installed or not functional")
    def test_loadMultipleTifsAsSeriesWithShuffle(self):
        self.__run_loadMultipleTifsAsSeries(True)<|endoftext|>


<commit_before>"""
Wrapper for the Spark EC2 launch script that additionally
installs Thunder and its dependencies, and optionally
loads an example data set
"""

from boto import ec2
import sys
import os
import time
import random
import subprocess
from sys import stderr
from optparse import OptionParser
from spark_ec2 import ssh, launch_cluster, get_existing_cluster, wait_for_cluster, deploy_files, setup_spark_cluster, \
    get_spark_ami, ssh_command, ssh_read, ssh_write, get_or_make_group

def get_s3_keys():
    """ Get user S3 keys from environmental variables"""
    if os.getenv('S3_AWS_ACCESS_KEY_ID') is not None:
        s3_access_key = os.getenv("S3_AWS_ACCESS_KEY_ID")
    else:
        s3_access_key = os.getenv("AWS_ACCESS_KEY_ID")
    if os.getenv('S3_AWS_SECRET_ACCESS_KEY') is not None:
        s3_secret_key = os.getenv("S3_AWS_SECRET_ACCESS_KEY")
    else:
        s3_secret_key = os.getenv("AWS_SECRET_ACCESS_KEY")
    return s3_access_key, s3_secret_key

def install_thunder(master, opts):
    """ Install Thunder and dependencies on a Spark EC2 cluster"""
    print "Installing Thunder on the cluster..."
    # download and build thunder
    ssh(master, opts, "rm -rf thunder && git clone https://github.com/freeman-lab/thunder.git")
    ssh(master, opts, "chmod u+x thunder/python/bin/build")
    ssh(master, opts, "thunder/python/bin/build")
    # install pip
    ssh(master, opts, "wget http://pypi.python.org/packages/source/p/pip/pip-1.1.tar.gz"
                      "#md5=62a9f08dd5dc69d76734568a6c040508")
    ssh(master, opts, "tar -xvf pip*.gz")
    ssh(master, opts, "cd pip* && sudo python setup.py install")
    # install libraries
    ssh(master, opts, "source ~/.bash_profile && pip install mpld3 && pip install seaborn "
                      "&& pip install jinja2 && pip install -U scikit-learn")
    # install ipython 1.1
    ssh(master, opts, "pip uninstall -y ipython")
    ssh(master, opts, "git clone https://github.com/ipython/ipython.git")
    ssh(master, opts, "cd ipython && git checkout tags/rel-1.1.0")
    ssh(master, opts, "cd ipython && sudo python setup.py install")
    # set environmental variables
    ssh(master, opts, "echo 'export SPARK_HOME=/root/spark' >> /root/.bash_profile")
    ssh(master, opts, "echo 'export PYTHONPATH=/root/thunder/python' >> /root/.bash_profile")
    ssh(master, opts, "echo 'export IPYTHON=1' >> /root/.bash_profile")
    ssh(master, opts, "echo 'export PATH=/root/thunder/python/bin:$PATH' >> /root/.bash_profile")
    # customize spark configuration parameters
    ssh(master, opts, "echo 'spark.akka.frameSize=10000' >> /root/spark/conf/spark-defaults.conf")
    ssh(master, opts, "echo 'export SPARK_DRIVER_MEMORY=20g' >> /root/spark/conf/spark-env.sh")
    # add AWS credentials to core-site.xml
    configstring = "<property><name>fs.s3n.awsAccessKeyId</name><value>ACCESS</value></property><property>" \
                   "<name>fs.s3n.awsSecretAccessKey</name><value>SECRET</value></property>"
    access, secret = get_s3_keys()
    filled = configstring.replace('ACCESS', access).replace('SECRET', secret)
    ssh(master, opts, "sed -i'f' 's,.*</configuration>.*,"+filled+"&,' /root/ephemeral-hdfs/conf/core-site.xml")

    print "\n\n"
    print "-------------------------------"
    print "Thunder successfully installed!"
    print "-------------------------------"
    print "\n"

def setup_cluster(conn, master_nodes, slave_nodes, opts, deploy_ssh_key):
    """Modified version of the setup_cluster function (borrowed from spark-ec.py)
    in order to manually set the folder with the deploy code"""
    master = master_nodes[0].public_dns_name
    if deploy_ssh_key:
        print "Generating cluster's SSH key on master..."
        key_setup = """
      [ -f ~/.ssh/id_rsa ] ||
        (ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa &&
         cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys)
        """
        ssh(master, opts, key_setup)
        dot_ssh_tar = ssh_read(master, opts, ['tar', 'c', '.ssh'])
        print "Transferring cluster's SSH key to slaves..."
        for slave in slave_nodes:
            print slave.public_dns_name
            ssh_write(slave.public_dns_name, opts, ['tar', 'x'], dot_ssh_tar)

    modules = ['spark', 'shark', 'ephemeral-hdfs', 'persistent-hdfs',
             'mapreduce', 'spark-standalone', 'tachyon']

    if opts.hadoop_major_version == "1":
        modules = filter(lambda x: x != "mapreduce", modules)

    if opts.ganglia:
        modules.append('ganglia')

    ssh(master, opts, "rm -rf spark-ec2 && git clone https://github.com/mesos/spark-ec2.git -b v3")

    print "Deploying files to master..."
    deploy_folder = os.path.join(os.environ['SPARK_HOME'], "ec2", "deploy.generic")
    deploy_files(conn, deploy_folder, opts, master_nodes, slave_nodes, modules)

    print "Running setup on master..."
    setup_spark_cluster(master, opts)
    print "Done!"

if __name__ == "__main__":
    parser = OptionParser(usage="thunder-ec2 [options] <action> <clustername>",  add_help_option=False)
    parser.add_option("-h", "--help", action="help", help="Show this help message and exit")
    parser.add_option("-k", "--key-pair", help="Key pair to use on instances")
    parser.add_option("-s", "--slaves", type="int", default=1, help="Number of slaves to launch (default: 1)")
    parser.add_option("-i", "--identity-file", help="SSH private key file to use for logging into instances")
    parser.add_option("-r", "--region", default="us-east-1", help="EC2 region zone to launch instances "
                                                                  "in (default: us-east-1)")
    parser.add_option("-t", "--instance-type", default="m3.2xlarge", help="Type of instance to launch (default: m3.2xlarge)."
                                                                        " WARNING: must be 64-bit; small instances "
                                                                        "won't work")
    parser.add_option("-u", "--user", default="root", help="User name for cluster (default: root)")
    parser.add_option("-z", "--zone", default="", help="Availability zone to launch instances in, or 'all' to spread "
                                                       "slaves across multiple (an additional $0.01/Gb for "
                                                       "bandwidth between zones applies)")
    parser.add_option("--resume", default=False, action="store_true", help="Resume installation on a previously "
                                                        "launched cluster (for debugging)")

    (opts, args) = parser.parse_args()
    if len(args) != 2:
        parser.print_help()
        sys.exit(1)
    (action, cluster_name) = args

    # Launch a cluster, setting several options to defaults
    # (use spark-ec2.py included with Spark for more control)
    if action == "launch":
        try:
            conn = ec2.connect_to_region(opts.region)
        except Exception as e:
            print >> stderr, (e)
            sys.exit(1)

        if opts.zone == "":
            opts.zone = random.choice(conn.get_all_zones()).name

        opts.ami = get_spark_ami(opts) #"ami-3ecd0c56"
        opts.ebs_vol_size = 0
        opts.spot_price = None
        opts.master_instance_type = ""
        opts.wait = 160
        opts.hadoop_major_version = "1"
        opts.ganglia = True
        opts.spark_version = "1.0.2"
        opts.swap = 1024
        opts.worker_instances = 1
        opts.master_opts = ""

        if opts.resume:
            (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name)
        else:
            (master_nodes, slave_nodes) = launch_cluster(conn, opts, cluster_name)

        wait_for_cluster(conn, opts.wait, master_nodes, slave_nodes)
        setup_cluster(conn, master_nodes, slave_nodes, opts, True)
        master = master_nodes[0].public_dns_name
        install_thunder(master, opts)
        print "\n\n"
        print "-------------------------------"
        print "Cluster successfully launched!"
        print "Go to http://%s:8080 to see the web UI for your cluster" % master
        print "-------------------------------"
        print "\n"

    if action != "launch":
        conn = ec2.connect_to_region(opts.region)
        (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name)
        master = master_nodes[0].public_dns_name

        # Login to the cluster
        if action == "login":
            print "Logging into master " + master + "..."
            proxy_opt = []
            subprocess.check_call(ssh_command(opts) + proxy_opt + ['-t', '-t', "%s@%s" % (opts.user, master)])

        # Install thunder on the cluster
        elif action == "install":
            install_thunder(master, opts)

        # Destroy the cluster
        elif action == "destroy":
            response = raw_input("Are you sure you want to destroy the cluster " + cluster_name +
                                 "?\nALL DATA ON ALL NODES WILL BE LOST!!\n" +
                                 "Destroy cluster " + cluster_name + " (y/N): ")
            if response == "y":
                (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name, die_on_error=False)
            print "Terminating master..."
            for inst in master_nodes:
                inst.terminate()
            print "Terminating slaves..."
            for inst in slave_nodes:
                inst.terminate()

        else:
            raise NotImplementedError("action: " + action + "not recognized")

<commit_msg>Add requester pays configuration<commit_after>"""
Wrapper for the Spark EC2 launch script that additionally
installs Thunder and its dependencies, and optionally
loads an example data set
"""

from boto import ec2
import sys
import os
import time
import random
import subprocess
from sys import stderr
from optparse import OptionParser
from spark_ec2 import ssh, launch_cluster, get_existing_cluster, wait_for_cluster, deploy_files, setup_spark_cluster, \
    get_spark_ami, ssh_command, ssh_read, ssh_write, get_or_make_group

def get_s3_keys():
    """ Get user S3 keys from environmental variables"""
    if os.getenv('S3_AWS_ACCESS_KEY_ID') is not None:
        s3_access_key = os.getenv("S3_AWS_ACCESS_KEY_ID")
    else:
        s3_access_key = os.getenv("AWS_ACCESS_KEY_ID")
    if os.getenv('S3_AWS_SECRET_ACCESS_KEY') is not None:
        s3_secret_key = os.getenv("S3_AWS_SECRET_ACCESS_KEY")
    else:
        s3_secret_key = os.getenv("AWS_SECRET_ACCESS_KEY")
    return s3_access_key, s3_secret_key

def install_thunder(master, opts):
    """ Install Thunder and dependencies on a Spark EC2 cluster"""
    print "Installing Thunder on the cluster..."
    # download and build thunder
    ssh(master, opts, "rm -rf thunder && git clone https://github.com/freeman-lab/thunder.git")
    ssh(master, opts, "chmod u+x thunder/python/bin/build")
    ssh(master, opts, "thunder/python/bin/build")
    # install pip
    ssh(master, opts, "wget http://pypi.python.org/packages/source/p/pip/pip-1.1.tar.gz"
                      "#md5=62a9f08dd5dc69d76734568a6c040508")
    ssh(master, opts, "tar -xvf pip*.gz")
    ssh(master, opts, "cd pip* && sudo python setup.py install")
    # install libraries
    ssh(master, opts, "source ~/.bash_profile && pip install mpld3 && pip install seaborn "
                      "&& pip install jinja2 && pip install -U scikit-learn")
    # install ipython 1.1
    ssh(master, opts, "pip uninstall -y ipython")
    ssh(master, opts, "git clone https://github.com/ipython/ipython.git")
    ssh(master, opts, "cd ipython && git checkout tags/rel-1.1.0")
    ssh(master, opts, "cd ipython && sudo python setup.py install")
    # set environmental variables
    ssh(master, opts, "echo 'export SPARK_HOME=/root/spark' >> /root/.bash_profile")
    ssh(master, opts, "echo 'export PYTHONPATH=/root/thunder/python' >> /root/.bash_profile")
    ssh(master, opts, "echo 'export IPYTHON=1' >> /root/.bash_profile")
    ssh(master, opts, "echo 'export PATH=/root/thunder/python/bin:$PATH' >> /root/.bash_profile")
    # customize spark configuration parameters
    ssh(master, opts, "echo 'spark.akka.frameSize=10000' >> /root/spark/conf/spark-defaults.conf")
    ssh(master, opts, "echo 'export SPARK_DRIVER_MEMORY=20g' >> /root/spark/conf/spark-env.sh")
    # add AWS credentials to core-site.xml
    configstring = "<property><name>fs.s3n.awsAccessKeyId</name><value>ACCESS</value></property><property>" \
                   "<name>fs.s3n.awsSecretAccessKey</name><value>SECRET</value></property>"
    access, secret = get_s3_keys()
    filled = configstring.replace('ACCESS', access).replace('SECRET', secret)
    ssh(master, opts, "sed -i'f' 's,.*</configuration>.*,"+filled+"&,' /root/ephemeral-hdfs/conf/core-site.xml")
    # configure requester pays
    ssh(master, opts, "touch /root/spark/conf/jets3t.properties")
    ssh(master, opts, "echo 'httpclient.requester-pays-buckets-enabled = true' >> /root/spark/conf/jets3t.properties")
    ssh(master, opts, "~/spark-ec2/copy-dir /root/spark/conf")

    print "\n\n"
    print "-------------------------------"
    print "Thunder successfully installed!"
    print "-------------------------------"
    print "\n"

def setup_cluster(conn, master_nodes, slave_nodes, opts, deploy_ssh_key):
    """Modified version of the setup_cluster function (borrowed from spark-ec.py)
    in order to manually set the folder with the deploy code"""
    master = master_nodes[0].public_dns_name
    if deploy_ssh_key:
        print "Generating cluster's SSH key on master..."
        key_setup = """
      [ -f ~/.ssh/id_rsa ] ||
        (ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa &&
         cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys)
        """
        ssh(master, opts, key_setup)
        dot_ssh_tar = ssh_read(master, opts, ['tar', 'c', '.ssh'])
        print "Transferring cluster's SSH key to slaves..."
        for slave in slave_nodes:
            print slave.public_dns_name
            ssh_write(slave.public_dns_name, opts, ['tar', 'x'], dot_ssh_tar)

    modules = ['spark', 'shark', 'ephemeral-hdfs', 'persistent-hdfs',
             'mapreduce', 'spark-standalone', 'tachyon']

    if opts.hadoop_major_version == "1":
        modules = filter(lambda x: x != "mapreduce", modules)

    if opts.ganglia:
        modules.append('ganglia')

    ssh(master, opts, "rm -rf spark-ec2 && git clone https://github.com/mesos/spark-ec2.git -b v3")

    print "Deploying files to master..."
    deploy_folder = os.path.join(os.environ['SPARK_HOME'], "ec2", "deploy.generic")
    deploy_files(conn, deploy_folder, opts, master_nodes, slave_nodes, modules)

    print "Running setup on master..."
    setup_spark_cluster(master, opts)
    print "Done!"

if __name__ == "__main__":
    parser = OptionParser(usage="thunder-ec2 [options] <action> <clustername>",  add_help_option=False)
    parser.add_option("-h", "--help", action="help", help="Show this help message and exit")
    parser.add_option("-k", "--key-pair", help="Key pair to use on instances")
    parser.add_option("-s", "--slaves", type="int", default=1, help="Number of slaves to launch (default: 1)")
    parser.add_option("-i", "--identity-file", help="SSH private key file to use for logging into instances")
    parser.add_option("-r", "--region", default="us-east-1", help="EC2 region zone to launch instances "
                                                                  "in (default: us-east-1)")
    parser.add_option("-t", "--instance-type", default="m3.2xlarge", help="Type of instance to launch (default: m3.2xlarge)."
                                                                        " WARNING: must be 64-bit; small instances "
                                                                        "won't work")
    parser.add_option("-u", "--user", default="root", help="User name for cluster (default: root)")
    parser.add_option("-z", "--zone", default="", help="Availability zone to launch instances in, or 'all' to spread "
                                                       "slaves across multiple (an additional $0.01/Gb for "
                                                       "bandwidth between zones applies)")
    parser.add_option("--resume", default=False, action="store_true", help="Resume installation on a previously "
                                                        "launched cluster (for debugging)")

    (opts, args) = parser.parse_args()
    if len(args) != 2:
        parser.print_help()
        sys.exit(1)
    (action, cluster_name) = args

    # Launch a cluster, setting several options to defaults
    # (use spark-ec2.py included with Spark for more control)
    if action == "launch":
        try:
            conn = ec2.connect_to_region(opts.region)
        except Exception as e:
            print >> stderr, (e)
            sys.exit(1)

        if opts.zone == "":
            opts.zone = random.choice(conn.get_all_zones()).name

        opts.ami = get_spark_ami(opts) #"ami-3ecd0c56"
        opts.ebs_vol_size = 0
        opts.spot_price = None
        opts.master_instance_type = ""
        opts.wait = 160
        opts.hadoop_major_version = "1"
        opts.ganglia = True
        opts.spark_version = "1.0.2"
        opts.swap = 1024
        opts.worker_instances = 1
        opts.master_opts = ""

        if opts.resume:
            (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name)
        else:
            (master_nodes, slave_nodes) = launch_cluster(conn, opts, cluster_name)

        wait_for_cluster(conn, opts.wait, master_nodes, slave_nodes)
        setup_cluster(conn, master_nodes, slave_nodes, opts, True)
        master = master_nodes[0].public_dns_name
        install_thunder(master, opts)
        print "\n\n"
        print "-------------------------------"
        print "Cluster successfully launched!"
        print "Go to http://%s:8080 to see the web UI for your cluster" % master
        print "-------------------------------"
        print "\n"

    if action != "launch":
        conn = ec2.connect_to_region(opts.region)
        (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name)
        master = master_nodes[0].public_dns_name

        # Login to the cluster
        if action == "login":
            print "Logging into master " + master + "..."
            proxy_opt = []
            subprocess.check_call(ssh_command(opts) + proxy_opt + ['-t', '-t', "%s@%s" % (opts.user, master)])

        # Install thunder on the cluster
        elif action == "install":
            install_thunder(master, opts)

        # Destroy the cluster
        elif action == "destroy":
            response = raw_input("Are you sure you want to destroy the cluster " + cluster_name +
                                 "?\nALL DATA ON ALL NODES WILL BE LOST!!\n" +
                                 "Destroy cluster " + cluster_name + " (y/N): ")
            if response == "y":
                (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name, die_on_error=False)
            print "Terminating master..."
            for inst in master_nodes:
                inst.terminate()
            print "Terminating slaves..."
            for inst in slave_nodes:
                inst.terminate()

        else:
            raise NotImplementedError("action: " + action + "not recognized")

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
import os
from fnmatch import fnmatch
import collections

from urchin.fs.plugin import MetadataMatcher, MetadataMunger, MetadataMerger, Formatter

class DefaultMetadataMatcher(MetadataMatcher):
    """Minimal matcher which returns the item path"""
    name = "default"
    def __init__(self, config):
        pass
    def match(self, path):
        return set(path,)

class DefaultMerger(MetadataMerger):
    """
    Minimal merger which does as little as possible to the data.
    """
    name = "default"
    def __init__(self, config):
        pass
    def merge(self, metadata):
        merged = dict()
        for source,d in metadata.iteritems():
            for k,v in d.iteritems():
                if k not in merged:
                    merged[k] = v
                else:
                    cur_val_type = type(merged[k])
                    val_type = type(v)
                    if cur_val_type == set and isinstance(v, collections.Hashable):
                        merged[k].update(v)
                    elif cur_val_type == tuple and val_type == tuple:
                        # i guess?
                        merged[k] = merged[k] + val_type
                    elif cur_val_type == list and val_type == list:
                        merged[k].extend(v)
                    elif cur_val_type == list:
                        merged[k].add(v)
                    else:
                        merged[k] = [merged[k], v]
                    #else:
                    #    logger.debug("could not merge key '%s' value types '%s' and '%s'" % (k, cur_val_type, val_type))
        return merged

class DefaultMunger(MetadataMunger):
    """Minimal munger which does nothing to the data"""
    name = "default"
    def __init__(self, config):
        pass
    def mung(self, metadata):
        return metadata

class DefaultFormatter(Formatter):
    """Default formatter which returns the original item file/directory name"""
    name = "default"
    def __init__(self, config):
        pass
    def format(self, original_name, metadata):
        return set([original_name])

<commit_msg>correct typing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
import os
from fnmatch import fnmatch
import collections

from urchin.fs.plugin import MetadataMatcher, MetadataMunger, MetadataMerger, Formatter

class DefaultMetadataMatcher(MetadataMatcher):
    """Minimal matcher which returns the item path"""
    name = "default"
    def __init__(self, config):
        pass
    def match(self, path):
        return set(path,)

class DefaultMerger(MetadataMerger):
    """
    Minimal merger which does as little as possible to the data.
    """
    name = "default"
    def __init__(self, config):
        pass
    def merge(self, metadata):
        merged = dict()
        for source,d in metadata.iteritems():
            for k,v in d.iteritems():
                if k not in merged:
                    merged[k] = v
                else:
                    cur_val_type = type(merged[k])
                    val_type = type(v)
                    if cur_val_type == set and isinstance(v, collections.Hashable):
                        merged[k].update(v)
                    elif cur_val_type == tuple and val_type == tuple:
                        # i guess?
                        merged[k] = merged[k] + val_type
                    elif cur_val_type == list and val_type == list:
                        merged[k].extend(v)
                    elif cur_val_type == list:
                        merged[k].add(v)
                    else:
                        merged[k] = [merged[k], v]
                    #else:
                    #    logger.debug("could not merge key '%s' value types '%s' and '%s'" % (k, cur_val_type, val_type))
        return merged

class DefaultMunger(MetadataMunger):
    """Minimal munger which does nothing to the data"""
    name = "default"
    def __init__(self, config):
        pass
    def mung(self, metadata):
        return metadata

class DefaultFormatter(Formatter):
    """Default formatter which returns the original item file/directory name"""
    name = "default"
    def __init__(self, config):
        pass
    def format(self, original_name, metadata):
        return set([os.path.basename(original_name)])

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutNil in the Ruby Koans
#

from runner.koan import *

class AboutNone(Koan):

    def test_none_is_an_object(self):
        "Unlike NULL in a lot of languages"
        self.assertEqual(__, isinstance(None, object))

    def test_none_is_universal(self):
        "There is only one None"
        self.assertEqual(__, None is None)

    def test_what_exception_do_you_get_when_calling_nonexistent_methods(self):
        """
        What is the Exception that is thrown when you call a method that does
        not exist?

        Hint: launch python command console and try the code in the
        block below.

        Don't worry about what 'try' and 'except' do, we'll talk about
        this later
        """
        try:
            None.some_method_none_does_not_know_about()
        except Exception as ex:
            # What exception has been caught?
            self.assertEqual(__, ex.__class__.__name__)

            # What message was attached to the exception?
            # (HINT: replace __ with part of the error message.)
            self.assertMatch(__, ex.args[0])

    def test_none_is_distinct(self):
        """
        None is distinct from other things which are False.
        """
        self.assertEqual(____, None is not 0)
        self.assertEqual(____, None is not False)
<commit_msg>Learn about None<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutNil in the Ruby Koans
#

from runner.koan import *

class AboutNone(Koan):

    def test_none_is_an_object(self):
        "Unlike NULL in a lot of languages"
        self.assertEqual(True, isinstance(None, object))

    def test_none_is_universal(self):
        "There is only one None"
        self.assertEqual(True, None is None)

    def test_what_exception_do_you_get_when_calling_nonexistent_methods(self):
        """
        What is the Exception that is thrown when you call a method that does
        not exist?

        Hint: launch python command console and try the code in the
        block below.

        Don't worry about what 'try' and 'except' do, we'll talk about
        this later
        """
        try:
            None.some_method_none_does_not_know_about()
        except Exception as ex:
            # What exception has been caught?
            self.assertEqual("AttributeError", ex.__class__.__name__)

            # What message was attached to the exception?
            # (HINT: replace __ with part of the error message.)
            self.assertMatch("'NoneType' object has no attribute 'some_method_none_does_not_know_about'", ex.args[0])

    def test_none_is_distinct(self):
        """
        None is distinct from other things which are False.
        """
        self.assertEqual(True, None is not 0)
        self.assertEqual(True, None is not False)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *

class AboutSets(Koan):
    def test_sets_make_keep_lists_unique(self):
        highlanders = ['MacLeod', 'Ramirez', 'MacLeod', 'Matunas',
            'MacLeod', 'Malcolm', 'MacLeod']

        there_can_only_be_only_one = set(highlanders)

        self.assertEqual(__, there_can_only_be_only_one)

    def test_sets_are_unordered(self):
        self.assertEqual(set([__, __, __, __, __]), set('12345'))

    def test_convert_the_set_into_a_list_to_sort_it(self):
        self.assertEqual(__, sorted(set('13245')))

    # ------------------------------------------------------------------

    def test_set_have_arithmetic_operators(self):
        scotsmen = set(['MacLeod', 'Wallace', 'Willie'])
        warriors = set(['MacLeod', 'Wallace', 'Leonidas'])

        self.assertEqual(__, scotsmen - warriors)
        self.assertEqual(__, scotsmen | warriors)
        self.assertEqual(__, scotsmen & warriors)
        self.assertEqual(__, scotsmen ^ warriors)

    # ------------------------------------------------------------------

    def test_we_can_query_set_membership(self):
        self.assertEqual(__, 127 in set([127, 0, 0, 1]))
        self.assertEqual(__, 'cow' not in set('apocalypse now'))

    def test_we_can_compare_subsets(self):
        self.assertEqual(__, set('cake') <= set('cherry cake'))
        self.assertEqual(__, set('cake').issubset(set('cherry cake')))

        self.assertEqual(__, set('cake') > set('pie'))
<commit_msg>You have completed 107 koans and 12 lessons.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *

class AboutSets(Koan):
    def test_sets_make_keep_lists_unique(self):
        highlanders = ['MacLeod', 'Ramirez', 'MacLeod', 'Matunas',
            'MacLeod', 'Malcolm', 'MacLeod']

        there_can_only_be_only_one = set(highlanders)

        self.assertEqual({'MacLeod', 'Ramirez', 'Matunas', 'Malcolm'}, there_can_only_be_only_one)

    def test_sets_are_unordered(self):
        self.assertEqual(set(['1', '5', '2', '3', '4']), set('12345'))

    def test_convert_the_set_into_a_list_to_sort_it(self):
        self.assertEqual(['1', '2', '3', '4', '5'], sorted(set('13245')))

    # ------------------------------------------------------------------

    def test_set_have_arithmetic_operators(self):
        scotsmen = set(['MacLeod', 'Wallace', 'Willie'])
        warriors = set(['MacLeod', 'Wallace', 'Leonidas'])

        self.assertEqual({'Willie'}, scotsmen - warriors)
        self.assertEqual({'Leonidas', 'MacLeod', 'Wallace', 'Willie'}, scotsmen | warriors)
        self.assertEqual({'MacLeod', 'Wallace'}, scotsmen & warriors)
        self.assertEqual({'Leonidas', 'Willie'}, scotsmen ^ warriors)

    # ------------------------------------------------------------------

    def test_we_can_query_set_membership(self):
        self.assertEqual(True, 127 in set([127, 0, 0, 1]))
        self.assertEqual(True, 'cow' not in set('apocalypse now'))

    def test_we_can_compare_subsets(self):
        self.assertEqual(True, set('cake') <= set('cherry cake'))
        self.assertEqual(True, set('cake').issubset(set('cherry cake')))

        self.assertEqual(False, set('cake') > set('pie'))
<|endoftext|>


<commit_before>from ArduinoMoistureSensor import ArduinoMoistureSensor
from datetime import datetime, time, timedelta
import pickle

cans = [ArduinoMoistureSensor(1,4),ArduinoMoistureSensor(1,8)]

with open('./sensors.log', "a") as logFile:

	logFile.write(datetime.now().isoformat())

	for can in cans:
		can.read()

		for sensor in can.sensors:
	                logFile.write("\t"+str(sensor))

	logFile.write("\n")
<commit_msg>added time of read + time of measure in log script<commit_after>from ArduinoMoistureSensor import ArduinoMoistureSensor
from datetime import datetime, time, timedelta
import pickle

cans = [ArduinoMoistureSensor(1,4),ArduinoMoistureSensor(1,8)]

with open('./sensors.log', "a") as logFile:

	logFile.write(datetime.now().isoformat())

	for can in cans:
		can.read()
		logFile.write("\t"+can.time.isoformat())

		for sensor in can.sensors:
	                logFile.write("\t"+str(sensor))

	logFile.write("\n")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import pywren
import boto3
import click
import shutil
import os
import json
import zipfile
import glob2
import io
import time
import botocore
from pywren import ec2standalone

@click.group()
def cli():
    pass

SOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
@cli.command()
@click.option('--filename', default=pywren.wrenconfig.get_default_home_filename(),
              help='create a default config and populate with sane values')
@click.option('--lambda_role', default='pywren_exec_role',
              help='name of the IAM role we are creating')
@click.option('--function_name', default='pywren1',
              help='lambda function name')
@click.option('--bucket_name', default='BUCKET_NAME',
              help='s3 bucket name for intermediates')
@click.option('--sqs_queue', default='pywren-queue',
              help='sqs queue name for standalone execution')
@click.option('--force', is_flag=True, default=False,
              help='force overwrite an existing file')
def create_config(filename, force, lambda_role, function_name, bucket_name,
                  sqs_queue):
    """
    Create a config file initialized with the defaults, and
    put it in your ~/.pywren_config

    """

    # copy default config file

    default_yaml = open(os.path.join(SOURCE_DIR, "../default_config.yaml")).read()

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]

    # perform substitutions -- get your AWS account ID and auto-populate

    default_yaml = default_yaml.replace('AWS_ACCOUNT_ID', account_id)
    default_yaml = default_yaml.replace('pywren_exec_role', lambda_role)
    default_yaml = default_yaml.replace('pywren1', function_name)
    default_yaml = default_yaml.replace('BUCKET_NAME', bucket_name)
    default_yaml = default_yaml.replace('pywren-queue', sqs_queue)

    # print out message about the stuff you need to do
    if os.path.exists(filename) and not force:
        raise ValueError("{} already exists; not overwriting (did you need --force?)".format(filename))

    open(filename, 'w').write(default_yaml)
    click.echo("new default file created in {}".format(filename))
    click.echo("lambda role is {}".format(lambda_role))
    click.echo("remember to set your s3 bucket and preferred AWS region")

@cli.command()
def test_config():
    """
    Test that you have properly filled in the necessary
    aws fields, your boto install is working correctly, your s3 bucket is
    readable and writable (also by your indicated role), etc.
    """

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]
    print "The accountID is ", account_id
    # make sure the bucket exists
    #config = pywren.wrenconfig.default()

@cli.command()
def create_role():
    """

    """
    config = pywren.wrenconfig.default()

    iamclient = boto3.resource('iam')
    json_policy = json.dumps(pywren.wrenconfig.basic_role_policy)
    role_name = config['account']['aws_lambda_role']
    role = iamclient.create_role(RoleName=role_name,
                                 AssumeRolePolicyDocument=json_policy)
    more_json_policy = json.dumps(pywren.wrenconfig.more_permissions_policy)

    AWS_ACCOUNT_ID = config['account']['aws_account_id']
    AWS_REGION = config['account']['aws_region']
    more_json_policy = more_json_policy.replace("AWS_ACCOUNT_ID", str(AWS_ACCOUNT_ID))
    more_json_policy = more_json_policy.replace("AWS_REGION", AWS_REGION)

    iamclient.RolePolicy(role_name, '{}-more-permissions'.format(role_name)).put(
        PolicyDocument=more_json_policy)

@cli.command()
def deploy_lambda(update_if_exists = True):
    """
    Package up the source code and deploy to aws. Only creates the new
    function if it doesn't already exist
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']
    MEMORY = config['lambda']['memory']
    TIMEOUT = config['lambda']['timeout']
    AWS_LAMBDA_ROLE = config['account']['aws_lambda_role']
    AWS_ACCOUNT_ID = config['account']['aws_account_id']

    file_like_object = io.BytesIO()
    zipfile_obj = zipfile.ZipFile(file_like_object, mode='w')

    # FIXME see if role exists
    files = glob2.glob(os.path.join(SOURCE_DIR, "../**/*.py"))
    for f in files:
        a = os.path.relpath(f, SOURCE_DIR + "/..")

        zipfile_obj.write(f, arcname=a)
    zipfile_obj.close()
    #open("/tmp/deploy.zip", 'w').write(file_like_object.getvalue())

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    ROLE = "arn:aws:iam::{}:role/{}".format(AWS_ACCOUNT_ID, AWS_LAMBDA_ROLE)

    b = lambclient.list_functions()
    function_exists = False

    if FUNCTION_NAME in [f['FunctionName'] for f in b['Functions']]:
        function_exists = True

    retries = 0
    while retries < 10:
        try:
            if function_exists:
                print "function exists, updating"
                if update_if_exists:

                    response = lambclient.update_function_code(FunctionName=FUNCTION_NAME,
                                                               ZipFile=file_like_object.getvalue())
                    return True
                else:
                    raise Exception() # FIXME will this work?
            else:

                lambclient.create_function(FunctionName = FUNCTION_NAME,
                                           Handler = pywren.wrenconfig.AWS_LAMBDA_HANDLER_NAME,
                                           Runtime = "python2.7",
                                           MemorySize = MEMORY,
                                           Timeout = TIMEOUT,
                                           Role = ROLE,
                                           Code = {'ZipFile' : file_like_object.getvalue()})
                print "Create successful"
                break
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == "InvalidParameterValueException":

                print "attempt", retries
                retries += 1

                # FIXME actually check for "botocore.exceptions.ClientError: An error occurred (InvalidParameterValueException) when calling the CreateFunction operation: The role defined for the function cannot be assumed by Lambda."
                print "sleeping for 5"
                time.sleep(5)
                print "done"
                continue
            else:
                raise e
    if retries == 10:
        raise ValueError("could not register funciton after 10 tries")

@cli.command()
def delete_lambda():
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    b = lambclient.list_functions()

    if FUNCTION_NAME not in [f['FunctionName'] for f in b['Functions']]:
        raise Exception()
    lambclient.delete_function(FunctionName = FUNCTION_NAME)

@cli.command()
def delete_role():
    """

    """

    config = pywren.wrenconfig.default()
    iamclient = boto3.client('iam')
    role_name = config['account']['aws_lambda_role']

    iamclient.delete_role_policy(RoleName = role_name,
                                 PolicyName = '{}-more-permissions'.format(role_name))
    iamclient.delete_role(RoleName = role_name)

@cli.command()
def create_queue():
    """
    Create the SQS queue
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    SQS_QUEUE_NAME = config['standalone']['sqs_queue_name']

    sqs = boto3.resource('sqs',  region_name=AWS_REGION)

    queue = sqs.create_queue(QueueName=SQS_QUEUE_NAME,
                             Attributes={'VisibilityTimeout' : "20"})

@cli.command()
def delete_queue():
    """
    Delete the SQS queue
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    SQS_QUEUE_NAME = config['standalone']['sqs_queue_name']

    sqs = boto3.resource('sqs',  region_name=AWS_REGION)
    queue = sqs.get_queue_by_name(QueueName=SQS_QUEUE_NAME)
    queue.delete()

def test_lambda():
    """
    Simple single-function test
    """

    config = pywren.wrenconfig.default()

@cli.command()
def print_latest_logs():
    """
    Print the latest log group and log stream.

    Note this does not contain support for going back further in history,
    use the CloudWatch Logs web GUI for that.
    """

    config = pywren.wrenconfig.default()

    logclient = boto3.client('logs', region_name=config['account']['aws_region'])

    logGroupName = "/aws/lambda/{}".format(config['lambda']['function_name'])

    response = logclient.describe_log_streams(
        logGroupName=logGroupName,
        orderBy='LastEventTime',
        descending=True)

    latest_logStreamName = response['logStreams'][0]['logStreamName']

    response = logclient.get_log_events(
        logGroupName=logGroupName,
        logStreamName=latest_logStreamName,)

    for event in response['events']:
        print "{} : {}".format(event['timestamp'], event['message'].strip())

@cli.command()
def log_url():
    """
    return the cloudwatch log URL
    """
    config = pywren.wrenconfig.default()
    function_name = config['lambda']['function_name']
    aws_region = config['account']['aws_region']
    url = "https://{}.console.aws.amazon.com/cloudwatch/home?region={}#logStream:group=/aws/lambda/{}".format(aws_region, aws_region, function_name)
    print url

@cli.command()
@click.option('--number', default=1, type=int,
              help='number of instances to launch')
def standalone_launch_instances(number):
    config = pywren.wrenconfig.default()
    sc= config['standalone']
    aws_region = config['account']['aws_region']

    ec2standalone.launch_instances(sc['target_ami'], aws_region,
                                   sc['ec2_ssh_key'], sc['ec2_instance_type'],
                                   sc['instance_name'], sc['instance_profile_name'])

@cli.command()
def standalone_list_instances():
    config = pywren.wrenconfig.default()
    aws_region = config['account']['aws_region']
    sc= config['standalone']

    inst_list = ec2standalone.list_instances(aws_region, sc['instance_name'])
    ec2standalone.prettyprint_instances(inst_list)

@cli.command()
def standalone_instance_uptime():
    pass

@cli.command()
def standalone_terminate_instances():
    config = pywren.wrenconfig.default()
    aws_region = config['account']['aws_region']
    sc= config['standalone']

    inst_list = ec2standalone.list_instances(aws_region, sc['instance_name'])
    print "terminate"
    ec2standalone.prettyprint_instances(inst_list)
    ec2standalone.terminate_instances(inst_list)
<commit_msg>change cli setup<commit_after>#!/usr/bin/env python

import pywren
import boto3
import click
import shutil
import os
import json
import zipfile
import glob2
import io
import time
import botocore
from pywren import ec2standalone

@click.group()
def cli():
    pass

@click.group("standalone")
def standalone():
    """
    Standalone commands to control groups of servers
    """

# FIXME use pywren main module SOURCE_DIR
SOURCE_DIR = os.path.dirname(os.path.abspath(__file__))

@click.command()
@click.option('--filename', default=pywren.wrenconfig.get_default_home_filename(),
              help='create a default config and populate with sane values')
@click.option('--lambda_role', default='pywren_exec_role',
              help='name of the IAM role we are creating')
@click.option('--function_name', default='pywren1',
              help='lambda function name')
@click.option('--bucket_name', default='BUCKET_NAME',
              help='s3 bucket name for intermediates')
@click.option('--sqs_queue', default='pywren-queue',
              help='sqs queue name for standalone execution')
@click.option('--force', is_flag=True, default=False,
              help='force overwrite an existing file')
def create_config(filename, force, lambda_role, function_name, bucket_name,
                  sqs_queue):
    """
    Create a config file initialized with the defaults, and
    put it in your ~/.pywren_config

    """

    # copy default config file

    default_yaml = open(os.path.join(SOURCE_DIR, "../default_config.yaml")).read()

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]

    # perform substitutions -- get your AWS account ID and auto-populate

    default_yaml = default_yaml.replace('AWS_ACCOUNT_ID', account_id)
    default_yaml = default_yaml.replace('pywren_exec_role', lambda_role)
    default_yaml = default_yaml.replace('pywren1', function_name)
    default_yaml = default_yaml.replace('BUCKET_NAME', bucket_name)
    default_yaml = default_yaml.replace('pywren-queue', sqs_queue)

    # print out message about the stuff you need to do
    if os.path.exists(filename) and not force:
        raise ValueError("{} already exists; not overwriting (did you need --force?)".format(filename))

    open(filename, 'w').write(default_yaml)
    click.echo("new default file created in {}".format(filename))
    click.echo("lambda role is {}".format(lambda_role))
    click.echo("remember to set your s3 bucket and preferred AWS region")

@click.command()
def test_config():
    """
    Test that you have properly filled in the necessary
    aws fields, your boto install is working correctly, your s3 bucket is
    readable and writable (also by your indicated role), etc.
    """

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]
    print "The accountID is ", account_id
    # make sure the bucket exists
    #config = pywren.wrenconfig.default()

@click.command()
def create_role():
    """

    """
    config = pywren.wrenconfig.default()

    iamclient = boto3.resource('iam')
    json_policy = json.dumps(pywren.wrenconfig.basic_role_policy)
    role_name = config['account']['aws_lambda_role']
    role = iamclient.create_role(RoleName=role_name,
                                 AssumeRolePolicyDocument=json_policy)
    more_json_policy = json.dumps(pywren.wrenconfig.more_permissions_policy)

    AWS_ACCOUNT_ID = config['account']['aws_account_id']
    AWS_REGION = config['account']['aws_region']
    more_json_policy = more_json_policy.replace("AWS_ACCOUNT_ID", str(AWS_ACCOUNT_ID))
    more_json_policy = more_json_policy.replace("AWS_REGION", AWS_REGION)

    iamclient.RolePolicy(role_name, '{}-more-permissions'.format(role_name)).put(
        PolicyDocument=more_json_policy)

@click.command()
def deploy_lambda(update_if_exists = True):
    """
    Package up the source code and deploy to aws. Only creates the new
    function if it doesn't already exist
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']
    MEMORY = config['lambda']['memory']
    TIMEOUT = config['lambda']['timeout']
    AWS_LAMBDA_ROLE = config['account']['aws_lambda_role']
    AWS_ACCOUNT_ID = config['account']['aws_account_id']

    file_like_object = io.BytesIO()
    zipfile_obj = zipfile.ZipFile(file_like_object, mode='w')

    # FIXME see if role exists
    files = glob2.glob(os.path.join(SOURCE_DIR, "../**/*.py"))
    for f in files:
        a = os.path.relpath(f, SOURCE_DIR + "/..")

        zipfile_obj.write(f, arcname=a)
    zipfile_obj.close()
    #open("/tmp/deploy.zip", 'w').write(file_like_object.getvalue())

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    ROLE = "arn:aws:iam::{}:role/{}".format(AWS_ACCOUNT_ID, AWS_LAMBDA_ROLE)

    b = lambclient.list_functions()
    function_exists = False

    if FUNCTION_NAME in [f['FunctionName'] for f in b['Functions']]:
        function_exists = True

    retries = 0
    while retries < 10:
        try:
            if function_exists:
                print "function exists, updating"
                if update_if_exists:

                    response = lambclient.update_function_code(FunctionName=FUNCTION_NAME,
                                                               ZipFile=file_like_object.getvalue())
                    return True
                else:
                    raise Exception() # FIXME will this work?
            else:

                lambclient.create_function(FunctionName = FUNCTION_NAME,
                                           Handler = pywren.wrenconfig.AWS_LAMBDA_HANDLER_NAME,
                                           Runtime = "python2.7",
                                           MemorySize = MEMORY,
                                           Timeout = TIMEOUT,
                                           Role = ROLE,
                                           Code = {'ZipFile' : file_like_object.getvalue()})
                print "Create successful"
                break
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == "InvalidParameterValueException":

                print "attempt", retries
                retries += 1

                # FIXME actually check for "botocore.exceptions.ClientError: An error occurred (InvalidParameterValueException) when calling the CreateFunction operation: The role defined for the function cannot be assumed by Lambda."
                print "sleeping for 5"
                time.sleep(5)
                print "done"
                continue
            else:
                raise e
    if retries == 10:
        raise ValueError("could not register funciton after 10 tries")

@click.command()
def delete_lambda():
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    b = lambclient.list_functions()

    if FUNCTION_NAME not in [f['FunctionName'] for f in b['Functions']]:
        raise Exception()
    lambclient.delete_function(FunctionName = FUNCTION_NAME)

@click.command()
def delete_role():
    """

    """

    config = pywren.wrenconfig.default()
    iamclient = boto3.client('iam')
    role_name = config['account']['aws_lambda_role']

    iamclient.delete_role_policy(RoleName = role_name,
                                 PolicyName = '{}-more-permissions'.format(role_name))
    iamclient.delete_role(RoleName = role_name)

@click.command()
def create_queue():
    """
    Create the SQS queue
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    SQS_QUEUE_NAME = config['standalone']['sqs_queue_name']

    sqs = boto3.resource('sqs',  region_name=AWS_REGION)

    queue = sqs.create_queue(QueueName=SQS_QUEUE_NAME,
                             Attributes={'VisibilityTimeout' : "20"})

@click.command()
def delete_queue():
    """
    Delete the SQS queue
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    SQS_QUEUE_NAME = config['standalone']['sqs_queue_name']

    sqs = boto3.resource('sqs',  region_name=AWS_REGION)
    queue = sqs.get_queue_by_name(QueueName=SQS_QUEUE_NAME)
    queue.delete()

def test_lambda():
    """
    Simple single-function test
    """

    config = pywren.wrenconfig.default()

@click.command()
def print_latest_logs():
    """
    Print the latest log group and log stream.

    Note this does not contain support for going back further in history,
    use the CloudWatch Logs web GUI for that.
    """

    config = pywren.wrenconfig.default()

    logclient = boto3.client('logs', region_name=config['account']['aws_region'])

    logGroupName = "/aws/lambda/{}".format(config['lambda']['function_name'])

    response = logclient.describe_log_streams(
        logGroupName=logGroupName,
        orderBy='LastEventTime',
        descending=True)

    latest_logStreamName = response['logStreams'][0]['logStreamName']

    response = logclient.get_log_events(
        logGroupName=logGroupName,
        logStreamName=latest_logStreamName,)

    for event in response['events']:
        print "{} : {}".format(event['timestamp'], event['message'].strip())

@click.command()
def log_url():
    """
    return the cloudwatch log URL
    """
    config = pywren.wrenconfig.default()
    function_name = config['lambda']['function_name']
    aws_region = config['account']['aws_region']
    url = "https://{}.console.aws.amazon.com/cloudwatch/home?region={}#logStream:group=/aws/lambda/{}".format(aws_region, aws_region, function_name)
    print url

@standalone.command('launch_instances')
@click.option('--number', default=1, type=int,
              help='number of instances to launch')
def standalone_launch_instances(number):
    config = pywren.wrenconfig.default()
    sc= config['standalone']
    aws_region = config['account']['aws_region']

    ec2standalone.launch_instances(sc['target_ami'], aws_region,
                                   sc['ec2_ssh_key'], sc['ec2_instance_type'],
                                   sc['instance_name'], sc['instance_profile_name'])

@standalone.command("list_instances")
def standalone_list_instances():
    config = pywren.wrenconfig.default()
    aws_region = config['account']['aws_region']
    sc= config['standalone']

    inst_list = ec2standalone.list_instances(aws_region, sc['instance_name'])
    ec2standalone.prettyprint_instances(inst_list)

@standalone.command("instance_uptime")
def standalone_instance_uptime():
    pass

@standalone.command("terminate_instances")
def standalone_terminate_instances():
    config = pywren.wrenconfig.default()
    aws_region = config['account']['aws_region']
    sc= config['standalone']

    inst_list = ec2standalone.list_instances(aws_region, sc['instance_name'])
    print "terminate"
    ec2standalone.prettyprint_instances(inst_list)
    ec2standalone.terminate_instances(inst_list)

cli.add_command(create_config)
cli.add_command(test_config)
cli.add_command(create_role)
cli.add_command(deploy_lambda)
cli.add_command(delete_lambda)
cli.add_command(delete_role)
cli.add_command(create_queue)
cli.add_command(delete_queue)
cli.add_command(print_latest_logs)
cli.add_command(log_url)
cli.add_command(standalone)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import pywren
import boto3
import click
import shutil
import os
import json
import zipfile
import glob2
import io

@click.group()
def cli():
    pass

SOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
@cli.command()
@click.option('--filename', default=pywren.wrenconfig.get_default_home_filename(),
              help='create a default config and populate with sane values')
@click.option('--lambda_role', default='pywren_exec_role',
              help='name of the IAM role we are creating')
@click.option('--function_name', default='pywren1',
              help='lambda function name')
@click.option('--bucket_name', default='BUCKET_NAME',
              help='s3 bucket name for intermediates')
@click.option('--force', is_flag=True, default=False,
              help='force overwrite an existing file')
def create_config(filename, force, lambda_role, function_name, bucket_name):
    """
    Create a config file initialized with the defaults, and
    put it in your ~/.pywren_config

    """

    # copy default config file

    default_yaml = open(os.path.join(SOURCE_DIR, "../default_config.yaml")).read()

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]

    # perform substitutions -- get your AWS account ID and auto-populate

    default_yaml = default_yaml.replace('AWS_ACCOUNT_ID', account_id)
    default_yaml = default_yaml.replace('pywren_exec_role', lambda_role)
    default_yaml = default_yaml.replace('pywren1', function_name)
    default_yaml = default_yaml.replace('BUCKET_NAME', bucket_name)

    # print out message about the stuff you need to do
    if os.path.exists(filename) and not force:
        raise ValueError("{} already exists; not overwriting (did you need --force?)".format(filename))

    open(filename, 'w').write(default_yaml)
    click.echo("new default file created in {}".format(filename))
    click.echo("lambda role is {}".format(lambda_role))
    click.echo("remember to set your s3 bucket and preferred AWS region")

@cli.command()
def test_config():
    """
    Test that you have properly filled in the necessary
    aws fields, your boto install is working correctly, your s3 bucket is
    readable and writable (also by your indicated role), etc.
    """

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]
    print "The accountID is ", account_id
    # make sure the bucket exists
    #config = pywren.wrenconfig.default()

@cli.command()
def create_role():
    """

    """

    config = pywren.wrenconfig.default()
    print "config=", config
    iamclient = boto3.resource('iam')
    json_policy = json.dumps(pywren.wrenconfig.basic_role_policy)
    role_name = config['account']['aws_lambda_role']
    role = iamclient.create_role(RoleName=role_name,
                                 AssumeRolePolicyDocument=json_policy)
    more_json_policy = json.dumps(pywren.wrenconfig.more_permissions_policy)

    iamclient.RolePolicy(role_name, '{}-more-permissions'.format(role_name)).put(
        PolicyDocument=more_json_policy)

@cli.command()
def deploy_lambda(update_if_exists = True):
    """
    Package up the source code and deploy to aws. Only creates the new
    function if it doesn't already exist
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']
    MEMORY = config['lambda']['memory']
    TIMEOUT = config['lambda']['timeout']
    AWS_LAMBDA_ROLE = config['account']['aws_lambda_role']
    AWS_ACCOUNT_ID = config['account']['aws_account_id']

    file_like_object = io.BytesIO()
    zipfile_obj = zipfile.ZipFile(file_like_object, mode='w')

    files = glob2.glob(os.path.join(SOURCE_DIR, "../**/*.py"))
    for f in files:
        a = os.path.relpath(f, SOURCE_DIR + "/..")

        zipfile_obj.write(f, arcname=a)
    zipfile_obj.close()
    #open("/tmp/deploy.zip", 'w').write(file_like_object.getvalue())

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    ROLE = "arn:aws:iam::{}:role/{}".format(AWS_ACCOUNT_ID, AWS_LAMBDA_ROLE)

    b = lambclient.list_functions()
    function_exists = False

    if FUNCTION_NAME in [f['FunctionName'] for f in b['Functions']]:
        function_exists = True

    if function_exists:
        if update_if_exists:

            response = lambclient.update_function_code(FunctionName=FUNCTION_NAME,
                                                       ZipFile=file_like_object.getvalue())
        else:
            raise Exception()
    else:

        lambclient.create_function(FunctionName = FUNCTION_NAME,
                                   Handler = pywren.wrenconfig.HANDLER_NAME,
                                   Runtime = "python2.7",
                                   MemorySize = MEMORY,
                                   Timeout = TIMEOUT,
                                   Role = ROLE,
                                   Code = {'ZipFile' : file_like_object.getvalue()})

@cli.command()
def delete_lambda():
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    b = lambclient.list_functions()

    if FUNCTION_NAME not in [f['FunctionName'] for f in b['Functions']]:
        raise Exception()
    lambclient.delete_function(FunctionName = FUNCTION_NAME)

@cli.command()
def delete_role():
    """

    """

    config = pywren.wrenconfig.default()
    iamclient = boto3.client('iam')
    role_name = config['account']['aws_lambda_role']

    iamclient.delete_role_policy(RoleName = role_name,
                                 PolicyName = '{}-more-permissions'.format(role_name))
    iamclient.delete_role(RoleName = role_name)

def test_lambda():
    """
    Simple single-function test
    """

    config = pywren.wrenconfig.default()
<commit_msg>lambda function create retry logic<commit_after>#!/usr/bin/env python

import pywren
import boto3
import click
import shutil
import os
import json
import zipfile
import glob2
import io
import time
import botocore

@click.group()
def cli():
    pass

SOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
@cli.command()
@click.option('--filename', default=pywren.wrenconfig.get_default_home_filename(),
              help='create a default config and populate with sane values')
@click.option('--lambda_role', default='pywren_exec_role',
              help='name of the IAM role we are creating')
@click.option('--function_name', default='pywren1',
              help='lambda function name')
@click.option('--bucket_name', default='BUCKET_NAME',
              help='s3 bucket name for intermediates')
@click.option('--force', is_flag=True, default=False,
              help='force overwrite an existing file')
def create_config(filename, force, lambda_role, function_name, bucket_name):
    """
    Create a config file initialized with the defaults, and
    put it in your ~/.pywren_config

    """

    # copy default config file

    default_yaml = open(os.path.join(SOURCE_DIR, "../default_config.yaml")).read()

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]

    # perform substitutions -- get your AWS account ID and auto-populate

    default_yaml = default_yaml.replace('AWS_ACCOUNT_ID', account_id)
    default_yaml = default_yaml.replace('pywren_exec_role', lambda_role)
    default_yaml = default_yaml.replace('pywren1', function_name)
    default_yaml = default_yaml.replace('BUCKET_NAME', bucket_name)

    # print out message about the stuff you need to do
    if os.path.exists(filename) and not force:
        raise ValueError("{} already exists; not overwriting (did you need --force?)".format(filename))

    open(filename, 'w').write(default_yaml)
    click.echo("new default file created in {}".format(filename))
    click.echo("lambda role is {}".format(lambda_role))
    click.echo("remember to set your s3 bucket and preferred AWS region")

@cli.command()
def test_config():
    """
    Test that you have properly filled in the necessary
    aws fields, your boto install is working correctly, your s3 bucket is
    readable and writable (also by your indicated role), etc.
    """

    client = boto3.client("sts")
    account_id = client.get_caller_identity()["Account"]
    print "The accountID is ", account_id
    # make sure the bucket exists
    #config = pywren.wrenconfig.default()

@cli.command()
def create_role():
    """

    """

    config = pywren.wrenconfig.default()
    print "config=", config
    iamclient = boto3.resource('iam')
    json_policy = json.dumps(pywren.wrenconfig.basic_role_policy)
    role_name = config['account']['aws_lambda_role']
    role = iamclient.create_role(RoleName=role_name,
                                 AssumeRolePolicyDocument=json_policy)
    more_json_policy = json.dumps(pywren.wrenconfig.more_permissions_policy)

    iamclient.RolePolicy(role_name, '{}-more-permissions'.format(role_name)).put(
        PolicyDocument=more_json_policy)

@cli.command()
def deploy_lambda(update_if_exists = True):
    """
    Package up the source code and deploy to aws. Only creates the new
    function if it doesn't already exist
    """
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']
    MEMORY = config['lambda']['memory']
    TIMEOUT = config['lambda']['timeout']
    AWS_LAMBDA_ROLE = config['account']['aws_lambda_role']
    AWS_ACCOUNT_ID = config['account']['aws_account_id']

    file_like_object = io.BytesIO()
    zipfile_obj = zipfile.ZipFile(file_like_object, mode='w')

    files = glob2.glob(os.path.join(SOURCE_DIR, "../**/*.py"))
    for f in files:
        a = os.path.relpath(f, SOURCE_DIR + "/..")

        zipfile_obj.write(f, arcname=a)
    zipfile_obj.close()
    #open("/tmp/deploy.zip", 'w').write(file_like_object.getvalue())

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    ROLE = "arn:aws:iam::{}:role/{}".format(AWS_ACCOUNT_ID, AWS_LAMBDA_ROLE)

    b = lambclient.list_functions()
    function_exists = False

    if FUNCTION_NAME in [f['FunctionName'] for f in b['Functions']]:
        function_exists = True

    retries = 0
    while retries < 4:
        try:
            if function_exists:
                if update_if_exists:

                    response = lambclient.update_function_code(FunctionName=FUNCTION_NAME,
                                                               ZipFile=file_like_object.getvalue())
                else:
                    raise Exception() # FIXME will this work?
            else:

                lambclient.create_function(FunctionName = FUNCTION_NAME,
                                           Handler = pywren.wrenconfig.HANDLER_NAME,
                                           Runtime = "python2.7",
                                           MemorySize = MEMORY,
                                           Timeout = TIMEOUT,
                                           Role = ROLE,
                                           Code = {'ZipFile' : file_like_object.getvalue()})
                print "Create successful"
                break
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == "InvalidParameterValueException":

                print "attempt", retries
                retries += 1

                # FIXME actually check for "botocore.exceptions.ClientError: An error occurred (InvalidParameterValueException) when calling the CreateFunction operation: The role defined for the function cannot be assumed by Lambda."
                print "sleeping for 5"
                time.sleep(5)
                print "done"
                continue
            else:
                raise e

@cli.command()
def delete_lambda():
    config = pywren.wrenconfig.default()
    AWS_REGION = config['account']['aws_region']
    FUNCTION_NAME = config['lambda']['function_name']

    lambclient = boto3.client('lambda', region_name=AWS_REGION)

    b = lambclient.list_functions()

    if FUNCTION_NAME not in [f['FunctionName'] for f in b['Functions']]:
        raise Exception()
    lambclient.delete_function(FunctionName = FUNCTION_NAME)

@cli.command()
def delete_role():
    """

    """

    config = pywren.wrenconfig.default()
    iamclient = boto3.client('iam')
    role_name = config['account']['aws_lambda_role']

    iamclient.delete_role_policy(RoleName = role_name,
                                 PolicyName = '{}-more-permissions'.format(role_name))
    iamclient.delete_role(RoleName = role_name)

def test_lambda():
    """
    Simple single-function test
    """

    config = pywren.wrenconfig.default()
<|endoftext|>