<commit_before>#!/usr/bin/env python

#Copyright (C) 2011 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
"""
"""

import unittest
import os
import sys

from sonLib.bioio import TestStatus
from sonLib.bioio import getTempDirectory
from sonLib.bioio import logger
from sonLib.bioio import system

from cactus.progressive.multiCactusTree import MultiCactusTree
from sonLib.nxnewick import NXNewick

class TestCase(unittest.TestCase):

    def setUp(self):
        unittest.TestCase.setUp(self)
        self.mcTree1 = None
        self.mcTree1a = None
        self.mcTree2 = None
        self.__generateTrees()

    def testSanity(self):
        parser = NXNewick()
        mcTree1 = MultiCactusTree(parser.parseString(self.tree1, addImpliedRoots = False))
        tree1String = NXNewick().writeString(mcTree1)
        assert tree1String == self.tree1
        mcTree2 = MultiCactusTree(parser.parseString(self.tree2, addImpliedRoots = False), subtreeSize = 3)
        tree2String = NXNewick().writeString(mcTree2)
        assert tree2String == self.tree2

    def testSubtrees(self):
        roots1 = ["Anc0", "Anc1", "Anc2", "Anc3", "Anc4", "Anc5", "Anc6", "Anc7"]
        roots1a = ["Anc0", "Anc2", "Anc3", "Anc5", "Anc6"]
        roots2 = ["Anc0", "Anc1", "Anc2", "Anc4", "Anc5"]

        subTree1_a6 = '(Anc7:0.025291,BABOON:0.044568)Anc6;'
        subTree1a_a0 = '((Anc6:0.11,Anc5:0.260342)Anc4:0.02326,(Anc3:0.087381,Anc2:0.104728)Anc1:0.04)Anc0;'
        subTree2_a3 = '(monkey:100.8593,cat:47.14069)Anc3;'

        trueRoots = [roots1, roots1a, roots2]
        trueSubtrees = [subTree1_a6, subTree1a_a0, subTree2_a3]
        trees = [self.mcTree1, self.mcTree1a, self.mcTree2]
        ancs = ["Anc6", "Anc0", "Anc3"]

        for i in range(0, 3):
            roots = trees[i].getSubtreeRootNames()
            assert sorted(roots) == sorted(trueRoots[i])
            subtree = trees[i].extractSubTree(ancs[i])
            subtree = NXNewick().writeString(subtree)
            assert subtree == trueSubtrees[i]

    def testAddSelf(self):
        trueSelf = '((((((COW:0.06)COW_self:0.06,(PIG:0.06)PIG_self:0.06)Anc2:0.104728)Anc2_self:0.104728,(((CAT:0.07)CAT_self:0.07,(DOG:0.07)DOG_self:0.07)Anc3:0.087381)Anc3_self:0.087381)Anc1:0.04)Anc1_self:0.04,(((((RAT:0.081244)RAT_self:0.081244,(MOUSE:0.072818)MOUSE_self:0.072818)Anc5:0.260342)Anc5_self:0.260342,(((BABOON:0.044568)BABOON_self:0.044568,(((CHIMP:0.009727)CHIMP_self:0.009727,(HUMAN:0.006969)HUMAN_self:0.006969)Anc7:0.025291)Anc7_self:0.025291)Anc6:0.11)Anc6_self:0.11)Anc4:0.02326)Anc4_self:0.02326)Anc0;'
        tree = MultiCactusTree(self.mcTree1)
        tree.nameUnlabeledInternalNodes()
        tree.computeSubtreeRoots()
        tree.addSelfEdges()
        treeString = NXNewick().writeString(tree)
        assert treeString == trueSelf

    def testAddOutgroup(self):
        trueOg = '((((HUMAN:0.006969,CHIMP:0.009727)Anc7:0.025291,BABOON:0.044568)Anc6:0.11,(MOUSE:0.072818,RAT:0.081244)Anc5:0.260342)Anc4:0.02326,((DOG:0.07,CAT:0.07)Anc3:0.087381,(PIG:0.06,COW:0.06)Anc2:0.104728)Anc1:0.04,outgroup:1.7)Anc0;'
        tree = MultiCactusTree(self.mcTree1)
        tree.nameUnlabeledInternalNodes()
        tree.computeSubtreeRoots()
        tree.addOutgroup("outgroup", 1.7)
        treeString = NXNewick().writeString(tree)
        assert treeString == trueOg

        trueLeafOg = "(A:1.1,outgroup:1.1);"
        leafTreeString = "A;"
        parser = NXNewick()
        leafTree = MultiCactusTree(parser.parseString(leafTreeString, addImpliedRoots = False))
        leafTree.nameUnlabeledInternalNodes()
        leafTree.computeSubtreeRoots()
        leafTree.addOutgroup("outgroup", 2.2)
        leafTreeOutString = NXNewick().writeString(leafTree)
        assert leafTreeOutString == trueLeafOg

    def __generateTrees(self):
        self.tree1 = '((((HUMAN:0.006969,CHIMP:0.009727):0.025291,BABOON:0.044568):0.11,(MOUSE:0.072818,RAT:0.081244):0.260342):0.02326,((DOG:0.07,CAT:0.07):0.087381,(PIG:0.06,COW:0.06):0.104728):0.04);'
        self.tree2 = '((raccoon:19.19959,bear:6.80041):0.846,((sea_lion:11.997,seal:12.003):7.52973,((monkey:100.8593,cat:47.14069):20.59201,weasel:18.87953):2.0946):3.87382,dog:25.46154);'
        parser = NXNewick()
        self.mcTree1 = MultiCactusTree(parser.parseString(self.tree1, addImpliedRoots = False))
        self.mcTree1a = MultiCactusTree(parser.parseString(self.tree1, addImpliedRoots = False), subtreeSize = 4)
        self.mcTree2 = MultiCactusTree(parser.parseString(self.tree2, addImpliedRoots = False), subtreeSize = 3)
        self.mcTree1.nameUnlabeledInternalNodes()
        self.mcTree1a.nameUnlabeledInternalNodes()
        self.mcTree2.nameUnlabeledInternalNodes()
        self.mcTree1.computeSubtreeRoots()
        self.mcTree1a.computeSubtreeRoots()
        self.mcTree2.computeSubtreeRoots()

def main():
    unittest.main()

if __name__ == '__main__':
    main()
<commit_msg>A couple tests for spanning-tree extraction<commit_after>#!/usr/bin/env python

#Copyright (C) 2011 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
"""
"""

import unittest
import os
import sys

from sonLib.bioio import TestStatus
from sonLib.bioio import getTempDirectory
from sonLib.bioio import logger
from sonLib.bioio import system

from cactus.progressive.multiCactusTree import MultiCactusTree
from sonLib.nxnewick import NXNewick

class TestCase(unittest.TestCase):

    def setUp(self):
        unittest.TestCase.setUp(self)
        self.mcTree1 = None
        self.mcTree1a = None
        self.mcTree2 = None
        self.__generateTrees()

    def testSanity(self):
        parser = NXNewick()
        mcTree1 = MultiCactusTree(parser.parseString(self.tree1, addImpliedRoots = False))
        tree1String = NXNewick().writeString(mcTree1)
        assert tree1String == self.tree1
        mcTree2 = MultiCactusTree(parser.parseString(self.tree2, addImpliedRoots = False), subtreeSize = 3)
        tree2String = NXNewick().writeString(mcTree2)
        assert tree2String == self.tree2

    def testSubtrees(self):
        roots1 = ["Anc0", "Anc1", "Anc2", "Anc3", "Anc4", "Anc5", "Anc6", "Anc7"]
        roots1a = ["Anc0", "Anc2", "Anc3", "Anc5", "Anc6"]
        roots2 = ["Anc0", "Anc1", "Anc2", "Anc4", "Anc5"]

        subTree1_a6 = '(Anc7:0.025291,BABOON:0.044568)Anc6;'
        subTree1a_a0 = '((Anc6:0.11,Anc5:0.260342)Anc4:0.02326,(Anc3:0.087381,Anc2:0.104728)Anc1:0.04)Anc0;'
        subTree2_a3 = '(monkey:100.8593,cat:47.14069)Anc3;'

        trueRoots = [roots1, roots1a, roots2]
        trueSubtrees = [subTree1_a6, subTree1a_a0, subTree2_a3]
        trees = [self.mcTree1, self.mcTree1a, self.mcTree2]
        ancs = ["Anc6", "Anc0", "Anc3"]

        for i in range(0, 3):
            roots = trees[i].getSubtreeRootNames()
            assert sorted(roots) == sorted(trueRoots[i])
            subtree = trees[i].extractSubTree(ancs[i])
            subtree = NXNewick().writeString(subtree)
            assert subtree == trueSubtrees[i]

    def testAddSelf(self):
        trueSelf = '((((((COW:0.06)COW_self:0.06,(PIG:0.06)PIG_self:0.06)Anc2:0.104728)Anc2_self:0.104728,(((CAT:0.07)CAT_self:0.07,(DOG:0.07)DOG_self:0.07)Anc3:0.087381)Anc3_self:0.087381)Anc1:0.04)Anc1_self:0.04,(((((RAT:0.081244)RAT_self:0.081244,(MOUSE:0.072818)MOUSE_self:0.072818)Anc5:0.260342)Anc5_self:0.260342,(((BABOON:0.044568)BABOON_self:0.044568,(((CHIMP:0.009727)CHIMP_self:0.009727,(HUMAN:0.006969)HUMAN_self:0.006969)Anc7:0.025291)Anc7_self:0.025291)Anc6:0.11)Anc6_self:0.11)Anc4:0.02326)Anc4_self:0.02326)Anc0;'
        tree = MultiCactusTree(self.mcTree1)
        tree.nameUnlabeledInternalNodes()
        tree.computeSubtreeRoots()
        tree.addSelfEdges()
        treeString = NXNewick().writeString(tree)
        assert treeString == trueSelf

    def testAddOutgroup(self):
        trueOg = '((((HUMAN:0.006969,CHIMP:0.009727)Anc7:0.025291,BABOON:0.044568)Anc6:0.11,(MOUSE:0.072818,RAT:0.081244)Anc5:0.260342)Anc4:0.02326,((DOG:0.07,CAT:0.07)Anc3:0.087381,(PIG:0.06,COW:0.06)Anc2:0.104728)Anc1:0.04,outgroup:1.7)Anc0;'
        tree = MultiCactusTree(self.mcTree1)
        tree.nameUnlabeledInternalNodes()
        tree.computeSubtreeRoots()
        tree.addOutgroup("outgroup", 1.7)
        treeString = NXNewick().writeString(tree)
        assert treeString == trueOg

        trueLeafOg = "(A:1.1,outgroup:1.1);"
        leafTreeString = "A;"
        parser = NXNewick()
        leafTree = MultiCactusTree(parser.parseString(leafTreeString, addImpliedRoots = False))
        leafTree.nameUnlabeledInternalNodes()
        leafTree.computeSubtreeRoots()
        leafTree.addOutgroup("outgroup", 2.2)
        leafTreeOutString = NXNewick().writeString(leafTree)
        assert leafTreeOutString == trueLeafOg

    def testExtractSpanningTree(self):
        """Tests whether extracting a binary spanning tree works correctly."""
        prevNewick1 = NXNewick().writeString(self.mcTree1)
        # Check a dead-simple spanning tree with 3 closely related leaves.
        spanHCB = self.mcTree1.extractSpanningTree(["HUMAN", "CHIMP", "BABOON"])
        # Check that the existing tree hasn't been modified (OK, a bit
        # silly, but just in case).
        self.assertEqual(NXNewick().writeString(self.mcTree1), prevNewick1)
        # Check the actual spanning tree.
        self.assertEqual(NXNewick().writeString(spanHCB), "((HUMAN:0.006969,CHIMP:0.009727)Anc7:0.025291,BABOON:0.044568)Anc6;")

        # Now test a more complicated tree, where we should remove as
        # many of the ancestors as possible (they will add extra
        # losses for no reason!).
        spanHCC = self.mcTree1.extractSpanningTree(["HUMAN", "CHIMP", "CAT"])
        self.assertEqual(NXNewick().writeString(self.mcTree1), prevNewick1)
        self.assertEqual(NXNewick().writeString(spanHCC), "((HUMAN:0.006969,CHIMP:0.009727)Anc7:0.158551,CAT:0.197381)Anc0;")

        # TODO: get rid of the binary-only restriction at some point.
        # Finally, test getting a spanning tree that must be binarized
        # by adding another leaf.
        spanHAnc3C = self.mcTree1.extractSpanningTree(["HUMAN", "Anc3", "CAT"])
        self.assertEqual(NXNewick().writeString(self.mcTree1), prevNewick1)
        self.assertEqual(NXNewick().writeString(spanHAnc3C), "(HUMAN:0.16552,(DOG:0.07,CAT:0.07)Anc3:0.127381)Anc0;")

    def __generateTrees(self):
        self.tree1 = '((((HUMAN:0.006969,CHIMP:0.009727):0.025291,BABOON:0.044568):0.11,(MOUSE:0.072818,RAT:0.081244):0.260342):0.02326,((DOG:0.07,CAT:0.07):0.087381,(PIG:0.06,COW:0.06):0.104728):0.04);'
        self.tree2 = '((raccoon:19.19959,bear:6.80041):0.846,((sea_lion:11.997,seal:12.003):7.52973,((monkey:100.8593,cat:47.14069):20.59201,weasel:18.87953):2.0946):3.87382,dog:25.46154);'
        parser = NXNewick()
        self.mcTree1 = MultiCactusTree(parser.parseString(self.tree1, addImpliedRoots = False))
        self.mcTree1a = MultiCactusTree(parser.parseString(self.tree1, addImpliedRoots = False), subtreeSize = 4)
        self.mcTree2 = MultiCactusTree(parser.parseString(self.tree2, addImpliedRoots = False), subtreeSize = 3)
        self.mcTree1.nameUnlabeledInternalNodes()
        self.mcTree1a.nameUnlabeledInternalNodes()
        self.mcTree2.nameUnlabeledInternalNodes()
        self.mcTree1.computeSubtreeRoots()
        self.mcTree1a.computeSubtreeRoots()
        self.mcTree2.computeSubtreeRoots()

def main():
    unittest.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import logging
import os
import hmac
from hashlib import sha1
import json
import re
import boto3

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)

def lambda_handler(event, context):
    logger.info(json.dumps(event, indent=4))
    try:
        # Get the sha_name and sig from the headers
        sha_name, sig = event['headers']['X-Hub-Signature'].split('=')
        # Get the body of the message
        body = event['body']
        # Only support sha1
        if sha_name != 'sha1':
            return {
                "statusCode": 501,
                "body": "Something is wrong, unsupported sha type?\n"
            }
    except Exception as e:
        logger.exception(e)
        return {
            "statusCode": 501,
            "body": "Something is wrong, not from GitHub?\n"
        }

    secret = os.environ['GHSECRET']
    # HMAC requires the key to be bytes, but data is string
    mac = hmac.new(secret, body, sha1)
    if not hmac.compare_digest(unicode(mac.hexdigest()), unicode(sig)):
        logger.critical("Signature doesn't match")
        return {
            'body': "Signature doesn't match",
            'statusCode': 403
            }

    githookbody = json.loads(body)
    logger.debug(json.dumps(githookbody, indent=4))
    # some sanity checking, only support pushes
    if event['headers']['X-GitHub-Event'] != "push":
        logger.critical("hook event is not supported")
        return {
            'body': "hook event is not supported",
            'statusCode': 501
            }

    repo_url = githookbody['repository']['url'] + ".git"
    username = githookbody['repository']['owner']['name']
    builds_list = []
    for i in githookbody['commits']:
        for a in i['added']:
            if re.search("/", a):
                builds_list.append(a.split("/")[0])
        for m in i['modified']:
            if re.search("/", m):
                builds_list.append(m.split("/")[0])
    # This is slow but don't care for our list size
    # Removes dupes to prevent spawning duplicate jobs
    builds = sorted(set(builds_list),key=builds_list.index)

    # Spawn the CodeBuild Job
    lambdac = boto3.client('lambda')
    message_input = {
        'repo_url': repo_url,
        'builds': builds,
        'username': username,
        }
    print message_input
    if builds: # False if empty
        response = lambdac.invoke(
            FunctionName=os.environ['SpawnCodeBuildFunctionArn'],
            InvocationType='Event', # async
            LogType='None',
            Payload=json.dumps(message_input)
        )
    else:
        logger.info("Not spawning a codebuild job")

    # Everything is good
    return {
        "statusCode": 200,
        "body": "worked"
    }
<commit_msg>Exit early for private repos, not supported<commit_after>import logging
import os
import hmac
from hashlib import sha1
import json
import re
import boto3

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)

def lambda_handler(event, context):
    logger.info(json.dumps(event, indent=4))
    try:
        # Get the sha_name and sig from the headers
        sha_name, sig = event['headers']['X-Hub-Signature'].split('=')
        # Get the body of the message
        body = event['body']
        # Only support sha1
        if sha_name != 'sha1':
            return {
                "statusCode": 501,
                "body": "Something is wrong, unsupported sha type?\n"
            }
    except Exception as e:
        logger.exception(e)
        return {
            "statusCode": 501,
            "body": "Something is wrong, not from GitHub?\n"
        }

    secret = os.environ['GHSECRET']
    # HMAC requires the key to be bytes, but data is string
    mac = hmac.new(secret, body, sha1)
    if not hmac.compare_digest(unicode(mac.hexdigest()), unicode(sig)):
        logger.critical("Signature doesn't match")
        return {
            'body': "Signature doesn't match",
            'statusCode': 403
            }

    githookbody = json.loads(body)
    logger.debug(json.dumps(githookbody, indent=4))
    # some sanity checking, only support pushes
    if event['headers']['X-GitHub-Event'] != "push":
        logger.critical("hook event is not supported")
        return {
            'body': "hook event is not supported",
            'statusCode': 501
            }

    if githookbody['repository']['private']: # bool
        logger.info("Event Accepted but Private Repos are not supported")
        return {
            'body': "Event Accepted but Private Repos are not supported",
            'statusCode': 200
            }

    repo_url = githookbody['repository']['url'] + ".git"
    username = githookbody['repository']['owner']['name']
    builds_list = []
    for i in githookbody['commits']:
        for a in i['added']:
            if re.search("/", a):
                builds_list.append(a.split("/")[0])
        for m in i['modified']:
            if re.search("/", m):
                builds_list.append(m.split("/")[0])
    # This is slow but don't care for our list size
    # Removes dupes to prevent spawning duplicate jobs
    builds = sorted(set(builds_list),key=builds_list.index)

    # Spawn the CodeBuild Job
    lambdac = boto3.client('lambda')
    message_input = {
        'repo_url': repo_url,
        'builds': builds,
        'username': username,
        }
    print message_input
    if builds: # False if empty
        response = lambdac.invoke(
            FunctionName=os.environ['SpawnCodeBuildFunctionArn'],
            InvocationType='Event', # async
            LogType='None',
            Payload=json.dumps(message_input)
        )
    else:
        logger.info("Not spawning a codebuild job due to input/commit")

    # Everything is good
    return {
        "statusCode": 200,
        "body": "worked"
    }
<|endoftext|>


<commit_before>from llvmlite import binding as ll
from llvmlite import ir
from warnings import warn

from numba.core import config, serialize
from numba.core.codegen import Codegen, CodeLibrary
from numba.core.errors import NumbaInvalidConfigWarning
from .cudadrv import devices, driver, nvvm

import ctypes
import numpy as np
import os
import subprocess
import tempfile

CUDA_TRIPLE = 'nvptx64-nvidia-cuda'

def disassemble_cubin(cubin):
    # nvdisasm only accepts input from a file, so we need to write out to a
    # temp file and clean up afterwards.
    fd = None
    fname = None
    try:
        fd, fname = tempfile.mkstemp()
        with open(fname, 'wb') as f:
            f.write(cubin)

        try:
            cp = subprocess.run(['nvdisasm', fname], check=True,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
        except FileNotFoundError as e:
            if e.filename == 'nvdisasm':
                msg = ("nvdisasm is required for SASS inspection, and has not "
                       "been found.\n\nYou may need to install the CUDA "
                       "toolkit and ensure that it is available on your "
                       "PATH.\n")
                raise RuntimeError(msg)
        return cp.stdout.decode('utf-8')
    finally:
        if fd is not None:
            os.close(fd)
        if fname is not None:
            os.unlink(fname)

class CUDACodeLibrary(serialize.ReduceMixin, CodeLibrary):
    """
    The CUDACodeLibrary generates PTX, SASS, cubins for multiple different
    compute capabilities. It also loads cubins to multiple devices (via
    get_cufunc), which may be of different compute capabilities.
    """

    def __init__(self, codegen, name, entry_name=None, max_registers=None,
                 nvvm_options=None):
        """
        codegen:
            Codegen object.
        name:
            Name of the function in the source.
        entry_name:
            Name of the kernel function in the binary, if this is a global
            kernel and not a device function.
        max_registers:
            The maximum register usage to aim for when linking.
        nvvm_options:
                Dict of options to pass to NVVM.
        """
        super().__init__(codegen, name)

        # The llvmlite module for this library.
        self._module = None
        # CodeLibrary objects that will be "linked" into this library. The
        # modules within them are compiled from NVVM IR to PTX along with the
        # IR from this module - in that sense they are "linked" by NVVM at PTX
        # generation time, rather than at link time.
        self._linking_libraries = set()
        # Files to link with the generated PTX. These are linked using the
        # Driver API at link time.
        self._linking_files = set()

        # Maps CC -> PTX string
        self._ptx_cache = {}
        # Maps CC -> cubin
        self._cubin_cache = {}
        # Maps CC -> linker info output for cubin
        self._linkerinfo_cache = {}
        # Maps Device numeric ID -> cufunc
        self._cufunc_cache = {}

        self._max_registers = max_registers
        if nvvm_options is None:
            nvvm_options = {}
        self._nvvm_options = nvvm_options
        self._entry_name = entry_name

    def get_llvm_str(self):
        return str(self._module)

    def get_asm_str(self, cc=None):
        return self._join_ptxes(self._get_ptxes(cc=cc))

    def _get_ptxes(self, cc=None):
        if not cc:
            ctx = devices.get_context()
            device = ctx.device
            cc = device.compute_capability

        ptxes = self._ptx_cache.get(cc, None)
        if ptxes:
            return ptxes

        arch = nvvm.get_arch_option(*cc)
        options = self._nvvm_options.copy()
        options['arch'] = arch
        if not nvvm.NVVM().is_nvvm70:
            # Avoid enabling debug for NVVM 3.4 as it has various issues. We
            # need to warn the user that we're doing this if any of the
            # functions that they're compiling have `debug=True` set, which we
            # can determine by checking the NVVM options.
            for lib in self.linking_libraries:
                if lib._nvvm_options.get('debug'):
                    msg = ("debuginfo is not generated for CUDA versions "
                           f"< 11.2 (debug=True on {lib.name})")
                    warn(NumbaInvalidConfigWarning(msg))
            options['debug'] = False

        irs = [str(mod) for mod in self.modules]

        if options.get('debug', False):
            # If we're compiling with debug, we need to compile modules with
            # NVVM one at a time, because it does not support multiple modules
            # with debug enabled:
            # https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#source-level-debugging-support
            ptxes = [nvvm.llvm_to_ptx(ir, **options) for ir in irs]
        else:
            # Otherwise, we compile all modules with NVVM at once because this
            # results in better optimization than separate compilation.
            ptxes = [nvvm.llvm_to_ptx(irs, **options)]

        # Sometimes the result from NVVM contains trailing whitespace and
        # nulls, which we strip so that the assembly dump looks a little
        # tidier.
        ptxes = [x.decode().strip('\x00').strip() for x in ptxes]

        if config.DUMP_ASSEMBLY:
            print(("ASSEMBLY %s" % self._name).center(80, '-'))
            print(self._join_ptxes(ptxes))
            print('=' * 80)

        self._ptx_cache[cc] = ptxes

        return ptxes

    def _join_ptxes(self, ptxes):
        return "\n\n".join(ptxes)

    def get_cubin(self, cc=None):
        if cc is None:
            ctx = devices.get_context()
            device = ctx.device
            cc = device.compute_capability

        cubin = self._cubin_cache.get(cc, None)
        if cubin:
            return cubin

        linker = driver.Linker(max_registers=self._max_registers, cc=cc)

        ptxes = self._get_ptxes(cc=cc)
        for ptx in ptxes:
            linker.add_ptx(ptx.encode())
        for path in self._linking_files:
            linker.add_file_guess_ext(path)

        cubin_buf, size = linker.complete()

        # We take a copy of the cubin because it's owned by the linker
        cubin_ptr = ctypes.cast(cubin_buf, ctypes.POINTER(ctypes.c_char))
        cubin = bytes(np.ctypeslib.as_array(cubin_ptr, shape=(size,)))
        self._cubin_cache[cc] = cubin
        self._linkerinfo_cache[cc] = linker.info_log

        return cubin

    def get_cufunc(self):
        if self._entry_name is None:
            msg = "Missing entry_name - are you trying to get the cufunc " \
                  "for a device function?"
            raise RuntimeError(msg)

        ctx = devices.get_context()
        device = ctx.device

        cufunc = self._cufunc_cache.get(device.id, None)
        if cufunc:
            return cufunc

        cubin = self.get_cubin(cc=device.compute_capability)
        module = ctx.create_module_image(cubin)

        # Load
        cufunc = module.get_function(self._entry_name)

        # Populate caches
        self._cufunc_cache[device.id] = cufunc

        return cufunc

    def get_linkerinfo(self, cc):
        try:
            return self._linkerinfo_cache[cc]
        except KeyError:
            raise KeyError(f'No linkerinfo for CC {cc}')

    def get_sass(self, cc=None):
        return disassemble_cubin(self.get_cubin(cc=cc))

    def add_ir_module(self, mod):
        self._raise_if_finalized()
        if self._module is not None:
            raise RuntimeError('CUDACodeLibrary only supports one module')
        self._module = mod

    def add_linking_library(self, library):
        library._ensure_finalized()

        # We don't want to allow linking more libraries in after finalization
        # because our linked libraries are modified by the finalization, and we
        # won't be able to finalize again after adding new ones
        self._raise_if_finalized()

        self._linking_libraries.add(library)

    def add_linking_file(self, filepath):
        self._linking_files.add(filepath)

    def get_function(self, name):
        for fn in self._module.functions:
            if fn.name == name:
                return fn
        raise KeyError(f'Function {name} not found')

    @property
    def modules(self):
        return [self._module] + [mod for lib in self._linking_libraries
                                 for mod in lib.modules]

    @property
    def linking_libraries(self):
        # Libraries we link to may link to other libraries, so we recursively
        # traverse the linking libraries property to build up a list of all
        # linked libraries.
        libs = []
        for lib in self._linking_libraries:
            libs.extend(lib.linking_libraries)
            libs.append(lib)
        return libs

    def finalize(self):
        # Unlike the CPUCodeLibrary, we don't invoke the binding layer here -
        # we only adjust the linkage of functions. Global kernels (with
        # external linkage) have their linkage untouched. Device functions are
        # set linkonce_odr to prevent them appearing in the PTX.

        self._raise_if_finalized()

        # Note in-place modification of the linkage of functions in linked
        # libraries. This presently causes no issues as only device functions
        # are shared across code libraries, so they would always need their
        # linkage set to linkonce_odr. If in a future scenario some code
        # libraries require linkonce_odr linkage of functions in linked
        # modules, and another code library requires another linkage, each code
        # library will need to take its own private copy of its linked modules.
        #
        # See also discussion on PR #890:
        # https://github.com/numba/numba/pull/890
        #
        # We don't adjust the linkage of functions when compiling for debug -
        # because the device functions are in separate modules, we need them to
        # be externally visible.
        for library in self._linking_libraries:
            for mod in library.modules:
                for fn in mod.functions:
                    if not fn.is_declaration:
                        if self._nvvm_options.get('debug', False):
                            fn.linkage = 'weak_odr'
                        else:
                            fn.linkage = 'linkonce_odr'

        self._finalized = True

    def _reduce_states(self):
        """
        Reduce the instance for serialization. We retain the PTX and cubins,
        but loaded functions are discarded. They are recreated when needed
        after deserialization.
        """
        if self._linking_files:
            msg = ('cannot pickle CUDACodeLibrary function with additional '
                   'libraries to link against')
            raise RuntimeError(msg)
        return dict(
            codegen=self._codegen,
            name=self.name,
            entry_name=self._entry_name,
            module=self._module,
            linking_libraries=self._linking_libraries,
            ptx_cache=self._ptx_cache,
            cubin_cache=self._cubin_cache,
            linkerinfo_cache=self._linkerinfo_cache,
            max_registers=self._max_registers,
            nvvm_options=self._nvvm_options
        )

    @classmethod
    def _rebuild(cls, codegen, name, entry_name, module, linking_libraries,
                 ptx_cache, cubin_cache, linkerinfo_cache, max_registers,
                 nvvm_options):
        """
        Rebuild an instance.
        """
        instance = cls.__new__(cls)
        super(cls, instance).__init__(codegen, name)
        instance._entry_name = entry_name

        instance._module = module
        instance._linking_libraries = linking_libraries
        instance._linking_files = set()

        instance._ptx_cache = ptx_cache
        instance._cubin_cache = cubin_cache
        instance._linkerinfo_cache = linkerinfo_cache
        instance._cufunc_cache = {}

        instance._max_registers = max_registers
        instance._nvvm_options = nvvm_options

class JITCUDACodegen(Codegen):
    """
    This codegen implementation for CUDA only generates optimized LLVM IR.
    Generation of PTX code is done separately (see numba.cuda.compiler).
    """

    _library_class = CUDACodeLibrary

    def __init__(self, module_name):
        self._data_layout = nvvm.default_data_layout
        self._target_data = ll.create_target_data(self._data_layout)

    def _create_empty_module(self, name):
        ir_module = ir.Module(name)
        ir_module.triple = CUDA_TRIPLE
        if self._data_layout:
            ir_module.data_layout = self._data_layout
        nvvm.add_ir_version(ir_module)
        return ir_module

    def _add_module(self, module):
        pass
<commit_msg>Add "function" to debug=True warning (PR #7162 feedback)<commit_after>from llvmlite import binding as ll
from llvmlite import ir
from warnings import warn

from numba.core import config, serialize
from numba.core.codegen import Codegen, CodeLibrary
from numba.core.errors import NumbaInvalidConfigWarning
from .cudadrv import devices, driver, nvvm

import ctypes
import numpy as np
import os
import subprocess
import tempfile

CUDA_TRIPLE = 'nvptx64-nvidia-cuda'

def disassemble_cubin(cubin):
    # nvdisasm only accepts input from a file, so we need to write out to a
    # temp file and clean up afterwards.
    fd = None
    fname = None
    try:
        fd, fname = tempfile.mkstemp()
        with open(fname, 'wb') as f:
            f.write(cubin)

        try:
            cp = subprocess.run(['nvdisasm', fname], check=True,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
        except FileNotFoundError as e:
            if e.filename == 'nvdisasm':
                msg = ("nvdisasm is required for SASS inspection, and has not "
                       "been found.\n\nYou may need to install the CUDA "
                       "toolkit and ensure that it is available on your "
                       "PATH.\n")
                raise RuntimeError(msg)
        return cp.stdout.decode('utf-8')
    finally:
        if fd is not None:
            os.close(fd)
        if fname is not None:
            os.unlink(fname)

class CUDACodeLibrary(serialize.ReduceMixin, CodeLibrary):
    """
    The CUDACodeLibrary generates PTX, SASS, cubins for multiple different
    compute capabilities. It also loads cubins to multiple devices (via
    get_cufunc), which may be of different compute capabilities.
    """

    def __init__(self, codegen, name, entry_name=None, max_registers=None,
                 nvvm_options=None):
        """
        codegen:
            Codegen object.
        name:
            Name of the function in the source.
        entry_name:
            Name of the kernel function in the binary, if this is a global
            kernel and not a device function.
        max_registers:
            The maximum register usage to aim for when linking.
        nvvm_options:
                Dict of options to pass to NVVM.
        """
        super().__init__(codegen, name)

        # The llvmlite module for this library.
        self._module = None
        # CodeLibrary objects that will be "linked" into this library. The
        # modules within them are compiled from NVVM IR to PTX along with the
        # IR from this module - in that sense they are "linked" by NVVM at PTX
        # generation time, rather than at link time.
        self._linking_libraries = set()
        # Files to link with the generated PTX. These are linked using the
        # Driver API at link time.
        self._linking_files = set()

        # Maps CC -> PTX string
        self._ptx_cache = {}
        # Maps CC -> cubin
        self._cubin_cache = {}
        # Maps CC -> linker info output for cubin
        self._linkerinfo_cache = {}
        # Maps Device numeric ID -> cufunc
        self._cufunc_cache = {}

        self._max_registers = max_registers
        if nvvm_options is None:
            nvvm_options = {}
        self._nvvm_options = nvvm_options
        self._entry_name = entry_name

    def get_llvm_str(self):
        return str(self._module)

    def get_asm_str(self, cc=None):
        return self._join_ptxes(self._get_ptxes(cc=cc))

    def _get_ptxes(self, cc=None):
        if not cc:
            ctx = devices.get_context()
            device = ctx.device
            cc = device.compute_capability

        ptxes = self._ptx_cache.get(cc, None)
        if ptxes:
            return ptxes

        arch = nvvm.get_arch_option(*cc)
        options = self._nvvm_options.copy()
        options['arch'] = arch
        if not nvvm.NVVM().is_nvvm70:
            # Avoid enabling debug for NVVM 3.4 as it has various issues. We
            # need to warn the user that we're doing this if any of the
            # functions that they're compiling have `debug=True` set, which we
            # can determine by checking the NVVM options.
            for lib in self.linking_libraries:
                if lib._nvvm_options.get('debug'):
                    msg = ("debuginfo is not generated for CUDA versions "
                           f"< 11.2 (debug=True on function: {lib.name})")
                    warn(NumbaInvalidConfigWarning(msg))
            options['debug'] = False

        irs = [str(mod) for mod in self.modules]

        if options.get('debug', False):
            # If we're compiling with debug, we need to compile modules with
            # NVVM one at a time, because it does not support multiple modules
            # with debug enabled:
            # https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#source-level-debugging-support
            ptxes = [nvvm.llvm_to_ptx(ir, **options) for ir in irs]
        else:
            # Otherwise, we compile all modules with NVVM at once because this
            # results in better optimization than separate compilation.
            ptxes = [nvvm.llvm_to_ptx(irs, **options)]

        # Sometimes the result from NVVM contains trailing whitespace and
        # nulls, which we strip so that the assembly dump looks a little
        # tidier.
        ptxes = [x.decode().strip('\x00').strip() for x in ptxes]

        if config.DUMP_ASSEMBLY:
            print(("ASSEMBLY %s" % self._name).center(80, '-'))
            print(self._join_ptxes(ptxes))
            print('=' * 80)

        self._ptx_cache[cc] = ptxes

        return ptxes

    def _join_ptxes(self, ptxes):
        return "\n\n".join(ptxes)

    def get_cubin(self, cc=None):
        if cc is None:
            ctx = devices.get_context()
            device = ctx.device
            cc = device.compute_capability

        cubin = self._cubin_cache.get(cc, None)
        if cubin:
            return cubin

        linker = driver.Linker(max_registers=self._max_registers, cc=cc)

        ptxes = self._get_ptxes(cc=cc)
        for ptx in ptxes:
            linker.add_ptx(ptx.encode())
        for path in self._linking_files:
            linker.add_file_guess_ext(path)

        cubin_buf, size = linker.complete()

        # We take a copy of the cubin because it's owned by the linker
        cubin_ptr = ctypes.cast(cubin_buf, ctypes.POINTER(ctypes.c_char))
        cubin = bytes(np.ctypeslib.as_array(cubin_ptr, shape=(size,)))
        self._cubin_cache[cc] = cubin
        self._linkerinfo_cache[cc] = linker.info_log

        return cubin

    def get_cufunc(self):
        if self._entry_name is None:
            msg = "Missing entry_name - are you trying to get the cufunc " \
                  "for a device function?"
            raise RuntimeError(msg)

        ctx = devices.get_context()
        device = ctx.device

        cufunc = self._cufunc_cache.get(device.id, None)
        if cufunc:
            return cufunc

        cubin = self.get_cubin(cc=device.compute_capability)
        module = ctx.create_module_image(cubin)

        # Load
        cufunc = module.get_function(self._entry_name)

        # Populate caches
        self._cufunc_cache[device.id] = cufunc

        return cufunc

    def get_linkerinfo(self, cc):
        try:
            return self._linkerinfo_cache[cc]
        except KeyError:
            raise KeyError(f'No linkerinfo for CC {cc}')

    def get_sass(self, cc=None):
        return disassemble_cubin(self.get_cubin(cc=cc))

    def add_ir_module(self, mod):
        self._raise_if_finalized()
        if self._module is not None:
            raise RuntimeError('CUDACodeLibrary only supports one module')
        self._module = mod

    def add_linking_library(self, library):
        library._ensure_finalized()

        # We don't want to allow linking more libraries in after finalization
        # because our linked libraries are modified by the finalization, and we
        # won't be able to finalize again after adding new ones
        self._raise_if_finalized()

        self._linking_libraries.add(library)

    def add_linking_file(self, filepath):
        self._linking_files.add(filepath)

    def get_function(self, name):
        for fn in self._module.functions:
            if fn.name == name:
                return fn
        raise KeyError(f'Function {name} not found')

    @property
    def modules(self):
        return [self._module] + [mod for lib in self._linking_libraries
                                 for mod in lib.modules]

    @property
    def linking_libraries(self):
        # Libraries we link to may link to other libraries, so we recursively
        # traverse the linking libraries property to build up a list of all
        # linked libraries.
        libs = []
        for lib in self._linking_libraries:
            libs.extend(lib.linking_libraries)
            libs.append(lib)
        return libs

    def finalize(self):
        # Unlike the CPUCodeLibrary, we don't invoke the binding layer here -
        # we only adjust the linkage of functions. Global kernels (with
        # external linkage) have their linkage untouched. Device functions are
        # set linkonce_odr to prevent them appearing in the PTX.

        self._raise_if_finalized()

        # Note in-place modification of the linkage of functions in linked
        # libraries. This presently causes no issues as only device functions
        # are shared across code libraries, so they would always need their
        # linkage set to linkonce_odr. If in a future scenario some code
        # libraries require linkonce_odr linkage of functions in linked
        # modules, and another code library requires another linkage, each code
        # library will need to take its own private copy of its linked modules.
        #
        # See also discussion on PR #890:
        # https://github.com/numba/numba/pull/890
        #
        # We don't adjust the linkage of functions when compiling for debug -
        # because the device functions are in separate modules, we need them to
        # be externally visible.
        for library in self._linking_libraries:
            for mod in library.modules:
                for fn in mod.functions:
                    if not fn.is_declaration:
                        if self._nvvm_options.get('debug', False):
                            fn.linkage = 'weak_odr'
                        else:
                            fn.linkage = 'linkonce_odr'

        self._finalized = True

    def _reduce_states(self):
        """
        Reduce the instance for serialization. We retain the PTX and cubins,
        but loaded functions are discarded. They are recreated when needed
        after deserialization.
        """
        if self._linking_files:
            msg = ('cannot pickle CUDACodeLibrary function with additional '
                   'libraries to link against')
            raise RuntimeError(msg)
        return dict(
            codegen=self._codegen,
            name=self.name,
            entry_name=self._entry_name,
            module=self._module,
            linking_libraries=self._linking_libraries,
            ptx_cache=self._ptx_cache,
            cubin_cache=self._cubin_cache,
            linkerinfo_cache=self._linkerinfo_cache,
            max_registers=self._max_registers,
            nvvm_options=self._nvvm_options
        )

    @classmethod
    def _rebuild(cls, codegen, name, entry_name, module, linking_libraries,
                 ptx_cache, cubin_cache, linkerinfo_cache, max_registers,
                 nvvm_options):
        """
        Rebuild an instance.
        """
        instance = cls.__new__(cls)
        super(cls, instance).__init__(codegen, name)
        instance._entry_name = entry_name

        instance._module = module
        instance._linking_libraries = linking_libraries
        instance._linking_files = set()

        instance._ptx_cache = ptx_cache
        instance._cubin_cache = cubin_cache
        instance._linkerinfo_cache = linkerinfo_cache
        instance._cufunc_cache = {}

        instance._max_registers = max_registers
        instance._nvvm_options = nvvm_options

class JITCUDACodegen(Codegen):
    """
    This codegen implementation for CUDA only generates optimized LLVM IR.
    Generation of PTX code is done separately (see numba.cuda.compiler).
    """

    _library_class = CUDACodeLibrary

    def __init__(self, module_name):
        self._data_layout = nvvm.default_data_layout
        self._target_data = ll.create_target_data(self._data_layout)

    def _create_empty_module(self, name):
        ir_module = ir.Module(name)
        ir_module.triple = CUDA_TRIPLE
        if self._data_layout:
            ir_module.data_layout = self._data_layout
        nvvm.add_ir_version(ir_module)
        return ir_module

    def _add_module(self, module):
        pass
<|endoftext|>


<commit_before>"""
Alexa Skill for Getting
Navy Federal Credit Union
Account Data
"""
from __future__ import print_function
import nfcu

def build_speechlet_response(title, output, reprompt_text, should_end_session):
    """
    Build alexa speechlet response
    """
    return {
        'outputSpeech': {
            'type': 'PlainText',
            'text': output
        },
        'card': {
            'type': 'Simple',
            'title': "SessionSpeechlet - " + title,
            'content': "SessionSpeechlet - " + output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'PlainText',
                'text': reprompt_text
            }
        },
        'shouldEndSession': should_end_session
    }

def build_response(session_attributes, speechlet_response):
    """
    Build response for Alexa
    """
    return {
        'version': '1.0',
        'sessionAttributes': session_attributes,
        'response': speechlet_response
    }

def get_welcome_response():
    """
    Set welcome message from alexa
    """
    session_attributes = {}
    card_title = "Welcome"
    speech_output = "Do you want to check your balance?"

    # If the user either does not reply to the
    # welcome message or says something
    # that is not understood, they will be prompted
    # again with this text.
    reprompt_text = "Do you want to check your balance?"

    should_end_session = True
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_account_summary():
    """
    Get all balances
    """
    session_attributes = {}
    card_title = "AccountSummary"

    # Calculate Total in All Accounts
    access_number, password = 1, 2

    api = nfcu.NFCU(access_number, password)
    data = api.get_account_summary()

    total = 0
    for item in data["accountSummary"]["data"]["accountCategories"]:
        total += item["totalBalance"]

    speech_output = "Your total account balance is ${total}".format(
        total=total)

    reprompt_text = speech_output
    should_end_session = True

    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_specific_account_balance():
    """
    Get specific account balance
    """
    session_attributes = {}
    card_title = "AccountDetailedSummary"

    # Calculate Total in All Accounts
    access_number, password = 1, 2

    api = nfcu.NFCU(access_number, password)
    data = api.get_account_summary()
    total = 0
    for item in data["accountSummary"]["data"]["accountCategories"]:
        total += item["totalBalance"]

    speech_output = "Your total account balance is ${total}".format(
        total=total)

    reprompt_text = speech_output
    should_end_session = True

    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def handle_session_end_request():
    """
    Set alexa response on close app
    """
    card_title = "Session Ended"
    speech_output = "Bye now."

    # Setting this to true ends the session and exits the skill.
    should_end_session = True
    return build_response({}, build_speechlet_response(
        card_title, speech_output, None, should_end_session))

# --------------- Events ------------------
def on_session_started(session_started_request, session):
    """
    Called when the session starts
    """

    print("on_session_started requestId=" +
          session_started_request['requestId'] +
          ", sessionId=" + session['sessionId'])

def on_launch(launch_request, session):
    """
    Called when the user launches the skill
    without specifying what they want
    """

    print("on_launch requestId=" + launch_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # Dispatch to your skill's launch
    return get_welcome_response()

def on_intent(intent_request, session):
    """
    Called when the user specifies an intent for this skill
    """

    print("on_intent requestId=" + intent_request['requestId'] +
          ", sessionId=" + session['sessionId'])

    intent_name = intent_request['intent']['name']

    # Dispatch to your skill's intent handlers
    if intent_name == "GetAccountSummary":
        return get_account_summary()
    elif intent_name == "AMAZON.HelpIntent":
        return get_welcome_response()
    elif intent_name == "AMAZON.CancelIntent" or \
            intent_name == "AMAZON.StopIntent":
        return handle_session_end_request()
    else:
        raise ValueError("Invalid intent")

def on_session_ended(session_ended_request, session):
    """
    Called when the user ends the session.
    Is not called when the skill
    :return: should_end_session=true
    """
    print("on_session_ended requestId=" + session_ended_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # add cleanup logic here

# --------------- Main handler ------------------
def lambda_handler(event):
    """
    Route the incoming request based on type
    (LaunchRequest, IntentRequest, etc.)
    The JSON body of the request is provided in the event parameter.
    """
    print("event.session.application.applicationId=" +
          event['session']['application']['applicationId'])

    """
    Uncomment this if statement and populate with
    your skill's application ID to prevent someone
    else from configuring a skill that sends requests to this
    function.
    """
    if event['session']['new']:
        on_session_started(
            {
                'requestId': event['request']['requestId']
            },
            event['session']
        )

    if event['request']['type'] == "LaunchRequest":
        return on_launch(event['request'], event['session'])
    elif event['request']['type'] == "IntentRequest":
        return on_intent(event['request'], event['session'])
    elif event['request']['type'] == "SessionEndedRequest":
        return on_session_ended(event['request'], event['session'])
<commit_msg>rm comment<commit_after>"""
Alexa Skill for Getting
Navy Federal Credit Union
Account Data
"""
from __future__ import print_function
import nfcu

def build_speechlet_response(title, output, reprompt_text, should_end_session):
    """
    Build alexa speechlet response
    """
    return {
        'outputSpeech': {
            'type': 'PlainText',
            'text': output
        },
        'card': {
            'type': 'Simple',
            'title': "SessionSpeechlet - " + title,
            'content': "SessionSpeechlet - " + output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'PlainText',
                'text': reprompt_text
            }
        },
        'shouldEndSession': should_end_session
    }

def build_response(session_attributes, speechlet_response):
    """
    Build response for Alexa
    """
    return {
        'version': '1.0',
        'sessionAttributes': session_attributes,
        'response': speechlet_response
    }

def get_welcome_response():
    """
    Set welcome message from alexa
    """
    session_attributes = {}
    card_title = "Welcome"
    speech_output = "Do you want to check your balance?"

    # If the user either does not reply to the
    # welcome message or says something
    # that is not understood, they will be prompted
    # again with this text.
    reprompt_text = "Do you want to check your balance?"

    should_end_session = True
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_account_summary():
    """
    Get all balances
    """
    session_attributes = {}
    card_title = "AccountSummary"

    # Calculate Total in All Accounts
    access_number, password = 1, 2

    api = nfcu.NFCU(access_number, password)
    data = api.get_account_summary()

    total = 0
    for item in data["accountSummary"]["data"]["accountCategories"]:
        total += item["totalBalance"]

    speech_output = "Your total account balance is ${total}".format(
        total=total)

    reprompt_text = speech_output
    should_end_session = True

    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_specific_account_balance():
    """
    Get specific account balance
    """
    session_attributes = {}
    card_title = "AccountDetailedSummary"

    # Calculate Total in All Accounts
    access_number, password = 1, 2

    api = nfcu.NFCU(access_number, password)
    data = api.get_account_summary()
    total = 0
    for item in data["accountSummary"]["data"]["accountCategories"]:
        total += item["totalBalance"]

    speech_output = "Your total account balance is ${total}".format(
        total=total)

    reprompt_text = speech_output
    should_end_session = True

    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def handle_session_end_request():
    """
    Set alexa response on close app
    """
    card_title = "Session Ended"
    speech_output = "Bye now."

    # Setting this to true ends the session and exits the skill.
    should_end_session = True
    return build_response({}, build_speechlet_response(
        card_title, speech_output, None, should_end_session))

# --------------- Events ------------------
def on_session_started(session_started_request, session):
    """
    Called when the session starts
    """

    print("on_session_started requestId=" +
          session_started_request['requestId'] +
          ", sessionId=" + session['sessionId'])

def on_launch(launch_request, session):
    """
    Called when the user launches the skill
    without specifying what they want
    """

    print("on_launch requestId=" + launch_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # Dispatch to your skill's launch
    return get_welcome_response()

def on_intent(intent_request, session):
    """
    Called when the user specifies an intent for this skill
    """

    print("on_intent requestId=" + intent_request['requestId'] +
          ", sessionId=" + session['sessionId'])

    intent_name = intent_request['intent']['name']

    # Dispatch to your skill's intent handlers
    if intent_name == "GetAccountSummary":
        return get_account_summary()
    elif intent_name == "AMAZON.HelpIntent":
        return get_welcome_response()
    elif intent_name == "AMAZON.CancelIntent" or \
            intent_name == "AMAZON.StopIntent":
        return handle_session_end_request()
    else:
        raise ValueError("Invalid intent")

def on_session_ended(session_ended_request, session):
    """
    Called when the user ends the session.
    Is not called when the skill
    :return: should_end_session=true
    """
    print("on_session_ended requestId=" + session_ended_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # add cleanup logic here

# --------------- Main handler ------------------
def lambda_handler(event):
    """
    Route the incoming request based on type
    (LaunchRequest, IntentRequest, etc.)
    The JSON body of the request is provided in the event parameter.
    """
    print("event.session.application.applicationId=" +
          event['session']['application']['applicationId'])

    if event['session']['new']:
        on_session_started(
            {
                'requestId': event['request']['requestId']
            },
            event['session']
        )

    if event['request']['type'] == "LaunchRequest":
        return on_launch(event['request'], event['session'])
    elif event['request']['type'] == "IntentRequest":
        return on_intent(event['request'], event['session'])
    elif event['request']['type'] == "SessionEndedRequest":
        return on_session_ended(event['request'], event['session'])
<|endoftext|>


<commit_before>#!/usr/bin/env python

# USGS Landsat Imagery Util
#
#
# Author: developmentseed
# Contributer: scisco, KAPPS-
#
# License: CC0 1.0 Universal

from __future__ import print_function
import sys
import subprocess
import argparse
import textwrap
import json

from dateutil.parser import parse

from gs_helper import GsHelper
from clipper_helper import Clipper
from search_helper import Search
from general_helper import reformat_date, Verbosity
from image_helper import Process
import settings

DESCRIPTION = """Landsat-util is a command line utility that makes it easy to
search, download, and process Landsat imagery.

    Commands:
        Search:
            landsat.py search [-h] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [--imageprocess]
                         {pr,shapefile,country}

            positional arguments:
                {pr,shapefile,country}
                                    Search commands
                pr                  Activate paths and rows
                shapefile           Activate Shapefile
                country             Activate country

                optional arguments:
                -h, --help            show this help message and exit
                -l LIMIT, --limit LIMIT
                                    Search return results limit default is 100

                -s START, --start START
                                    Start Date - Most formats are accepted e.g.
                                    Jun 12 2014 OR 06/12/2014

                -e END, --end END   End Date - Most formats are accepted e.g.
                                    Jun 12 2014 OR 06/12/2014

                -c CLOUD, --cloud CLOUD
                                    Maximum cloud percentage default is 20 perct

                -d, --download        Use this flag to download found images

                --imageprocess      If this flag is used, the images are downloaded
                                    and process. Be cautious as it might take a
                                    long time to both download and process large
                                    batches of images

                --pansharpen        Whether to also pansharpen the process image.
                                    Pansharpening takes a long time

        Download:
            landsat download [-h] sceneID [sceneID ...]

            positional arguments:
                sceneID     Provide Full sceneID, e.g. LC81660392014196LGN00

        Process:
            landsat.py process [-h] [--pansharpen] path

            positional arguments:
                path          Path to the compressed image file

            optional arguments:
                --pansharpen  Whether to also pansharpen the process image.
                              Pansharpening takes a long time

                -v, --verbose
"""

def args_options():
    parser = argparse.ArgumentParser(prog='landsat',
                                     formatter_class=argparse.RawDescriptionHelpFormatter,
                                     description=textwrap.dedent(DESCRIPTION))

    subparsers = parser.add_subparsers(help='Landsat Utility',
                                       dest='subs')

    # Search Logic
    parser_search = subparsers.add_parser('search',
                                          help='Search Landsat metdata')

    # Global search options
    parser_search.add_argument('-l', '--limit', default=100, type=int,
                               help='Search return results limit\n'
                               'default is 100')
    parser_search.add_argument('-s', '--start',
                               help='Start Date - Most formats are accepted '
                               'e.g. Jun 12 2014 OR 06/12/2014')
    parser_search.add_argument('-e', '--end',
                               help='End Date - Most formats are accepted '
                               'e.g. Jun 12 2014 OR 06/12/2014')
    parser_search.add_argument('-c', '--cloud', type=float, default=20.0,
                               help='Maximum cloud percentage '
                               'default is 20 perct')
    parser_search.add_argument('-d', '--download', action='store_true',
                               help='Use this flag to download found images')
    parser_search.add_argument('--imageprocess', action='store_true',
                               help='If this flag is used, the images are '
                               'downloaded and process. Be cautious as it '
                               'might take a long time to both download and '
                               'process large batches of images')
    parser_search.add_argument('--pansharpen', action='store_true',
                               help='Whether to also pansharpen the process '
                               'image. Pan sharpening takes a long time')

    search_subparsers = parser_search.add_subparsers(help='Search commands',
                                                     dest='search_subs')

    search_pr = search_subparsers.add_parser('pr',
                                             help="Activate paths and rows")
    search_pr.add_argument('paths_rows',
                           metavar='path_row',
                           type=int,
                           nargs="+",
                           help="Provide paths and rows")

    search_shapefile = search_subparsers.add_parser('shapefile',
                                                    help="Activate Shapefile")
    search_shapefile.add_argument('path',
                                  help="Path to shapefile")

    search_country = search_subparsers.add_parser('country',
                                                  help="Activate country")
    search_country.add_argument('name', help="Country name e.g. ARE")

    parser_download = subparsers.add_parser('download',
                                            help='Download images from Google Storage')
    parser_download.add_argument('scenes',
                                 metavar='sceneID',
                                 nargs="+",
                                 help="Provide Full sceneID, e.g. "
                                 "LC81660392014196LGN00")

    parser_process = subparsers.add_parser('process',
                                           help='Process Landsat imagery')
    parser_process.add_argument('path',
                                help='Path to the compressed image file')
    parser_process.add_argument('--pansharpen', action='store_true',
                                help='Whether to also pansharpen the process '
                                'image. Pan sharpening takes a long time')
    parser_process.add_argument('-v', '--verbose', action='store_true',
                                help='Turn on verbosity')

    return parser

def main(args):
    """
    Main function - launches the program
    """

    v = Verbosity(False)

    if args:
        if args.subs == 'process':
            verbose = True if args.verbose else False
            p = Process(args.path, verbose=verbose)
            if args.pansharpen:
                p.full_with_pansharpening()
            else:
                p.full()

            exit("The output is stored at %s." % settings.PROCESSED_IMAGE)

        elif args.subs == 'search':

            try:
                if args.start:
                    args.start = reformat_date(parse(args.start))

                if args.end:
                    args.end = reformat_date(parse(args.end))
            except TypeError:
                exit("You date format is incorrect. Please try again!", 1)

            s = Search()
            if args.search_subs == 'pr':
                result = s.search(row_paths=args.paths_rows,
                                  limit=args.limit,
                                  start_date=args.start,
                                  end_date=args.end,
                                  cloud_max=args.cloud)

            elif args.search_subs == 'shapefile':
                clipper = Clipper()
                prs = clipper.shapefile(args.path)
                if prs:
                    result = s.search(prs,
                                      limit=args.limit,
                                      start_date=args.start,
                                      end_date=args.end,
                                      cloud_max=args.cloud)
                else:
                    result = {'status': 'error',
                              'message': 'There was a problem reading the shapefile!'}
            elif args.search_subs == 'country':
                clipper = Clipper()
                prs = clipper.country(args.name)
                if prs:
                    result = s.search(prs,
                                      limit=args.limit,
                                      start_date=args.start,
                                      end_date=args.end,
                                      cloud_max=args.cloud)
            try:
                if result['status'] == 'SUCCESS':
                    v.output('%s items were found' % result['total'], normal=True, arrow=True)
                    if result['total'] > 100:
                        exit('Too many results. Please narrow your search')
                    else:
                        v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green')
                    # If only search
                    if args.download:
                        gs = GsHelper()
                        v.output('Starting the download:', normal=True, arrow=True)
                        for item in result['results']:
                            gs.single_download(row=item['row'],
                                               path=item['path'],
                                               name=item['sceneID'])
                        v.output("%s images were downloaded"
                                 % result['total_returned'], normal=True, arrow=True)
                        if args.imageprocess:
                            for item in result['results']:
                                p = Process('%s/%s.tar.bz' % (gs.zip_dir,
                                                              item['sceneID']))
                                if args.pansharpen:
                                    p.full_with_pansharpening()
                                else:
                                    p.full()
                        else:
                            exit("The downloaded images are located here: %s" %
                                 gs.zip_dir)
                    else:
                        exit('Search completed!')
                elif result['status'] == 'error':
                    exit(result['message'], 1)
            except KeyError:
                exit('Too Many API queries. You can only query DevSeed\'s '
                     'API 5 times per minute', 1)
        elif args.subs == 'download':
            gs = GsHelper()
            v.output('Starting the download:', normal=True, arrow=True)
            for scene in args.scenes:
                if gs.single_download(row=gs.extract_row_path(scene)[1],
                                      path=gs.extract_row_path(scene)[0],
                                      name=scene):
                    exit("Downloaded images are located here: %s" % gs.zip_dir)
                else:
                    exit("Download error!", 1)

def exit(message, code=0):

    v = Verbosity(True)
    if code == 0:
        v.output(message, arrow=True)
        v.output('Done!', arrow=True)
    else:
        v.output(message, error=True)
    sys.exit(code)

def package_installed(package):
    """
    Check if a package is installed on the machine
    """

    v = Verbosity(True)

    v.output("Checking if %s is installed on the system" % package, arrow=True)
    installed = not subprocess.call(["which", package])
    if installed:
        v.output("%s is installed" % package)
        return True
    else:
        v.output("You have to install %s first!" % package, error=True)
        return False

def __main__():

    global parser
    parser = args_options()
    args = parser.parse_args()
    main(args)

if __name__ == "__main__":
    try:
        __main__()
    except KeyboardInterrupt:
        exit('Received Ctrl + C... Exiting! Bye.', 1)
    # except:
    #     exit('Unexpected Error: %s' % (sys.exc_info()[0]), 1)
<commit_msg>catch error if zip file is corrupted<commit_after>#!/usr/bin/env python

# USGS Landsat Imagery Util
#
#
# Author: developmentseed
# Contributer: scisco, KAPPS-
#
# License: CC0 1.0 Universal

from __future__ import print_function
import sys
import subprocess
import argparse
import textwrap
import json

from dateutil.parser import parse

from gs_helper import GsHelper
from clipper_helper import Clipper
from search_helper import Search
from general_helper import reformat_date, Verbosity
from image_helper import Process
import settings

DESCRIPTION = """Landsat-util is a command line utility that makes it easy to
search, download, and process Landsat imagery.

    Commands:
        Search:
            landsat.py search [-h] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [--imageprocess]
                         {pr,shapefile,country}

            positional arguments:
                {pr,shapefile,country}
                                    Search commands
                pr                  Activate paths and rows
                shapefile           Activate Shapefile
                country             Activate country

                optional arguments:
                -h, --help            show this help message and exit
                -l LIMIT, --limit LIMIT
                                    Search return results limit default is 100

                -s START, --start START
                                    Start Date - Most formats are accepted e.g.
                                    Jun 12 2014 OR 06/12/2014

                -e END, --end END   End Date - Most formats are accepted e.g.
                                    Jun 12 2014 OR 06/12/2014

                -c CLOUD, --cloud CLOUD
                                    Maximum cloud percentage default is 20 perct

                -d, --download        Use this flag to download found images

                --imageprocess      If this flag is used, the images are downloaded
                                    and process. Be cautious as it might take a
                                    long time to both download and process large
                                    batches of images

                --pansharpen        Whether to also pansharpen the process image.
                                    Pansharpening takes a long time

        Download:
            landsat download [-h] sceneID [sceneID ...]

            positional arguments:
                sceneID     Provide Full sceneID, e.g. LC81660392014196LGN00

        Process:
            landsat.py process [-h] [--pansharpen] path

            positional arguments:
                path          Path to the compressed image file

            optional arguments:
                --pansharpen  Whether to also pansharpen the process image.
                              Pansharpening takes a long time

                -v, --verbose
"""

def args_options():
    parser = argparse.ArgumentParser(prog='landsat',
                                     formatter_class=argparse.RawDescriptionHelpFormatter,
                                     description=textwrap.dedent(DESCRIPTION))

    subparsers = parser.add_subparsers(help='Landsat Utility',
                                       dest='subs')

    # Search Logic
    parser_search = subparsers.add_parser('search',
                                          help='Search Landsat metdata')

    # Global search options
    parser_search.add_argument('-l', '--limit', default=100, type=int,
                               help='Search return results limit\n'
                               'default is 100')
    parser_search.add_argument('-s', '--start',
                               help='Start Date - Most formats are accepted '
                               'e.g. Jun 12 2014 OR 06/12/2014')
    parser_search.add_argument('-e', '--end',
                               help='End Date - Most formats are accepted '
                               'e.g. Jun 12 2014 OR 06/12/2014')
    parser_search.add_argument('-c', '--cloud', type=float, default=20.0,
                               help='Maximum cloud percentage '
                               'default is 20 perct')
    parser_search.add_argument('-d', '--download', action='store_true',
                               help='Use this flag to download found images')
    parser_search.add_argument('--imageprocess', action='store_true',
                               help='If this flag is used, the images are '
                               'downloaded and process. Be cautious as it '
                               'might take a long time to both download and '
                               'process large batches of images')
    parser_search.add_argument('--pansharpen', action='store_true',
                               help='Whether to also pansharpen the process '
                               'image. Pan sharpening takes a long time')

    search_subparsers = parser_search.add_subparsers(help='Search commands',
                                                     dest='search_subs')

    search_pr = search_subparsers.add_parser('pr',
                                             help="Activate paths and rows")
    search_pr.add_argument('paths_rows',
                           metavar='path_row',
                           type=int,
                           nargs="+",
                           help="Provide paths and rows")

    search_shapefile = search_subparsers.add_parser('shapefile',
                                                    help="Activate Shapefile")
    search_shapefile.add_argument('path',
                                  help="Path to shapefile")

    search_country = search_subparsers.add_parser('country',
                                                  help="Activate country")
    search_country.add_argument('name', help="Country name e.g. ARE")

    parser_download = subparsers.add_parser('download',
                                            help='Download images from Google Storage')
    parser_download.add_argument('scenes',
                                 metavar='sceneID',
                                 nargs="+",
                                 help="Provide Full sceneID, e.g. "
                                 "LC81660392014196LGN00")

    parser_process = subparsers.add_parser('process',
                                           help='Process Landsat imagery')
    parser_process.add_argument('path',
                                help='Path to the compressed image file')
    parser_process.add_argument('--pansharpen', action='store_true',
                                help='Whether to also pansharpen the process '
                                'image. Pan sharpening takes a long time')
    parser_process.add_argument('-v', '--verbose', action='store_true',
                                help='Turn on verbosity')

    return parser

def main(args):
    """
    Main function - launches the program
    """

    v = Verbosity(False)

    if args:
        if args.subs == 'process':
            verbose = True if args.verbose else False
            try:
                p = Process(args.path, verbose=verbose)
            except IOError:
                exit("Zip file corrupted", 1)
            if args.pansharpen:
                p.full_with_pansharpening()
            else:
                p.full()

            exit("The output is stored at %s." % settings.PROCESSED_IMAGE)

        elif args.subs == 'search':

            try:
                if args.start:
                    args.start = reformat_date(parse(args.start))

                if args.end:
                    args.end = reformat_date(parse(args.end))
            except TypeError:
                exit("You date format is incorrect. Please try again!", 1)

            s = Search()
            if args.search_subs == 'pr':
                result = s.search(row_paths=args.paths_rows,
                                  limit=args.limit,
                                  start_date=args.start,
                                  end_date=args.end,
                                  cloud_max=args.cloud)

            elif args.search_subs == 'shapefile':
                clipper = Clipper()
                prs = clipper.shapefile(args.path)
                if prs:
                    result = s.search(prs,
                                      limit=args.limit,
                                      start_date=args.start,
                                      end_date=args.end,
                                      cloud_max=args.cloud)
                else:
                    result = {'status': 'error',
                              'message': 'There was a problem reading the shapefile!'}
            elif args.search_subs == 'country':
                clipper = Clipper()
                prs = clipper.country(args.name)
                if prs:
                    result = s.search(prs,
                                      limit=args.limit,
                                      start_date=args.start,
                                      end_date=args.end,
                                      cloud_max=args.cloud)
            try:
                if result['status'] == 'SUCCESS':
                    v.output('%s items were found' % result['total'], normal=True, arrow=True)
                    if result['total'] > 100:
                        exit('Too many results. Please narrow your search')
                    else:
                        v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green')
                    # If only search
                    if args.download:
                        gs = GsHelper()
                        v.output('Starting the download:', normal=True, arrow=True)
                        for item in result['results']:
                            gs.single_download(row=item['row'],
                                               path=item['path'],
                                               name=item['sceneID'])
                        v.output("%s images were downloaded"
                                 % result['total_returned'], normal=True, arrow=True)
                        if args.imageprocess:
                            for item in result['results']:
                                p = Process('%s/%s.tar.bz' % (gs.zip_dir,
                                                              item['sceneID']))
                                if args.pansharpen:
                                    p.full_with_pansharpening()
                                else:
                                    p.full()
                        else:
                            exit("The downloaded images are located here: %s" %
                                 gs.zip_dir)
                    else:
                        exit('Search completed!')
                elif result['status'] == 'error':
                    exit(result['message'], 1)
            except KeyError:
                exit('Too Many API queries. You can only query DevSeed\'s '
                     'API 5 times per minute', 1)
        elif args.subs == 'download':
            gs = GsHelper()
            v.output('Starting the download:', normal=True, arrow=True)
            for scene in args.scenes:
                if gs.single_download(row=gs.extract_row_path(scene)[1],
                                      path=gs.extract_row_path(scene)[0],
                                      name=scene):
                    exit("Downloaded images are located here: %s" % gs.zip_dir)
                else:
                    exit("Download error!", 1)

def exit(message, code=0):

    v = Verbosity(True)
    if code == 0:
        v.output(message, arrow=True)
        v.output('Done!', arrow=True)
    else:
        v.output(message, error=True)
    sys.exit(code)

def package_installed(package):
    """
    Check if a package is installed on the machine
    """

    v = Verbosity(True)

    v.output("Checking if %s is installed on the system" % package, arrow=True)
    installed = not subprocess.call(["which", package])
    if installed:
        v.output("%s is installed" % package)
        return True
    else:
        v.output("You have to install %s first!" % package, error=True)
        return False

def __main__():

    global parser
    parser = args_options()
    args = parser.parse_args()
    main(args)

if __name__ == "__main__":
    try:
        __main__()
    except KeyboardInterrupt:
        exit('Received Ctrl + C... Exiting! Bye.', 1)
    # except:
    #     exit('Unexpected Error: %s' % (sys.exc_info()[0]), 1)
<|endoftext|>