<commit_before>"""
This module is the new command of bowl.

Created on 15 March 2014
@author: Charlie Lewis
"""

import curses
import docker
import importlib
import inspect
import os
import uuid
from bowl.containers import oses

# !! TODO this needs to be refactored when there are multiple versions
from bowl.containers.ubuntu.precise.databases import databases
from bowl.containers.ubuntu.precise.environment import environment
from bowl.containers.ubuntu.precise.services import services

class new(object):
    """
    This class is responsible for the new command of the cli.
    """
    @staticmethod
    def build_dockerfile(self, dockerfile):
        c = docker.Client(base_url='tcp://localhost:4243', version='1.9',
                          timeout=10)
        uuid_dir = str(uuid.uuid4())

        # try/catch
        os.mkdir('/tmp/'+uuid_dir)
        with open('/tmp/'+uuid_dir+'/Dockerfile', 'w') as f:
            for line in dockerfile:
                f.write(line+'\n')
        c.build(tag="bowl-"+uuid_dir, quiet=False, path='/tmp/'+uuid_dir,
                nocache=False, rm=False, stream=False)
        os.remove('/tmp/'+uuid_dir+'/Dockerfile')
        os.rmdir('/tmp/'+uuid_dir)
        return c, uuid_dir

    @staticmethod
    def run_dockerfile(self, c, image_tag):
        container = c.create_container(image_tag)
        c.start(container, publish_all_ports=True)

    @staticmethod
    def build_options(self):
        menu_dict = {
         'title': "Build your bowl",
         'type': "menu",
         'subtitle': "Please select a choice...",
         'options': [
          {
           'title': "Build Container",
           'type': "menu",
           'subtitle': "Please select an Operating System...",
           'options': []
          },
          {
           'title': "Choose Image",
           'type': "menu",
           'subtitle': "Please select an Image...",
           'options': []
          }
         ]
        }
        os_dict = {}
        base = "bowl.containers."
        os_list = inspect.getmembers(oses.oses, predicate=inspect.ismethod)
        for item in os_list:
            package = base + item[0]
            os_dict[item[0]+'_versions'] = getattr(__import__(package,
                                                              fromlist=['versions']),
                                                              'versions')
            menu_dict['options'][0]['options'].append(getattr(oses.oses(), item[0])())
        os_num = 0
        for key,os_inst in os_dict.iteritems():
            version_list = inspect.getmembers(os_inst.versions,
                                              predicate=inspect.ismethod)
            version_num = 0
            for version in version_list:
                menu_dict['options'][0]['options'][os_num]['options'].append(getattr(os_inst.versions(), version[0])())

                os_name = key.split("_", 1)
                package = base + os_name[0] + "." + version[0] + "."

                database_dict = {
                 'title': "Databases",
                 'type': "menu",
                 'subtitle': "Please select databases...",
                 'options': []
                }
                db_package = package + "databases"
                db_inst = getattr(__import__(db_package, fromlist=['databases']), 'databases')
                database_list = inspect.getmembers(db_inst.databases, predicate=inspect.ismethod)
                for database in database_list:
                    database_dict['options'].append(getattr(db_inst.databases(), database[0])())

                environment_dict = {
                 'title': "Environment Tools",
                 'type': "menu",
                 'subtitle': "Please select environment tools...",
                 'options': []
                }
                env_package = package + "environment"
                env_inst = getattr(__import__(env_package, fromlist=['environment']), 'environment')
                environment_list = inspect.getmembers(env_inst.environment, predicate=inspect.ismethod)
                for environ in environment_list:
                    environment_dict['options'].append(getattr(env_inst.environment(), environ[0])())

                service_dict = {
                 'title': "Services",
                 'type': "menu",
                 'subtitle': "Please select services...",
                 'options': []
                }
                serv_package = package + "services"
                serv_inst = getattr(__import__(serv_package, fromlist=['services']), 'services')
                service_list = inspect.getmembers(serv_inst.services, predicate=inspect.ismethod)
                for service in service_list:
                    service_dict['options'].append(getattr(serv_inst.services(), service[0])())

                launch_dict = {
                 'title': "Launch Container",
                 'type': "launch"
                }

                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(database_dict)
                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(environment_dict)
                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(service_dict)
                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(launch_dict)
                version_num += 1
            os_num += 1
        return menu_dict

    @classmethod
    def main(self, args):
        # build dictionary of available container options
        menu_dict = self.build_options(self)

        self.win = curses.initscr()
        self.win.keypad(1)
        self.build_dict = {}
        self.build_dict['services'] = []
        self.launch = False

        # init curses stuff
        curses.noecho()
        curses.cbreak()
        curses.start_color()
        curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_WHITE)

        self.process_menu(self, menu_dict)
        curses.endwin()
        # !! TODO use this to build the dockerfile
        # !! TODO if no services were seleted, don't create a container
        # !! TODO if contains an ADD line, be sure and copy additional files
        this_dir, this_filename = os.path.split(__file__)
        services = self.build_dict['services']
        dockerfile = []
        for service in services:
            # !! error check that the array is this size
            service_name = service.split(':', 3)
            os_flavor = "/".join(service_name[0:3])
            path = os.path.join(os.path.dirname(this_dir),
                                "containers/"+os_flavor+"/dockerfiles/"+service_name[3]+"/Dockerfile")
            with open(path, 'r') as f:
                for line in f:
                    # remove duplicate lines
                    if line.strip() not in dockerfile:
                        # combine EXPOSE commands
                        if line.startswith("EXPOSE"):
                            if any(cmd.startswith("EXPOSE") for cmd in dockerfile):
                                line = ' '.join(line.strip().split(' ', 1)[1:])
                                # !! TODO make sure this is what the command starts with
                                # !! TODO display a warning to the user if there is overlapping ports
                                matching = [s for s in dockerfile if "EXPOSE" in s]
                                matching.append(line)
                                dockerfile.remove(matching[0])
                                dockerfile.append(' '.join(matching))
                            else:
                                dockerfile.append(line.strip())
                        # !! TODO
                        #    check for multiple CMD commands
                        #    check for multiple ENTRYPOINT commands
                        #    check for multiple USER commands
                        #    check WORKDIR since CMD and ENTRYPOINT are modified
                        else:
                            dockerfile.append(line.strip())

        c, uuid_dir = self.build_dockerfile(self, dockerfile)
        self.run_dockerfile(self, c, 'bowl-'+uuid_dir)

    @staticmethod
    def display_menu(self, menu, parent):
        if parent is None:
            back = "Exit"
        else:
            back = "Return to %s menu" % parent['title']

        option_size = len(menu['options'])
        position = 0
        key = None
        highlighted = curses.color_pair(1)
        normal = curses.A_NORMAL
        choice = [" "]*(option_size+1)

        while key != ord('\n'):
            self.win.clear()
            self.win.border(0)
            self.win.addstr(2,2, menu['title'], curses.A_STANDOUT)
            self.win.addstr(4,2, menu['subtitle'], curses.A_BOLD)

            for index in range(option_size):
                textstyle = normal
                if position == index:
                    textstyle = highlighted
                if menu['options'][index]['type'] == "choice_menu":
                    self.win.addstr(index+5, 4, "%d - [%s] %s" % (index+1, choice[index], menu['options'][index]['title']), textstyle)
                elif menu['options'][index]['type'] == "launch":
                    self.win.addstr(index+6, 4, "%d - %s" % (index+1, menu['options'][index]['title']), textstyle)
                else:
                    self.win.addstr(index+5, 4, "%d - %s" % (index+1, menu['options'][index]['title']), textstyle)
            textstyle = normal
            if position == option_size:
                textstyle = highlighted
            self.win.addstr(option_size+6, 4, "%d - %s" % (option_size+1, back), textstyle)
            self.win.refresh()

            key = self.win.getch()
            try:
                if key >= ord('1') and key <= ord(str(option_size+1)):
                    position = key - ord('0') - 1
                elif key == 258:
                    if position < option_size:
                        position += 1
                    else:
                        position = 0
                elif key == 259:
                    if position > 0:
                        position += -1
                    else:
                        position = option_size
                # !! TODO error check this!!
                elif key == 32 and menu['options'][position]['type'] == "choice_menu":
                    if choice[position] == " ":
                        choice[position] = "x"
                        # add to build_dict
                        self.build_dict['services'].append(menu['options'][position]['command'])
                    else:
                        choice[position] = " "
                        # remove from build_dict
                        # TODO check if exists first
                        self.build_dict['services'].remove(menu['options'][position]['command'])
                elif key != ord('\n'):
                    curses.flash()
            except:
                curses.flash()
        return position

    @staticmethod
    def process_menu(self, menu, parent=None):
        option_size = len(menu['options'])
        exit_menu = False
        while not exit_menu and not self.launch:
            position = self.display_menu(self, menu, parent)
            if position == option_size:
                exit_menu = True
            elif menu['options'][position]['type'] == "menu":
                self.process_menu(self, menu['options'][position], menu)
            elif menu['options'][position]['type'] == "launch":
                self.launch = True
<commit_msg>stub out commands that need modification in dockerfile<commit_after>"""
This module is the new command of bowl.

Created on 15 March 2014
@author: Charlie Lewis
"""

import curses
import docker
import importlib
import inspect
import os
import uuid
from bowl.containers import oses

# !! TODO this needs to be refactored when there are multiple versions
from bowl.containers.ubuntu.precise.databases import databases
from bowl.containers.ubuntu.precise.environment import environment
from bowl.containers.ubuntu.precise.services import services

class new(object):
    """
    This class is responsible for the new command of the cli.
    """
    @staticmethod
    def build_dockerfile(self, dockerfile):
        c = docker.Client(base_url='tcp://localhost:4243', version='1.9',
                          timeout=10)
        uuid_dir = str(uuid.uuid4())

        # try/catch
        os.mkdir('/tmp/'+uuid_dir)
        with open('/tmp/'+uuid_dir+'/Dockerfile', 'w') as f:
            for line in dockerfile:
                f.write(line+'\n')
        c.build(tag="bowl-"+uuid_dir, quiet=False, path='/tmp/'+uuid_dir,
                nocache=False, rm=False, stream=False)
        os.remove('/tmp/'+uuid_dir+'/Dockerfile')
        os.rmdir('/tmp/'+uuid_dir)
        return c, uuid_dir

    @staticmethod
    def run_dockerfile(self, c, image_tag):
        container = c.create_container(image_tag)
        c.start(container, publish_all_ports=True)

    @staticmethod
    def build_options(self):
        menu_dict = {
         'title': "Build your bowl",
         'type': "menu",
         'subtitle': "Please select a choice...",
         'options': [
          {
           'title': "Build Container",
           'type': "menu",
           'subtitle': "Please select an Operating System...",
           'options': []
          },
          {
           'title': "Choose Image",
           'type': "menu",
           'subtitle': "Please select an Image...",
           'options': []
          }
         ]
        }
        os_dict = {}
        base = "bowl.containers."
        os_list = inspect.getmembers(oses.oses, predicate=inspect.ismethod)
        for item in os_list:
            package = base + item[0]
            os_dict[item[0]+'_versions'] = getattr(__import__(package,
                                                              fromlist=['versions']),
                                                              'versions')
            menu_dict['options'][0]['options'].append(getattr(oses.oses(), item[0])())
        os_num = 0
        for key,os_inst in os_dict.iteritems():
            version_list = inspect.getmembers(os_inst.versions,
                                              predicate=inspect.ismethod)
            version_num = 0
            for version in version_list:
                menu_dict['options'][0]['options'][os_num]['options'].append(getattr(os_inst.versions(), version[0])())

                os_name = key.split("_", 1)
                package = base + os_name[0] + "." + version[0] + "."

                database_dict = {
                 'title': "Databases",
                 'type': "menu",
                 'subtitle': "Please select databases...",
                 'options': []
                }
                db_package = package + "databases"
                db_inst = getattr(__import__(db_package, fromlist=['databases']), 'databases')
                database_list = inspect.getmembers(db_inst.databases, predicate=inspect.ismethod)
                for database in database_list:
                    database_dict['options'].append(getattr(db_inst.databases(), database[0])())

                environment_dict = {
                 'title': "Environment Tools",
                 'type': "menu",
                 'subtitle': "Please select environment tools...",
                 'options': []
                }
                env_package = package + "environment"
                env_inst = getattr(__import__(env_package, fromlist=['environment']), 'environment')
                environment_list = inspect.getmembers(env_inst.environment, predicate=inspect.ismethod)
                for environ in environment_list:
                    environment_dict['options'].append(getattr(env_inst.environment(), environ[0])())

                service_dict = {
                 'title': "Services",
                 'type': "menu",
                 'subtitle': "Please select services...",
                 'options': []
                }
                serv_package = package + "services"
                serv_inst = getattr(__import__(serv_package, fromlist=['services']), 'services')
                service_list = inspect.getmembers(serv_inst.services, predicate=inspect.ismethod)
                for service in service_list:
                    service_dict['options'].append(getattr(serv_inst.services(), service[0])())

                launch_dict = {
                 'title': "Launch Container",
                 'type': "launch"
                }

                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(database_dict)
                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(environment_dict)
                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(service_dict)
                menu_dict['options'][0]['options'][os_num]['options'][version_num]['options'].append(launch_dict)
                version_num += 1
            os_num += 1
        return menu_dict

    @classmethod
    def main(self, args):
        # build dictionary of available container options
        menu_dict = self.build_options(self)

        self.win = curses.initscr()
        self.win.keypad(1)
        self.build_dict = {}
        self.build_dict['services'] = []
        self.launch = False

        # init curses stuff
        curses.noecho()
        curses.cbreak()
        curses.start_color()
        curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_WHITE)

        self.process_menu(self, menu_dict)
        curses.endwin()
        # !! TODO use this to build the dockerfile
        # !! TODO if no services were seleted, don't create a container
        # !! TODO if contains an ADD line, be sure and copy additional files
        this_dir, this_filename = os.path.split(__file__)
        services = self.build_dict['services']
        dockerfile = []
        for service in services:
            # !! error check that the array is this size
            service_name = service.split(':', 3)
            os_flavor = "/".join(service_name[0:3])
            path = os.path.join(os.path.dirname(this_dir),
                                "containers/"+os_flavor+"/dockerfiles/"+service_name[3]+"/Dockerfile")
            with open(path, 'r') as f:
                for line in f:
                    # remove duplicate lines
                    if line.strip() not in dockerfile:
                        # combine EXPOSE commands
                        if line.startswith("EXPOSE"):
                            if any(cmd.startswith("EXPOSE") for cmd in dockerfile):
                                line = ' '.join(line.strip().split(' ', 1)[1:])
                                # !! TODO make sure this is what the command starts with
                                # !! TODO display a warning to the user if there is overlapping ports
                                matching = [s for s in dockerfile if "EXPOSE" in s]
                                matching.append(line)
                                dockerfile.remove(matching[0])
                                dockerfile.append(' '.join(matching))
                            else:
                                dockerfile.append(line.strip())
                        elif line.startswith("ADD"):
                            # !! TODO
                            # copy context directory to tmp directory for building
                            dockerfile.append(line.strip())
                        # check for multiple USER commands
                        elif line.startswith("USER"):
                            # !! TODO
                            dockerfile.append(line.strip())
                        # check for multiple ENTRYPOINT commands
                        elif line.startswith("ENTRYPOINT"):
                            # !! TODO
                            # check WORKDIR since CMD and ENTRYPOINT are modified
                            dockerfile.append(line.strip())
                        # check for multiple CMD commands
                        elif line.startswith("CMD"):
                            # !! TODO
                            # check WORKDIR since CMD and ENTRYPOINT are modified
                            dockerfile.append(line.strip())
                        else:
                            dockerfile.append(line.strip())

        c, uuid_dir = self.build_dockerfile(self, dockerfile)
        self.run_dockerfile(self, c, 'bowl-'+uuid_dir)

    @staticmethod
    def display_menu(self, menu, parent):
        if parent is None:
            back = "Exit"
        else:
            back = "Return to %s menu" % parent['title']

        option_size = len(menu['options'])
        position = 0
        key = None
        highlighted = curses.color_pair(1)
        normal = curses.A_NORMAL
        choice = [" "]*(option_size+1)

        while key != ord('\n'):
            self.win.clear()
            self.win.border(0)
            self.win.addstr(2,2, menu['title'], curses.A_STANDOUT)
            self.win.addstr(4,2, menu['subtitle'], curses.A_BOLD)

            for index in range(option_size):
                textstyle = normal
                if position == index:
                    textstyle = highlighted
                if menu['options'][index]['type'] == "choice_menu":
                    self.win.addstr(index+5, 4, "%d - [%s] %s" % (index+1, choice[index], menu['options'][index]['title']), textstyle)
                elif menu['options'][index]['type'] == "launch":
                    self.win.addstr(index+6, 4, "%d - %s" % (index+1, menu['options'][index]['title']), textstyle)
                else:
                    self.win.addstr(index+5, 4, "%d - %s" % (index+1, menu['options'][index]['title']), textstyle)
            textstyle = normal
            if position == option_size:
                textstyle = highlighted
            self.win.addstr(option_size+6, 4, "%d - %s" % (option_size+1, back), textstyle)
            self.win.refresh()

            key = self.win.getch()
            try:
                if key >= ord('1') and key <= ord(str(option_size+1)):
                    position = key - ord('0') - 1
                elif key == 258:
                    if position < option_size:
                        position += 1
                    else:
                        position = 0
                elif key == 259:
                    if position > 0:
                        position += -1
                    else:
                        position = option_size
                # !! TODO error check this!!
                elif key == 32 and menu['options'][position]['type'] == "choice_menu":
                    if choice[position] == " ":
                        choice[position] = "x"
                        # add to build_dict
                        self.build_dict['services'].append(menu['options'][position]['command'])
                    else:
                        choice[position] = " "
                        # remove from build_dict
                        # TODO check if exists first
                        self.build_dict['services'].remove(menu['options'][position]['command'])
                elif key != ord('\n'):
                    curses.flash()
            except:
                curses.flash()
        return position

    @staticmethod
    def process_menu(self, menu, parent=None):
        option_size = len(menu['options'])
        exit_menu = False
        while not exit_menu and not self.launch:
            position = self.display_menu(self, menu, parent)
            if position == option_size:
                exit_menu = True
            elif menu['options'][position]['type'] == "menu":
                self.process_menu(self, menu['options'][position], menu)
            elif menu['options'][position]['type'] == "launch":
                self.launch = True
<|endoftext|>


<commit_before># Copyright (c) 2015 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import

import random
import threading
from functools import wraps

from tornado import gen
from tornado.ioloop import IOLoop

from tchannel.errors import TChannelError
from tchannel.tornado import TChannel

from .proxy import VCRProxy

def wrap_uncaught(func=None, reraise=None):
    """Catches uncaught exceptions and raises VCRServiceErrors instead.

    :param reraise:
        Collection of exception clasess that should be re-raised as-is.
    """
    reraise = reraise or ()

    def decorator(f):

        @wraps(f)
        @gen.coroutine
        def new_f(*args, **kwargs):
            try:
                result = yield gen.maybe_future(f(*args, **kwargs))
            except Exception as e:
                if any(isinstance(e, cls) for cls in reraise):
                    # TODO maybe use traceback.format_exc to also send a
                    # traceback?
                    raise e
                raise VCRProxy.VCRServiceError(e.message)
            else:
                raise gen.Return(result)

        return new_f

    if func is not None:
        return decorator(func)
    else:
        return decorator

class VCRProxyService(object):

    def __init__(self, cassette, unpatch):
        """
        :param unpatch:
            A function returning a context manager which temporarily unpatches
            any monkey patched code so that a real request can be made.
        :param cassette:
            Cassette being played.
        """
        self.unpatch = unpatch
        self.cassette = cassette

        self.io_loop = None
        self.thread = None
        self.tchannel = None

        self._running = threading.Event()

    @wrap_uncaught(reraise=(
        VCRProxy.CannotRecordInteractionsError,
        VCRProxy.NoPeersAvailableError,
        VCRProxy.RemoteServiceError,
        VCRProxy.VCRServiceError,
    ))
    @gen.coroutine
    def send(self, request, response):
        cassette = self.cassette
        request = request.args.request

        # TODO decode requests and responses based on arg scheme into more
        # readable formats.

        # Because Thrift doesn't handle UTF-8 correctly right now
        request.serviceName = request.serviceName.decode('utf-8')
        request.endpoint = request.endpoint.decode('utf-8')

        # TODO do we care about hostport being the same?
        if cassette.can_replay(request):
            vcr_response = cassette.replay(request)
            raise gen.Return(vcr_response)

        if cassette.write_protected:
            raise VCRProxy.CannotRecordInteractionsError(
                'Could not find a matching response for request %s and the '
                'record mode %s prevents new interactions from being '
                'recorded. Your test may be performing an uenxpected '
                'request.' % (str(request), cassette.record_mode)
            )

        peers = []
        if request.hostPort:
            peers = [request.hostPort]
        else:
            peers = request.knownPeers

        if not peers:
            raise VCRProxy.NoPeersAvailableError(
                'Both, hostPort and knownPeers were empty or unset. '
                'One of them must be specified and non-empty.'
            )

        arg_scheme = VCRProxy.ArgScheme.to_name(request.argScheme).lower()

        with self.unpatch():
            # TODO propagate other request and response parameters
            # TODO might make sense to tag all VCR requests with a protocol
            # header of some kind
            response_future = self.tchannel.request(
                service=request.serviceName,
                arg_scheme=arg_scheme,
                hostport=random.choice(peers),
            ).send(
                request.endpoint,
                request.headers,
                request.body,
                headers={h.key: h.value for h in request.transportHeaders},
            )

        # Don't actually yield while everything is unpatched.
        try:
            response = yield response_future
        except TChannelError as e:
            raise VCRProxy.RemoteServiceError(
                code=e.code,
                message=e.message,
            )
        response_headers = yield response.get_header()
        response_body = yield response.get_body()

        vcr_response = VCRProxy.Response(
            response.status_code,
            response_headers,
            response_body,
        )
        cassette.record(request, vcr_response)
        raise gen.Return(vcr_response)

    @property
    def hostport(self):
        return self.tchannel.hostport

    def _run(self):
        self.io_loop = IOLoop()
        self.io_loop.make_current()

        self.tchannel = TChannel('proxy-server')
        self.tchannel.register(VCRProxy, handler=self.send)

        self.tchannel.listen()
        self._running.set()
        self.io_loop.start()

    def start(self):
        self.thread = threading.Thread(target=self._run)
        self.thread.start()

        self._running.wait(1)

    def stop(self):
        self.tchannel.close()
        self.tchannel = None

        self.io_loop.stop()
        self.io_loop = None

        self.thread.join(1)  # seconds
        self.thread = None

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args):
        self.stop()
<commit_msg>fix typo<commit_after># Copyright (c) 2015 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import

import random
import threading
from functools import wraps

from tornado import gen
from tornado.ioloop import IOLoop

from tchannel.errors import TChannelError
from tchannel.tornado import TChannel

from .proxy import VCRProxy

def wrap_uncaught(func=None, reraise=None):
    """Catches uncaught exceptions and raises VCRServiceErrors instead.

    :param reraise:
        Collection of exception clasess that should be re-raised as-is.
    """
    reraise = reraise or ()

    def decorator(f):

        @wraps(f)
        @gen.coroutine
        def new_f(*args, **kwargs):
            try:
                result = yield gen.maybe_future(f(*args, **kwargs))
            except Exception as e:
                if any(isinstance(e, cls) for cls in reraise):
                    # TODO maybe use traceback.format_exc to also send a
                    # traceback?
                    raise e
                raise VCRProxy.VCRServiceError(e.message)
            else:
                raise gen.Return(result)

        return new_f

    if func is not None:
        return decorator(func)
    else:
        return decorator

class VCRProxyService(object):

    def __init__(self, cassette, unpatch):
        """
        :param unpatch:
            A function returning a context manager which temporarily unpatches
            any monkey patched code so that a real request can be made.
        :param cassette:
            Cassette being played.
        """
        self.unpatch = unpatch
        self.cassette = cassette

        self.io_loop = None
        self.thread = None
        self.tchannel = None

        self._running = threading.Event()

    @wrap_uncaught(reraise=(
        VCRProxy.CannotRecordInteractionsError,
        VCRProxy.NoPeersAvailableError,
        VCRProxy.RemoteServiceError,
        VCRProxy.VCRServiceError,
    ))
    @gen.coroutine
    def send(self, request, response):
        cassette = self.cassette
        request = request.args.request

        # TODO decode requests and responses based on arg scheme into more
        # readable formats.

        # Because Thrift doesn't handle UTF-8 correctly right now
        request.serviceName = request.serviceName.decode('utf-8')
        request.endpoint = request.endpoint.decode('utf-8')

        # TODO do we care about hostport being the same?
        if cassette.can_replay(request):
            vcr_response = cassette.replay(request)
            raise gen.Return(vcr_response)

        if cassette.write_protected:
            raise VCRProxy.CannotRecordInteractionsError(
                'Could not find a matching response for request %s and the '
                'record mode %s prevents new interactions from being '
                'recorded. Your test may be performing an unexpected '
                'request.' % (str(request), cassette.record_mode)
            )

        peers = []
        if request.hostPort:
            peers = [request.hostPort]
        else:
            peers = request.knownPeers

        if not peers:
            raise VCRProxy.NoPeersAvailableError(
                'Both, hostPort and knownPeers were empty or unset. '
                'One of them must be specified and non-empty.'
            )

        arg_scheme = VCRProxy.ArgScheme.to_name(request.argScheme).lower()

        with self.unpatch():
            # TODO propagate other request and response parameters
            # TODO might make sense to tag all VCR requests with a protocol
            # header of some kind
            response_future = self.tchannel.request(
                service=request.serviceName,
                arg_scheme=arg_scheme,
                hostport=random.choice(peers),
            ).send(
                request.endpoint,
                request.headers,
                request.body,
                headers={h.key: h.value for h in request.transportHeaders},
            )

        # Don't actually yield while everything is unpatched.
        try:
            response = yield response_future
        except TChannelError as e:
            raise VCRProxy.RemoteServiceError(
                code=e.code,
                message=e.message,
            )
        response_headers = yield response.get_header()
        response_body = yield response.get_body()

        vcr_response = VCRProxy.Response(
            response.status_code,
            response_headers,
            response_body,
        )
        cassette.record(request, vcr_response)
        raise gen.Return(vcr_response)

    @property
    def hostport(self):
        return self.tchannel.hostport

    def _run(self):
        self.io_loop = IOLoop()
        self.io_loop.make_current()

        self.tchannel = TChannel('proxy-server')
        self.tchannel.register(VCRProxy, handler=self.send)

        self.tchannel.listen()
        self._running.set()
        self.io_loop.start()

    def start(self):
        self.thread = threading.Thread(target=self._run)
        self.thread.start()

        self._running.wait(1)

    def stop(self):
        self.tchannel.close()
        self.tchannel = None

        self.io_loop.stop()
        self.io_loop = None

        self.thread.join(1)  # seconds
        self.thread = None

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args):
        self.stop()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from collections import Counter
from paasta_tools import marathon_tools
from paasta_tools.mesos_tools import fetch_mesos_stats
from paasta_tools.mesos_tools import fetch_mesos_state_from_leader
from paasta_tools.mesos_tools import get_mesos_quorum
from paasta_tools.mesos_tools import get_zookeeper_config
from paasta_tools.mesos_tools import get_number_of_mesos_masters
from paasta_tools.utils import PaastaColors
from paasta_tools.mesos_tools import MissingMasterException
import sys

def get_num_masters(state):
    """ Gets the number of masters from mesos state """
    return get_number_of_mesos_masters(get_zookeeper_config(state))

def get_mesos_cpu_status(metrics):
    """Takes in the mesos metrics and analyzes them, returning the status
    :param metrics: mesos metrics dictionary
    :returns: Tuple of the output array and is_ok bool
    """

    total = metrics['master/cpus_total']
    used = metrics['master/cpus_used']
    available = total - used
    return total, used, available

def quorum_ok(masters, quorum):
    return masters >= quorum

def check_threshold(percent_available, threshold):
    return percent_available > threshold

def percent_available(total, available):
    return round(available / float(total) * 100.0, 2)

def assert_cpu_health(metrics, threshold=10):
    total, used, available = get_mesos_cpu_status(metrics)
    perc_available = percent_available(total, available)
    if check_threshold(perc_available, threshold):
        return ("cpus: total: %d used: %d available: %d percent_available: %d"
                % (total, used, available, perc_available),
                True)
    else:
        return (PaastaColors.red(
                "CRITICAL: Less than %d%% CPUs available. (Currently at %.2f%%)"
                % (threshold, perc_available)),
                False)

def assert_memory_health(metrics, threshold=10):
    total = metrics['master/mem_total']/float(1024)
    used = metrics['master/mem_used']/float(1024)
    available = total - used
    perc_available = percent_available(total, available)

    if check_threshold(perc_available, threshold):
        return ("memory: total: %0.2f GB used: %0.2f GB available: %0.2f GB"
                % (total, used, available),
                True)
    else:
        return (PaastaColors.red(
                "CRITICAL: Less than %d%% memory available. (Currently at %.2f%%)"
                % (threshold, perc_available)),
                False)

def assert_tasks_running(metrics):
    running = metrics['master/tasks_running']
    staging = metrics['master/tasks_staging']
    starting = metrics['master/tasks_starting']
    return ("tasks: running: %d staging: %d starting: %d"
            % (running, staging, starting),
            True)

def assert_no_duplicate_frameworks(state):
    """A predicate which asserts that there are no duplicate frameworks running, where
    frameworks are identified by their name.

    :param state: the state info from the Mesos master
    :return a tuple containing (output, ok): output is a log of the state of frameworks, ok a boolean
    indicating if there are any duplicate frameworks.
    """
    frameworks = state['frameworks']
    framework_counts = Counter([fw['name'] for fw in frameworks])
    output = ["frameworks:"]
    ok = True

    for framework, count in framework_counts.iteritems():
        if count > 1:
            ok = False
            output.append(PaastaColors.red(
                          "  CRITICAL: Framework %s has %d instances running--expected no more than 1."
                          % (framework, count)))
        else:
            output.append("  framework: %s count: %d" % (framework, count))
    return (("\n").join(output), ok)

def assert_slave_health(metrics):
    active, inactive = metrics['master/slaves_active'], metrics['master/slaves_inactive']
    return ("slaves: active: %d inactive: %d"
            % (active, inactive),
            True)

def assert_quorum_size(state):
    masters, quorum = get_num_masters(state), get_mesos_quorum(state)
    if quorum_ok(masters, quorum):
        return ("quorum: masters: %d configured quorum: %d "
                % (masters, quorum),
                True)
    else:
        return (PaastaColors.red(
                "CRITICAL: Number of masters (%d) less than configured quorum(%d)."
                % (masters, quorum)),
                False)

def get_mesos_status():
    """Gathers information about the mesos cluster.
       :return: tuple of a string containing the status and a bool representing if it is ok or not
    """

    state = fetch_mesos_state_from_leader()
    cluster_outputs, cluster_ok = run_healthchecks_with_param(state, [assert_quorum_size,
                                                                      assert_no_duplicate_frameworks])

    metrics = fetch_mesos_stats()
    metrics_outputs, metrics_ok = run_healthchecks_with_param(metrics, [
        assert_cpu_health,
        assert_memory_health,
        assert_slave_health,
        assert_tasks_running])

    cluster_outputs.extend(metrics_outputs)
    cluster_ok.extend(metrics_ok)
    return cluster_outputs, cluster_ok

def run_healthchecks_with_param(param, healthcheck_functions):
    outputs, oks = [], []
    for healthcheck in healthcheck_functions:
        output, ok = healthcheck(param)
        outputs.append(output)
        oks.append(ok)
    return outputs, oks

def assert_marathon_apps(client):
    num_apps = len(client.list_apps())
    if num_apps < 1:
        return (PaastaColors.red(
            "CRITICAL: No marathon apps running"),
            False)
    else:
        return ("marathon apps: %d"
                % num_apps,
                True)

def assert_marathon_tasks(client):
    num_tasks = len(client.list_tasks())
    return ("marathon tasks: %d"
            % num_tasks,
            True)

def assert_marathon_deployments(client):
    num_deployments = len(client.list_deployments())
    return ("marathon deployments: %d"
            % num_deployments,
            True)

def get_marathon_status(client):
    """ Gathers information about marathon.
    :return: string containing the status.  """
    outputs, oks = run_healthchecks_with_param(client, [
        assert_marathon_apps,
        assert_marathon_tasks,
        assert_marathon_deployments])
    return outputs, oks

def get_marathon_client(marathon_config):
    """ Given a MarathonConfig object, return
    a client.
    :param marathon_config: a MarathonConfig object
    :returns client: a marathon client
    """
    return marathon_tools.get_marathon_client(
        marathon_config.get_url(),
        marathon_config.get_username(),
        marathon_config.get_password()
    )

def main():
    marathon_config = marathon_tools.load_marathon_config()
    marathon_client = get_marathon_client(marathon_config)
    try:
        mesos_outputs, mesos_oks = get_mesos_status()
        marathon_outputs, marathon_oks = get_marathon_status(marathon_client)
    except MissingMasterException as e:
        # if we can't connect to master at all,
        # then bomb out early
        print(PaastaColors.red("CRITICAL:  %s" % e.message))
        sys.exit(2)

    print("Mesos Status:")
    print(("\n").join(map(lambda x: "  %s" % x, mesos_outputs)))
    print("Marathon Status:")
    print(("\n").join(map(lambda x: "  %s" % x, marathon_outputs)))

    if False in mesos_oks or False in marathon_oks:
        sys.exit(2)

if __name__ == '__main__':
    main()
<commit_msg>extract printing function<commit_after>#!/usr/bin/env python

from collections import Counter
from paasta_tools import marathon_tools
from paasta_tools.mesos_tools import fetch_mesos_stats
from paasta_tools.mesos_tools import fetch_mesos_state_from_leader
from paasta_tools.mesos_tools import get_mesos_quorum
from paasta_tools.mesos_tools import get_zookeeper_config
from paasta_tools.mesos_tools import get_number_of_mesos_masters
from paasta_tools.utils import PaastaColors
from paasta_tools.mesos_tools import MissingMasterException
import sys

def get_num_masters(state):
    """ Gets the number of masters from mesos state """
    return get_number_of_mesos_masters(get_zookeeper_config(state))

def get_mesos_cpu_status(metrics):
    """Takes in the mesos metrics and analyzes them, returning the status
    :param metrics: mesos metrics dictionary
    :returns: Tuple of the output array and is_ok bool
    """

    total = metrics['master/cpus_total']
    used = metrics['master/cpus_used']
    available = total - used
    return total, used, available

def quorum_ok(masters, quorum):
    return masters >= quorum

def check_threshold(percent_available, threshold):
    return percent_available > threshold

def percent_available(total, available):
    return round(available / float(total) * 100.0, 2)

def assert_cpu_health(metrics, threshold=10):
    total, used, available = get_mesos_cpu_status(metrics)
    perc_available = percent_available(total, available)
    if check_threshold(perc_available, threshold):
        return ("cpus: total: %d used: %d available: %d percent_available: %d"
                % (total, used, available, perc_available),
                True)
    else:
        return (PaastaColors.red(
                "CRITICAL: Less than %d%% CPUs available. (Currently at %.2f%%)"
                % (threshold, perc_available)),
                False)

def assert_memory_health(metrics, threshold=10):
    total = metrics['master/mem_total']/float(1024)
    used = metrics['master/mem_used']/float(1024)
    available = total - used
    perc_available = percent_available(total, available)

    if check_threshold(perc_available, threshold):
        return ("memory: total: %0.2f GB used: %0.2f GB available: %0.2f GB"
                % (total, used, available),
                True)
    else:
        return (PaastaColors.red(
                "CRITICAL: Less than %d%% memory available. (Currently at %.2f%%)"
                % (threshold, perc_available)),
                False)

def assert_tasks_running(metrics):
    running = metrics['master/tasks_running']
    staging = metrics['master/tasks_staging']
    starting = metrics['master/tasks_starting']
    return ("tasks: running: %d staging: %d starting: %d"
            % (running, staging, starting),
            True)

def assert_no_duplicate_frameworks(state):
    """A predicate which asserts that there are no duplicate frameworks running, where
    frameworks are identified by their name.

    :param state: the state info from the Mesos master
    :return a tuple containing (output, ok): output is a log of the state of frameworks, ok a boolean
    indicating if there are any duplicate frameworks.
    """
    frameworks = state['frameworks']
    framework_counts = Counter([fw['name'] for fw in frameworks])
    output = ["frameworks:"]
    ok = True

    for framework, count in framework_counts.iteritems():
        if count > 1:
            ok = False
            output.append(PaastaColors.red(
                          "  CRITICAL: Framework %s has %d instances running--expected no more than 1."
                          % (framework, count)))
        else:
            output.append("  framework: %s count: %d" % (framework, count))
    return (("\n").join(output), ok)

def assert_slave_health(metrics):
    active, inactive = metrics['master/slaves_active'], metrics['master/slaves_inactive']
    return ("slaves: active: %d inactive: %d"
            % (active, inactive),
            True)

def assert_quorum_size(state):
    masters, quorum = get_num_masters(state), get_mesos_quorum(state)
    if quorum_ok(masters, quorum):
        return ("quorum: masters: %d configured quorum: %d "
                % (masters, quorum),
                True)
    else:
        return (PaastaColors.red(
                "CRITICAL: Number of masters (%d) less than configured quorum(%d)."
                % (masters, quorum)),
                False)

def get_mesos_status():
    """Gathers information about the mesos cluster.
       :return: tuple of a string containing the status and a bool representing if it is ok or not
    """

    state = fetch_mesos_state_from_leader()
    cluster_outputs, cluster_ok = run_healthchecks_with_param(state, [assert_quorum_size,
                                                                      assert_no_duplicate_frameworks])

    metrics = fetch_mesos_stats()
    metrics_outputs, metrics_ok = run_healthchecks_with_param(metrics, [
        assert_cpu_health,
        assert_memory_health,
        assert_slave_health,
        assert_tasks_running])

    cluster_outputs.extend(metrics_outputs)
    cluster_ok.extend(metrics_ok)
    return cluster_outputs, cluster_ok

def run_healthchecks_with_param(param, healthcheck_functions):
    outputs, oks = [], []
    for healthcheck in healthcheck_functions:
        output, ok = healthcheck(param)
        outputs.append(output)
        oks.append(ok)
    return outputs, oks

def assert_marathon_apps(client):
    num_apps = len(client.list_apps())
    if num_apps < 1:
        return (PaastaColors.red(
            "CRITICAL: No marathon apps running"),
            False)
    else:
        return ("marathon apps: %d"
                % num_apps,
                True)

def assert_marathon_tasks(client):
    num_tasks = len(client.list_tasks())
    return ("marathon tasks: %d"
            % num_tasks,
            True)

def assert_marathon_deployments(client):
    num_deployments = len(client.list_deployments())
    return ("marathon deployments: %d"
            % num_deployments,
            True)

def get_marathon_status(client):
    """ Gathers information about marathon.
    :return: string containing the status.  """
    outputs, oks = run_healthchecks_with_param(client, [
        assert_marathon_apps,
        assert_marathon_tasks,
        assert_marathon_deployments])
    return outputs, oks

def get_marathon_client(marathon_config):
    """ Given a MarathonConfig object, return
    a client.
    :param marathon_config: a MarathonConfig object
    :returns client: a marathon client
    """
    return marathon_tools.get_marathon_client(
        marathon_config.get_url(),
        marathon_config.get_username(),
        marathon_config.get_password()
    )

def main():
    marathon_config = marathon_tools.load_marathon_config()
    marathon_client = get_marathon_client(marathon_config)
    try:
        mesos_outputs, mesos_oks = get_mesos_status()
        marathon_outputs, marathon_oks = get_marathon_status(marathon_client)
    except MissingMasterException as e:
        # if we can't connect to master at all,
        # then bomb out early
        print(PaastaColors.red("CRITICAL:  %s" % e.message))
        sys.exit(2)

    def print_outputs(outputs):
        print(("\n").join(map(lambda x: "  %s" % x, outputs)))

    print("Mesos Status:")
    print_outputs(mesos_outputs)
    print("Marathon Status:")
    print_outputs(marathon_outputs)

    if False in mesos_oks or False in marathon_oks:
        sys.exit(2)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
import os.path as op

try:
    from typing import List, Sequence
except:
    raise ImportError('`typing` module not found, please install it.')

import pandas as pd
import xlrd
from   openpyxl import load_workbook

def _openpyxl_read_xl(xl_path: str):
    """ Use openpyxl to read an Excel file. """
    try:
        wb = load_workbook(filename=xl_path, read_only=True)
    except:
        raise
    else:
        return wb

def _xlrd_read_xl(xl_path: str):
    """ Use xlrd to get the list of sheet names from `xl_path`."""
    try:
        wb = xlrd.open_workbook(xl_path)
    except:
        raise
    else:
        return wb

XL_READERS = {'xlrd': _xlrd_read_xl,
              'openpyxl': _openpyxl_read_xl,
             }

def _use_openpyxl_or_xlrf(xl_path: str):
    fails   = []
    choices = ['openpyxl', 'xlrd']
    for m in choices:
        reader = XL_READERS[m]

        try:
            reader(xl_path)
        except:
            fails.append(m)
            pass
        else:
            return m

    raise RuntimeError("Could not open {} with {}.".format(xl_path,
                                                           ' nor '.join(fails)))

def _check_xl_path(xl_path: str):
    """ Return the expanded absolute path of `xl_path` if
    if exists and 'xlrd' or 'openpyxl' depending on
    which module should be used for the Excel file in `xl_path`.

    Parameters
    ----------
    xl_path: str
        Path to an Excel file

    Returns
    -------
    xl_path: str
        User expanded and absolute path to `xl_path`

    module: str
        The name of the module you should use to process the
        Excel file.
        Choices: 'xlrd', 'pyopenxl'

    Raises
    ------
    IOError
        If the file does not exist

    RuntimError
        If a suitable reader for xl_path is not found
    """
    xl_path = op.abspath(op.expanduser(xl_path))

    if not op.isfile(xl_path):
        raise IOError("Could not find file in {}.".format(xl_path))

    return xl_path, _use_openpyxl_or_xlrf(xl_path)

def read_xl(xl_path: str):
    """ Return the workbook from the Excel file in `xl_path`."""
    xl_path, choice = _check_xl_path(xl_path)
    reader = XL_READERS[choice]

    return reader(xl_path)

def get_sheet_list(xl_path: str) -> List:
    """Return a list with the name of the sheets in
    the Excel file in `xl_path`.
    """
    wb = read_xl(xl_path)

    if hasattr(wb, 'sheetnames'):
        return wb.sheetnames
    else:
        return wb.sheet_names()

def concat_sheets(xl_path: str, sheetnames=None):
    """ Return a pandas DataFrame with the concat'ed
    content of the `sheetnames` from the Excel file in
    `xl_path`.

    Parameters
    ----------
    xl_path: str
        Path to the Excel file

    sheetnames: list of str
        List of existing sheet names of `xl_path`.
        If None, will use all sheets from `xl_path`.

    Returns
    -------
    df: pandas.DataFrame
    """
    xl_path, choice = _check_xl_path(xl_path)

    if sheetnames is None:
        sheetnames = get_sheet_list(xl_path)

    sheets = pd.read_excel(xl_path, sheetname=sheetnames)
    return pd.concat([sheets[tab] for tab in sheets])

def _check_cols(df, col_names):
    """ Raise an AttributeError if `df` does not have a column named as an item of
    the list of strings `col_names`.
    """
    for col in col_names:
        if not hasattr(df, col):
            raise AttributeError("DataFrame does not have a '{}' column, got {}.".format(col,
                                                                                df.columns))

def col_values(df, col_name):
    """ Return a list of not null values from the `col_name` column of `df`."""
    _check_cols(df, [col_name])

    return [nom.lower() for nom in df[pd.notnull(df)][col_name] if not pd.isnull(nom)]

def duplicated_rows(df, col_name):
    """ Return a DataFrame with the duplicated values of the column `col_name`
    in `df`."""
    _check_cols(df, [col_name])

    dups = df[pd.notnull(df[col_name]) & df.duplicated(subset=[col_name])]
    return dups

def duplicated(values: Sequence):
    """ Return the duplicated items in `values`"""
    vals = pd.Series(values)
    return vals[vals.duplicated()]
<commit_msg>fix col_values in excel_utils<commit_after>
import os.path as op

try:
    from typing import List, Sequence
except:
    raise ImportError('`typing` module not found, please install it.')

import pandas as pd
import xlrd
from   openpyxl import load_workbook

def _openpyxl_read_xl(xl_path: str):
    """ Use openpyxl to read an Excel file. """
    try:
        wb = load_workbook(filename=xl_path, read_only=True)
    except:
        raise
    else:
        return wb

def _xlrd_read_xl(xl_path: str):
    """ Use xlrd to get the list of sheet names from `xl_path`."""
    try:
        wb = xlrd.open_workbook(xl_path)
    except:
        raise
    else:
        return wb

XL_READERS = {'xlrd': _xlrd_read_xl,
              'openpyxl': _openpyxl_read_xl,
             }

def _use_openpyxl_or_xlrf(xl_path: str):
    fails   = []
    choices = ['openpyxl', 'xlrd']
    for m in choices:
        reader = XL_READERS[m]

        try:
            reader(xl_path)
        except:
            fails.append(m)
            pass
        else:
            return m

    raise RuntimeError("Could not open {} with {}.".format(xl_path,
                                                           ' nor '.join(fails)))

def _check_xl_path(xl_path: str):
    """ Return the expanded absolute path of `xl_path` if
    if exists and 'xlrd' or 'openpyxl' depending on
    which module should be used for the Excel file in `xl_path`.

    Parameters
    ----------
    xl_path: str
        Path to an Excel file

    Returns
    -------
    xl_path: str
        User expanded and absolute path to `xl_path`

    module: str
        The name of the module you should use to process the
        Excel file.
        Choices: 'xlrd', 'pyopenxl'

    Raises
    ------
    IOError
        If the file does not exist

    RuntimError
        If a suitable reader for xl_path is not found
    """
    xl_path = op.abspath(op.expanduser(xl_path))

    if not op.isfile(xl_path):
        raise IOError("Could not find file in {}.".format(xl_path))

    return xl_path, _use_openpyxl_or_xlrf(xl_path)

def read_xl(xl_path: str):
    """ Return the workbook from the Excel file in `xl_path`."""
    xl_path, choice = _check_xl_path(xl_path)
    reader = XL_READERS[choice]

    return reader(xl_path)

def get_sheet_list(xl_path: str) -> List:
    """Return a list with the name of the sheets in
    the Excel file in `xl_path`.
    """
    wb = read_xl(xl_path)

    if hasattr(wb, 'sheetnames'):
        return wb.sheetnames
    else:
        return wb.sheet_names()

def concat_sheets(xl_path: str, sheetnames=None):
    """ Return a pandas DataFrame with the concat'ed
    content of the `sheetnames` from the Excel file in
    `xl_path`.

    Parameters
    ----------
    xl_path: str
        Path to the Excel file

    sheetnames: list of str
        List of existing sheet names of `xl_path`.
        If None, will use all sheets from `xl_path`.

    Returns
    -------
    df: pandas.DataFrame
    """
    xl_path, choice = _check_xl_path(xl_path)

    if sheetnames is None:
        sheetnames = get_sheet_list(xl_path)

    sheets = pd.read_excel(xl_path, sheetname=sheetnames)
    return pd.concat([sheets[tab] for tab in sheets])

def _check_cols(df, col_names):
    """ Raise an AttributeError if `df` does not have a column named as an item of
    the list of strings `col_names`.
    """
    for col in col_names:
        if not hasattr(df, col):
            raise AttributeError("DataFrame does not have a '{}' column, got {}.".format(col,
                                                                                df.columns))

def col_values(df, col_name):
    """ Return a list of not null values from the `col_name` column of `df`."""
    _check_cols(df, [col_name])

    if 'O' in df[col_name] or pd.np.issubdtype(df[col_name].dtype, str): # if the column is of strings
        return [nom.lower() for nom in df[pd.notnull(df)][col_name] if not pd.isnull(nom)]
    else:
        return [nom for nom in df[pd.notnull(df)][col_name] if not pd.isnull(nom)]

def duplicated_rows(df, col_name):
    """ Return a DataFrame with the duplicated values of the column `col_name`
    in `df`."""
    _check_cols(df, [col_name])

    dups = df[pd.notnull(df[col_name]) & df.duplicated(subset=[col_name])]
    return dups

def duplicated(values: Sequence):
    """ Return the duplicated items in `values`"""
    vals = pd.Series(values)
    return vals[vals.duplicated()]
<|endoftext|>


<commit_before># Copyright (c) 2016 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import

import sys
import random
import threading
from functools import wraps
from concurrent.futures import Future as ConcFuture

from tornado import gen
from tornado.ioloop import IOLoop

from tchannel.errors import TChannelError
from tchannel import TChannel

from . import proxy

def wrap_uncaught(func=None, reraise=None):
    """Catches uncaught exceptions and raises VCRServiceErrors instead.

    :param reraise:
        Collection of exception clasess that should be re-raised as-is.
    """
    reraise = reraise or ()

    def decorator(f):

        @wraps(f)
        @gen.coroutine
        def new_f(*args, **kwargs):
            try:
                result = yield gen.maybe_future(f(*args, **kwargs))
            except Exception as e:
                if any(isinstance(e, cls) for cls in reraise):
                    # TODO maybe use traceback.format_exc to also send a
                    # traceback?
                    raise e
                raise proxy.VCRServiceError(str(e))
            else:
                raise gen.Return(result)

        return new_f

    if func is not None:
        return decorator(func)
    else:
        return decorator

class VCRProxyService(object):

    def __init__(self, cassette, unpatch):
        """
        :param unpatch:
            A function returning a context manager which temporarily unpatches
            any monkey patched code so that a real request can be made.
        :param cassette:
            Cassette being played.
        """
        self.unpatch = unpatch
        self.cassette = cassette

        self.io_loop = None
        self.thread = None
        self.tchannel = None

        self._running = ConcFuture()

    @wrap_uncaught(reraise=(
        proxy.CannotRecordInteractionsError,
        proxy.NoPeersAvailableError,
        proxy.RemoteServiceError,
        proxy.VCRServiceError,
    ))
    @gen.coroutine
    def send(self, request):
        cassette = self.cassette
        request = request.body.request

        # TODO decode requests and responses based on arg scheme into more
        # readable formats.

        # Because Thrift doesn't handle UTF-8 correctly right now
        if isinstance(request.serviceName, bytes):
            request.serviceName = request.serviceName.decode('utf-8')
        if isinstance(request.endpoint, bytes):
            request.endpoint = request.endpoint.decode('utf-8')

        # TODO do we care about hostport being the same?
        if cassette.can_replay(request):
            vcr_response = cassette.replay(request)
            raise gen.Return(vcr_response)

        if cassette.write_protected:
            raise proxy.CannotRecordInteractionsError(
                'Could not find a matching response for request %s and the '
                'record mode %s prevents new interactions from being '
                'recorded. Your test may be performing an unexpected '
                'request.' % (str(request), cassette.record_mode)
            )

        peers = []
        if request.hostPort:
            peers = [request.hostPort]
        else:
            peers = request.knownPeers

        if not peers:
            raise proxy.NoPeersAvailableError(
                'Could not find a recorded response for request %s and was '
                'unable to make a new request because both, hostPort and '
                'knownPeers were unspecified. One of them must be specified '
                'for me to make new requests. Make sure you specified a '
                'hostport in the original request or are advertising '
                'on Hyperbahn.' % (str(request),)
            )

        arg_scheme = proxy.ArgScheme.name_of(request.argScheme).lower()

        with self.unpatch():
            # TODO propagate other request and response parameters
            # TODO might make sense to tag all VCR requests with a protocol
            # header of some kind
            response_future = self.tchannel._dep_tchannel.request(
                service=request.serviceName,
                arg_scheme=arg_scheme,
                hostport=random.choice(peers),
            ).send(
                request.endpoint,
                request.headers,
                request.body,
                headers={h.key: h.value for h in request.transportHeaders},
            )

        # Don't actually yield while everything is unpatched.
        try:
            response = yield response_future
        except TChannelError as e:
            raise proxy.RemoteServiceError(
                code=e.code,
                message=str(e),
            )
        response_headers = yield response.get_header()
        response_body = yield response.get_body()

        vcr_response = proxy.Response(
            code=response.status_code,
            headers=response_headers,
            body=response_body,
        )
        cassette.record(request, vcr_response)
        raise gen.Return(vcr_response)

    @property
    def hostport(self):
        return self.tchannel.hostport

    def _run(self):
        self.io_loop = IOLoop()
        self.io_loop.make_current()

        self.tchannel = TChannel('proxy-server')

        # Hack around legacy TChannel
        from tchannel.thrift import rw as thriftrw
        thriftrw.register(
            self.tchannel._dep_tchannel._handler,
            proxy.VCRProxy,
            handler=self.send,
        )

        try:
            self.tchannel.listen()
            self._running.set_result(None)
        except Exception:
            self._running.set_exception_info(*sys.exc_info()[1:])
        else:
            self.io_loop.start()

    def start(self):
        self.thread = threading.Thread(target=self._run)
        self.thread.start()

        self._running.result(1)  # seconds

    def stop(self):
        self.tchannel._dep_tchannel.close()
        self.tchannel = None

        self.io_loop.stop()
        self.io_loop = None

        self.thread.join(1)  # seconds
        self.thread = None

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args):
        self.stop()
<commit_msg>Adding traceback to server for easier debugging<commit_after># Copyright (c) 2016 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import

import sys
import random
import threading
from functools import wraps
import traceback
from concurrent.futures import Future as ConcFuture

from tornado import gen
from tornado.ioloop import IOLoop

from tchannel.errors import TChannelError
from tchannel import TChannel

from . import proxy

def wrap_uncaught(func=None, reraise=None):
    """Catches uncaught exceptions and raises VCRServiceErrors instead.

    :param reraise:
        Collection of exception clasess that should be re-raised as-is.
    """
    reraise = reraise or ()

    def decorator(f):

        @wraps(f)
        @gen.coroutine
        def new_f(*args, **kwargs):
            try:
                result = yield gen.maybe_future(f(*args, **kwargs))
            except Exception as e:
                if any(isinstance(e, cls) for cls in reraise):
                    # TODO maybe use traceback.format_exc to also send a
                    # traceback?
                    raise e
                raise proxy.VCRServiceError(str(e) + ' ' + traceback.format_exc())
            else:
                raise gen.Return(result)

        return new_f

    if func is not None:
        return decorator(func)
    else:
        return decorator

class VCRProxyService(object):

    def __init__(self, cassette, unpatch):
        """
        :param unpatch:
            A function returning a context manager which temporarily unpatches
            any monkey patched code so that a real request can be made.
        :param cassette:
            Cassette being played.
        """
        self.unpatch = unpatch
        self.cassette = cassette

        self.io_loop = None
        self.thread = None
        self.tchannel = None

        self._running = ConcFuture()

    @wrap_uncaught(reraise=(
        proxy.CannotRecordInteractionsError,
        proxy.NoPeersAvailableError,
        proxy.RemoteServiceError,
        proxy.VCRServiceError,
    ))
    @gen.coroutine
    def send(self, request):
        cassette = self.cassette
        request = request.body.request

        # TODO decode requests and responses based on arg scheme into more
        # readable formats.

        # Because Thrift doesn't handle UTF-8 correctly right now
        if isinstance(request.serviceName, bytes):
            request.serviceName = request.serviceName.decode('utf-8')
        if isinstance(request.endpoint, bytes):
            request.endpoint = request.endpoint.decode('utf-8')

        # TODO do we care about hostport being the same?
        if cassette.can_replay(request):
            vcr_response = cassette.replay(request)
            raise gen.Return(vcr_response)

        if cassette.write_protected:
            raise proxy.CannotRecordInteractionsError(
                'Could not find a matching response for request %s and the '
                'record mode %s prevents new interactions from being '
                'recorded. Your test may be performing an unexpected '
                'request.' % (str(request), cassette.record_mode)
            )

        peers = []
        if request.hostPort:
            peers = [request.hostPort]
        else:
            peers = request.knownPeers

        if not peers:
            raise proxy.NoPeersAvailableError(
                'Could not find a recorded response for request %s and was '
                'unable to make a new request because both, hostPort and '
                'knownPeers were unspecified. One of them must be specified '
                'for me to make new requests. Make sure you specified a '
                'hostport in the original request or are advertising '
                'on Hyperbahn.' % (str(request),)
            )

        arg_scheme = proxy.ArgScheme.name_of(request.argScheme).lower()

        with self.unpatch():
            # TODO propagate other request and response parameters
            # TODO might make sense to tag all VCR requests with a protocol
            # header of some kind
            response_future = self.tchannel._dep_tchannel.request(
                service=request.serviceName,
                arg_scheme=arg_scheme,
                hostport=random.choice(peers),
            ).send(
                request.endpoint,
                request.headers,
                request.body,
                headers={h.key: h.value for h in request.transportHeaders},
            )

        # Don't actually yield while everything is unpatched.
        try:
            response = yield response_future
        except TChannelError as e:
            raise proxy.RemoteServiceError(
                code=e.code,
                message=str(e),
            )
        response_headers = yield response.get_header()
        response_body = yield response.get_body()

        vcr_response = proxy.Response(
            code=response.status_code,
            headers=response_headers,
            body=response_body,
        )
        cassette.record(request, vcr_response)
        raise gen.Return(vcr_response)

    @property
    def hostport(self):
        return self.tchannel.hostport

    def _run(self):
        self.io_loop = IOLoop()
        self.io_loop.make_current()

        self.tchannel = TChannel('proxy-server')

        # Hack around legacy TChannel
        from tchannel.thrift import rw as thriftrw
        thriftrw.register(
            self.tchannel._dep_tchannel._handler,
            proxy.VCRProxy,
            handler=self.send,
        )

        try:
            self.tchannel.listen()
            self._running.set_result(None)
        except Exception:
            self._running.set_exception_info(*sys.exc_info()[1:])
        else:
            self.io_loop.start()

    def start(self):
        self.thread = threading.Thread(target=self._run)
        self.thread.start()

        self._running.result(1)  # seconds

    def stop(self):
        self.tchannel._dep_tchannel.close()
        self.tchannel = None

        self.io_loop.stop()
        self.io_loop = None

        self.thread.join(1)  # seconds
        self.thread = None

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args):
        self.stop()
<|endoftext|>


<commit_before>import glob
import os

import numpy as np
import caffe
import plyvel
import skimage
from caffe_pb2 import Datum

import constants

def predict(image_path):
    """
    Takes a single image, and makes a prediction whether it has a cloud or not.
    """

    print "Generating prediction for %s..." % image_path

    _initialize_caffe()
    im = caffe.io.load_image(image_path)
    prob = _predict_image(im)
    print "Probability this image has a cloud: {}%".format(prob)

def test_validation():
    """
    Takes validation images and runs them through a trained model to see how
    well they do. Generates statistics like precision and recall, F1, and a confusion matrix,
    in order to gauge progress.
    """
    print "Testing trained model against validation data..."

    validation_data = _load_validation_data()
    results = _run_through_caffe(validation_data)

    # TODO: Write these out to a file as well as the screen.
    # statistics = _calculate_positives_negatives(results)

    # accuracy = _calculate_accuracy(statistics)
    # precision = _calculate_precision(statistics)
    # recall = _calculate_recall(statistics)
    # f1 = _calculate_f1(statistics)

    # print "Accuracy: %f" % accuracy
    # print "Precision: %f" % precision
    # print "Recall: %f" % recall
    # print "F1 Score: %f" % f1_score

    # _draw_confusion_matrix(statistics)

def _load_validation_data():
    """
    Loads all of our validation data from our leveldb database, producing unrolled numpy input
    vectors ready to test along with their correct, expected target values.
    """

    print "\tLoading validation data..."
    input_vectors = []
    expected_targets = []

    # TODO: Change this back to validation_file.
    db = plyvel.DB(constants.TRAINING_FILE)
    for key, value in db:
        datum = Datum()
        datum.ParseFromString(value)

        # TODO: Make sure this works.
        data = np.fromstring(datum.data, dtype=np.uint8)
        data = np.reshape(data, (3, constants.HEIGHT, constants.WIDTH))
        data = skimage.img_as_float(data).astype(np.float32)

        input_vectors.append(data)
        expected_targets.append(datum.label)

    db.close()

    print "\tValidation data has %d images" % len(input_vectors)

    return {
        "input_vectors": np.array(input_vectors),
        "expected_targets": np.array(expected_targets)
    }

def _initialize_caffe():
    """
    Initializes Caffe to prepare to run some data through the model for inference.
    """
    caffe.set_mode_gpu()
    net = caffe.Net(constants.DEPLOY_FILE, constants.WEIGHTS_FINETUNED, caffe.TEST)

    # input preprocessing: 'data' is the name of the input blob == net.inputs[0]
    transformer = caffe.io.Transformer({"data": net.blobs["data"].data.shape})
    # PIL.Image loads the data with the channel last.
    transformer.set_transpose("data", (2, 0, 1))
    # Mean pixel.
    transformer.set_mean("data", np.load(constants.TRAINING_MEAN_PICKLE).mean(1).mean(1))
    # The reference model operates on images in [0, 255] range instead of [0, 1].
    transformer.set_raw_scale("data", 255)
    # The reference model has channels in BGR order instead of RGB.
    transformer.set_channel_swap("data", (2, 1, 0))

    # Deal with only a single image to predict.
    net.blobs["data"].reshape(1, 3, constants.INFERENCE_HEIGHT, constants.INFERENCE_WIDTH)

def _run_through_caffe(validation_data):
    """
    Runs our validation images through Caffe.
    """
    # TODO: Run through all the results using Caffe, then return their actual values.

def _predict_image(im):
    """
    Given a caffe.io.load_image, returns the probability that it contains a cloud.
    """

    net.blobs["data"].data[...] = transformer.preprocess("data", im)
    out = net.forward()

    probs = out["prob"][0]
    prob_cloud = probs[0] * 100.0
    return prob_cloud

def _calculate_positives_negatives(target_details):
    """
    Takes expected and actual target values, generating true and false positives and negatives,
    including the actual correct # of positive and negative values.
    """

    # TODO

    return {
        "positive": {
            "true": float(true_positive),
            "false": float(false_positive),
            "actual": float(actual_positive)
        },
        "negative": {
            "true": float(true_negative),
            "false": float(false_negative),
            "actual": float(actual_negative)
        }
    }

# def _calculate_accuracy(statistics):
#     # TODO

# def _calculate_precision(statistics):
#     # TODO

# def _calculate_recall(statistics):
#     # TODO

# def _calculate_f1(statistics):
#     # TODO

# def _generate_confusion_matrix(statistics):
#     # TODO
<commit_msg>Pass the net and transformer through.<commit_after>import glob
import os

import numpy as np
import caffe
import plyvel
import skimage
from caffe_pb2 import Datum

import constants

def predict(image_path):
    """
    Takes a single image, and makes a prediction whether it has a cloud or not.
    """

    print "Generating prediction for %s..." % image_path

    net, transformer = _initialize_caffe()
    im = caffe.io.load_image(image_path)
    prob = _predict_image(im, net, transformer)
    print "Probability this image has a cloud: {}%".format(prob)

def test_validation():
    """
    Takes validation images and runs them through a trained model to see how
    well they do. Generates statistics like precision and recall, F1, and a confusion matrix,
    in order to gauge progress.
    """
    print "Testing trained model against validation data..."

    validation_data = _load_validation_data()
    results = _run_through_caffe(validation_data)

    # TODO: Write these out to a file as well as the screen.
    # statistics = _calculate_positives_negatives(results)

    # accuracy = _calculate_accuracy(statistics)
    # precision = _calculate_precision(statistics)
    # recall = _calculate_recall(statistics)
    # f1 = _calculate_f1(statistics)

    # print "Accuracy: %f" % accuracy
    # print "Precision: %f" % precision
    # print "Recall: %f" % recall
    # print "F1 Score: %f" % f1_score

    # _draw_confusion_matrix(statistics)

def _load_validation_data():
    """
    Loads all of our validation data from our leveldb database, producing unrolled numpy input
    vectors ready to test along with their correct, expected target values.
    """

    print "\tLoading validation data..."
    input_vectors = []
    expected_targets = []

    # TODO: Change this back to validation_file.
    db = plyvel.DB(constants.TRAINING_FILE)
    for key, value in db:
        datum = Datum()
        datum.ParseFromString(value)

        # TODO: Make sure this works.
        data = np.fromstring(datum.data, dtype=np.uint8)
        data = np.reshape(data, (3, constants.HEIGHT, constants.WIDTH))
        data = skimage.img_as_float(data).astype(np.float32)

        input_vectors.append(data)
        expected_targets.append(datum.label)

    db.close()

    print "\tValidation data has %d images" % len(input_vectors)

    return {
        "input_vectors": np.array(input_vectors),
        "expected_targets": np.array(expected_targets)
    }

def _initialize_caffe():
    """
    Initializes Caffe to prepare to run some data through the model for inference.
    """
    caffe.set_mode_gpu()
    net = caffe.Net(constants.DEPLOY_FILE, constants.WEIGHTS_FINETUNED, caffe.TEST)

    # input preprocessing: 'data' is the name of the input blob == net.inputs[0]
    transformer = caffe.io.Transformer({"data": net.blobs["data"].data.shape})
    # PIL.Image loads the data with the channel last.
    transformer.set_transpose("data", (2, 0, 1))
    # Mean pixel.
    transformer.set_mean("data", np.load(constants.TRAINING_MEAN_PICKLE).mean(1).mean(1))
    # The reference model operates on images in [0, 255] range instead of [0, 1].
    transformer.set_raw_scale("data", 255)
    # The reference model has channels in BGR order instead of RGB.
    transformer.set_channel_swap("data", (2, 1, 0))

    # Deal with only a single image to predict.
    net.blobs["data"].reshape(1, 3, constants.INFERENCE_HEIGHT, constants.INFERENCE_WIDTH)

    return (net, transformer)

def _run_through_caffe(validation_data):
    """
    Runs our validation images through Caffe.
    """
    # TODO: Run through all the results using Caffe, then return their actual values.

def _predict_image(im, net, transformer):
    """
    Given a caffe.io.load_image, returns the probability that it contains a cloud.
    """

    net.blobs["data"].data[...] = transformer.preprocess("data", im)
    out = net.forward()

    probs = out["prob"][0]
    prob_cloud = probs[0] * 100.0
    return prob_cloud

def _calculate_positives_negatives(target_details):
    """
    Takes expected and actual target values, generating true and false positives and negatives,
    including the actual correct # of positive and negative values.
    """

    # TODO

    return {
        "positive": {
            "true": float(true_positive),
            "false": float(false_positive),
            "actual": float(actual_positive)
        },
        "negative": {
            "true": float(true_negative),
            "false": float(false_negative),
            "actual": float(actual_negative)
        }
    }

# def _calculate_accuracy(statistics):
#     # TODO

# def _calculate_precision(statistics):
#     # TODO

# def _calculate_recall(statistics):
#     # TODO

# def _calculate_f1(statistics):
#     # TODO

# def _generate_confusion_matrix(statistics):
#     # TODO
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse, logging

def main(args, loglevel):
    logging.basicConfig(format="%(levelname)s: %(message)s", level=loglevel)

    print "Hello there."
    logging.debug("Your Arguments: %s" % args.argument)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Does a thing to some stuff.")
    parser.add_argument('argument', nargs='*')
    parser.add_argument("-v",
                        "--verbose",
                        help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args()

    # Setup logging
    if args.verbose:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO

    main(args, loglevel)
<commit_msg>Count lines for given files<commit_after>#!/usr/bin/env python
import argparse, logging

def calculateFileRasa(filePath):
    rowCount = 0
    for line in open(filePath):
        rowCount += 1
    return (rowCount, 0)

def calculateRasa(filePaths):
    totalRows = 0
    totalRasa = 0
    for filePath in filePaths:
        rowCount, rasa = calculateFileRasa(filePath)
        totalRows += rowCount
        totalRasa += rasa
        print '%s: lines %d, RaSa: TBD' % (filePath, rowCount)
    print 'total: lines %d, RaSa: TBD' % (totalRows)

def main(args, loglevel):
    logging.basicConfig(format="%(levelname)s: %(message)s", level=loglevel)
    logging.debug("Your Arguments: %s" % args.argument)
    calculateRasa(args.argument)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Does a thing to some stuff.")
    parser.add_argument('argument', nargs='*')
    parser.add_argument("-v",
                        "--verbose",
                        help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args()

    # Setup logging
    if args.verbose:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO

    main(args, loglevel)
<|endoftext|>


<commit_before>from resource_management import *
import os

def create_hdfs_dir(path, owner, perms):
  Execute('hadoop fs -mkdir -p '+path, user='hdfs')
  Execute('hadoop fs -chown ' + owner + ' ' + path, user='hdfs')
  Execute('hadoop fs -chmod ' + str(perms) + ' ' + path, user='hdfs')

def package(name):
  import params
  Execute(params.package_mgr + ' install -y ' + name, user='root')

def add_repo(source, dest):
  import params
  if not os.path.isfile(dest + params.repo_file):
    Execute('cp ' + source + ' ' + dest)
    Execute(params.key_cmd)
    Execute(params.cache_cmd)

def cdap_config(name=None):
  import params
  print 'Setting up CDAP configuration for ' + name
  # We're only setup for *NIX, for now
  Directory(params.etc_prefix_dir,
            mode=0755
  )

  Directory(params.cdap_conf_dir,
            owner = params.cdap_user,
            group = params.user_group,
            recursive = True
  )

  XmlConfig("cdap-site.xml",
            conf_dir = params.cdap_conf_dir,
            configurations = params.config['configurations']['cdap-site'],
            owner = params.cdap_user,
            group = params.user_group
  )

  File(format("{params.cdap_conf_dir}/cdap-env.sh"),
       owner = params.cdap_user,
       content=InlineTemplate(params.cdap_env_sh_template)
  )

  cleanup_opts()

  # Copy logback.xml and logback-container.xml
  for i in 'logback.xml', 'logback-container.xml':
    no_op_test = 'ls ' + params.cdap_conf_dir + '/' + i + ' 2>/dev/null'
    Execute('cp -f /etc/cdap/conf.dist/' + i + ' ' + params.cdap_conf_dir, not_if=no_op_test)

  Execute('update-alternatives --install /etc/cdap/conf cdap-conf ' + params.cdap_conf_dir + ' 50')

def has_hive():
  import params
  if len(params.hive_metastore_host) > 0:
    return true
  else:
    return false

### TODO: Update this with a safe version, as hdp-select doesn't exist on HDP < 2.2
def get_hdp_version():
  try:
    command = 'hdp-select status hadoop-client'
    return_code, hdp_output = shell.call(command, timeout=20)
  except Exception, e:
    Logger.error(str(e))
    raise Fail('Unable to execute hdp-select command to retrieve the version.')

  if return_code != 0:
    raise Fail(
      'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))

  hdp_version = re.sub('hadoop-client - ', '', hdp_output)
  hdp_version = hdp_version.rstrip()
  match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)

  if match is None:
    raise Fail('Failed to get extracted version')

  return hdp_version

def cleanup_opts():
  Execute('sed -i \'s/"$OPTS"/$OPTS/g\' /opt/cdap/*/bin/service')
<commit_msg>Switch from hdp-select to hadoop version for get_hdp_version<commit_after>from resource_management import *
import os

def create_hdfs_dir(path, owner, perms):
  Execute('hadoop fs -mkdir -p '+path, user='hdfs')
  Execute('hadoop fs -chown ' + owner + ' ' + path, user='hdfs')
  Execute('hadoop fs -chmod ' + str(perms) + ' ' + path, user='hdfs')

def package(name):
  import params
  Execute(params.package_mgr + ' install -y ' + name, user='root')

def add_repo(source, dest):
  import params
  if not os.path.isfile(dest + params.repo_file):
    Execute('cp ' + source + ' ' + dest)
    Execute(params.key_cmd)
    Execute(params.cache_cmd)

def cdap_config(name=None):
  import params
  print 'Setting up CDAP configuration for ' + name
  # We're only setup for *NIX, for now
  Directory(params.etc_prefix_dir,
            mode=0755
  )

  Directory(params.cdap_conf_dir,
            owner = params.cdap_user,
            group = params.user_group,
            recursive = True
  )

  XmlConfig("cdap-site.xml",
            conf_dir = params.cdap_conf_dir,
            configurations = params.config['configurations']['cdap-site'],
            owner = params.cdap_user,
            group = params.user_group
  )

  File(format("{params.cdap_conf_dir}/cdap-env.sh"),
       owner = params.cdap_user,
       content=InlineTemplate(params.cdap_env_sh_template)
  )

  cleanup_opts()

  # Copy logback.xml and logback-container.xml
  for i in 'logback.xml', 'logback-container.xml':
    no_op_test = 'ls ' + params.cdap_conf_dir + '/' + i + ' 2>/dev/null'
    Execute('cp -f /etc/cdap/conf.dist/' + i + ' ' + params.cdap_conf_dir, not_if=no_op_test)

  Execute('update-alternatives --install /etc/cdap/conf cdap-conf ' + params.cdap_conf_dir + ' 50')

def has_hive():
  import params
  if len(params.hive_metastore_host) > 0:
    return true
  else:
    return false

def get_hdp_version():
  command = 'hadoop version | head -n 1 | cut -d. -f4-'
  return_code, hdp_output = shell.call(command, timeout=20)

  if return_code != 0:
    raise Fail(
      'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))

  hdp_version = hdp_output.rstrip()
  match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)

  if match is None:
    raise Fail('Failed to get extracted version')

  return hdp_version

def cleanup_opts():
  Execute('sed -i \'s/"$OPTS"/$OPTS/g\' /opt/cdap/*/bin/service')
<|endoftext|>