<commit_before># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

"""FogLAMP Monitor module"""

import asyncio
import aiohttp
import json
from foglamp.common import logger
from foglamp.common.audit_logger import AuditLogger
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.services.core.service_registry.service_registry import ServiceRegistry
from foglamp.common.service_record import ServiceRecord
from foglamp.services.core import connect

__author__ = "Ashwin Gopalakrishnan, Amarendra K Sinha"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

class Monitor(object):

    _DEFAULT_SLEEP_INTERVAL = 5
    """The time (in seconds) to sleep between health checks"""

    _DEFAULT_PING_TIMEOUT = 1
    """Timeout for a response from any given micro-service"""

    _DEFAULT_MAX_ATTEMPTS = 15

    _DEFAULT_RESTART_FAILED = "auto"
    """Restart failed microservice - manual/auto"""

    _logger = None

    def __init__(self):
        self._logger = logger.setup(__name__, level=20)

        self._monitor_loop_task = None  # type: asyncio.Task
        """Task for :meth:`_monitor_loop`, to ensure it has finished"""
        self._sleep_interval = None  # type: int
        """The time (in seconds) to sleep between health checks"""
        self._ping_timeout = None  # type: int
        """Timeout for a response from any given micro-service"""
        self._max_attempts = None  # type: int
        """Number of max attempts for finding a heartbeat of service"""
        self._restart_failed = None  # type: str
        """Restart failed microservice - manual/auto"""

        self.restarted_services = []

    async def _sleep(self, sleep_time):
        await asyncio.sleep(sleep_time)

    async def _monitor_loop(self):
        """async Monitor loop to monitor registered services"""
        # check health of all micro-services every N seconds
        round_cnt = 0
        check_count = {}  # dict to hold current count of current status.
                          # In case of ok and running status, count will always be 1.
                          # In case of of non running statuses, count shows since when this status is set.
        while True:
            round_cnt += 1
            self._logger.debug("Starting next round#{} of service monitoring, sleep/i:{} ping/t:{} max/a:{}".format(
                round_cnt, self._sleep_interval, self._ping_timeout, self._max_attempts))
            for service_record in ServiceRegistry.all():
                if service_record._id not in check_count:
                    check_count.update({service_record._id: 1})

                # Try ping if service status is either running or doubtful (i.e. give service a chance to recover)
                if service_record._status not in [ServiceRecord.Status.Running,
                                                  ServiceRecord.Status.Unresponsive,
                                                  ServiceRecord.Status.Failed]:
                    continue

                self._logger.debug("Service: {} Status: {}".format(service_record._name, service_record._status))

                if service_record._status == ServiceRecord.Status.Failed:
                    if self._restart_failed == "auto":
                        if service_record._id not in self.restarted_services:
                            self.restarted_services.append(service_record._id)
                            asyncio.ensure_future(self.restart_service(service_record))
                    continue

                try:
                    url = "{}://{}:{}/foglamp/service/ping".format(
                        service_record._protocol, service_record._address, service_record._management_port)
                    async with aiohttp.ClientSession() as session:
                        async with session.get(url, timeout=self._ping_timeout) as resp:
                            text = await resp.text()
                            res = json.loads(text)
                            if res["uptime"] is None:
                                raise ValueError('res.uptime is None')
                except aiohttp.client_exceptions.ClientError as ex:
                    service_record._status = ServiceRecord.Status.Unresponsive
                    check_count[service_record._id] += 1
                    self._logger.info("ClientError: %s, %s", str(ex), service_record.__repr__())
                except ValueError as ex:
                    service_record._status = ServiceRecord.Status.Unresponsive
                    check_count[service_record._id] += 1
                    self._logger.info("Invalid response: %s, %s", str(ex), service_record.__repr__())
                except Exception as ex:
                    service_record._status = ServiceRecord.Status.Unresponsive
                    check_count[service_record._id] += 1
                    self._logger.info("Exception occurred: %s, %s", str(ex), service_record.__repr__())
                else:
                    service_record._status = ServiceRecord.Status.Running
                    check_count[service_record._id] = 1

                if check_count[service_record._id] > self._max_attempts:
                    ServiceRegistry.mark_as_failed(service_record._id)
                    check_count[service_record._id] = 0
                    try:
                        audit = AuditLogger(connect.get_storage_async())
                        await audit.failure('SRVFL', {'name':service_record._name})
                    except Exception as ex:
                        self._logger.info("Failed to audit service failure %s", str(ex))
            await self._sleep(self._sleep_interval)

    async def _read_config(self):
        """Reads configuration"""
        default_config = {
            "sleep_interval": {
                "description": "Time in seconds to sleep between health checks. (must be greater than 5)",
                "type": "integer",
                "default": str(self._DEFAULT_SLEEP_INTERVAL)
            },
            "ping_timeout": {
                "description": "Timeout for a response from any given micro-service. (must be greater than 0)",
                "type": "integer",
                "default": str(self._DEFAULT_PING_TIMEOUT)
            },
            "max_attempts": {
                "description": "Maximum number of attempts for finding a heartbeat of service",
                "type": "integer",
                "default": str(self._DEFAULT_MAX_ATTEMPTS)
            },
            "restart_failed": {
                "description": "Restart failed microservice - manual/auto",
                "type": "string",
                "default": self._DEFAULT_RESTART_FAILED
            }
        }

        storage_client = connect.get_storage_async()
        cfg_manager = ConfigurationManager(storage_client)
        await cfg_manager.create_category('SMNTR', default_config, 'Service Monitor')

        config = await cfg_manager.get_category_all_items('SMNTR')

        self._sleep_interval = int(config['sleep_interval']['value'])
        self._ping_timeout = int(config['ping_timeout']['value'])
        self._max_attempts = int(config['max_attempts']['value'])
        self._restart_failed = config['restart_failed']['value']

    async def restart_service(self, service_record):
        from foglamp.services.core import server  # To avoid cyclic import as server also imports monitor
        schedule = await server.Server.scheduler.get_schedule_by_name(service_record._name)
        await server.Server.scheduler.queue_task(schedule.schedule_id)
        self.restarted_services.remove(service_record._id)

    async def start(self):
        await self._read_config()
        self._monitor_loop_task = asyncio.ensure_future(self._monitor_loop())

    async def stop(self):
        self._monitor_loop_task.cancel()
<commit_msg>More refactoring and fixes<commit_after># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

"""FogLAMP Monitor module"""

import asyncio
import aiohttp
import json
from foglamp.common import logger
from foglamp.common.audit_logger import AuditLogger
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.services.core.service_registry.service_registry import ServiceRegistry
from foglamp.common.service_record import ServiceRecord
from foglamp.services.core import connect

__author__ = "Ashwin Gopalakrishnan, Amarendra K Sinha"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

class Monitor(object):

    _DEFAULT_SLEEP_INTERVAL = 5
    """The time (in seconds) to sleep between health checks"""

    _DEFAULT_PING_TIMEOUT = 1
    """Timeout for a response from any given micro-service"""

    _DEFAULT_MAX_ATTEMPTS = 15

    _DEFAULT_RESTART_FAILED = "auto"
    """Restart failed microservice - manual/auto"""

    _logger = None

    def __init__(self):
        self._logger = logger.setup(__name__, level=20)

        self._monitor_loop_task = None  # type: asyncio.Task
        """Task for :meth:`_monitor_loop`, to ensure it has finished"""
        self._sleep_interval = None  # type: int
        """The time (in seconds) to sleep between health checks"""
        self._ping_timeout = None  # type: int
        """Timeout for a response from any given micro-service"""
        self._max_attempts = None  # type: int
        """Number of max attempts for finding a heartbeat of service"""
        self._restart_failed = None  # type: str
        """Restart failed microservice - manual/auto"""

        self.restarted_services = []

    async def _sleep(self, sleep_time):
        await asyncio.sleep(sleep_time)

    async def _monitor_loop(self):
        """async Monitor loop to monitor registered services"""
        # check health of all micro-services every N seconds
        round_cnt = 0
        check_count = {}  # dict to hold current count of current status.
                          # In case of ok and running status, count will always be 1.
                          # In case of of non running statuses, count shows since when this status is set.
        while True:
            round_cnt += 1
            self._logger.debug("Starting next round#{} of service monitoring, sleep/i:{} ping/t:{} max/a:{}".format(
                round_cnt, self._sleep_interval, self._ping_timeout, self._max_attempts))
            for service_record in ServiceRegistry.all():
                if service_record._id not in check_count:
                    check_count.update({service_record._id: 1})

                # Try ping if service status is either running or doubtful (i.e. give service a chance to recover)
                if service_record._status not in [ServiceRecord.Status.Running,
                                                  ServiceRecord.Status.Unresponsive,
                                                  ServiceRecord.Status.Failed]:
                    continue

                self._logger.debug("Service: {} Status: {}".format(service_record._name, service_record._status))

                if service_record._status == ServiceRecord.Status.Failed:
                    if self._restart_failed == "auto":
                        if service_record._id not in self.restarted_services:
                            self.restarted_services.append(service_record._id)
                            asyncio.ensure_future(self.restart_service(service_record))
                    continue

                try:
                    url = "{}://{}:{}/foglamp/service/ping".format(
                        service_record._protocol, service_record._address, service_record._management_port)
                    async with aiohttp.ClientSession() as session:
                        async with session.get(url, timeout=self._ping_timeout) as resp:
                            text = await resp.text()
                            res = json.loads(text)
                            if res["uptime"] is None:
                                raise ValueError('res.uptime is None')
                except (asyncio.TimeoutError, aiohttp.client_exceptions.ServerTimeoutError) as ex:
                    service_record._status = ServiceRecord.Status.Unresponsive
                    check_count[service_record._id] += 1
                    self._logger.info("ServerTimeoutError: %s, %s", str(ex), service_record.__repr__())
                except aiohttp.client_exceptions.ClientConnectorError as ex:
                    service_record._status = ServiceRecord.Status.Unresponsive
                    check_count[service_record._id] += 1
                    self._logger.info("ClientConnectorError: %s, %s", str(ex), service_record.__repr__())
                except ValueError as ex:
                    service_record._status = ServiceRecord.Status.Unresponsive
                    check_count[service_record._id] += 1
                    self._logger.info("Invalid response: %s, %s", str(ex), service_record.__repr__())
                except Exception as ex:
                    service_record._status = ServiceRecord.Status.Unresponsive
                    check_count[service_record._id] += 1
                    self._logger.info("Exception occurred: %s, %s", str(ex), service_record.__repr__())
                else:
                    service_record._status = ServiceRecord.Status.Running
                    check_count[service_record._id] = 1

                if check_count[service_record._id] > self._max_attempts:
                    ServiceRegistry.mark_as_failed(service_record._id)
                    check_count[service_record._id] = 0
                    try:
                        audit = AuditLogger(connect.get_storage_async())
                        await audit.failure('SRVFL', {'name':service_record._name})
                    except Exception as ex:
                        self._logger.info("Failed to audit service failure %s", str(ex))
            await self._sleep(self._sleep_interval)

    async def _read_config(self):
        """Reads configuration"""
        default_config = {
            "sleep_interval": {
                "description": "Time in seconds to sleep between health checks. (must be greater than 5)",
                "type": "integer",
                "default": str(self._DEFAULT_SLEEP_INTERVAL)
            },
            "ping_timeout": {
                "description": "Timeout for a response from any given micro-service. (must be greater than 0)",
                "type": "integer",
                "default": str(self._DEFAULT_PING_TIMEOUT)
            },
            "max_attempts": {
                "description": "Maximum number of attempts for finding a heartbeat of service",
                "type": "integer",
                "default": str(self._DEFAULT_MAX_ATTEMPTS)
            },
            "restart_failed": {
                "description": "Restart failed microservice - manual/auto",
                "type": "string",
                "default": self._DEFAULT_RESTART_FAILED
            }
        }

        storage_client = connect.get_storage_async()
        cfg_manager = ConfigurationManager(storage_client)
        await cfg_manager.create_category('SMNTR', default_config, 'Service Monitor')

        config = await cfg_manager.get_category_all_items('SMNTR')

        self._sleep_interval = int(config['sleep_interval']['value'])
        self._ping_timeout = int(config['ping_timeout']['value'])
        self._max_attempts = int(config['max_attempts']['value'])
        self._restart_failed = config['restart_failed']['value']

    async def restart_service(self, service_record):
        from foglamp.services.core import server  # To avoid cyclic import as server also imports monitor
        schedule = await server.Server.scheduler.get_schedule_by_name(service_record._name)
        await server.Server.scheduler.queue_task(schedule.schedule_id)
        self.restarted_services.remove(service_record._id)

    async def start(self):
        await self._read_config()
        self._monitor_loop_task = asyncio.ensure_future(self._monitor_loop())

    async def stop(self):
        self._monitor_loop_task.cancel()
<|endoftext|>


<commit_before># -*- mode: python; -*-
# Execute this script using Bash script of the same name but without a file
# extension. Use ../pdbwrapper/pdbwrapper instead of pdb for debugging.

import os
import sys
import argparse
import re
import time
from contextlib import contextmanager
import subprocess

# Example of contextmanager (http://tinyurl.com/llogvwu):
#
#   with time_print("processes"):
#       [doproc() for _ in range(500)]
#
@contextmanager
def time_print(task_name):
    t = time.time()
    try:
        yield
    finally:
        print task_name, "took", time.time() - t, "seconds."

description = r"""
the_name_of_script_goes_here -- Short description line goes here

Multi-line description goes here.

Multi-line description goes here.

Multi-line description goes here.
"""

def main():
    """
    Main function
    """
    # Parse command-line arguments:
    #    https://docs.python.org/2/howto/argparse.html
    parser = argparse.ArgumentParser(
        description=description,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("-theint", type=int,
                        help="some integer")

    # Warning: because of this nargs='+' item below, this is valid:
    #
    #     pythonTemplate.py -runmod one two -simplearg bar lonely
    #
    # but this is invalid:
    #
    #     pythonTemplate.py -simplearg bar -runmod one two lonely
    #     usage: pythonTemplate.py [-h] [-theint THEINT]
    #            [-runmod RUNMOD [RUNMOD ...]]
    #            [-simplearg SIMPLEARG] [-v] lonearg
    #     pythonTemplate.py: error: too few arguments
    #
    parser.add_argument('-runmod', nargs='+',
                        help='runner and its arguments')
    parser.add_argument('-simplearg', help='a simple arg')
    parser.add_argument('lonearg', help='a lone arg')
    # -v is an example of an argument that does not take a value
    # (http://stackoverflow.com/a/5271692/257924):
    parser.add_argument("-v", "--verbose", help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args(sys.argv[1:])

    print 'args.runmod {0}'.format(args.runmod)
    print 'args.simplearg {0}'.format(args.simplearg)
    print 'args.lonearg {0}'.format(args.lonearg)
    print 'args.theint {0}'.format(args.theint)

    # --------------------------------------------------------------------------------
    # Lists:
    # --------------------------------------------------------------------------------
    thelist = ["foo", "bar"]
    thejoinedstring = " ".join(thelist)
    print "thejoinedstring<" + thejoinedstring + ">"

    # --------------------------------------------------------------------------------
    # Environment variables:
    # --------------------------------------------------------------------------------

    env = "SOME_VAR||unset"
    # env = "SOME_VAR|var"
    vals = env.split('|')
    print 'vals' + str(vals)
    if len(vals) == 3:
        var = vals[0]
        print "Unsetting environment variable:", var
        if var in os.environ:
            del os.environ[var]
    else:
        var, val = vals
        print "Setting environment variable: {0}={1}".format(var, val)
        os.environ[var] = val

    print 'Left aligned example:  x{0:<20}x'.format('foo')
    print 'Right aligned example: x{0:>20}x'.format('foo')
    print 'Floating point format: {:.6f}'.format(12.1234)

    # --------------------------------------------------------------------------------
    # Reading from subprocesses:
    # --------------------------------------------------------------------------------

    # For more advanced methods that are Unix specific, See:
    #
    #  1. handling stdout and stderr from Popen:
    #     http://stackoverflow.com/a/7730201
    #  2. http://stackoverflow.com/a/17698359/257924
    cmd = 'ls -ld /etc /etc/does_not_exist'
    #
    #
    # This does not work: it prints each character in the stdout_line,
    # not each line:
    #
    if False:
        print 'Example: Reading stdout and stderr from a process based upon http://stackoverflow.com/a/17698359/257924'
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        for stdout_line in stdout:
            print 'stdout_line=="{}"'.format(stdout_line)
        for stderr_line in stderr:
            print 'stderr_line=="{}"'.format(stderr_line)
    #
    # This variation works.
    #   See http://stackoverflow.com/a/17698359/257924
    #
    if True:
        print 'Example: Reading stdout and stderr from a process based upon http://stackoverflow.com/a/17698359/257924'
        # shell=True as otherwise "ls ..." raises exception of "no such file
        # or directory":
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             bufsize=1)
        #
        # Read all of stdout:
        #
        with p.stdout:
            for stdout_line in iter(p.stdout.readline, b''):
                print 'stdout_line=="{}"'.format(stdout_line)
        #
        # Read all of stderr:
        #
        with p.stderr:
            for stderr_line in iter(p.stderr.readline, b''):
                print 'stderr_line=="{}"'.format(stderr_line)
        p.wait()  # wait for the subprocess to exit

        print "Example: Simplified Reading combined stdout and stderr from a process."
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             bufsize=1)

        #
        # Read all of stdout (which should contain stderr due to the stderr
        # directive above):
        #
        with p.stdout:
            for line in iter(p.stdout.readline, b''):
                line = line.rstrip()
                print line
        p.wait()  # wait for the subprocess to exit and set return code
        print "exit code: {}".format(p.returncode)

    # --------------------------------------------------------------------------------
    # Reading/writing from files and matching on regular expressions:
    # --------------------------------------------------------------------------------

    print "Example: Reading a file and matching on regular expressions and subgroups:"
    in_file = "/etc/issue"
    with open(in_file, 'r') as f:
        for line in f:
            line = line.rstrip()
            m = re.search("release *([0-9.]+)", line)
            if m:
                release_num = m.group(1)
                print "release_num == <{}>".format(release_num)

    print "Example: Reading a file and searching/replacing on a regular expression:"
    with open(in_file, 'r') as f:
        for line in f:
            line = line.rstrip()
            line = re.sub(r'release', '<REPLACEMENT>', line)
            print line

    print "Example: Writing a file and searching/replacing on a regular expression"
    that_file = "/tmp/somefile.for.pythonTemplate"
    queue = ["testing", "one", "two", "three"]
    with open(that_file, 'w') as f:
        for line in queue:
            f.write(line + "\n")

    # --------------------------------------------------------------------------------
    # Context managers:
    # --------------------------------------------------------------------------------

    with time_print("something to do"):
        print 'sleeping ...'
        time.sleep(1)

    # --------------------------------------------------------------------------------
    # Working with dictionaries:
    # --------------------------------------------------------------------------------

    # Formatting dictionary using lambda:
    line_style = {"fill": "none", "fill-capacity": 1}
    print ";".join(map(
        lambda (k, v): "{}:{}".format(k, str(v)),
        line_style.iteritems()))

    # --------------------------------------------------------------------------------
    # Try/except clauses:
    # --------------------------------------------------------------------------------

    try:
        raise argparse.ArgumentTypeError('{} was not set in the environment.'.format("THE_ENV_VAR"))
    except argparse.ArgumentTypeError, e:
        # raise Exception, "%s [%d]" % (e.strerror, e.errno)
        print "The exception {}".format(e)

if __name__ == '__main__':
    main()
<commit_msg>Demonstrate that multiple shell commands can be included in the cmd. Add if True statements to allow enable/disabling sections of code for debugging.<commit_after># -*- mode: python; -*-
# Execute this script using Bash script of the same name but without a file
# extension. Use ../pdbwrapper/pdbwrapper instead of pdb for debugging.

import os
import sys
import argparse
import re
import time
from contextlib import contextmanager
import subprocess

# Example of contextmanager (http://tinyurl.com/llogvwu):
#
#   with time_print("processes"):
#       [doproc() for _ in range(500)]
#
@contextmanager
def time_print(task_name):
    t = time.time()
    try:
        yield
    finally:
        print task_name, "took", time.time() - t, "seconds."

description = r"""
the_name_of_script_goes_here -- Short description line goes here

Multi-line description goes here.

Multi-line description goes here.

Multi-line description goes here.
"""

def main():
    """
    Main function
    """
    # Parse command-line arguments:
    #    https://docs.python.org/2/howto/argparse.html
    parser = argparse.ArgumentParser(
        description=description,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("-theint", type=int,
                        help="some integer")

    # Warning: because of this nargs='+' item below, this is valid:
    #
    #     pythonTemplate.py -runmod one two -simplearg bar lonely
    #
    # but this is invalid:
    #
    #     pythonTemplate.py -simplearg bar -runmod one two lonely
    #     usage: pythonTemplate.py [-h] [-theint THEINT]
    #            [-runmod RUNMOD [RUNMOD ...]]
    #            [-simplearg SIMPLEARG] [-v] lonearg
    #     pythonTemplate.py: error: too few arguments
    #
    parser.add_argument('-runmod', nargs='+',
                        help='runner and its arguments')
    parser.add_argument('-simplearg', help='a simple arg')
    parser.add_argument('lonearg', help='a lone arg')
    # -v is an example of an argument that does not take a value
    # (http://stackoverflow.com/a/5271692/257924):
    parser.add_argument("-v", "--verbose", help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args(sys.argv[1:])

    print 'args.runmod {0}'.format(args.runmod)
    print 'args.simplearg {0}'.format(args.simplearg)
    print 'args.lonearg {0}'.format(args.lonearg)
    print 'args.theint {0}'.format(args.theint)

    # --------------------------------------------------------------------------------
    # Lists:
    # --------------------------------------------------------------------------------
    thelist = ["foo", "bar"]
    thejoinedstring = " ".join(thelist)
    print "thejoinedstring<" + thejoinedstring + ">"

    # --------------------------------------------------------------------------------
    # Environment variables:
    # --------------------------------------------------------------------------------

    env = "SOME_VAR||unset"
    # env = "SOME_VAR|var"
    vals = env.split('|')
    print 'vals' + str(vals)
    if len(vals) == 3:
        var = vals[0]
        print "Unsetting environment variable:", var
        if var in os.environ:
            del os.environ[var]
    else:
        var, val = vals
        print "Setting environment variable: {0}={1}".format(var, val)
        os.environ[var] = val

    print 'Left aligned example:  x{0:<20}x'.format('foo')
    print 'Right aligned example: x{0:>20}x'.format('foo')
    print 'Floating point format: {:.6f}'.format(12.1234)

    # --------------------------------------------------------------------------------
    # Reading from subprocesses:
    # --------------------------------------------------------------------------------

    # For more advanced methods that are Unix specific, See:
    #
    #  1. handling stdout and stderr from Popen:
    #     http://stackoverflow.com/a/7730201
    #  2. http://stackoverflow.com/a/17698359/257924
    #
    # Demonstrate that multiple shell commands can be included in the cmd:
    cmd = 'export THE_ENV_VAR=the_env_var_value; ls -ld /etc /etc/does_not_exist; echo THE_ENV_VAR==$THE_ENV_VAR'
    #
    #
    # This does not work: it prints each character in the stdout_line,
    # not each line:
    #
    if False:
        print 'Example: Reading stdout and stderr from a process based upon http://stackoverflow.com/a/17698359/257924'
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        for stdout_line in stdout:
            print 'stdout_line=="{}"'.format(stdout_line)
        for stderr_line in stderr:
            print 'stderr_line=="{}"'.format(stderr_line)
    #
    # This variation works.
    #   See http://stackoverflow.com/a/17698359/257924
    #
    if True:
        print 'Example: Reading stdout and stderr from a process based upon http://stackoverflow.com/a/17698359/257924'
        # shell=True as otherwise "ls ..." raises exception of "no such file
        # or directory":
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             bufsize=1)
        #
        # Read all of stdout:
        #
        with p.stdout:
            for stdout_line in iter(p.stdout.readline, b''):
                print 'stdout_line=="{}"'.format(stdout_line)
        #
        # Read all of stderr:
        #
        with p.stderr:
            for stderr_line in iter(p.stderr.readline, b''):
                print 'stderr_line=="{}"'.format(stderr_line)
        p.wait()  # wait for the subprocess to exit

        print "Example: Simplified Reading combined stdout and stderr from a process."
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             bufsize=1)

        #
        # Read all of stdout (which should contain stderr due to the stderr
        # directive above):
        #
        with p.stdout:
            for line in iter(p.stdout.readline, b''):
                line = line.rstrip()
                print line
        p.wait()  # wait for the subprocess to exit and set return code
        print "exit code: {}".format(p.returncode)

    # --------------------------------------------------------------------------------
    # Reading/writing from files and matching on regular expressions:
    # --------------------------------------------------------------------------------

    if True:
        print "Example: Reading a file and matching on regular expressions and subgroups:"
        in_file = "/etc/issue"
        with open(in_file, 'r') as f:
            for line in f:
                line = line.rstrip()
                m = re.search("release *([0-9.]+)", line)
                if m:
                    release_num = m.group(1)
                    print "release_num == <{}>".format(release_num)

        print "Example: Reading a file and searching/replacing on a regular expression:"
        with open(in_file, 'r') as f:
            for line in f:
                line = line.rstrip()
                line = re.sub(r'release', '<REPLACEMENT>', line)
                print line

        print "Example: Writing a file and searching/replacing on a regular expression"
        that_file = "/tmp/somefile.for.pythonTemplate"
        queue = ["testing", "one", "two", "three"]
        with open(that_file, 'w') as f:
            for line in queue:
                f.write(line + "\n")

    # --------------------------------------------------------------------------------
    # Context managers:
    # --------------------------------------------------------------------------------

    if True:
        with time_print("something to do"):
            print 'sleeping ...'
            time.sleep(1)

    # --------------------------------------------------------------------------------
    # Working with dictionaries:
    # --------------------------------------------------------------------------------

    if True:
        # Formatting dictionary using lambda:
        line_style = {"fill": "none", "fill-capacity": 1}
        print ";".join(map(
            lambda (k, v): "{}:{}".format(k, str(v)),
            line_style.iteritems()))

    # --------------------------------------------------------------------------------
    # Try/except clauses:
    # --------------------------------------------------------------------------------

    if True:
        try:
            raise argparse.ArgumentTypeError('{} was not set in the environment.'.format("THE_ENV_VAR"))
        except argparse.ArgumentTypeError, e:
            # raise Exception, "%s [%d]" % (e.strerror, e.errno)
            print "The exception {}".format(e)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os

from media.monitor.exceptions import NoConfigFile
from media.monitor.pure import LazyProperty
from media.monitor.config import MMConfig
from api_clients import AirtimeApiClient

class AirtimeInstance(object):
    """ AirtimeInstance is a class that abstracts away every airtime
    instance by providing all the necessary objects required to interact
    with the instance. ApiClient, configs, root_directory """

    def __init__(self,name, root_path, config_paths):
        """ name is an internal name only """
        for cfg in ['api_client','media_monitor', 'logging']:
            if cfg not in config_paths: raise NoConfigFile(config_paths)
            elif not os.path.exists(config_paths[cfg]):
                raise NoConfigFile(config_paths[cfg])
        self.name         = name
        self.config_paths = config_paths
        self.root_path    = root_path

    def __str__(self):
        return "%s,%s(%s)" % (self.name, self.root_path, self.config_paths)

    @LazyProperty
    def api_client(self):
        return AirtimeApiClient(config_path=self.config_paths['api_client'])

    @LazyProperty
    def mm_config(self):
        return MMConfig(self.config_paths['media_monitor'])
<commit_msg>fixed import typo<commit_after>import os

from media.monitor.exceptions import NoConfigFile
from media.monitor.pure import LazyProperty
from media.monitor.config import MMConfig
from api_clients.api_client import AirtimeApiClient

class AirtimeInstance(object):
    """ AirtimeInstance is a class that abstracts away every airtime
    instance by providing all the necessary objects required to interact
    with the instance. ApiClient, configs, root_directory """

    def __init__(self,name, root_path, config_paths):
        """ name is an internal name only """
        for cfg in ['api_client','media_monitor', 'logging']:
            if cfg not in config_paths: raise NoConfigFile(config_paths)
            elif not os.path.exists(config_paths[cfg]):
                raise NoConfigFile(config_paths[cfg])
        self.name         = name
        self.config_paths = config_paths
        self.root_path    = root_path

    def __str__(self):
        return "%s,%s(%s)" % (self.name, self.root_path, self.config_paths)

    @LazyProperty
    def api_client(self):
        return AirtimeApiClient(config_path=self.config_paths['api_client'])

    @LazyProperty
    def mm_config(self):
        return MMConfig(self.config_paths['media_monitor'])
<|endoftext|>


<commit_before>import unittest
from media.monitor.eventcontractor import EventContractor
#from media.monitor.exceptions import BadSongFile
from media.monitor.events import FakePyinotify, NewFile, MoveFile, \
DeleteFile
from mock import patch

class TestMMP(unittest.TestCase):
    def test_event_registered(self):
        ev = EventContractor()
        e1 = NewFile( FakePyinotify('bullshit.mp3') )
        e2 = MoveFile( FakePyinotify('bullshit.mp3') )
        ev.register(e1)
        self.assertTrue( ev.event_registered(e2) )

    def test_get_old_event(self):
        ev = EventContractor()
        e1 = NewFile( FakePyinotify('bullshit.mp3') )
        e2 = MoveFile( FakePyinotify('bullshit.mp3') )
        ev.register(e1)
        self.assertEqual( ev.get_old_event(e2), e1 )

    def test_register(self):
        ev = EventContractor()
        e1 = NewFile( FakePyinotify('bullshit.mp3') )
        e2 = DeleteFile( FakePyinotify('bullshit.mp3') )
        self.assertTrue( ev.register(e1) )

        # Check that morph_into is called when it should be
        with patch.object(NewFile, 'morph_into', return_value='kimchi') \
        as mock_method:
            ret = ev.register(e2)
            self.assertFalse(ret)
            mock_method.assert_called_once_with(e2)

        # This time we are not patching morph
        self.assertFalse( ev.register(e2) )
        # We did not an element
        self.assertTrue( len(ev.store.keys()) == 1 )
        morphed = ev.get_old_event(e2)
        self.assertTrue( isinstance(morphed, DeleteFile) )

        delete_ev = e1.safe_pack()[0]
        print( ev.store )
        self.assertEqual( delete_ev['mode'], u'delete')
        self.assertTrue( len(ev.store.keys()) == 0 )

        e3 = DeleteFile( FakePyinotify('horseshit.mp3') )
        self.assertTrue( ev.register(e3) )
        self.assertTrue( ev.register(e2) )

    def test_register2(self):
        ev = EventContractor()
        p = 'bullshit.mp3'
        events = [
                NewFile( FakePyinotify(p) ),
                NewFile( FakePyinotify(p) ),
                DeleteFile( FakePyinotify(p) ),
                NewFile( FakePyinotify(p) ),
                NewFile( FakePyinotify(p) ), ]
        actual_events = []
        for e in events:
            if ev.register(e):
                actual_events.append(e)
        self.assertEqual( len(ev.store.keys()), 1 )
        packed = [ x.safe_pack() for x in actual_events ]
        print(packed)

if __name__ == '__main__': unittest.main()
<commit_msg>mm2: removed useless print statement<commit_after>import unittest
from media.monitor.eventcontractor import EventContractor
#from media.monitor.exceptions import BadSongFile
from media.monitor.events import FakePyinotify, NewFile, MoveFile, \
DeleteFile
from mock import patch

class TestMMP(unittest.TestCase):
    def test_event_registered(self):
        ev = EventContractor()
        e1 = NewFile( FakePyinotify('bullshit.mp3') )
        e2 = MoveFile( FakePyinotify('bullshit.mp3') )
        ev.register(e1)
        self.assertTrue( ev.event_registered(e2) )

    def test_get_old_event(self):
        ev = EventContractor()
        e1 = NewFile( FakePyinotify('bullshit.mp3') )
        e2 = MoveFile( FakePyinotify('bullshit.mp3') )
        ev.register(e1)
        self.assertEqual( ev.get_old_event(e2), e1 )

    def test_register(self):
        ev = EventContractor()
        e1 = NewFile( FakePyinotify('bullshit.mp3') )
        e2 = DeleteFile( FakePyinotify('bullshit.mp3') )
        self.assertTrue( ev.register(e1) )

        # Check that morph_into is called when it should be
        with patch.object(NewFile, 'morph_into', return_value='kimchi') \
        as mock_method:
            ret = ev.register(e2)
            self.assertFalse(ret)
            mock_method.assert_called_once_with(e2)

        # This time we are not patching morph
        self.assertFalse( ev.register(e2) )
        # We did not an element
        self.assertTrue( len(ev.store.keys()) == 1 )
        morphed = ev.get_old_event(e2)
        self.assertTrue( isinstance(morphed, DeleteFile) )

        delete_ev = e1.safe_pack()[0]
        self.assertEqual( delete_ev['mode'], u'delete')
        self.assertTrue( len(ev.store.keys()) == 0 )

        e3 = DeleteFile( FakePyinotify('horseshit.mp3') )
        self.assertTrue( ev.register(e3) )
        self.assertTrue( ev.register(e2) )

    def test_register2(self):
        ev = EventContractor()
        p = 'bullshit.mp3'
        events = [
                NewFile( FakePyinotify(p) ),
                NewFile( FakePyinotify(p) ),
                DeleteFile( FakePyinotify(p) ),
                NewFile( FakePyinotify(p) ),
                NewFile( FakePyinotify(p) ), ]
        actual_events = []
        for e in events:
            if ev.register(e):
                actual_events.append(e)
        self.assertEqual( len(ev.store.keys()), 1 )
        packed = [ x.safe_pack() for x in actual_events ]
        print(packed)

if __name__ == '__main__': unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vispy: gallery 30
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Vispy Development Team. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------

from vispy import scene, gloo
from vispy.visuals import Text

class Canvas(scene.SceneCanvas):
    def __init__(self):
        scene.SceneCanvas.__init__(self, title='Glyphs', keys='interactive')
        self.font_size = 48.
        self.text = Text('', bold=True)
        self.apply_zoom()

    def on_draw(self, event):
        gloo.clear(color='white')
        self.draw_visual(self.text)

    def on_mouse_wheel(self, event):
        """Use the mouse wheel to zoom."""
        self.font_size *= 1.25 if event.delta[1] > 0 else 0.8
        self.font_size = max(min(self.font_size, 160.), 6.)
        self.apply_zoom()

    def on_resize(self, event):
        self.apply_zoom()

    def apply_zoom(self):
        self.text.text = '%s pt' % round(self.font_size, 1)
        self.text.font_size = self.font_size
        self.text.pos = self.size[0] // 2, self.size[1] // 2
        self.update()

if __name__ == '__main__':
    c = Canvas()
    c.show()
    c.app.run()
<commit_msg>text example works<commit_after># -*- coding: utf-8 -*-
# vispy: gallery 30
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Vispy Development Team. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------

from vispy import app, gloo, visuals

class Canvas(app.Canvas):
    def __init__(self):
        app.Canvas.__init__(self, title='Glyphs', keys='interactive')
        self.font_size = 48.
        self.text = visuals.TextVisual('', bold=True)
        self.tr_sys = visuals.transforms.TransformSystem(self)
        self.apply_zoom()

    def on_draw(self, event):
        gloo.clear(color='white')
        gloo.set_viewport(0, 0, *self.size)
        self.text.draw(self.tr_sys)

    def on_mouse_wheel(self, event):
        """Use the mouse wheel to zoom."""
        self.font_size *= 1.25 if event.delta[1] > 0 else 0.8
        self.font_size = max(min(self.font_size, 160.), 6.)
        self.apply_zoom()

    def on_resize(self, event):
        self.apply_zoom()

    def apply_zoom(self):
        self.text.text = '%s pt' % round(self.font_size, 1)
        self.text.font_size = self.font_size
        self.text.pos = self.size[0] // 2, self.size[1] // 2
        self.update()

if __name__ == '__main__':
    c = Canvas()
    c.show()
    c.app.run()
<|endoftext|>


<commit_before>import os
import re

from django.conf import settings
from django.db import transaction
from django.http import *
from django.utils.hashcompat import md5_constructor
from django.utils.translation import ugettext_lazy as _
from django.utils.datastructures import MultiValueDict
from django.contrib.auth import login

from djangopypi.decorators import basic_auth
from djangopypi.forms import PackageForm, ReleaseForm
from djangopypi.models import Package, Release, Distribution, Classifier

ALREADY_EXISTS_FMT = _(
    "A file named '%s' already exists for %s. Please create a new release.")

def submit_package_or_release(user, post_data, files):
    """Registers/updates a package or release"""
    try:
        package = Package.objects.get(name=post_data['name'])
        if package.owner != user:
            return HttpResponseForbidden(
                    "That package is owned by someone else!")
    except Package.DoesNotExist:
        package = None

    package_form = PackageForm(post_data, instance=package)
    if package_form.is_valid():
        package = package_form.save(commit=False)
        package.owner = user
        package.save()
        for c in post_data.getlist('classifiers'):
            classifier, created = Classifier.objects.get_or_create(name=c)
            package.classifiers.add(classifier)
        if files:
            allow_overwrite = getattr(settings,
                "DJANGOPYPI_ALLOW_VERSION_OVERWRITE", False)
            try:
                release = Release.objects.get(version=post_data['version'],
                                              package=package,
                                              distribution=UPLOAD_TO + '/' +
                                              files['distribution']._name)
                if not allow_overwrite:
                    return HttpResponseForbidden(ALREADY_EXISTS_FMT % (
                                release.filename, release))
            except Release.DoesNotExist:
                release = None

            # If the old file already exists, django will append a _ after the
            # filename, however with .tar.gz files django does the "wrong"
            # thing and saves it as package-0.1.2.tar_.gz. So remove it before
            # django sees anything.
            release_form = ReleaseForm(post_data, files, instance=release)
            if release_form.is_valid():
                if release and os.path.exists(release.distribution.path):
                    os.remove(release.distribution.path)
                release = release_form.save(commit=False)
                release.package = package
                release.save()
            else:
                return HttpResponseBadRequest(
                        "ERRORS: %s" % release_form.errors)
    else:
        return HttpResponseBadRequest("ERRORS: %s" % package_form.errors)

    return HttpResponse()

@basic_auth
@transaction.commit_manually
def register_or_upload(request):
    if request.method != 'POST':
        return HttpResponseBadRequest('Only post requests are supported')

    name = request.POST.get('name',None).strip()

    if not name:
        return HttpResponseBadRequest('No package name specified')

    try:
        package = Package.objects.get(name=name)
    except Package.DoesNotExist:
        package = Package.objects.create(name=name)
        package.owners.add(request.user)

    if (request.user not in package.owners.all() and
        request.user not in package.maintainers.all()):

        return HttpResponseForbidden('You are not an owner/maintainer of %s' % (package.name,))

    version = request.POST.get('version',None).strip()
    metadata_version = request.POST.get('metadata_version', None).strip()

    if not version or not metadata_version:
        transaction.rollback()
        return HttpResponseBadRequest('Release version and metadata version must be specified')

    if not metadata_version in settings.DJANGOPYPI_METADATA_FIELDS:
        transaction.rollback()
        return HttpResponseBadRequest('Metadata version must be one of: %s'
                                      (', '.join(settings.DJANGOPYPI_METADATA_FIELDS.keys()),))

    release, created = Release.objects.get_or_create(package=package,
                                                     version=version)

    if (('classifiers' in request.POST or 'download_url' in request.POST) and
        metadata_version == '1.0'):
        metadata_version = '1.1'

    release.metadata_version = metadata_version

    fields = settings.DJANGOPYPI_METADATA_FIELDS[metadata_version]

    if 'classifiers' in request.POST:
        request.POST.setlist('classifier',request.POST.getlist('classifiers'))

    release.package_info = MultiValueDict(dict(filter(lambda t: t[0] in fields,
                                                      request.POST.iterlists())))

    release.save()
    if not 'content' in request.FILES:
        transaction.commit()
        return HttpResponse('release registered')

    uploaded = request.FILES.get('content')

    for dist in release.distributions.all():
        if os.path.basename(dist.content.name) == uploaded.name:
            """ Need to add handling optionally deleting old and putting up new """
            transaction.rollback()
            return HttpResponseBadRequest('That file has already been uploaded...')

    md5_digest = request.POST.get('md5_digest','')

    if not md5_digest:
        digest = md5_constructor()
        map(digest.update,uploaded)
        md5_digest = digest.hexdigest()

    try:
        new_file = Distribution.objects.create(release=release,
                                               content=uploaded,
                                               filetype=request.POST.get('filetype','sdist'),
                                               pyversion=request.POST.get('pyversion',''),
                                               uploader=request.user,
                                               comment=request.POST.get('comment',''),
                                               signature=request.POST.get('gpg_signature',''),
                                               md5_digest=md5_digest)
    except Exception, e:
        transaction.rollback()
        print str(e)

    transaction.commit()

    return HttpResponse('upload accepted')
<commit_msg>Filtering incoming data to remove 'UNKNOWN' that register command likes to add<commit_after>import os
import re

from django.conf import settings
from django.db import transaction
from django.http import *
from django.utils.hashcompat import md5_constructor
from django.utils.translation import ugettext_lazy as _
from django.utils.datastructures import MultiValueDict
from django.contrib.auth import login

from djangopypi.decorators import basic_auth
from djangopypi.forms import PackageForm, ReleaseForm
from djangopypi.models import Package, Release, Distribution, Classifier

ALREADY_EXISTS_FMT = _(
    "A file named '%s' already exists for %s. Please create a new release.")

def submit_package_or_release(user, post_data, files):
    """Registers/updates a package or release"""
    try:
        package = Package.objects.get(name=post_data['name'])
        if package.owner != user:
            return HttpResponseForbidden(
                    "That package is owned by someone else!")
    except Package.DoesNotExist:
        package = None

    package_form = PackageForm(post_data, instance=package)
    if package_form.is_valid():
        package = package_form.save(commit=False)
        package.owner = user
        package.save()
        for c in post_data.getlist('classifiers'):
            classifier, created = Classifier.objects.get_or_create(name=c)
            package.classifiers.add(classifier)
        if files:
            allow_overwrite = getattr(settings,
                "DJANGOPYPI_ALLOW_VERSION_OVERWRITE", False)
            try:
                release = Release.objects.get(version=post_data['version'],
                                              package=package,
                                              distribution=UPLOAD_TO + '/' +
                                              files['distribution']._name)
                if not allow_overwrite:
                    return HttpResponseForbidden(ALREADY_EXISTS_FMT % (
                                release.filename, release))
            except Release.DoesNotExist:
                release = None

            # If the old file already exists, django will append a _ after the
            # filename, however with .tar.gz files django does the "wrong"
            # thing and saves it as package-0.1.2.tar_.gz. So remove it before
            # django sees anything.
            release_form = ReleaseForm(post_data, files, instance=release)
            if release_form.is_valid():
                if release and os.path.exists(release.distribution.path):
                    os.remove(release.distribution.path)
                release = release_form.save(commit=False)
                release.package = package
                release.save()
            else:
                return HttpResponseBadRequest(
                        "ERRORS: %s" % release_form.errors)
    else:
        return HttpResponseBadRequest("ERRORS: %s" % package_form.errors)

    return HttpResponse()

@basic_auth
@transaction.commit_manually
def register_or_upload(request):
    if request.method != 'POST':
        return HttpResponseBadRequest('Only post requests are supported')

    name = request.POST.get('name',None).strip()

    if not name:
        return HttpResponseBadRequest('No package name specified')

    try:
        package = Package.objects.get(name=name)
    except Package.DoesNotExist:
        package = Package.objects.create(name=name)
        package.owners.add(request.user)

    if (request.user not in package.owners.all() and
        request.user not in package.maintainers.all()):

        return HttpResponseForbidden('You are not an owner/maintainer of %s' % (package.name,))

    version = request.POST.get('version',None).strip()
    metadata_version = request.POST.get('metadata_version', None).strip()

    if not version or not metadata_version:
        transaction.rollback()
        return HttpResponseBadRequest('Release version and metadata version must be specified')

    if not metadata_version in settings.DJANGOPYPI_METADATA_FIELDS:
        transaction.rollback()
        return HttpResponseBadRequest('Metadata version must be one of: %s'
                                      (', '.join(settings.DJANGOPYPI_METADATA_FIELDS.keys()),))

    release, created = Release.objects.get_or_create(package=package,
                                                     version=version)

    if (('classifiers' in request.POST or 'download_url' in request.POST) and
        metadata_version == '1.0'):
        metadata_version = '1.1'

    release.metadata_version = metadata_version

    fields = settings.DJANGOPYPI_METADATA_FIELDS[metadata_version]

    if 'classifiers' in request.POST:
        request.POST.setlist('classifier',request.POST.getlist('classifiers'))

    release.package_info = MultiValueDict(dict(filter(lambda t: t[0] in fields,
                                                      request.POST.iterlists())))

    for key, value in release.package_info.iterlists():
        release.package_info.setlist(key,
                                     filter(lambda v: v != 'UNKNOWN', value))

    release.save()
    if not 'content' in request.FILES:
        transaction.commit()
        return HttpResponse('release registered')

    uploaded = request.FILES.get('content')

    for dist in release.distributions.all():
        if os.path.basename(dist.content.name) == uploaded.name:
            """ Need to add handling optionally deleting old and putting up new """
            transaction.rollback()
            return HttpResponseBadRequest('That file has already been uploaded...')

    md5_digest = request.POST.get('md5_digest','')

    if not md5_digest:
        digest = md5_constructor()
        map(digest.update,uploaded)
        md5_digest = digest.hexdigest()

    try:
        new_file = Distribution.objects.create(release=release,
                                               content=uploaded,
                                               filetype=request.POST.get('filetype','sdist'),
                                               pyversion=request.POST.get('pyversion',''),
                                               uploader=request.user,
                                               comment=request.POST.get('comment',''),
                                               signature=request.POST.get('gpg_signature',''),
                                               md5_digest=md5_digest)
    except Exception, e:
        transaction.rollback()
        print str(e)

    transaction.commit()

    return HttpResponse('upload accepted')
<|endoftext|>


<commit_before># coding: UTF-8
"""
Copyright (c) 2009 Marian Tietz
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

1. Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
"""

import gtk
from lib.htmlbuffer import HTMLBuffer
from helper import URLHandler
import config

class OutputTextView(gtk.TextView):

	def __init__(self):
		gtk.TextView.__init__(self,
			HTMLBuffer(handler = URLHandler.URLHandler))

		self.set_property("editable", False)
		self.set_property("can-focus", False)
		self.set_property("wrap-mode", gtk.WRAP_WORD_CHAR)
		self.set_property("cursor-visible", False)

		self.read_line = ()

	def scroll_to_bottom(self):
		tb = self.get_buffer()

		mark = tb.create_mark("end", tb.get_end_iter(), False)
		self.scroll_to_mark(mark, 0.05, True, 0.0, 1.0)
		tb.delete_mark(mark)

	def set_read_line(self):
		buffer = self.get_buffer()

		if self.read_line:
			offsetA ,offsetB = self.read_line[1:]
			iterA = buffer.get_iter_at_offset(offsetA)
			iterB = buffer.get_iter_at_offset(offsetB)
			buffer.delete(iterA, iterB)
			buffer.remove_tag(self.read_line[0], iterA, iterB)

		tag = buffer.create_tag(None, justification = gtk.JUSTIFY_CENTER,
			strikethrough = True)
		end_iter = buffer.get_end_iter()
		offset = end_iter.get_offset()

		buffer.insert_with_tags(end_iter,
			"\n"+" "*int(config.get("tekka","divider_length")), tag)

		self.read_line = (tag, offset, buffer.get_end_iter().get_offset())
<commit_msg>Added check for empty iter<commit_after># coding: UTF-8
"""
Copyright (c) 2009 Marian Tietz
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

1. Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
"""

import gtk
from lib.htmlbuffer import HTMLBuffer
from helper import URLHandler
import config

class OutputTextView(gtk.TextView):

	def __init__(self):
		gtk.TextView.__init__(self,
			HTMLBuffer(handler = URLHandler.URLHandler))

		self.set_property("editable", False)
		self.set_property("can-focus", False)
		self.set_property("wrap-mode", gtk.WRAP_WORD_CHAR)
		self.set_property("cursor-visible", False)

		self.read_line = ()

	def scroll_to_bottom(self):
		tb = self.get_buffer()

		mark = tb.create_mark("end", tb.get_end_iter(), False)
		self.scroll_to_mark(mark, 0.05, True, 0.0, 1.0)
		tb.delete_mark(mark)

	def set_read_line(self):
		buffer = self.get_buffer()

		if self.read_line:
			offsetA ,offsetB = self.read_line[1:]
			iterA = buffer.get_iter_at_offset(offsetA)
			iterB = buffer.get_iter_at_offset(offsetB)

			if None in (iterA, iterB):
				return

			buffer.delete(iterA, iterB)
			buffer.remove_tag(self.read_line[0], iterA, iterB)

		tag = buffer.create_tag(None, justification = gtk.JUSTIFY_CENTER,
			strikethrough = True)
		end_iter = buffer.get_end_iter()
		offset = end_iter.get_offset()

		buffer.insert_with_tags(end_iter,
			"\n"+" "*int(config.get("tekka","divider_length")), tag)

		self.read_line = (tag, offset, buffer.get_end_iter().get_offset())
<|endoftext|>


<commit_before>""" SmaSh plugin describing support and shortcuts
    for the duckduckgo search engine.

    TODO: 'requires' is not currently enforced
"""
import threading
from smashlib.util import report
from smashlib.smash_plugin import SmashPlugin

class Plugin(SmashPlugin):
    """ TODO: caching, integrate with bookmarks """

    requires = [ 'pynotify', 'duckduckgo' ]

    class q(object):
        """

        Simple operations:

           Excuting plain duckduckgo searches is asynchronous.

              q('multiple', 'words')
              q('multiple words')
              ,q singleWord

           execute "bang" search, aka "exclusive" aka site-search.
           if possible, these are opened in an existing webbrowser immediately.

              q.wiki('world war 2')
              ,q.wiki world war 2
              ,q.stacktrace python quicksort

        """

        def __getattr__(self, name):
            return lambda *x: self( *(('!'+name,) + x) )

        def note(self, msg):
            import pynotify
            pynotify.init("Test Capabilities")
            caps = pynotify.get_server_caps()
            note = pynotify.Notification('Search "{0}" finished'.format(self.last_search), msg)
            return note

        def __call__(self, *search_string):
            search_string = ' '.join(search_string)
            self.last_search = search_string
            def func():
                import webbrowser
                import duckduckgo
                result = duckduckgo.query(search_string)
                if result.type=='exclusive':
                    note = self.note("Opening {0}".format(result.redirect.url))
                    webbrowser.open_new_tab(result.redirect.url)
                else:
                    note   = self.note('type={0} related={1}'.format(result.type, len(result.related)))
                    note.set_timeout(120*1000) # dont auto-hide for two minutes
                    def callback(*args, **kargs):
                        print 'args/kargs', args, kargs
                        for r in result:
                            print r.text
                    note.add_action('show results', callback)
                note.show()
                __IPYTHON__.user_ns.update(result = result)
                return result, note

            threading.Thread(target=func).start()
    q = q()

    def install(self):
        self.contribute('q', self.q)
        """
        if 'q' in __IPYTHON__.user_ns:
            report.quacker('"q" variable is taken in user namespace.  refusing to proceed')
        else:
            #self.q.stackoverflow = lambda *search_string: self.q('!stackoverflow',*search_string)
            __IPYTHON__.user_ns.update(q=self.q)
            report.quacker("finished installing.  type 'q?' for help with search")
            """
<commit_msg>handle urllib errors more gracefully<commit_after>""" SmaSh plugin describing support and shortcuts
    for the duckduckgo search engine.

    TODO: 'requires' is not currently enforced
"""
import threading
import urllib2
import webbrowser
from smashlib.util import report
from smashlib.smash_plugin import SmashPlugin

class Plugin(SmashPlugin):
    """ TODO: caching, integrate with bookmarks """

    requires = [ 'pynotify', 'duckduckgo' ]

    class q(object):
        """

        Simple operations:

           Excuting plain duckduckgo searches is asynchronous.

              q('multiple', 'words')
              q('multiple words')
              ,q singleWord

           execute "bang" search, aka "exclusive" aka site-search.
           if possible, these are opened in an existing webbrowser immediately.

              q.wiki('world war 2')
              ,q.wiki world war 2
              ,q.stacktrace python quicksort

        """

        def __getattr__(self, name):
            return lambda *x: self( *(('!'+name,) + x) )

        @property
        def pynotify(self):
            try:
                import pynotify
                return pynotify
            except:
                return None

        def note(self, msg):
            self.pynotify.init("Test Capabilities")
            caps = self.pynotify.get_server_caps()
            note = self.pynotify.Notification('Search "{0}" finished'.format(self.last_search), msg)
            return note

        def __call__(self, *search_string):
            search_string = ' '.join(search_string)
            self.last_search = search_string
            def func():
                import duckduckgo
                result = duckduckgo.query(search_string)
                if result.type=='exclusive':
                    note = self.note("Opening {0}".format(result.redirect.url))
                    try:
                        webbrowser.open_new_tab(result.redirect.url)
                    except urllib2.URLError:
                        note=self.note('caught URLError.  is the internet turned on?')
                        note.show()
                else:
                    note   = self.note('type={0} related={1}'.format(result.type, len(result.related)))
                    note.set_timeout(120*1000) # dont auto-hide for two minutes
                    def callback(*args, **kargs):
                        print 'args/kargs', args, kargs
                        for r in result:
                            print r.text
                    note.add_action('show results', callback)
                note.show()
                __IPYTHON__.user_ns.update(result = result)
                return result, note
            if self.pynotify is not None:
                threading.Thread(target=func).start()
            else:
                report('pynotify is not available')
    q = q()

    def install(self):
        self.contribute('q', self.q)
        """
        if 'q' in __IPYTHON__.user_ns:
            report.quacker('"q" variable is taken in user namespace.  refusing to proceed')
        else:
            #self.q.stackoverflow = lambda *search_string: self.q('!stackoverflow',*search_string)
            __IPYTHON__.user_ns.update(q=self.q)
            report.quacker("finished installing.  type 'q?' for help with search")
            """
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Handles a single Python breakpoint."""

from datetime import datetime
from datetime import timedelta
import os
from threading import Lock

import capture_collector
import cdbg_native as native
import deferred_modules
import module_explorer
import module_lookup

# TODO(vlif): move to messages.py module.
BREAKPOINT_ONLY_SUPPORTS_PY_FILES = (
    'Only files with .py or .pyc extension are supported')
MODULE_NOT_FOUND = (
    'Python module not found')
NO_CODE_FOUND_AT_LINE = (
    'No code found at line $0')
GLOBAL_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation cost for all active '
    'snapshots might affect the application performance.')
BREAKPOINT_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation at this location might '
    'affect application performance. Please simplify the condition or move '
    'the snapshot to a less frequently called statement.')
MUTABLE_CONDITION = (
    'Only immutable expressions can be used in snapshot conditions')
SNAPSHOT_EXPIRED = (
    'The snapshot has expired')
LOGPOINT_EXPIRED = (
    'The logpoint has expired')
INTERNAL_ERROR = (
    'Internal error occurred')

# Status messages for different breakpoint events (except of "hit").
_BREAKPOINT_EVENT_STATUS = dict(
    [(native.BREAKPOINT_EVENT_ERROR,
      {'isError': True,
       'description': {'format': INTERNAL_ERROR}}),
     (native.BREAKPOINT_EVENT_GLOBAL_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': GLOBAL_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_BREAKPOINT_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': BREAKPOINT_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_CONDITION_EXPRESSION_MUTABLE,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': MUTABLE_CONDITION}})])

class PythonBreakpoint(object):
  """Handles a single Python breakpoint.

  Taking care of a breakpoint starts with setting one and evaluating
  condition. When a breakpoint we need to evaluate all the watched expressions
  and take an action. The action can be either to collect all the data or
  to log a statement.
  """

  def __init__(self, definition, hub_client, breakpoints_manager):
    """Class constructor.

    Tries to set the breakpoint. If the source location is invalid, the
    breakpoint is completed with an error message. If the source location is
    valid, but the module hasn't been loaded yet, the breakpoint is initialized
    as deferred.

    Args:
      definition: breakpoint definition as it came from the backend.
      hub_client: asynchronously sends breakpoint updates to the backend.
      breakpoints_manager: parent object managing active breakpoints.
    """
    self.definition = definition

    # Breakpoint expiration time.
    self.expiration_period = timedelta(hours=24)

    self._hub_client = hub_client
    self._breakpoints_manager = breakpoints_manager
    self._cookie = None
    self._import_hook_cleanup = None

    self._lock = Lock()
    self._completed = False

    if not self._TryActivateBreakpoint() and not self._completed:
      self._DeferBreakpoint()

  def Clear(self):
    """Clears the breakpoint and releases all breakpoint resources.

    This function is assumed to be called by BreakpointsManager. Therefore we
    don't call CompleteBreakpoint from here.
    """
    self._RemoveImportHook()
    if self._cookie is not None:
      native.LogInfo('Clearing breakpoint %s' % self.GetBreakpointId())
      native.ClearConditionalBreakpoint(self._cookie)
      self._cookie = None

    self._completed = True  # Never again send updates for this breakpoint.

  def GetBreakpointId(self):
    return self.definition['id']

  def GetExpirationTime(self):
    """Computes the timestamp at which this breakpoint will expire."""
    # TODO(emrekultursay): Move this to a common method.
    if '.' not in self.definition['createTime']:
      fmt = '%Y-%m-%dT%H:%M:%S%Z'
    else:
      fmt = '%Y-%m-%dT%H:%M:%S.%f%Z'

    create_datetime = datetime.strptime(
        self.definition['createTime'].replace('Z', 'UTC'), fmt)
    return create_datetime + self.expiration_period

  def ExpireBreakpoint(self):
    """Expires this breakpoint."""
    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    if self.definition.get('action') == 'LOG':
      message = LOGPOINT_EXPIRED
    else:
      message = SNAPSHOT_EXPIRED
    self._CompleteBreakpoint({
        'status': {
            'isError': True,
            'refersTo': 'BREAKPOINT_AGE',
            'description': {'format': message}}})

  def _TryActivateBreakpoint(self):
    """Sets the breakpoint if the module has already been loaded.

    This function will complete the breakpoint with error if breakpoint
    definition is incorrect. Examples: invalid line or bad condition.

    If the code object corresponding to the source path can't be found,
    this function returns False. In this case, the breakpoint is not
    completed, since the breakpoint may be deferred.

    Returns:
      True if breakpoint was set or false otherwise. False can be returned
      for potentially deferred breakpoints or in case of a bad breakpoint
      definition. The self._completed flag distinguishes between the two cases.
    """

    # Find the code object in which the breakpoint is being set.
    code_object = self._FindCodeObject()
    if not code_object:
      return False

    # Compile the breakpoint condition.
    condition = None
    if self.definition.get('condition'):
      try:
        condition = compile(self.definition.get('condition'),
                            '<condition_expression>',
                            'eval')
      except TypeError as e:  # condition string contains null bytes.
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Invalid expression',
                    'parameters': [str(e)]}}})
        return False
      except SyntaxError as e:
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Expression could not be compiled: $0',
                    'parameters': [e.msg]}}})
        return False

    line = self.definition['location']['line']

    native.LogInfo('Creating new Python breakpoint %s in %s, line %d' % (
        self.GetBreakpointId(), code_object, line))

    self._cookie = native.SetConditionalBreakpoint(
        code_object,
        line,
        condition,
        self._BreakpointEvent)

    self._RemoveImportHook()
    return True

  def _FindCodeObject(self):
    """Finds the target code object for the breakpoint.

    This function completes breakpoint with error if the module was found,
    but the line number is invalid. When code object is not found for the
    breakpoint source location, this function just returns None. It does not
    assume error, because it might be a deferred breakpoint.

    Returns:
      Python code object object in which the breakpoint will be set or None if
      module not found or if there is no code at the specified line.
    """
    path = self.definition['location']['path']
    line = self.definition['location']['line']

    module = module_lookup.FindModule(path)
    if not module:
      return None

    code_object = module_explorer.GetCodeObjectAtLine(module, line)
    if code_object is None:
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {
                  'format': NO_CODE_FOUND_AT_LINE,
                  'parameters': [str(line)]}}})
      return None

    return code_object

  # Enables deferred breakpoints.
  def _DeferBreakpoint(self):
    """Defers breakpoint activation until the module has been loaded.

    This function first verifies that a module corresponding to breakpoint
    location exists. This way if the user sets breakpoint in a file that
    doesn't even exist, the debugger will not be waiting forever. If there
    is definitely no module that matches this breakpoint, this function
    completes the breakpoint with error status.

    Otherwise the debugger assumes that the module corresponding to breakpoint
    location hasn't been loaded yet. The debugger will then start waiting for
    the module to get loaded. Once the module is loaded, the debugger
    will automatically try to activate the breakpoint.
    """
    path = self.definition['location']['path']

    if os.path.splitext(path)[1] != '.py':
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': BREAKPOINT_ONLY_SUPPORTS_PY_FILES}}})
      return

    if not deferred_modules.IsValidSourcePath(path):
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': MODULE_NOT_FOUND}}})

    assert not self._import_hook_cleanup
    self._import_hook_cleanup = deferred_modules.AddImportCallback(
        self.definition['location']['path'],
        lambda unused_module_name: self._TryActivateBreakpoint())

  def _RemoveImportHook(self):
    """Removes the import hook if one was installed."""
    if self._import_hook_cleanup:
      self._import_hook_cleanup()
      self._import_hook_cleanup = None

  def _CompleteBreakpoint(self, data, is_incremental=True):
    """Sends breakpoint update and deactivates the breakpoint."""
    if is_incremental:
      data = dict(self.definition, **data)
    data['isFinalState'] = True

    self._hub_client.EnqueueBreakpointUpdate(data)
    self._breakpoints_manager.CompleteBreakpoint(self.GetBreakpointId())
    self.Clear()

  def _SetCompleted(self):
    """Atomically marks the breakpoint as completed.

    Returns:
      True if the breakpoint wasn't marked already completed or False if the
      breakpoint was already completed.
    """
    with self._lock:
      if self._completed:
        return False
      self._completed = True
      return True

  def _BreakpointEvent(self, event, frame):
    """Callback invoked by cdbg_native when breakpoint hits.

    Args:
      event: breakpoint event (see kIntegerConstants in native_module.cc).
      frame: Python stack frame of breakpoint hit or None for other events.
    """
    error_status = None

    if event != native.BREAKPOINT_EVENT_HIT:
      error_status = _BREAKPOINT_EVENT_STATUS[event]
    elif self.definition.get('action') == 'LOG':
      collector = capture_collector.LogCollector(self.definition)
      error_status = collector.Log(frame)
      if not error_status:
        return  # Log action successful, no need to clear the breakpoint.

    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    self.Clear()

    if error_status:
      self._CompleteBreakpoint({'status': error_status})
      return

    collector = capture_collector.CaptureCollector(self.definition)
    collector.Collect(frame)

    self._CompleteBreakpoint(collector.breakpoint, is_incremental=False)
<commit_msg>Do a dummy call to strptime at startup to work around a concurrency issue on the first call to strptime.<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Handles a single Python breakpoint."""

from datetime import datetime
from datetime import timedelta
import os
from threading import Lock

import capture_collector
import cdbg_native as native
import deferred_modules
import module_explorer
import module_lookup

# TODO(vlif): move to messages.py module.
BREAKPOINT_ONLY_SUPPORTS_PY_FILES = (
    'Only files with .py or .pyc extension are supported')
MODULE_NOT_FOUND = (
    'Python module not found')
NO_CODE_FOUND_AT_LINE = (
    'No code found at line $0')
GLOBAL_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation cost for all active '
    'snapshots might affect the application performance.')
BREAKPOINT_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation at this location might '
    'affect application performance. Please simplify the condition or move '
    'the snapshot to a less frequently called statement.')
MUTABLE_CONDITION = (
    'Only immutable expressions can be used in snapshot conditions')
SNAPSHOT_EXPIRED = (
    'The snapshot has expired')
LOGPOINT_EXPIRED = (
    'The logpoint has expired')
INTERNAL_ERROR = (
    'Internal error occurred')

# Status messages for different breakpoint events (except of "hit").
_BREAKPOINT_EVENT_STATUS = dict(
    [(native.BREAKPOINT_EVENT_ERROR,
      {'isError': True,
       'description': {'format': INTERNAL_ERROR}}),
     (native.BREAKPOINT_EVENT_GLOBAL_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': GLOBAL_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_BREAKPOINT_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': BREAKPOINT_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_CONDITION_EXPRESSION_MUTABLE,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': MUTABLE_CONDITION}})])

# The implementation of datetime.strptime imports an undocumented module called
# _strptime. If it happens at the wrong time, we can get an exception about
# trying to import while another thread holds the import lock. This dummy call
# to strptime ensures that the module is loaded at startup.
# See http://bugs.python.org/issue7980 for discussion of the Python bug.
datetime.strptime('2017-01-01', '%Y-%m-%d')

class PythonBreakpoint(object):
  """Handles a single Python breakpoint.

  Taking care of a breakpoint starts with setting one and evaluating
  condition. When a breakpoint we need to evaluate all the watched expressions
  and take an action. The action can be either to collect all the data or
  to log a statement.
  """

  def __init__(self, definition, hub_client, breakpoints_manager):
    """Class constructor.

    Tries to set the breakpoint. If the source location is invalid, the
    breakpoint is completed with an error message. If the source location is
    valid, but the module hasn't been loaded yet, the breakpoint is initialized
    as deferred.

    Args:
      definition: breakpoint definition as it came from the backend.
      hub_client: asynchronously sends breakpoint updates to the backend.
      breakpoints_manager: parent object managing active breakpoints.
    """
    self.definition = definition

    # Breakpoint expiration time.
    self.expiration_period = timedelta(hours=24)

    self._hub_client = hub_client
    self._breakpoints_manager = breakpoints_manager
    self._cookie = None
    self._import_hook_cleanup = None

    self._lock = Lock()
    self._completed = False

    if not self._TryActivateBreakpoint() and not self._completed:
      self._DeferBreakpoint()

  def Clear(self):
    """Clears the breakpoint and releases all breakpoint resources.

    This function is assumed to be called by BreakpointsManager. Therefore we
    don't call CompleteBreakpoint from here.
    """
    self._RemoveImportHook()
    if self._cookie is not None:
      native.LogInfo('Clearing breakpoint %s' % self.GetBreakpointId())
      native.ClearConditionalBreakpoint(self._cookie)
      self._cookie = None

    self._completed = True  # Never again send updates for this breakpoint.

  def GetBreakpointId(self):
    return self.definition['id']

  def GetExpirationTime(self):
    """Computes the timestamp at which this breakpoint will expire."""
    # TODO(emrekultursay): Move this to a common method.
    if '.' not in self.definition['createTime']:
      fmt = '%Y-%m-%dT%H:%M:%S%Z'
    else:
      fmt = '%Y-%m-%dT%H:%M:%S.%f%Z'

    create_datetime = datetime.strptime(
        self.definition['createTime'].replace('Z', 'UTC'), fmt)
    return create_datetime + self.expiration_period

  def ExpireBreakpoint(self):
    """Expires this breakpoint."""
    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    if self.definition.get('action') == 'LOG':
      message = LOGPOINT_EXPIRED
    else:
      message = SNAPSHOT_EXPIRED
    self._CompleteBreakpoint({
        'status': {
            'isError': True,
            'refersTo': 'BREAKPOINT_AGE',
            'description': {'format': message}}})

  def _TryActivateBreakpoint(self):
    """Sets the breakpoint if the module has already been loaded.

    This function will complete the breakpoint with error if breakpoint
    definition is incorrect. Examples: invalid line or bad condition.

    If the code object corresponding to the source path can't be found,
    this function returns False. In this case, the breakpoint is not
    completed, since the breakpoint may be deferred.

    Returns:
      True if breakpoint was set or false otherwise. False can be returned
      for potentially deferred breakpoints or in case of a bad breakpoint
      definition. The self._completed flag distinguishes between the two cases.
    """

    # Find the code object in which the breakpoint is being set.
    code_object = self._FindCodeObject()
    if not code_object:
      return False

    # Compile the breakpoint condition.
    condition = None
    if self.definition.get('condition'):
      try:
        condition = compile(self.definition.get('condition'),
                            '<condition_expression>',
                            'eval')
      except TypeError as e:  # condition string contains null bytes.
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Invalid expression',
                    'parameters': [str(e)]}}})
        return False
      except SyntaxError as e:
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Expression could not be compiled: $0',
                    'parameters': [e.msg]}}})
        return False

    line = self.definition['location']['line']

    native.LogInfo('Creating new Python breakpoint %s in %s, line %d' % (
        self.GetBreakpointId(), code_object, line))

    self._cookie = native.SetConditionalBreakpoint(
        code_object,
        line,
        condition,
        self._BreakpointEvent)

    self._RemoveImportHook()
    return True

  def _FindCodeObject(self):
    """Finds the target code object for the breakpoint.

    This function completes breakpoint with error if the module was found,
    but the line number is invalid. When code object is not found for the
    breakpoint source location, this function just returns None. It does not
    assume error, because it might be a deferred breakpoint.

    Returns:
      Python code object object in which the breakpoint will be set or None if
      module not found or if there is no code at the specified line.
    """
    path = self.definition['location']['path']
    line = self.definition['location']['line']

    module = module_lookup.FindModule(path)
    if not module:
      return None

    code_object = module_explorer.GetCodeObjectAtLine(module, line)
    if code_object is None:
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {
                  'format': NO_CODE_FOUND_AT_LINE,
                  'parameters': [str(line)]}}})
      return None

    return code_object

  # Enables deferred breakpoints.
  def _DeferBreakpoint(self):
    """Defers breakpoint activation until the module has been loaded.

    This function first verifies that a module corresponding to breakpoint
    location exists. This way if the user sets breakpoint in a file that
    doesn't even exist, the debugger will not be waiting forever. If there
    is definitely no module that matches this breakpoint, this function
    completes the breakpoint with error status.

    Otherwise the debugger assumes that the module corresponding to breakpoint
    location hasn't been loaded yet. The debugger will then start waiting for
    the module to get loaded. Once the module is loaded, the debugger
    will automatically try to activate the breakpoint.
    """
    path = self.definition['location']['path']

    if os.path.splitext(path)[1] != '.py':
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': BREAKPOINT_ONLY_SUPPORTS_PY_FILES}}})
      return

    if not deferred_modules.IsValidSourcePath(path):
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': MODULE_NOT_FOUND}}})

    assert not self._import_hook_cleanup
    self._import_hook_cleanup = deferred_modules.AddImportCallback(
        self.definition['location']['path'],
        lambda unused_module_name: self._TryActivateBreakpoint())

  def _RemoveImportHook(self):
    """Removes the import hook if one was installed."""
    if self._import_hook_cleanup:
      self._import_hook_cleanup()
      self._import_hook_cleanup = None

  def _CompleteBreakpoint(self, data, is_incremental=True):
    """Sends breakpoint update and deactivates the breakpoint."""
    if is_incremental:
      data = dict(self.definition, **data)
    data['isFinalState'] = True

    self._hub_client.EnqueueBreakpointUpdate(data)
    self._breakpoints_manager.CompleteBreakpoint(self.GetBreakpointId())
    self.Clear()

  def _SetCompleted(self):
    """Atomically marks the breakpoint as completed.

    Returns:
      True if the breakpoint wasn't marked already completed or False if the
      breakpoint was already completed.
    """
    with self._lock:
      if self._completed:
        return False
      self._completed = True
      return True

  def _BreakpointEvent(self, event, frame):
    """Callback invoked by cdbg_native when breakpoint hits.

    Args:
      event: breakpoint event (see kIntegerConstants in native_module.cc).
      frame: Python stack frame of breakpoint hit or None for other events.
    """
    error_status = None

    if event != native.BREAKPOINT_EVENT_HIT:
      error_status = _BREAKPOINT_EVENT_STATUS[event]
    elif self.definition.get('action') == 'LOG':
      collector = capture_collector.LogCollector(self.definition)
      error_status = collector.Log(frame)
      if not error_status:
        return  # Log action successful, no need to clear the breakpoint.

    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    self.Clear()

    if error_status:
      self._CompleteBreakpoint({'status': error_status})
      return

    collector = capture_collector.CaptureCollector(self.definition)
    collector.Collect(frame)

    self._CompleteBreakpoint(collector.breakpoint, is_incremental=False)
<|endoftext|>