<commit_before>#! /usr/bin/env python3

# Copyright (c) 2017 Linaro Limited.
#
# SPDX-License-Identifier: Apache-2.0

"""Zephyr binary runner core interfaces

This provides the core ZephyrBinaryRunner class meant for public use,
as well as some other helpers for concrete runner classes.
"""

import abc
import os
import platform
import pprint
import shlex
import signal
import subprocess
import sys

def get_env_or_bail(env_var):
    '''Get an environment variable, or raise an error.

    In case of KeyError, an error message is printed, along with the
    environment, and the exception is re-raised.
    '''
    try:
        return os.environ[env_var]
    except KeyError:
        print('Variable {} not in environment:'.format(
                  env_var), file=sys.stderr)
        pprint.pprint(dict(os.environ), stream=sys.stderr)
        raise

def get_env_bool_or(env_var, default_value):
    '''Get an environment variable as a boolean, or return a default value.

    Get an environment variable, interpret it as a base ten
    integer, and convert that to a boolean.

    In case the environment variable is not defined, return default_value.
    '''
    try:
        return bool(int(os.environ[env_var]))
    except KeyError:
        return default_value

def get_env_strip_or(env_var, to_strip, default_value):
    '''Get and clean up an environment variable, or return a default value.

    Get the value of env_var from the environment. If it is
    defined, return that value with to_strip stripped off. If it
    is undefined, return default_value (without any stripping).
    '''
    value = os.environ.get(env_var, None)
    if value is not None:
        return value.strip(to_strip)
    else:
        return default_value

def quote_sh_list(cmd):
    '''Transform a command from list into shell string form.'''
    fmt = ' '.join('{}' for _ in cmd)
    args = [shlex.quote(s) for s in cmd]
    return fmt.format(*args)

MAX_PORT = 49151

class NetworkPortHelper:
    '''Helper class for dealing with local IP network ports.'''

    def get_unused_ports(self, starting_from):
        '''Find unused network ports, starting at given values.

        starting_from is an iterable of ports the caller would like to use.

        The return value is an iterable of ports, in the same order, using
        the given values if they were unused, or the next sequentially
        available unused port otherwise.

        Ports may be bound between this call's check and actual usage, so
        callers still need to handle errors involving returned ports.'''
        start = list(starting_from)
        used = self._used_now()
        ret = []

        for desired in start:
            port = desired
            while port in used:
                port += 1
                if port > MAX_PORT:
                    msg = "ports above {} are in use"
                    raise ValueError(msg.format(desired))
            used.add(port)
            ret.append(port)

        return ret

    def _used_now(self):
        handlers = {
            'Windows': self._used_now_windows,
            'Linux': self._used_now_linux,
            'Darwin': self._used_now_darwin,
        }
        handler = handlers[platform.system()]
        return handler()

    def _used_now_windows(self):
        cmd = ['netstat', '-a', '-n', '-p', 'tcp']
        return self._parser_windows(cmd)

    def _used_now_linux(self):
        cmd = ['ss', '-a', '-n', '-t']
        return self._parser_linux(cmd)

    def _used_now_darwin(self):
        cmd = ['netstat', '-a', '-n', '-p', 'tcp']
        return self._parser_darwin(cmd)

    def _parser_windows(self, cmd):
        out = subprocess.check_output(cmd).split(b'\r\n')
        used_bytes = [x.split()[1].rsplit(b':', 1)[1] for x in out
                      if x.startswith(b'  TCP')]
        return {int(b) for b in used_bytes}

    def _parser_linux(self, cmd):
        out = subprocess.check_output(cmd).splitlines()[1:]
        used_bytes = [s.split()[3].rsplit(b':', 1)[1] for s in out]
        return {int(b) for b in used_bytes}

    def _parser_darwin(self, cmd):
        out = subprocess.check_output(cmd).split(b'\n')
        used_bytes = [x.split()[3].rsplit(b':', 1)[1] for x in out
                      if x.startswith(b'tcp')]
        return {int(b) for b in used_bytes}

class ZephyrBinaryRunner(abc.ABC):
    '''Abstract superclass for binary runners (flashers, debuggers).

    With some exceptions, boards supported by Zephyr must provide
    generic means to be flashed (have a Zephyr firmware binary
    permanently installed on the device for running) and debugged
    (have a breakpoint debugger and program loader on a host
    workstation attached to a running target). This is supported by
    three top-level commands managed by the Zephyr build system:

    - 'flash': flash a previously configured binary to the board,
      start execution on the target, then return.

    - 'debug': connect to the board via a debugging protocol, then
      drop the user into a debugger interface with symbol tables
      loaded from the current binary, and block until it exits.

    - 'debugserver': connect via a board-specific debugging protocol,
      then reset and halt the target. Ensure the user is now able to
      connect to a debug server with symbol tables loaded from the
      binary.

    Runner functionality relies on a variety of target-specific tools
    and configuration values, the user interface to which is
    abstracted by this class. Each runner subclass should take any
    values it needs to execute one of these commands in its
    constructor.  The actual command execution is handled in the run()
    method.

    This functionality is also replacing the legacy Zephyr runners,
    which are shell scripts.

    At present, the Zephyr build system uses a variety of
    tool-specific environment variables to control runner behavior.
    To support a transition to ZephyrBinaryRunner and subclasses, this
    class provides a create_for_shell_script() static factory method.
    This method iterates over ZephyrBinaryRunner subclasses,
    determines which (if any) can provide equivalent functionality to
    the shell-based runner, and returns a subclass instance with its
    configuration determined from the environment.

    To support this, subclasess currently must provide a pair of
    static methods, replaces_shell_script() and create_from_env(). The
    first allows the runner subclass to declare which commands and
    scripts it can replace. The second is called by
    create_for_shell_script() to create a concrete runner instance.

    The environment-based factories are for legacy use *only*; the
    user must be able to construct and use a runner using only the
    constructor and run() method.'''

    def __init__(self, debug=False):
        self.debug = debug

    @staticmethod
    def create_for_shell_script(shell_script, command, debug):
        '''Factory for using as a drop-in replacement to a shell script.

        Command is one of 'flash', 'debug', 'debugserver'.

        Get runner instance to use in place of shell_script, deriving
        configuration from the environment.'''
        for sub_cls in ZephyrBinaryRunner.__subclasses__():
            if sub_cls.replaces_shell_script(shell_script, command):
                return sub_cls.create_from_env(command, debug)
        raise ValueError('cannot implement script {} command {}'.format(
                             shell_script, command))

    @staticmethod
    @abc.abstractmethod
    def replaces_shell_script(shell_script, command):
        '''Check if this class replaces shell_script for the given command.'''

    @staticmethod
    @abc.abstractmethod
    def create_from_env(command, debug):
        '''Create new flasher instance from environment variables.

        This class must be able to replace the current shell script
        (FLASH_SCRIPT or DEBUG_SCRIPT, depending on command). The
        environment variables expected by that script are used to build
        the flasher in a backwards-compatible manner.'''

    @abc.abstractmethod
    def run(self, command, **kwargs):
        '''Run a command ('flash', 'debug', 'debugserver').

        In case of an unsupported command, raise a ValueError.'''

    def run_server_and_client(self, server, client):
        '''Run a server that ignores SIGINT, and a client that handles it.

        This routine portably:

        - creates a Popen object for the `server' command which ignores SIGINT
        - runs `client' in a subprocess while temporarily ignoring SIGINT
        - cleans up the server after the client exits.

        It's useful to e.g. open a GDB server and client.'''
        server_proc = self.popen_ignore_int(server)
        previous = signal.signal(signal.SIGINT, signal.SIG_IGN)
        try:
            self.check_call(client)
        finally:
            signal.signal(signal.SIGINT, previous)
            server_proc.terminate()
            server_proc.wait()

    def check_call(self, cmd):
        '''Subclass subprocess.check_call() wrapper.

        Subclasses should use this command to run command in a
        subprocess and check that it executed correctly, rather than
        using subprocess directly, to keep accurate debug logs.
        '''
        if self.debug:
            print(quote_sh_list(cmd))
        subprocess.check_call(cmd)

    def check_output(self, cmd):
        '''Subclass subprocess.check_output() wrapper.

        Subclasses should use this command to run command in a
        subprocess and check that it executed correctly, rather than
        using subprocess directly, to keep accurate debug logs.
        '''
        if self.debug:
            print(quote_sh_list(cmd))
        return subprocess.check_output(cmd)

    def popen_ignore_int(self, cmd):
        '''Spawn a child command, ensuring it ignores SIGINT.

        The returned subprocess.Popen object must be manually terminated.'''
        cflags = 0
        preexec = None
        system = platform.system()

        if system == 'Windows':
            cflags |= subprocess.CREATE_NEW_PROCESS_GROUP
        elif system in {'Linux', 'Darwin'}:
            preexec = os.setsid

        if self.debug:
            print(quote_sh_list(cmd))

        return subprocess.Popen(cmd, creationflags=cflags, preexec_fn=preexec)
<commit_msg>scripts: runner: core: fix stale comments<commit_after>#! /usr/bin/env python3

# Copyright (c) 2017 Linaro Limited.
#
# SPDX-License-Identifier: Apache-2.0

"""Zephyr binary runner core interfaces

This provides the core ZephyrBinaryRunner class meant for public use,
as well as some other helpers for concrete runner classes.
"""

import abc
import os
import platform
import pprint
import shlex
import signal
import subprocess
import sys

def get_env_or_bail(env_var):
    '''Get an environment variable, or raise an error.

    In case of KeyError, an error message is printed, along with the
    environment, and the exception is re-raised.
    '''
    try:
        return os.environ[env_var]
    except KeyError:
        print('Variable {} not in environment:'.format(
                  env_var), file=sys.stderr)
        pprint.pprint(dict(os.environ), stream=sys.stderr)
        raise

def get_env_bool_or(env_var, default_value):
    '''Get an environment variable as a boolean, or return a default value.

    Get an environment variable, interpret it as a base ten
    integer, and convert that to a boolean.

    In case the environment variable is not defined, return default_value.
    '''
    try:
        return bool(int(os.environ[env_var]))
    except KeyError:
        return default_value

def get_env_strip_or(env_var, to_strip, default_value):
    '''Get and clean up an environment variable, or return a default value.

    Get the value of env_var from the environment. If it is
    defined, return that value with to_strip stripped off. If it
    is undefined, return default_value (without any stripping).
    '''
    value = os.environ.get(env_var, None)
    if value is not None:
        return value.strip(to_strip)
    else:
        return default_value

def quote_sh_list(cmd):
    '''Transform a command from list into shell string form.'''
    fmt = ' '.join('{}' for _ in cmd)
    args = [shlex.quote(s) for s in cmd]
    return fmt.format(*args)

MAX_PORT = 49151

class NetworkPortHelper:
    '''Helper class for dealing with local IP network ports.'''

    def get_unused_ports(self, starting_from):
        '''Find unused network ports, starting at given values.

        starting_from is an iterable of ports the caller would like to use.

        The return value is an iterable of ports, in the same order, using
        the given values if they were unused, or the next sequentially
        available unused port otherwise.

        Ports may be bound between this call's check and actual usage, so
        callers still need to handle errors involving returned ports.'''
        start = list(starting_from)
        used = self._used_now()
        ret = []

        for desired in start:
            port = desired
            while port in used:
                port += 1
                if port > MAX_PORT:
                    msg = "ports above {} are in use"
                    raise ValueError(msg.format(desired))
            used.add(port)
            ret.append(port)

        return ret

    def _used_now(self):
        handlers = {
            'Windows': self._used_now_windows,
            'Linux': self._used_now_linux,
            'Darwin': self._used_now_darwin,
        }
        handler = handlers[platform.system()]
        return handler()

    def _used_now_windows(self):
        cmd = ['netstat', '-a', '-n', '-p', 'tcp']
        return self._parser_windows(cmd)

    def _used_now_linux(self):
        cmd = ['ss', '-a', '-n', '-t']
        return self._parser_linux(cmd)

    def _used_now_darwin(self):
        cmd = ['netstat', '-a', '-n', '-p', 'tcp']
        return self._parser_darwin(cmd)

    def _parser_windows(self, cmd):
        out = subprocess.check_output(cmd).split(b'\r\n')
        used_bytes = [x.split()[1].rsplit(b':', 1)[1] for x in out
                      if x.startswith(b'  TCP')]
        return {int(b) for b in used_bytes}

    def _parser_linux(self, cmd):
        out = subprocess.check_output(cmd).splitlines()[1:]
        used_bytes = [s.split()[3].rsplit(b':', 1)[1] for s in out]
        return {int(b) for b in used_bytes}

    def _parser_darwin(self, cmd):
        out = subprocess.check_output(cmd).split(b'\n')
        used_bytes = [x.split()[3].rsplit(b':', 1)[1] for x in out
                      if x.startswith(b'tcp')]
        return {int(b) for b in used_bytes}

class ZephyrBinaryRunner(abc.ABC):
    '''Abstract superclass for binary runners (flashers, debuggers).

    With some exceptions, boards supported by Zephyr must provide
    generic means to be flashed (have a Zephyr firmware binary
    permanently installed on the device for running) and debugged
    (have a breakpoint debugger and program loader on a host
    workstation attached to a running target). This is supported by
    three top-level commands managed by the Zephyr build system:

    - 'flash': flash a previously configured binary to the board,
      start execution on the target, then return.

    - 'debug': connect to the board via a debugging protocol, then
      drop the user into a debugger interface with symbol tables
      loaded from the current binary, and block until it exits.

    - 'debugserver': connect via a board-specific debugging protocol,
      then reset and halt the target. Ensure the user is now able to
      connect to a debug server with symbol tables loaded from the
      binary.

    Runner functionality relies on a variety of target-specific tools
    and configuration values, the user interface to which is
    abstracted by this class. Each runner subclass should take any
    values it needs to execute one of these commands in its
    constructor.  The actual command execution is handled in the run()
    method.

    This functionality has replaced the legacy Zephyr runners,
    which were shell scripts.

    At present, the Zephyr build system uses a variety of
    tool-specific environment variables to control runner behavior.
    To support a transition to ZephyrBinaryRunner and subclasses, this
    class provides a create_for_shell_script() static factory method.
    This method iterates over ZephyrBinaryRunner subclasses,
    determines which (if any) can provide equivalent functionality to
    the old shell-based runner, and returns a subclass instance with its
    configuration determined from the environment.

    To support this, subclasess currently must provide a pair of
    static methods, replaces_shell_script() and create_from_env(). The
    first allows the runner subclass to declare which commands and
    scripts it can replace. The second is called by
    create_for_shell_script() to create a concrete runner instance.

    The environment-based factories are for legacy use *only*; the
    user must be able to construct and use a runner using only the
    constructor and run() method.'''

    def __init__(self, debug=False):
        self.debug = debug

    @staticmethod
    def create_for_shell_script(shell_script, command, debug):
        '''Factory for using as a drop-in replacement to a shell script.

        Command is one of 'flash', 'debug', 'debugserver'.

        Get runner instance to use in place of shell_script, deriving
        configuration from the environment.'''
        for sub_cls in ZephyrBinaryRunner.__subclasses__():
            if sub_cls.replaces_shell_script(shell_script, command):
                return sub_cls.create_from_env(command, debug)
        raise ValueError('cannot implement script {} command {}'.format(
                             shell_script, command))

    @staticmethod
    @abc.abstractmethod
    def replaces_shell_script(shell_script, command):
        '''Check if this class replaces shell_script for the given command.'''

    @staticmethod
    @abc.abstractmethod
    def create_from_env(command, debug):
        '''Create new flasher instance from environment variables.

        This class must be able to replace the current shell script
        (FLASH_SCRIPT or DEBUG_SCRIPT, depending on command). The
        environment variables expected by that script are used to build
        the flasher in a backwards-compatible manner.'''

    @abc.abstractmethod
    def run(self, command, **kwargs):
        '''Run a command ('flash', 'debug', 'debugserver').

        In case of an unsupported command, raise a ValueError.'''

    def run_server_and_client(self, server, client):
        '''Run a server that ignores SIGINT, and a client that handles it.

        This routine portably:

        - creates a Popen object for the `server' command which ignores SIGINT
        - runs `client' in a subprocess while temporarily ignoring SIGINT
        - cleans up the server after the client exits.

        It's useful to e.g. open a GDB server and client.'''
        server_proc = self.popen_ignore_int(server)
        previous = signal.signal(signal.SIGINT, signal.SIG_IGN)
        try:
            self.check_call(client)
        finally:
            signal.signal(signal.SIGINT, previous)
            server_proc.terminate()
            server_proc.wait()

    def check_call(self, cmd):
        '''Subclass subprocess.check_call() wrapper.

        Subclasses should use this command to run command in a
        subprocess and check that it executed correctly, rather than
        using subprocess directly, to keep accurate debug logs.
        '''
        if self.debug:
            print(quote_sh_list(cmd))
        subprocess.check_call(cmd)

    def check_output(self, cmd):
        '''Subclass subprocess.check_output() wrapper.

        Subclasses should use this command to run command in a
        subprocess and check that it executed correctly, rather than
        using subprocess directly, to keep accurate debug logs.
        '''
        if self.debug:
            print(quote_sh_list(cmd))
        return subprocess.check_output(cmd)

    def popen_ignore_int(self, cmd):
        '''Spawn a child command, ensuring it ignores SIGINT.

        The returned subprocess.Popen object must be manually terminated.'''
        cflags = 0
        preexec = None
        system = platform.system()

        if system == 'Windows':
            cflags |= subprocess.CREATE_NEW_PROCESS_GROUP
        elif system in {'Linux', 'Darwin'}:
            preexec = os.setsid

        if self.debug:
            print(quote_sh_list(cmd))

        return subprocess.Popen(cmd, creationflags=cflags, preexec_fn=preexec)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__version__ = '1.2.0'
<commit_msg>Fixed a bug in crispy forms<commit_after># -*- coding: utf-8 -*-

__version__ = '1.2.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Tests the classes in .../classes/Environment.py

import sys
import os
import copy
import random
import itertools

locationOfFiles = os.getcwd() + '/classes'  # add location of class files to PYTHONPATH
print 'File loc: ', locationOfFiles
sys.path.append(locationOfFiles)

from environment import Location, Bounds2D, World
from neuralNet import *

def initNNs(lenOfPool, numAgents):
    ''' Creates a pool of neuralNets
        Each sublist corresponds to an agent's pool of NNs, so its length is the number of nn in a pool
        Length of the list is the number of agents.

        @author Kory Kraft
    '''
    nnList = []
    for agent in range(numAgents):
        agentsNNList = []
        for i in range(lenOfPool):
            agentsNNList.append(createNN())
        nnList.append(agentsNNList)
    return nnList

def createNN():
    ''' Creates a 1 hidden layer ff-nn of input 8, hidden layer 10, and output 2
    @author Kory Kraft
    '''
    nn = NeuralNetwork(8, 10, 2)
    nn.createNodes()
    nn.createRandomWeights()
    return nn

def createTeams(nns):
    '''
    Creates a 'team' of agents.  Can be thought of as creating a team of agents' brains.
    In reality, it selects a previously unselected NN from each agent's NN pool.

    @author Kory Kraft
    '''
    allTeams = [] # list that will hold each team of agents' nns

    # Shuffle each sublist of agent's nns
    for agentNNs in nns:
        random.shuffle(agentNNs)

    # create k teams
    #   where the nn from each agent is "picked" from the shuffled list
    for k in range(lenOfPool):
        team = []
        for agentNNs in nns:
            team.append(agentNNs[k])
        allTeams.append(team)

    # Works but slow and could looop for a loooooong time
    # for agentPool in nns:
    #     print agentPool
    #     i = 0
    #     while True:
    #         i += 1
    #         if i == 100:
    #             print 'hit 100'
    #             sys.exit(1)

    #         agentNN = random.choice(agentPool)
    #         print 'Selected ', agentNN.selected
    #         if agentNN.selected == False:
    #             agentNN.selected = True
    #             team.append(agentNN)
    #             break

    #  Wrong!!!

    # Create randomized selection indices for each agent's nn to create the random teams
    # It is a list of lists.  Each sublist corresponds to the selection indices for the nn
    # agentSelChoices = [] # selection indices for the nn for each agent
    # for agent in range(numAgents):
    #     indices = [x for x in range(lenOfPool)]
    #     random.shuffle(indices)
    #     agentSelChoices.append(indices)
    # allTeams = [] # will hold a list of nns
    #             # indices of the list correspond to agents.  E.g. list[1] is agent 1's nn
    # for teamSel in teamSelChoices:
    #     print 'Team sel: ', teamSel
    #     team = []
    #     for selection, agentsNN in itertools.izip(teamSel, nns):
    #         print 'Selection: ', selection
    #         team.append(agentsNN[selection])
    #     allTeams.append(team)

    # print 'Number of team members ', len(allTeams[0])
    # print 'Number of teams ', len(allTeams)
    return allTeams

def updateNNS(nns, egreedy):
    ''' Selects the top performers of nns for each agent
        Mutates these and replaces the lower performers.

        Note: Keeps the same neural networks in memory, but with different weights.
    '''

    for agentNNs in nns:
        # Divide out the agent's nns in the the high performers and low performers
        topHalfNNs, lowHalfNNs = selectPerformers(agentNNs, egreedy)
        # mutate the top performers and replace the lowHalfNN's with these mutate weights
        mutateNNs(topHalfNNs, lowHalfNNs)

def selectPerformers(nns, egreedy):
    ''' Given a list a nueral networks,
         this method selects the top half performers,
         always keeping the highest performer,
         and e-greedy times switching other low-high performers

         Returns: List of nns that were the top performers
    @ author Kory Kraft
    '''
    # sort the agent's nns according to value from low to high
    sorted(nns, key=lambda nn: nn.value)

    # keep the highest, and e-greedy time swap out a low performer with a high performer
    # get the top half of the list (those are the highest performers)
    midPointIndex = int(len(nns)/2)
    topHalf = nns[midPointIndex:]
    lowHalf = nns[:midPointIndex]

    # take egreedy high performers (minus the highest), and swap out with the lower half
    for i in xrange(int(len(nns) * egreedy))
        # randomly insert switch one high performer with low performer (don't want to just assign low performer to high otherwise we might get weird linking errors)
        index = random.randint(0,midPointIndex-2) # <- prevent from indexing out of bounds and ensures that highest one is excluded (i.e. kept in top half)
        temp = lowHalf[index]
        lowHalf[index] = topHalf[index]
        topHalf[index] = temp

    return topHalf, lowHalf

def mutateNNs(topHalf, lowHalf):
    ''' Each nn in the low half is replaced by a mutated version of the nn in the topHalf.
        Leaves the low nn in the same memory address (for the sake of the larger nns matrix.
        Instead, does a deep copy of the weights in the mutated nn.

        @author: Kory Kraft
        '''

    for low, top in zip(lowHalf, topHalf):
        low.weights = top.mutateWeights(.1).weights
        low.value = sys.float_info.min

def calcDX(output, maxDist, noise):
    ''' Calculates dx taking into account the noise.
        The actual noise rate is taking from a uniform distribution between
            +- noise.

        dx = 2maxD(x - .5) * noise ; this came from Tumer's paper
    @author Kory Kraft
    '''
    dx = 2 * maxDist * (x - .5)
    noiseDist = random.uniform(-noise, noise) * dx
    dx = dx + noiseDist
    return dx

def doEpisode(world, team, headings, timesteps, maxDist, minDist, mvtNoise):
    # reset world
    # init/place rovs
    world.reset(headings)  # randomize POI locations and reset rover locations
    for t in range(timesteps):
        for rov, nn in itertools.izip(world.rovers, team):
            # Do a prediction with the rovs associated nn from the team
            x,y = nn.predict(rov.getNNInputs(world.POIs, minDist, world.rovers))
            dy = calcDX(y, maxDist, mvtNoise)

            # take the action with the pred. action
            rov.takeAction(dx, dy)
            # get reward for agent
    # Get rewards for nn's which correspond to system rewards
    # return
    pass

def main():
    # Evo Training
    #
    # Until convergence
    #     For num of NNs each agent has (length of the pool)
    #        Create team ( selected previously unselected NN from each row)
    #        Do an episode (do stuff in world for 15 time steps and get rewards)
    #        Assign each individual NN in the current team a value based on the
    #          corresponding agent performance/rewards
    #     Select top NN performers from each agent's NN pool e-greey, but keep top 1 each time
    #     Mutate the above
    #     Replace the unselected onces

    # Hyperparamters for training
    lenOfPool = 10 # Num of nn's for each agent
    numAgents = 5 # Num of agents in the system
    timesteps = 15
    maxDist = 10 # maximum distance the agent can move in one timestep
    minDist = 5 # minimum distance
    mvtNoise = .1 # the noise added to each actions outcome
    nns = initNNs(lenOfPool, numAgents) # Can be thought of as matrix of NNs, each sublist is an agents pool of nns
    # random headings for agent that will stay
    # the same for each agent through the runs and epochs
    #  so that the corresponding agent and nn pool will reflect the
    #   same initial values/assumptions
    agentInitHeadings = [(random.randint(0,359) * math.pi /180) for x in range(numAgents)]
    egreedy = .2 # number of weights to mutate starting out, this is decreased over time
    egreedyDecreaseRate = .9 # rate at which egreedy selection is decreased

    # World parameters
    world_bounds = Bounds2D((0, 115), (0, 100))  # world borders
    world_center = Location(60, 50)  # where the agents start at
    poi_ranges = (70, 70)
    poi_bounds = Bounds2D((world_center.x-poi_ranges[0]/2, world_center.x+poi_ranges[0]/2),
                          (world_center.y-poi_ranges[1]/2, world_center.y+poi_ranges[1]/2))  # bounds of where POIs can go

    world = World(world_bounds, 100, poi_bounds, 30, rover_start=world_center, agentInitHeadings)  # make a world

    # Create orientations for the agents outside so they will all be consist for agent i
    for i in range(3): # random definition of convergence

        # create random team of agent brains for the game
        teams = createTeams(nns)

        for team in teams:

            # init agents, world, etc and do episode

            # do the episode, get rovers or just rewards?
            doEpisode(world, team, agentInitHeadings, timesteps, maxDist, minDist, mvtNoise)

            # # assign each nn in team a value
            ## This can be done in do episode since each nn holds its own value
            # for nn, rover in team, rovers:
            #     # nn.value = rover.getReward()
            #     nn.value = random.randint(0,10)

        # select best nn performers
        #   and mutate and replace low performers
        updateNNS(nns, egreedy * egreedyDecreaseRate)

if __name__ == "__main__":
    main()

<commit_msg>Added reward assignment at end of an episode.<commit_after>#!/usr/bin/env python
# Tests the classes in .../classes/Environment.py

import sys
import os
import copy
import random
import itertools

locationOfFiles = os.getcwd() + '/classes'  # add location of class files to PYTHONPATH
print 'File loc: ', locationOfFiles
sys.path.append(locationOfFiles)

from environment import Location, Bounds2D, World
from neuralNet import *

def initNNs(lenOfPool, numAgents):
    ''' Creates a pool of neuralNets
        Each sublist corresponds to an agent's pool of NNs, so its length is the number of nn in a pool
        Length of the list is the number of agents.

        @author Kory Kraft
    '''
    nnList = []
    for agent in range(numAgents):
        agentsNNList = []
        for i in range(lenOfPool):
            agentsNNList.append(createNN())
        nnList.append(agentsNNList)
    return nnList

def createNN():
    ''' Creates a 1 hidden layer ff-nn of input 8, hidden layer 10, and output 2
    @author Kory Kraft
    '''
    nn = NeuralNetwork(8, 10, 2)
    nn.createNodes()
    nn.createRandomWeights()
    return nn

def createTeams(nns):
    '''
    Creates a 'team' of agents.  Can be thought of as creating a team of agents' brains.
    In reality, it selects a previously unselected NN from each agent's NN pool.

    @author Kory Kraft
    '''
    allTeams = [] # list that will hold each team of agents' nns

    # Shuffle each sublist of agent's nns
    for agentNNs in nns:
        random.shuffle(agentNNs)

    # create k teams
    #   where the nn from each agent is "picked" from the shuffled list
    for k in range(lenOfPool):
        team = []
        for agentNNs in nns:
            team.append(agentNNs[k])
        allTeams.append(team)

    # Works but slow and could looop for a loooooong time
    # for agentPool in nns:
    #     print agentPool
    #     i = 0
    #     while True:
    #         i += 1
    #         if i == 100:
    #             print 'hit 100'
    #             sys.exit(1)

    #         agentNN = random.choice(agentPool)
    #         print 'Selected ', agentNN.selected
    #         if agentNN.selected == False:
    #             agentNN.selected = True
    #             team.append(agentNN)
    #             break

    #  Wrong!!!

    # Create randomized selection indices for each agent's nn to create the random teams
    # It is a list of lists.  Each sublist corresponds to the selection indices for the nn
    # agentSelChoices = [] # selection indices for the nn for each agent
    # for agent in range(numAgents):
    #     indices = [x for x in range(lenOfPool)]
    #     random.shuffle(indices)
    #     agentSelChoices.append(indices)
    # allTeams = [] # will hold a list of nns
    #             # indices of the list correspond to agents.  E.g. list[1] is agent 1's nn
    # for teamSel in teamSelChoices:
    #     print 'Team sel: ', teamSel
    #     team = []
    #     for selection, agentsNN in itertools.izip(teamSel, nns):
    #         print 'Selection: ', selection
    #         team.append(agentsNN[selection])
    #     allTeams.append(team)

    # print 'Number of team members ', len(allTeams[0])
    # print 'Number of teams ', len(allTeams)
    return allTeams

def updateNNS(nns, egreedy):
    ''' Selects the top performers of nns for each agent
        Mutates these and replaces the lower performers.

        Note: Keeps the same neural networks in memory, but with different weights.
    '''

    for agentNNs in nns:
        # Divide out the agent's nns in the the high performers and low performers
        topHalfNNs, lowHalfNNs = selectPerformers(agentNNs, egreedy)
        # mutate the top performers and replace the lowHalfNN's with these mutate weights
        mutateNNs(topHalfNNs, lowHalfNNs)

def selectPerformers(nns, egreedy):
    ''' Given a list a nueral networks,
         this method selects the top half performers,
         always keeping the highest performer,
         and e-greedy times switching other low-high performers

         Returns: List of nns that were the top performers
    @ author Kory Kraft
    '''
    # sort the agent's nns according to value from low to high
    sorted(nns, key=lambda nn: nn.value)

    # keep the highest, and e-greedy time swap out a low performer with a high performer
    # get the top half of the list (those are the highest performers)
    midPointIndex = int(len(nns)/2)
    topHalf = nns[midPointIndex:]
    lowHalf = nns[:midPointIndex]

    # take egreedy high performers (minus the highest), and swap out with the lower half
    for i in xrange(int(len(nns) * egreedy))
        # randomly insert switch one high performer with low performer (don't want to just assign low performer to high otherwise we might get weird linking errors)
        index = random.randint(0,midPointIndex-2) # <- prevent from indexing out of bounds and ensures that highest one is excluded (i.e. kept in top half)
        temp = lowHalf[index]
        lowHalf[index] = topHalf[index]
        topHalf[index] = temp

    return topHalf, lowHalf

def mutateNNs(topHalf, lowHalf):
    ''' Each nn in the low half is replaced by a mutated version of the nn in the topHalf.
        Leaves the low nn in the same memory address (for the sake of the larger nns matrix.
        Instead, does a deep copy of the weights in the mutated nn.

        @author: Kory Kraft
        '''

    for low, top in zip(lowHalf, topHalf):
        low.weights = top.mutateWeights(.1).weights
        low.value = sys.float_info.min

def calcDX(output, maxDist, noise):
    ''' Calculates dx taking into account the noise.
        The actual noise rate is taking from a uniform distribution between
            +- noise.

        dx = 2maxD(x - .5) * noise ; this came from Tumer's paper
    @author Kory Kraft
    '''
    dx = 2 * maxDist * (x - .5)
    noiseDist = random.uniform(-noise, noise) * dx
    dx = dx + noiseDist
    return dx

def doEpisode(world, team, headings, timesteps, maxDist, minDist, mvtNoise):
    # reset world
    # init/place rovs
    world.reset()  # randomize POI locations and reset rover locations
    for t in range(timesteps):
        for rov, nn in itertools.izip(world.rovers, team):
            # Do a prediction with the rovs associated nn from the team
            x,y = nn.predict(rov.getNNInputs(world.POIs, minDist, world.rovers))
            dy = calcDX(y, maxDist, mvtNoise)

            # take the action with the pred. action
            rov.takeAction(dx, dy)
            # get reward for agent
    # Get rewards for nn's which correspond to system rewards
    rewards, rover_closest_list = world.get_rewards()
    for nn, reward in zip(team, rewards['DIFFERENCE']):
        nn.value = reward
    # return
    pass

def main():
    # Evo Training
    #
    # Until convergence
    #     For num of NNs each agent has (length of the pool)
    #        Create team ( selected previously unselected NN from each row)
    #        Do an episode (do stuff in world for 15 time steps and get rewards)
    #        Assign each individual NN in the current team a value based on the
    #          corresponding agent performance/rewards
    #     Select top NN performers from each agent's NN pool e-greey, but keep top 1 each time
    #     Mutate the above
    #     Replace the unselected onces

    # Hyperparamters for training
    lenOfPool = 10 # Num of nn's for each agent
    numAgents = 5 # Num of agents in the system
    timesteps = 15
    maxDist = 10 # maximum distance the agent can move in one timestep
    minDist = 5 # minimum distance
    mvtNoise = .1 # the noise added to each actions outcome
    nns = initNNs(lenOfPool, numAgents) # Can be thought of as matrix of NNs, each sublist is an agents pool of nns
    # random headings for agent that will stay
    # the same for each agent through the runs and epochs
    #  so that the corresponding agent and nn pool will reflect the
    #   same initial values/assumptions
    agentInitHeadings = [(random.randint(0,359) * math.pi /180) for x in range(numAgents)]
    egreedy = .2 # number of weights to mutate starting out, this is decreased over time
    egreedyDecreaseRate = .9 # rate at which egreedy selection is decreased

    # World parameters
    world_bounds = Bounds2D((0, 115), (0, 100))  # world borders
    world_center = Location(60, 50)  # where the agents start at
    poi_ranges = (70, 70)
    poi_bounds = Bounds2D((world_center.x-poi_ranges[0]/2, world_center.x+poi_ranges[0]/2),
                          (world_center.y-poi_ranges[1]/2, world_center.y+poi_ranges[1]/2))  # bounds of where POIs can go

    world = World(world_bounds, 100, poi_bounds, 30, rover_start=world_center, agentInitHeadings)  # make a world

    # Create orientations for the agents outside so they will all be consist for agent i
    for i in range(3): # random definition of convergence

        # create random team of agent brains for the game
        teams = createTeams(nns)

        for team in teams:

            # init agents, world, etc and do episode

            # do the episode, get rovers or just rewards?
            doEpisode(world, team, agentInitHeadings, timesteps, maxDist, minDist, mvtNoise)

            # # assign each nn in team a value
            ## This can be done in do episode since each nn holds its own value
            # for nn, rover in team, rovers:
            #     # nn.value = rover.getReward()
            #     nn.value = random.randint(0,10)

        # select best nn performers
        #   and mutate and replace low performers
        updateNNS(nns, egreedy * egreedyDecreaseRate)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import pika
import sys
import json
from configlib import getConfig,OptionParser
from logging.handlers import SysLogHandler
import logging
from datetime import datetime
from datetime import timedelta
from dateutil.parser import parse
import pytz
import pyes

logger = logging.getLogger(sys.argv[0])

def loggerTimeStamp(self, record, datefmt=None):
    return toUTC(datetime.now()).isoformat()

def initLogger():
    logger.level=logging.INFO
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    formatter.formatTime = loggerTimeStamp
    if options.output=='syslog':
        logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
    else:
        sh=logging.StreamHandler(sys.stderr)
        sh.setFormatter(formatter)
        logger.addHandler(sh)

def toUTC(suspectedDate,localTimeZone="US/Pacific"):
    '''make a UTC date out of almost anything'''
    utc=pytz.UTC
    objDate=None
    if type(suspectedDate)==str:
        objDate=parse(suspectedDate,fuzzy=True)
    elif type(suspectedDate)==datetime:
        objDate=suspectedDate

    if objDate.tzinfo is None:
        objDate=pytz.timezone(localTimeZone).localize(objDate)
        objDate=utc.normalize(objDate)
    else:
        objDate=utc.normalize(objDate)
    if objDate is not None:
        objDate=utc.normalize(objDate)

    return objDate

def alertToMessageQueue(alertDict):
    try:
        connection = pika.BlockingConnection(pika.ConnectionParameters(host=options.mqserver))
        channel = connection.channel()
        #declare the exchanges
        channel.exchange_declare(exchange=options.alertexchange,type='topic')

        #cherry pick items from the alertDict to send to the alerts messageQueue
        mqAlert=dict(severity='INFO',type='')
        if 'severity' in alertDict.keys():
            mqAlert['severity']=alertDict['severity']
        if 'category' in alertDict.keys():
            mqAlert['category']=alertDict['category']
        if 'utctimestamp' in alertDict.keys():
            mqAlert['utctimestamp']=alertDict['utctimestamp']
        if 'eventtimestamp' in alertDict.keys():
            mqAlert['eventtimestamp']=alertDict['eventtimestamp']
        mqAlert['summary']=alertDict['summary']
        channel.basic_publish(exchange=options.alertexchange,routing_key=options.alertqueue,body=json.dumps(mqAlert))
    except Exception as e:
        logger.error('Exception while sending alert to message queue: {0}'.format(e))

def alertToES(es,alertDict):
    try:
        res=es.index(index='alerts',doc_type='alert',doc=alertDict)
        return(res)
    except pyes.exceptions.NoServerAvailable:
        logger.error('Elastic Search server could not be reached, check network connectivity')

def esCloudTrailSearch(es,begindateUTC=None, enddateUTC=None):
    resultsList=list()
    if begindateUTC is None:
        begindateUTC=toUTC(datetime.now() - timedelta(hours=160))
    if enddateUTC is None:
        enddateUTC= toUTC(datetime.now())
    try:
        #search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp)
        qDate=pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',from_value=begindateUTC,to_value=enddateUTC))
        qcloud=pyes.TermFilter('_type','cloudtrail')
        qEvents=pyes.TermsFilter('eventName',['runinstances','stopinstances','startinstances'])
        qalerted=pyes.ExistsFilter('alerttimestamp')
        results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents],must_not=[qalerted])))
        #uncomment for debugging to recreate alerts for events that already have an alerttimestamp
        #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
        return(results)

    except pyes.exceptions.NoServerAvailable:
        logger.error('Elastic Search server could not be reached, check network connectivity')

def createAlerts(es,esResults):
    try:
        if len(esResults)>0:
            for r in esResults:
                alert=dict(utctimestamp=toUTC(datetime.now()).isoformat(),severity='INFO',summary='',category='AWSCloudtrail',tags=['cloudtrail','aws'],eventsource=[],events=[])
                alert['events'].append(dict(index=r.get_meta()['index'],category=r.get_meta()['category'],id=r.get_meta()['id']))
                alert['eventtimestamp']=r['eventTime']
                alert['severity']='INFO'
                alert['summary']=('{0} called {1} from {2}'.format(r['userIdentity']['userName'],r['eventName'],r['sourceIPAddress']))
                alert['eventsource']=r
                if r['eventName']=='RunInstances':
                    for i in r['responseElements']['instancesSet']['items']:
                        alert['summary'] += (' running {0} '.format(i['privateDnsName']))
                if r['eventName']=='StartInstances':
                    for i in r['requestParameters']['instancesSet']['items']:
                        alert['summary'] += (' starting {0} '.format(i['instanceId']))
                logger.debug(alert['summary'])
                #logger.debug(alert['events'])

                #save alert to alerts index, update events index with alert ID for cross reference
                alertResult=alertToES(es,alert)
                logger.debug(alertResult)
                if 'alerts' not in r.keys():
                    r['alerts']=[]
                r['alerts'].append(dict(index=alertResult['_index'],type=alertResult['_type'],id=alertResult['_id']))
                r['alerttimestamp']=toUTC(datetime.now()).isoformat()
                es.update(r.get_meta()['index'],r.get_meta()['type'],r.get_meta()['id'],document=r)
                alertToMessageQueue(alert)
    except Exception as e:
        logger.error("Exception %r when creating alerts "%e)

def main():
    logger.debug('starting')
    logger.debug(options)
    es=pyes.ES(("http",options.esserver,options.esport))
    results=esCloudTrailSearch(es)
    createAlerts(es,results)
    logger.debug('finished')

def initConfig():
    #change this to your default zone for when it's not specified
    options.defaultTimeZone=getConfig('defaulttimezone','US/Pacific',options.configfile)
    #msg queue settings
    options.mqserver=getConfig('mqserver','localhost',options.configfile)               #message queue server hostname
    options.alertqueue=getConfig('alertqueue','mozdef.alert',options.configfile)        #alert queue topic
    options.alertexchange=getConfig('alertexchange','alerts',options.configfile)        #alert queue exchange name
    #logging settings
    options.output=getConfig('output','stdout',options.configfile)                      #output our log to stdout or syslog
    options.sysloghostname=getConfig('sysloghostname','localhost',options.configfile)   #syslog hostname
    options.syslogport=getConfig('syslogport',514,options.configfile)                   #syslog port
    #elastic search server settings
    options.esserver=getConfig('esserver','localhost',options.configfile)
    options.esport=getConfig('esport',9200,options.configfile)

if __name__ == '__main__':
    parser=OptionParser()
    parser.add_option("-c", dest='configfile' , default=sys.argv[0].replace('.py','.conf'), help="configuration file to use")
    (options,args) = parser.parse_args()
    initConfig()
    initLogger()
    main()
<commit_msg>minor python version change<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import pika
import sys
import json
from configlib import getConfig,OptionParser
from logging.handlers import SysLogHandler
import logging
from datetime import datetime
from datetime import timedelta
from dateutil.parser import parse
import pytz
import pyes

logger = logging.getLogger(sys.argv[0])

def loggerTimeStamp(self, record, datefmt=None):
    return toUTC(datetime.now()).isoformat()

def initLogger():
    logger.level=logging.INFO
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    formatter.formatTime = loggerTimeStamp
    if options.output=='syslog':
        logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
    else:
        sh=logging.StreamHandler(sys.stderr)
        sh.setFormatter(formatter)
        logger.addHandler(sh)

def toUTC(suspectedDate,localTimeZone="US/Pacific"):
    '''make a UTC date out of almost anything'''
    utc=pytz.UTC
    objDate=None
    if type(suspectedDate)==str:
        objDate=parse(suspectedDate,fuzzy=True)
    elif type(suspectedDate)==datetime:
        objDate=suspectedDate

    if objDate.tzinfo is None:
        objDate=pytz.timezone(localTimeZone).localize(objDate)
        objDate=utc.normalize(objDate)
    else:
        objDate=utc.normalize(objDate)
    if objDate is not None:
        objDate=utc.normalize(objDate)

    return objDate

def alertToMessageQueue(alertDict):
    try:
        connection = pika.BlockingConnection(pika.ConnectionParameters(host=options.mqserver))
        channel = connection.channel()
        #declare the exchanges
        channel.exchange_declare(exchange=options.alertexchange,type='topic')

        #cherry pick items from the alertDict to send to the alerts messageQueue
        mqAlert=dict(severity='INFO',type='')
        if 'severity' in alertDict.keys():
            mqAlert['severity']=alertDict['severity']
        if 'category' in alertDict.keys():
            mqAlert['category']=alertDict['category']
        if 'utctimestamp' in alertDict.keys():
            mqAlert['utctimestamp']=alertDict['utctimestamp']
        if 'eventtimestamp' in alertDict.keys():
            mqAlert['eventtimestamp']=alertDict['eventtimestamp']
        mqAlert['summary']=alertDict['summary']
        channel.basic_publish(exchange=options.alertexchange,routing_key=options.alertqueue,body=json.dumps(mqAlert))
    except Exception as e:
        logger.error('Exception while sending alert to message queue: {0}'.format(e))

def alertToES(es,alertDict):
    try:
        res=es.index(index='alerts',doc_type='alert',doc=alertDict)
        return(res)
    except pyes.exceptions.NoServerAvailable:
        logger.error('Elastic Search server could not be reached, check network connectivity')

def esCloudTrailSearch(es,begindateUTC=None, enddateUTC=None):
    resultsList=list()
    if begindateUTC is None:
        begindateUTC=toUTC(datetime.now() - timedelta(hours=160))
    if enddateUTC is None:
        enddateUTC= toUTC(datetime.now())
    try:
        #search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp)
        qDate=pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',from_value=begindateUTC,to_value=enddateUTC))
        qcloud=pyes.TermFilter('_type','cloudtrail')
        qEvents=pyes.TermsFilter('eventName',['runinstances','stopinstances','startinstances'])
        qalerted=pyes.ExistsFilter('alerttimestamp')
        results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents],must_not=[qalerted])))
        #uncomment for debugging to recreate alerts for events that already have an alerttimestamp
        #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
        return(results)

    except pyes.exceptions.NoServerAvailable:
        logger.error('Elastic Search server could not be reached, check network connectivity')

def createAlerts(es,esResults):
    try:
        if len(esResults)>0:
            for r in esResults:
                alert=dict(utctimestamp=toUTC(datetime.now()).isoformat(),severity='INFO',summary='',category='AWSCloudtrail',tags=['cloudtrail','aws'],eventsource=[],events=[])
                alert['events'].append(dict(index=r.get_meta()['index'],category=r.get_meta()['category'],id=r.get_meta()['id']))
                alert['eventtimestamp']=r['eventTime']
                alert['severity']='INFO'
                alert['summary']=('{0} called {1} from {2}'.format(r['userIdentity']['userName'],r['eventName'],r['sourceIPAddress']))
                alert['eventsource']=r
                if r['eventName']=='RunInstances':
                    for i in r['responseElements']['instancesSet']['items']:
                        alert['summary'] += (' running {0} '.format(i['privateDnsName']))
                if r['eventName']=='StartInstances':
                    for i in r['requestParameters']['instancesSet']['items']:
                        alert['summary'] += (' starting {0} '.format(i['instanceId']))
                logger.debug(alert['summary'])
                #logger.debug(alert['events'])

                #save alert to alerts index, update events index with alert ID for cross reference
                alertResult=alertToES(es,alert)
                logger.debug(alertResult)
                if 'alerts' not in r.keys():
                    r['alerts']=[]
                r['alerts'].append(dict(index=alertResult['_index'],type=alertResult['_type'],id=alertResult['_id']))
                r['alerttimestamp']=toUTC(datetime.now()).isoformat()
                es.update(r.get_meta()['index'],r.get_meta()['type'],r.get_meta()['id'],document=r)
                alertToMessageQueue(alert)
    except Exception as e:
        logger.error("Exception %r when creating alerts "%e)

def main():
    logger.debug('starting')
    logger.debug(options)
    es=pyes.ES(("http",options.esserver,options.esport))
    results=esCloudTrailSearch(es)
    createAlerts(es,results)
    logger.debug('finished')

def initConfig():
    #change this to your default zone for when it's not specified
    options.defaultTimeZone=getConfig('defaulttimezone','US/Pacific',options.configfile)
    #msg queue settings
    options.mqserver=getConfig('mqserver','localhost',options.configfile)               #message queue server hostname
    options.alertqueue=getConfig('alertqueue','mozdef.alert',options.configfile)        #alert queue topic
    options.alertexchange=getConfig('alertexchange','alerts',options.configfile)        #alert queue exchange name
    #logging settings
    options.output=getConfig('output','stdout',options.configfile)                      #output our log to stdout or syslog
    options.sysloghostname=getConfig('sysloghostname','localhost',options.configfile)   #syslog hostname
    options.syslogport=getConfig('syslogport',514,options.configfile)                   #syslog port
    #elastic search server settings
    options.esserver=getConfig('esserver','localhost',options.configfile)
    options.esport=getConfig('esport',9200,options.configfile)

if __name__ == '__main__':
    parser=OptionParser()
    parser.add_option("-c", dest='configfile' , default=sys.argv[0].replace('.py','.conf'), help="configuration file to use")
    (options,args) = parser.parse_args()
    initConfig()
    initLogger()
    main()
<|endoftext|>


<commit_before>import datetime
from collections import defaultdict
import pytz
from xml.etree import ElementTree

from celery.task import periodic_task
from celery.schedules import crontab
from celery.utils.log import get_task_logger
from django.conf import settings
from django.utils.dateparse import parse_datetime, parse_date

from casexml.apps.case.mock import CaseBlock
from corehq import toggles
from corehq.apps.hqcase.utils import submit_case_blocks
from corehq.apps.fixtures.models import FixtureDataItem
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.util.soft_assert import soft_assert
from dimagi.utils.decorators.memoized import memoized

from .case_utils import CASE_TYPE_EPISODE
from .const import (
    DOSE_TAKEN_INDICATORS,
    DAILY_SCHEDULE_FIXTURE_NAME,
    DAILY_SCHEDULE_ID,
    SCHEDULE_ID_FIXTURE,
    HISTORICAL_CLOSURE_REASON,
)
from .exceptions import EnikshayTaskException
from .data_store import AdherenceDatastore

logger = get_task_logger(__name__)

@periodic_task(
    run_every=crontab(day_of_week=[1], hour=0, minute=0),  # every Monday
    queue=getattr(settings, 'CELERY_PERIODIC_QUEUE', 'celery')
)
def enikshay_adherence_task():
    # runs adherence calculations for all domains that have `toggles.UATBC_ADHERENCE_TASK` enabled
    domains = toggles.UATBC_ADHERENCE_TASK.get_enabled_domains()
    for domain in domains:
        updater = EpisodeAdherenceUpdater(domain)
        updater.run()

class EpisodeAdherenceUpdater(object):
    """This iterates over all open 'episode' cases and sets 'adherence' related properties
    according to this spec https://docs.google.com/document/d/1FjSdLYOYUCRBuW3aSxvu3Z5kvcN6JKbpDFDToCgead8/edit

    This is applicable to various enikshay domains. The domain can be specified in the initalization
    """

    def __init__(self, domain):
        self.domain = domain
        self.purge_date = pytz.UTC.localize(datetime.datetime.today() - datetime.timedelta(days=60))
        self.adherence_data_store = AdherenceDatastore(domain)

    def run(self):
        # iterate over all open 'episode' cases and set 'adherence' properties
        for episode in self._get_open_episode_cases():
            update = EpisodeUpdate(episode, self)
            try:
                case_block = update.case_block()
                if case_block:
                    submit_case_blocks(
                        [ElementTree.tostring(case_block.as_xml())],
                        self.domain
                    )
            except Exception, e:
                raise EnikshayTaskException(
                    "Error calculating adherence values for episode case_id({}): {}".format(
                        episode.case_id,
                        e
                    )
                )

    def _get_open_episode_cases(self):
        # return all open 'episode' cases
        case_accessor = CaseAccessors(self.domain)
        case_ids = case_accessor.get_open_case_ids_in_domain_by_type(CASE_TYPE_EPISODE)
        return case_accessor.iter_cases(case_ids)

    @memoized
    def get_doses_data(self):
        # return 'doses_per_week' by 'schedule_id' from the Fixture data
        fixtures = FixtureDataItem.get_indexed_items(self.domain, DAILY_SCHEDULE_FIXTURE_NAME, SCHEDULE_ID_FIXTURE)
        return dict((k, int(fixture['doses_per_week'])) for k, fixture in fixtures.items())

class EpisodeUpdate(object):
    """
    Class to capture adherence related calculations specific to an 'episode' case
    """
    def __init__(self, episode_case, case_updater):
        """
        Args:
            episode_case: An 'episode' case object
            case_repeater: EpisodeAdherenceUpdater object
        """
        self.episode = episode_case
        self.case_updater = case_updater
        self._cache_dose_taken_by_date = False

    def get_property(self, property):
        """
        Args:
            name of the case-property

        Returns:
            value of the episode case-property named 'property'
        """
        return self.episode.dynamic_case_properties().get(property)

    @memoized
    def get_valid_adherence_cases(self):
        # Returns list of 'adherence' cases of which 'adherence_value' is one of DOSE_KNOWN_INDICATORS
        return self.case_updater.adherence_data_store.dose_known_adherences(
            self.episode.case_id
        )

    def get_latest_adherence_date(self):
        """
        return open case of type 'adherence' reverse-indexed to episode that
            has the latest 'adherence_date' property of all
        """
        return self.case_updater.adherence_data_store.latest_adherence_date(
            self.episode.case_id
        )

    def calculate_doses_taken_by_day(self, adherence_cases):
        """
        Args:
            adherence_cases: list of 'adherence' case dicts

        Returns:
            dict indexed by date part of 'adherence_date' and whether a dose is taken as value.
            Below is criteria to calculate whether a dose is taken on a day or not

        If there are multiple cases on one day filter to a single case as per below
        1. Find most relevant case
            if only non-enikshay source cases
                consider the case with latest_modified - irrespective of case is closed/open

            if only enikshay source cases
                filter by '(closed and closure_reason == HISTORICAL_CLOSURE_REASON) or open'
                consider the case with latest modified after above filter

            if mix of enikshay and non-enikshay source cases
                ignore non-enikshay and apply above enikshay only condition
        2. Check if 'adherence_value' of most relevent case is one of DOSE_TAKEN_INDICATORS
        """
        def is_dose_taken(cases):
            # runs above discribed calculation and returns whether a dose is taken or not
            sources = set(map(lambda x: x["adherence_source"], cases))
            if 'enikshay' not in sources:
                valid_cases = cases
            else:
                valid_cases = filter(
                    lambda case: (
                        case['adherence_source'] is 'enikshay' and
                        (not case['closed'] or (case['closed'] and
                         case['adherence_closure_reason'] == HISTORICAL_CLOSURE_REASON))
                    ),
                    cases
                )
            if valid_cases:
                by_modified_on = sorted(valid_cases, key=lambda case: case['modified_on'])
                return by_modified_on[-1]['adherence_value'] in DOSE_TAKEN_INDICATORS
            else:
                return False

        # index by 'adherence_date'
        cases_by_date = defaultdict(list)
        for case in adherence_cases:
            adherence_date = parse_date(case['adherence_date']) or parse_datetime(case['adherence_date']).date()
            cases_by_date[adherence_date].append(case)

        # calculate whether adherence is taken on each day
        dose_taken_by_date = defaultdict(bool)
        for d, cases in cases_by_date.iteritems():
            dose_taken_by_date[d] = is_dose_taken(cases)

        return dose_taken_by_date

    def get_adherence_schedule_start_date(self):
        # return property 'adherence_schedule_date_start' of episode case, cast to datetime
        raw_date = self.get_property('adherence_schedule_date_start')
        if not raw_date:
            return None
        if parse_datetime(raw_date):
            return parse_datetime(raw_date)
        elif parse_date(raw_date):
            return datetime.datetime.combine(parse_date(raw_date), datetime.datetime.min.time())
        else:
            logger.error(
                "Episode case {case_id} has invalid format for 'adherence_schedule_date_start' {date}".format(
                    case_id=self.episode.case_id,
                    date=raw_date
                )
            )
            return None

    def count_doses_taken(self, dose_taken_by_date, lte=None, gte=None):
        """
        Args:
            dose_taken_by_date: result of self.calculate_doses_taken_by_day
        Returns:
            total count of adherence_cases excluding duplicates on a given day. If there are
            two adherence_cases on one day at different time, it will be counted as one
        """
        if bool(lte) != bool(gte):
            raise EnikshayTaskException("Both of lte and gte should be specified or niether of them")

        if not lte:
            return dose_taken_by_date.values().count(True)
        else:
            # any efficient way to do this - numpy, python bisect?
            return [
                is_taken
                for date, is_taken in dose_taken_by_date.iteritems()
                if lte.date() <= date <= gte.date()
            ].count(True)

    def update_json(self):
        """
        Evaluates adherence calculations on the 'episode' case and returns dict of values

        Returns:
            If no update is necessary, empty dict is returned, if not, dict with following
            keys is returned

            {
                'aggregated_score_date_calculated': value,
                'expected_doses_taken': value,
                'aggregated_score_count_taken': value
            }
        """

        debug_data = []
        adherence_schedule_date_start = self.get_adherence_schedule_start_date()
        debug_data.append("adherence_schedule_date_start: {}".format(adherence_schedule_date_start))
        debug_data.append("purge_date: {}".format(self.case_updater.purge_date))

        if not adherence_schedule_date_start:
            # adherence schedule hasn't been selected, so no update necessary
            return {'update': {}, 'debug_data': debug_data}

        default_update = {
            'aggregated_score_date_calculated': adherence_schedule_date_start - datetime.timedelta(days=1),
            'expected_doses_taken': 0,
            'aggregated_score_count_taken': 0,
            'adherence_total_doses_taken': 0,
            'adherence_latest_date_recorded': adherence_schedule_date_start - datetime.timedelta(days=1)
        }

        if adherence_schedule_date_start > self.case_updater.purge_date:
            update = default_update
        else:
            update = {}
            latest_adherence_date = self.get_latest_adherence_date()
            debug_data.append("latest_adherence_date: {}".format(latest_adherence_date))
            if not latest_adherence_date:
                update = default_update
            else:
                update["adherence_latest_date_recorded"] = latest_adherence_date
                if latest_adherence_date < self.case_updater.purge_date:
                    update["aggregated_score_date_calculated"] = latest_adherence_date
                else:
                    update["aggregated_score_date_calculated"] = self.case_updater.purge_date

                # calculate 'adherence_total_doses_taken'
                adherence_cases = self.get_valid_adherence_cases()
                dose_taken_by_date = self.calculate_doses_taken_by_day(adherence_cases)
                update["adherence_total_doses_taken"] = self.count_doses_taken(dose_taken_by_date)
                # calculate 'aggregated_score_count_taken'
                update["aggregated_score_count_taken"] = self.count_doses_taken(
                    dose_taken_by_date,
                    lte=adherence_schedule_date_start,
                    gte=update["aggregated_score_date_calculated"]
                )

                # calculate 'expected_doses_taken' score
                dose_data = self.case_updater.get_doses_data()
                adherence_schedule_id = self.get_property('adherence_schedule_id') or DAILY_SCHEDULE_ID
                doses_per_week = dose_data.get(adherence_schedule_id)
                if doses_per_week:
                    update['expected_doses_taken'] = int(((
                        (update['aggregated_score_date_calculated'] - adherence_schedule_date_start)).days / 7.0
                    ) * doses_per_week)
                else:
                    update['expected_doses_taken'] = 0
                    soft_assert(notify_admins=True)(
                        True,
                        "No fixture item found with schedule_id {}".format(adherence_schedule_id)
                    )
        # convert datetime -> date objects
        for key, val in update.iteritems():
            if isinstance(val, datetime.datetime):
                update[key] = val.date()
        return {'update': update, 'debug_data': debug_data}

    def case_block(self):
        """
        Returns:
            CaseBlock object with adherence updates. If no update is necessary, None is returned
        """
        update = self.update_json()['update']
        if update:
            return CaseBlock(**{
                'case_id': self.episode.case_id,
                'create': False,
                'update': update
            })
        else:
            return None
<commit_msg>correct logger call<commit_after>import datetime
from collections import defaultdict
import pytz
from xml.etree import ElementTree

from celery.task import periodic_task
from celery.schedules import crontab
from celery.utils.log import get_task_logger
from django.conf import settings
from django.utils.dateparse import parse_datetime, parse_date

from casexml.apps.case.mock import CaseBlock
from corehq import toggles
from corehq.apps.hqcase.utils import submit_case_blocks
from corehq.apps.fixtures.models import FixtureDataItem
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.util.soft_assert import soft_assert
from dimagi.utils.decorators.memoized import memoized

from .case_utils import CASE_TYPE_EPISODE
from .const import (
    DOSE_TAKEN_INDICATORS,
    DAILY_SCHEDULE_FIXTURE_NAME,
    DAILY_SCHEDULE_ID,
    SCHEDULE_ID_FIXTURE,
    HISTORICAL_CLOSURE_REASON,
)
from .exceptions import EnikshayTaskException
from .data_store import AdherenceDatastore

logger = get_task_logger(__name__)

@periodic_task(
    run_every=crontab(day_of_week=[1], hour=0, minute=0),  # every Monday
    queue=getattr(settings, 'CELERY_PERIODIC_QUEUE', 'celery')
)
def enikshay_adherence_task():
    # runs adherence calculations for all domains that have `toggles.UATBC_ADHERENCE_TASK` enabled
    domains = toggles.UATBC_ADHERENCE_TASK.get_enabled_domains()
    for domain in domains:
        updater = EpisodeAdherenceUpdater(domain)
        updater.run()

class EpisodeAdherenceUpdater(object):
    """This iterates over all open 'episode' cases and sets 'adherence' related properties
    according to this spec https://docs.google.com/document/d/1FjSdLYOYUCRBuW3aSxvu3Z5kvcN6JKbpDFDToCgead8/edit

    This is applicable to various enikshay domains. The domain can be specified in the initalization
    """

    def __init__(self, domain):
        self.domain = domain
        self.purge_date = pytz.UTC.localize(datetime.datetime.today() - datetime.timedelta(days=60))
        self.adherence_data_store = AdherenceDatastore(domain)

    def run(self):
        # iterate over all open 'episode' cases and set 'adherence' properties
        for episode in self._get_open_episode_cases():
            update = EpisodeUpdate(episode, self)
            try:
                case_block = update.case_block()
                if case_block:
                    submit_case_blocks(
                        [ElementTree.tostring(case_block.as_xml())],
                        self.domain
                    )
            except Exception, e:
                logger.error(
                    "Error calculating adherence values for episode case_id({}): {}".format(
                        episode.case_id,
                        e
                    )
                )

    def _get_open_episode_cases(self):
        # return all open 'episode' cases
        case_accessor = CaseAccessors(self.domain)
        case_ids = case_accessor.get_open_case_ids_in_domain_by_type(CASE_TYPE_EPISODE)
        return case_accessor.iter_cases(case_ids)

    @memoized
    def get_doses_data(self):
        # return 'doses_per_week' by 'schedule_id' from the Fixture data
        fixtures = FixtureDataItem.get_indexed_items(self.domain, DAILY_SCHEDULE_FIXTURE_NAME, SCHEDULE_ID_FIXTURE)
        return dict((k, int(fixture['doses_per_week'])) for k, fixture in fixtures.items())

class EpisodeUpdate(object):
    """
    Class to capture adherence related calculations specific to an 'episode' case
    """
    def __init__(self, episode_case, case_updater):
        """
        Args:
            episode_case: An 'episode' case object
            case_repeater: EpisodeAdherenceUpdater object
        """
        self.episode = episode_case
        self.case_updater = case_updater
        self._cache_dose_taken_by_date = False

    def get_property(self, property):
        """
        Args:
            name of the case-property

        Returns:
            value of the episode case-property named 'property'
        """
        return self.episode.dynamic_case_properties().get(property)

    @memoized
    def get_valid_adherence_cases(self):
        # Returns list of 'adherence' cases of which 'adherence_value' is one of DOSE_KNOWN_INDICATORS
        return self.case_updater.adherence_data_store.dose_known_adherences(
            self.episode.case_id
        )

    def get_latest_adherence_date(self):
        """
        return open case of type 'adherence' reverse-indexed to episode that
            has the latest 'adherence_date' property of all
        """
        return self.case_updater.adherence_data_store.latest_adherence_date(
            self.episode.case_id
        )

    def calculate_doses_taken_by_day(self, adherence_cases):
        """
        Args:
            adherence_cases: list of 'adherence' case dicts

        Returns:
            dict indexed by date part of 'adherence_date' and whether a dose is taken as value.
            Below is criteria to calculate whether a dose is taken on a day or not

        If there are multiple cases on one day filter to a single case as per below
        1. Find most relevant case
            if only non-enikshay source cases
                consider the case with latest_modified - irrespective of case is closed/open

            if only enikshay source cases
                filter by '(closed and closure_reason == HISTORICAL_CLOSURE_REASON) or open'
                consider the case with latest modified after above filter

            if mix of enikshay and non-enikshay source cases
                ignore non-enikshay and apply above enikshay only condition
        2. Check if 'adherence_value' of most relevent case is one of DOSE_TAKEN_INDICATORS
        """
        def is_dose_taken(cases):
            # runs above discribed calculation and returns whether a dose is taken or not
            sources = set(map(lambda x: x["adherence_source"], cases))
            if 'enikshay' not in sources:
                valid_cases = cases
            else:
                valid_cases = filter(
                    lambda case: (
                        case['adherence_source'] is 'enikshay' and
                        (not case['closed'] or (case['closed'] and
                         case['adherence_closure_reason'] == HISTORICAL_CLOSURE_REASON))
                    ),
                    cases
                )
            if valid_cases:
                by_modified_on = sorted(valid_cases, key=lambda case: case['modified_on'])
                return by_modified_on[-1]['adherence_value'] in DOSE_TAKEN_INDICATORS
            else:
                return False

        # index by 'adherence_date'
        cases_by_date = defaultdict(list)
        for case in adherence_cases:
            adherence_date = parse_date(case['adherence_date']) or parse_datetime(case['adherence_date']).date()
            cases_by_date[adherence_date].append(case)

        # calculate whether adherence is taken on each day
        dose_taken_by_date = defaultdict(bool)
        for d, cases in cases_by_date.iteritems():
            dose_taken_by_date[d] = is_dose_taken(cases)

        return dose_taken_by_date

    def get_adherence_schedule_start_date(self):
        # return property 'adherence_schedule_date_start' of episode case, cast to datetime
        raw_date = self.get_property('adherence_schedule_date_start')
        if not raw_date:
            return None
        if parse_datetime(raw_date):
            return parse_datetime(raw_date)
        elif parse_date(raw_date):
            return datetime.datetime.combine(parse_date(raw_date), datetime.datetime.min.time())
        else:
            raise EnikshayTaskException(
                "Episode case {case_id} has invalid format for 'adherence_schedule_date_start' {date}".format(
                    case_id=self.episode.case_id,
                    date=raw_date
                )
            )
            return None

    def count_doses_taken(self, dose_taken_by_date, lte=None, gte=None):
        """
        Args:
            dose_taken_by_date: result of self.calculate_doses_taken_by_day
        Returns:
            total count of adherence_cases excluding duplicates on a given day. If there are
            two adherence_cases on one day at different time, it will be counted as one
        """
        if bool(lte) != bool(gte):
            raise EnikshayTaskException("Both of lte and gte should be specified or niether of them")

        if not lte:
            return dose_taken_by_date.values().count(True)
        else:
            # any efficient way to do this - numpy, python bisect?
            return [
                is_taken
                for date, is_taken in dose_taken_by_date.iteritems()
                if lte.date() <= date <= gte.date()
            ].count(True)

    def update_json(self):
        """
        Evaluates adherence calculations on the 'episode' case and returns dict of values

        Returns:
            If no update is necessary, empty dict is returned, if not, dict with following
            keys is returned

            {
                'aggregated_score_date_calculated': value,
                'expected_doses_taken': value,
                'aggregated_score_count_taken': value
            }
        """

        debug_data = []
        adherence_schedule_date_start = self.get_adherence_schedule_start_date()
        debug_data.append("adherence_schedule_date_start: {}".format(adherence_schedule_date_start))
        debug_data.append("purge_date: {}".format(self.case_updater.purge_date))

        if not adherence_schedule_date_start:
            # adherence schedule hasn't been selected, so no update necessary
            return {'update': {}, 'debug_data': debug_data}

        default_update = {
            'aggregated_score_date_calculated': adherence_schedule_date_start - datetime.timedelta(days=1),
            'expected_doses_taken': 0,
            'aggregated_score_count_taken': 0,
            'adherence_total_doses_taken': 0,
            'adherence_latest_date_recorded': adherence_schedule_date_start - datetime.timedelta(days=1)
        }

        if adherence_schedule_date_start > self.case_updater.purge_date:
            update = default_update
        else:
            update = {}
            latest_adherence_date = self.get_latest_adherence_date()
            debug_data.append("latest_adherence_date: {}".format(latest_adherence_date))
            if not latest_adherence_date:
                update = default_update
            else:
                update["adherence_latest_date_recorded"] = latest_adherence_date
                if latest_adherence_date < self.case_updater.purge_date:
                    update["aggregated_score_date_calculated"] = latest_adherence_date
                else:
                    update["aggregated_score_date_calculated"] = self.case_updater.purge_date

                # calculate 'adherence_total_doses_taken'
                adherence_cases = self.get_valid_adherence_cases()
                dose_taken_by_date = self.calculate_doses_taken_by_day(adherence_cases)
                update["adherence_total_doses_taken"] = self.count_doses_taken(dose_taken_by_date)
                # calculate 'aggregated_score_count_taken'
                update["aggregated_score_count_taken"] = self.count_doses_taken(
                    dose_taken_by_date,
                    lte=adherence_schedule_date_start,
                    gte=update["aggregated_score_date_calculated"]
                )

                # calculate 'expected_doses_taken' score
                dose_data = self.case_updater.get_doses_data()
                adherence_schedule_id = self.get_property('adherence_schedule_id') or DAILY_SCHEDULE_ID
                doses_per_week = dose_data.get(adherence_schedule_id)
                if doses_per_week:
                    update['expected_doses_taken'] = int(((
                        (update['aggregated_score_date_calculated'] - adherence_schedule_date_start)).days / 7.0
                    ) * doses_per_week)
                else:
                    update['expected_doses_taken'] = 0
                    soft_assert(notify_admins=True)(
                        True,
                        "No fixture item found with schedule_id {}".format(adherence_schedule_id)
                    )
        # convert datetime -> date objects
        for key, val in update.iteritems():
            if isinstance(val, datetime.datetime):
                update[key] = val.date()
        return {'update': update, 'debug_data': debug_data}

    def case_block(self):
        """
        Returns:
            CaseBlock object with adherence updates. If no update is necessary, None is returned
        """
        update = self.update_json()['update']
        if update:
            return CaseBlock(**{
                'case_id': self.episode.case_id,
                'create': False,
                'update': update
            })
        else:
            return None
<|endoftext|>


<commit_before>"""Command to compare a Git tree against CVS."""

import re
import subprocess
from subprocess import PIPE
import sys

from cvsgit.cvs import split_cvs_source
from cvsgit.git import GitCommandError
from cvsgit.i18n import _
from cvsgit.main import Command, Conduit
from cvsgit.utils import Tempdir, stripnl

class verify(Command):
    __doc__ = _(
    """Verify the Git work tree against CVS.

    Usage: %prog

    Compares the work tree against a clean checkout from CVS with the
    same timestamp as the HEAD commit.
    """)

    def initialize_options(self):
        self.commit = 'HEAD'
        self.add_option('--history', action='store_true', help=\
            _("Walk backwards through the entire history."))
        self.add_option('--quiet', action='store_true', help=\
            _("Only report error and warning messages."))

    def finalize_options(self):
        if len(self.args) > 0:
            self.usage_error(_('too many arguments'))

    def run(self):
        conduit = Conduit()
        self.cvsroot, self.module = split_cvs_source(conduit.source)
        self.git = git = conduit.git
        with Tempdir() as tempdir:
            self.tempdir = tempdir

            while True:
                returncode = self._run()
                if returncode != 0:
                    return returncode
                elif not self.options.history:
                    return 0

                try:
                    self.git.checkout('-q', 'HEAD~1')
                except GitCommandError:
                    return 0

    def _run(self):
        returncode = 0
        date = self.commit_date(self.commit)
        head = self.git.rev_parse('--short', 'HEAD')
        command = ['cvs', '-Q', '-d', self.cvsroot, 'checkout',
                   '-P', '-D', date, '-d', self.tempdir, self.module]
        if not self.options.quiet:
            print "(%s) '%s'" % (head, "' '".join(command))
        subprocess.check_call(command)
        command = ['diff', '-r', self.tempdir, self.git.git_work_tree]
        pipe = subprocess.Popen(command, stdout=PIPE, stderr=PIPE)
        stdout, dummy = pipe.communicate()
        for line in stripnl(stdout).split('\n'):
            if re.match('^Only in .+: \.git$', line):
                continue
            if re.match('^Only in .+: CVS$', line):
                continue
            if not self.options.quiet:
                sys.stdout.write(line + '\n')
            returncode = 1
        return returncode

    def commit_date(self, commit):
        """Get the commit date as a string in UTC timezone.
        """
        command = ['git', 'log', '-1', commit]
        pipe = self.git._popen(command, stdout=PIPE, stderr=PIPE)
        stdout, stderr = pipe.communicate()
        if pipe.returncode != 0:
            self.fatal(_("can't get commit date of %s" % commit))
        match = re.search('Date: +[^ ]+ (.*) \+0000', stdout)
        if match:
            return match.group(1)
        else:
            self.fatal(_("couldn't match Date: in output of '%s'") % \
                           ' '.join(command))

if __name__ == '__main__':
    verify()
<commit_msg>Implemented --forward and --skip options to verify<commit_after>"""Command to compare a Git tree against CVS."""

import re
import subprocess
from subprocess import PIPE
import sys

from cvsgit.cvs import split_cvs_source
from cvsgit.git import GitCommandError
from cvsgit.i18n import _
from cvsgit.main import Command, Conduit
from cvsgit.utils import Tempdir, stripnl

class verify(Command):
    __doc__ = _(
    """Verify the Git work tree against CVS.

    Usage: %prog

    Compares the work tree against a clean checkout from CVS with the
    same timestamp as the HEAD commit.
    """)

    def initialize_options(self):
        self.commit = 'HEAD'
        self.add_option('--history', action='store_true', help=\
            _("Walk backwards through the entire history."))
        self.add_option('--forward', action='store_true', help=\
            _("Modifies the --history option to walk forward."))
        self.add_option('--skip', action='store_true', help=\
            _("Skip the first verification step and move HEAD "
              "backward instead, the opposite direction with "
              "--forward)."))
        self.add_option('--quiet', action='store_true', help=\
            _("Only report error and warning messages."))

    def finalize_options(self):
        if len(self.args) > 0:
            self.usage_error(_('too many arguments'))

    def run(self):
        conduit = Conduit()
        self.cvsroot, self.module = split_cvs_source(conduit.source)
        self.git = git = conduit.git
        with Tempdir() as tempdir:
            self.tempdir = tempdir

            while True:
                if self.options.skip:
                    self.options.skip = False
                else:
                    returncode = self._run()
                    if returncode != 0:
                        return returncode
                    elif not self.options.history:
                        return 0

                if self.options.forward:
                    wc_head = self.git.rev_parse('HEAD')
                    cvs_head = self.git.rev_parse(conduit.branch)
                    if wc_head == cvs_head:
                        return 0
                    subprocess.check_call(['/bin/sh', '-c',
                       'git checkout `git rev-list %s..%s|tail -1`' % \
                       (wc_head, cvs_head)])
                else:
                    try:
                        self.git.checkout('-q', 'HEAD~1')
                    except GitCommandError:
                        return 0

    def _run(self):
        returncode = 0
        date = self.commit_date(self.commit)
        head = self.git.rev_parse('--short', 'HEAD')
        command = ['cvs', '-Q', '-d', self.cvsroot, 'checkout',
                   '-P', '-D', date, '-d', self.tempdir, self.module]
        if not self.options.quiet:
            print "(%s) '%s'" % (head, "' '".join(command))
        subprocess.check_call(command)
        command = ['diff', '-r', self.tempdir, self.git.git_work_tree]
        pipe = subprocess.Popen(command, stdout=PIPE, stderr=PIPE)
        stdout, dummy = pipe.communicate()
        for line in stripnl(stdout).split('\n'):
            if re.match('^Only in .+: \.git$', line):
                continue
            if re.match('^Only in .+: CVS$', line):
                continue
            if not self.options.quiet:
                sys.stdout.write(line + '\n')
            returncode = 1
        return returncode

    def commit_date(self, commit):
        """Get the commit date as a string in UTC timezone.
        """
        command = ['git', 'log', '-1', commit]
        pipe = self.git._popen(command, stdout=PIPE, stderr=PIPE)
        stdout, stderr = pipe.communicate()
        if pipe.returncode != 0:
            self.fatal(_("can't get commit date of %s" % commit))
        match = re.search('Date: +[^ ]+ (.*) \+0000', stdout)
        if match:
            return match.group(1)
        else:
            self.fatal(_("couldn't match Date: in output of '%s'") % \
                           ' '.join(command))

if __name__ == '__main__':
    verify()
<|endoftext|>


<commit_before>"""
Copyright 2013 Steven Diamond

This file is part of CVXPY.

CVXPY is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

CVXPY is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with CVXPY.  If not, see <http://www.gnu.org/licenses/>.
"""

from .. import interface as intf
from ..expressions.expression import Expression
from ..expressions.constants import Constant
from norm2 import norm2
from square import square
from scipy import linalg as LA
import numpy as np

""" Alias for x.T*P*x """
def quad_form(x, P):
    x,P = map(Expression.cast_to_const, (x,P))
    # Check dimensions.
    n = P.size[0]
    if P.size[1] != n or x.size != (n,1):
        raise Exception("Invalid dimensions for arguments.")
    if x.curvature.is_constant():
        return x.T*P*x
    elif P.curvature.is_constant():
        np_intf = intf.get_matrix_interface(np.ndarray)
        P = np_intf.const_to_matrix(P.value)
        # Replace P with symmetric version.
        P = (P + P.T)/2
        # Check if P is PSD.
        eigvals = LA.eigvalsh(P)
        if min(eigvals) > 0:
            P_sqrt = Constant(LA.sqrtm(P).real)
            return square(norm2(P_sqrt*x))
        elif max(eigvals) < 0:
            P_sqrt = Constant(LA.sqrtm(-P).real)
            return -square(norm2(P_sqrt*x))
        else:
            raise Exception("P has both positive and negative eigenvalues.")
    else:
        raise Exception("At least one argument to quad_form must be constant.")<commit_msg>fixed import issue<commit_after>"""
Copyright 2013 Steven Diamond

This file is part of CVXPY.

CVXPY is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

CVXPY is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with CVXPY.  If not, see <http://www.gnu.org/licenses/>.
"""

from .. import interface as intf
from ..expressions.expression import Expression
from ..expressions.constants import Constant
from norm2 import norm2
from elementwise.square import square
from scipy import linalg as LA
import numpy as np

""" Alias for x.T*P*x """
def quad_form(x, P):
    x,P = map(Expression.cast_to_const, (x,P))
    # Check dimensions.
    n = P.size[0]
    if P.size[1] != n or x.size != (n,1):
        raise Exception("Invalid dimensions for arguments.")
    if x.curvature.is_constant():
        return x.T*P*x
    elif P.curvature.is_constant():
        np_intf = intf.get_matrix_interface(np.ndarray)
        P = np_intf.const_to_matrix(P.value)
        # Replace P with symmetric version.
        P = (P + P.T)/2
        # Check if P is PSD.
        eigvals = LA.eigvalsh(P)
        if min(eigvals) > 0:
            P_sqrt = Constant(LA.sqrtm(P).real)
            return square(norm2(P_sqrt*x))
        elif max(eigvals) < 0:
            P_sqrt = Constant(LA.sqrtm(-P).real)
            return -square(norm2(P_sqrt*x))
        else:
            raise Exception("P has both positive and negative eigenvalues.")
    else:
        raise Exception("At least one argument to quad_form must be constant.")<|endoftext|>