<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Mario Lassnig, mario.lassnig@cern.ch, 2016-2017
# - Daniel Drizhuk, d.drizhuk@gmail.com, 2017
# - Tobias Wegner, tobias.wegner@cern.ch, 2017
# - Paul Nilsson, paul.nilsson@cern.ch, 2017-8
# - Wen Guan, wen.guan@cern.ch, 2018

import time
import os
import signal
from subprocess import PIPE

from pilot.control.job import send_state
from pilot.util.auxiliary import get_logger
from pilot.util.container import execute
from pilot.util.constants import UTILITY_BEFORE_PAYLOAD, UTILITY_WITH_PAYLOAD, UTILITY_AFTER_PAYLOAD, \
    PILOT_PRE_SETUP, PILOT_POST_SETUP, PILOT_PRE_PAYLOAD, PILOT_POST_PAYLOAD, LOG_TRANSFER_NOT_DONE, PILOT_KILL_SIGNAL
from pilot.util.timing import add_to_pilot_timing
from pilot.common.errorcodes import ErrorCodes
from pilot.common.exception import PilotException

import logging
logger = logging.getLogger(__name__)

errors = ErrorCodes()

class Executor(object):
    def __init__(self, args, queues, job, out, err):
        self.__args = args
        self.__queues = queues
        self.__job = job
        self.__out = out
        self.__err = err

    def get_job(self):
        """
        Get the job object.

        :return: job object.
        """
        return self.__job

    def setup_payload(self, job, out, err):
        """
        (add description)

        :param job:
        :param out:
        :param err:
        :return:
        """
        # log = get_logger(job.jobid)

        # try:
        # create symbolic link for sqlite200 and geomDB in job dir
        #    for db_name in ['sqlite200', 'geomDB']:
        #         src = '/cvmfs/atlas.cern.ch/repo/sw/database/DBRelease/current/%s' % db_name
        #         link_name = 'job-%s/%s' % (job.jobid, db_name)
        #         os.symlink(src, link_name)
        # except Exception as e:
        #     log.error('could not create symbolic links to database files: %s' % e)
        #     return False

        return True

    def run_payload(self, job, out, err):
        """
        (add description)

        :param job: job object
        :param out: (currently not used; deprecated)
        :param err: (currently not used; deprecated)
        :return: proc (subprocess returned by Popen())
        """

        log = get_logger(job.jobid)

        # write time stamps to pilot timing file
        add_to_pilot_timing(job.jobid, PILOT_PRE_SETUP, time.time(), self.__args)

        # get the payload command from the user specific code
        pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
        user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)
        # for testing looping job:    cmd = user.get_payload_command(job) + ';sleep 240'
        try:
            cmd = user.get_payload_command(job)
        except PilotException as e:
            log.fatal('could not define payload command')
            return None

        log.info("payload execution command: %s" % cmd)

        # write time stamps to pilot timing file
        add_to_pilot_timing(job.jobid, PILOT_POST_SETUP, time.time(), self.__args)

        # should we run any additional commands? (e.g. special monitoring commands)
        cmds = user.get_utility_commands_list(order=UTILITY_BEFORE_PAYLOAD)
        if cmds != []:
            for utcmd in cmds:
                log.info('utility command to be executed before the payload: %s' % utcmd)
                # add execution code here
                # store pid in job object job.utilitypids = {<name>: <pid>}
                job.utilitypids[utcmd] = -1

        # should any additional commands be prepended to the payload execution string?
        cmds = user.get_utility_commands_list(order=UTILITY_WITH_PAYLOAD)
        if cmds != []:
            for utcmd in cmds:
                log.info('utility command to be executed with the payload: %s' % utcmd)
                # add execution code here

        # write time stamps to pilot timing file
        add_to_pilot_timing(job.jobid, PILOT_PRE_PAYLOAD, time.time(), self.__args)

        # replace platform and workdir with new function get_payload_options() or something from experiment specific
        # code
        try:
            proc = execute(cmd, workdir=job.workdir, returnproc=True,
                           usecontainer=True, stdout=out, stderr=err, cwd=job.workdir, job=job)
        except Exception as e:
            log.error('could not execute: %s' % str(e))
            return None

        log.info('started -- pid=%s executable=%s' % (proc.pid, cmd))
        job.pid = proc.pid

        # should any additional commands be executed after the payload?
        cmds = user.get_utility_commands_list(order=UTILITY_AFTER_PAYLOAD)
        if cmds != []:
            for utcmd in cmds:
                log.info('utility command to be executed after the payload: %s' % utcmd)

                # how should this command be executed?
                utilitycommand = user.get_utility_command_setup(utcmd, job)

                try:
                    proc1 = execute(utilitycommand, workdir=job.workdir, returnproc=True,
                                    usecontainer=True, stdout=PIPE, stderr=PIPE, cwd=job.workdir,
                                    job=job)
                except Exception as e:
                    log.error('could not execute: %s' % e)
                else:
                    # store process handle in job object, and keep track on how many times the command has been launched
                    # also store the full command in case it needs to be restarted later (by the job_monitor() thread)
                    job.utilities[utcmd] = [proc1, 1, utilitycommand]

        return proc

    def wait_graceful(self, args, proc, job):
        """
        (add description)

        :param args:
        :param proc:
        :param job:
        :return:
        """

        log = get_logger(job.jobid)

        breaker = False
        exit_code = None
        iteration = 0L
        while True:
            iteration += 1
            for i in xrange(100):
                if args.graceful_stop.is_set():
                    # does the job have a set error code?
                    if os.environ.get('REACHED_MAXTIME'):  # TODO: READ FROM WITH SINGLETON
                        job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(errors.REACHEDMAXTIME)
                    if not job.piloterrorcodes:
                        log.warning('received graceful stop but no pilot error code has been set yet')
                    log_transfer = job.get_status('LOG_TRANSFER')
                    log.info('job.status=%s, log_transfer=%s' % (job.status, log_transfer))
                    if job not in self.__queues.failed_jobs.queue and log_transfer == LOG_TRANSFER_NOT_DONE:
                        self.__queues.failed_jobs.put(job)
                        log.warning('added job object to failed_jobs queue')
                    breaker = True
                    add_to_pilot_timing(0, PILOT_KILL_SIGNAL, time.time(), args)
                    log.info('breaking -- sending SIGTERM pid=%s' % proc.pid)
                    os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
                    # proc.terminate()
                    break
                time.sleep(0.1)
            if breaker:
                delay = 30
                log.info('breaking -- sleep %d s before sending kill instruction to pid=%s' % (delay, proc.pid))
                time.sleep(delay)
                proc.kill()
                break

            exit_code = proc.poll()

            if iteration % 10 == 0:
                log.info('running: iteration=%d pid=%s exit_code=%s' % (iteration, proc.pid, exit_code))
            if exit_code is not None:
                break
            else:
                # send_state(job, args, 'running')
                continue

        return exit_code

    def run(self):
        """
        (add description)

        :return:
        """
        log = get_logger(str(self.__job.jobid))

        exit_code = 1
        if self.setup_payload(self.__job, self.__out, self.__err):
            log.debug('running payload')
            self.__job.state = 'running'
            send_state(self.__job, self.__args, self.__job.state)
            proc = self.run_payload(self.__job, self.__out, self.__err)
            if proc is not None:
                log.info('will wait for graceful exit')
                exit_code = self.wait_graceful(self.__args, proc, self.__job)
                self.__job.state = 'finished' if exit_code == 0 else 'failed'
                log.info('finished pid=%s exit_code=%s state=%s' % (proc.pid, exit_code, self.__job.state))

                if exit_code is None:
                    log.warning('detected unset exit_code from wait_graceful - reset to -1')
                    exit_code = -1

                # write time stamps to pilot timing file
                add_to_pilot_timing(self.__job.jobid, PILOT_POST_PAYLOAD, time.time(), self.__args)

                # stop any running utilities
                if self.__job.utilities != {}:
                    for utcmd in self.__job.utilities.keys():
                        utproc = self.__job.utilities[utcmd][0]
                        if utproc:
                            pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
                            user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user],
                                              -1)
                            sig = user.get_utility_command_kill_signal(utcmd)
                            log.info("stopping process \'%s\' with signal %d" % (utcmd, sig))
                            os.killpg(os.getpgid(utproc.pid), sig)

                            user.post_utility_command_action(utcmd, self.__job)

        return exit_code
<commit_msg>update generator executor to be reusable<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Mario Lassnig, mario.lassnig@cern.ch, 2016-2017
# - Daniel Drizhuk, d.drizhuk@gmail.com, 2017
# - Tobias Wegner, tobias.wegner@cern.ch, 2017
# - Paul Nilsson, paul.nilsson@cern.ch, 2017-8
# - Wen Guan, wen.guan@cern.ch, 2018

import time
import os
import signal
from subprocess import PIPE

from pilot.control.job import send_state
from pilot.util.auxiliary import get_logger
from pilot.util.container import execute
from pilot.util.constants import UTILITY_BEFORE_PAYLOAD, UTILITY_WITH_PAYLOAD, UTILITY_AFTER_PAYLOAD_STARTED, \
    UTILITY_AFTER_PAYLOAD_FINISHED, PILOT_PRE_SETUP, PILOT_POST_SETUP, PILOT_PRE_PAYLOAD, PILOT_POST_PAYLOAD
from pilot.util.timing import add_to_pilot_timing
from pilot.common.exception import PilotException

import logging
logger = logging.getLogger(__name__)

class Executor(object):
    def __init__(self, args, job, out, err):
        self.__args = args
        self.__job = job
        self.__out = out
        self.__err = err

    def get_job(self):
        """
        Get the job object.
        :return: job object.
        """
        return self.__job

    def setup_payload(self, job, out, err):
        """
        (add description)
        :param job:
        :param out:
        :param err:
        :return:
        """
        # log = get_logger(job.jobid, logger)

        # try:
        # create symbolic link for sqlite200 and geomDB in job dir
        #    for db_name in ['sqlite200', 'geomDB']:
        #         src = '/cvmfs/atlas.cern.ch/repo/sw/database/DBRelease/current/%s' % db_name
        #         link_name = 'job-%s/%s' % (job.jobid, db_name)
        #         os.symlink(src, link_name)
        # except Exception as e:
        #     log.error('could not create symbolic links to database files: %s' % e)
        #     return False

        return True

    def pre_setup(self, job):
        """
        Functions to run pre setup
        :param job: job object
        """
        # write time stamps to pilot timing file
        add_to_pilot_timing(job.jobid, PILOT_PRE_SETUP, time.time())

    def post_setup(self, job):
        """
        Functions to run post setup
        :param job: job object
        """
        # write time stamps to pilot timing file
        add_to_pilot_timing(job.jobid, PILOT_POST_SETUP, time.time())

    def utility_before_payload(self, job):
        """
        Functions to run before payload
        :param job: job object
        """
        log = get_logger(job.jobid, logger)

        # get the payload command from the user specific code
        pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
        user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)

        # should we run any additional commands? (e.g. special monitoring commands)
        cmds = user.get_utility_commands_list(order=UTILITY_BEFORE_PAYLOAD)
        if cmds != []:
            for utcmd in cmds:
                log.info('utility command to be executed before the payload: %s' % utcmd)
                # add execution code here
                # store pid in job object job.utilitypids = {<name>: <pid>}
                job.utilitypids[utcmd] = -1

    def utility_with_payload(self, job):
        """
        Functions to run with payload
        :param job: job object
        """
        log = get_logger(job.jobid, logger)

        # get the payload command from the user specific code
        pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
        user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)

        # should any additional commands be prepended to the payload execution string?
        cmds = user.get_utility_commands_list(order=UTILITY_WITH_PAYLOAD)
        if cmds != []:
            for utcmd in cmds:
                log.info('utility command to be executed with the payload: %s' % utcmd)
                # add execution code here

    def utility_after_payload_started(self, job):
        """
        Functions to run after payload started
        :param job: job object
        """
        log = get_logger(job.jobid, logger)

        # get the payload command from the user specific code
        pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
        user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)

        # should any additional commands be executed after the payload?
        cmds = user.get_utility_commands_list(order=UTILITY_AFTER_PAYLOAD_STARTED)
        if cmds != []:
            for utcmd in cmds:
                log.info('utility command to be executed after the payload: %s' % utcmd)

                # how should this command be executed?
                utilitycommand = user.get_utility_command_setup(utcmd, job)

                try:
                    proc1 = execute(utilitycommand, workdir=job.workdir, returnproc=True,
                                    usecontainer=True, stdout=PIPE, stderr=PIPE, cwd=job.workdir,
                                    job=job)
                except Exception as e:
                    log.error('could not execute: %s' % e)
                else:
                    # store process handle in job object, and keep track on how many times the command has been launched
                    # also store the full command in case it needs to be restarted later (by the job_monitor() thread)
                    job.utilities[utcmd] = [proc1, 1, utilitycommand]

    def utility_after_payload_finished(self, job):
        """
        Functions to run after payload finished
        :param job: job object
        """
        log = get_logger(job.jobid, logger)

        # get the payload command from the user specific code
        pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
        user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)

        # should any additional commands be prepended to the payload execution string?
        cmds = user.get_utility_commands_list(order=UTILITY_AFTER_PAYLOAD_FINISHED)
        if cmds != []:
            for utcmd in cmds:
                log.info('utility command to be executed with the payload: %s' % utcmd)
                # add execution code here

    def pre_payload(self, job):
        """
        Functions to run pilot pre payload
        :param job: job object
        """
        # write time stamps to pilot timing file
        add_to_pilot_timing(job.jobid, PILOT_PRE_PAYLOAD, time.time())

    def post_payload(self, job):
        """
        Functions to run pilot post payload
        :param job: job object
        """
        # write time stamps to pilot timing file
        add_to_pilot_timing(job.jobid, PILOT_POST_PAYLOAD, time.time())

    def run_payload(self, job, out, err):
        """
        (add description)
        :param job: job object
        :param out: (currently not used; deprecated)
        :param err: (currently not used; deprecated)
        :return: proc (subprocess returned by Popen())
        """

        log = get_logger(job.jobid, logger)

        self.pre_setup(job)

        self.post_setup(job)

        self.utility_before_payload(job)
        self.pre_payload(job)

        self.utility_with_payload(job)

        # get the payload command from the user specific code
        pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
        user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)
        # for testing looping job:    cmd = user.get_payload_command(job) + ';sleep 240'
        try:
            cmd = user.get_payload_command(job)
            #cmd = user.get_payload_command(job) + ';sleep 240'
        except PilotException as e:
            log.fatal('could not define payload command')
            return None

        log.info("payload execution command: %s" % cmd)

        # replace platform and workdir with new function get_payload_options() or something from experiment specific
        # code
        try:
            proc = execute(cmd, workdir=job.workdir, returnproc=True,
                           usecontainer=True, stdout=out, stderr=err, cwd=job.workdir, job=job)
        except Exception as e:
            log.error('could not execute: %s' % str(e))
            return None

        log.info('started -- pid=%s executable=%s' % (proc.pid, cmd))
        job.pid = proc.pid

        self.utility_after_payload_started(job)

        return proc

    def wait_graceful(self, args, proc, job):
        """
        (add description)
        :param args:
        :param proc:
        :param job:
        :return:
        """

        log = get_logger(job.jobid, logger)

        breaker = False
        exit_code = None
        iteration = 0L
        while True:
            iteration += 1
            for i in xrange(100):
                if args.graceful_stop.is_set():
                    breaker = True
                    log.info('breaking -- sending SIGTERM pid=%s' % proc.pid)
                    os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
                    # proc.terminate()
                    break
                time.sleep(0.1)
            if breaker:
                log.info('breaking -- sleep 3s before sending SIGKILL pid=%s' % proc.pid)
                time.sleep(3)
                proc.kill()
                break

            exit_code = proc.poll()

            if iteration % 10 == 0:
                log.info('running: iteration=%d pid=%s exit_code=%s' % (iteration, proc.pid, exit_code))
            if exit_code is not None:
                break
            else:
                # send_state(job, args, 'running')
                continue

        return exit_code

    def run(self):
        """
        (add description)
        :return:
        """
        log = get_logger(str(self.__job.jobid), logger)

        exit_code = 1
        if self.setup_payload(self.__job, self.__out, self.__err):
            log.debug('running payload')
            self.__job.state = 'running'
            send_state(self.__job, self.__args, self.__job.state)
            proc = self.run_payload(self.__job, self.__out, self.__err)
            if proc is not None:
                log.info('will wait for graceful exit')
                exit_code = self.wait_graceful(self.__args, proc, self.__job)
                self.__job.state = 'finished' if exit_code == 0 else 'failed'
                log.info('finished pid=%s exit_code=%s state=%s' % (proc.pid, exit_code, self.__job.state))

                if exit_code is None:
                    log.warning('detected unset exit_code from wait_graceful - reset to -1')
                    exit_code = -1

                self.utility_after_payload_finished(self.__job)
                self.post_payload(self.__job)

                # stop any running utilities
                if self.__job.utilities != {}:
                    for utcmd in self.__job.utilities.keys():
                        utproc = self.__job.utilities[utcmd][0]
                        if utproc:
                            pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
                            user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user],
                                              -1)
                            sig = user.get_utility_command_kill_signal(utcmd)
                            log.info("stopping process \'%s\' with signal %d" % (utcmd, sig))
                            os.killpg(os.getpgid(utproc.pid), sig)

                            user.post_utility_command_action(utcmd, self.__job)

        return exit_code
<|endoftext|>


<commit_before>#!/usr/bin/python

import serial
import os

ser = serial.Serial("/dev/ttyUSB1")  # open first serial port
print ser.portstr       # check which port was really used

try:
    line = ""
    while True:
        try:
            newLine = ser.readline()
            print newLine
            if newLine != line:
                line = newLine
                if "ObjectHit" in line:
                    parts = line.split(" ")
                    angleStr = parts[0]
                    angle = int(angleStr)

                    os.system("echo 'your size is {0}' | festival --tts".format(angle))

                    ser.write("start\n")
        except KeyboardInterrupt:
            break
finally:
    print "Closing"
    ser.close()<commit_msg>Basic version working, need to check diff in serial input more closely and calc size better<commit_after>#!/usr/bin/python

import serial
import os
from math import cos, sqrt, radians

BASE_ANGLE = 19
arm = 34.0
BASE_DIST = 11.0

def calc_size(angle):
    angle += BASE_ANGLE

    length = sqrt(arm**2 + arm**2 - 2*arm*arm*cos(radians(angle))) #c = sqrt(a^2 + b^2 - 2*a*b*cos(angle))
    size = (3.0/2) * (length + 2) #http://en.wikipedia.org/wiki/Shoe_size#Europe
    return size

ser = serial.Serial("/dev/ttyUSB1")  # open first serial port
print ser.portstr       # check which port was really used
try:
    line = ""
    while True:
        try:
            newLine = ser.readline()
            if newLine[3:] != line[3:]:
                print newLine
                line = newLine
                if "ObjectHit" in line:
                    parts = line.split(" ")
                    angleStr = parts[0]
                    angle = int(angleStr)
                    size = int(calc_size(angle))
                    text = "'your size is {0}'".format(size)
                    print text
                    os.system("echo '{0}' | festival --tts".format(text))

                    #ser.write("start\n")
        except KeyboardInterrupt:
            break
finally:
    print "Closing"
    ser.close()<|endoftext|>


<commit_before>#
#  Copyright 2018 Barcelona Supercomputing Center (www.bsc.es)
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
from pycompss.api.api import compss_wait_on as cwo
from pycompss.api.parameter import INOUT, IN
from pycompss.api.task import task

marker = "COMPSS_DEFAULT_VALUE_TO_BE_USED_AS_A_MARKER"

@task(iterator=IN, returns=list)
def get_next_partition(iterable, start, end):
    """
    Divide and retrieve the next partition.
    :return:
    """

    # If it's a dict
    if isinstance(iterable, dict):
        sorted_keys = sorted(iterable.keys())
        for key in sorted_keys[start:end]:
            yield key, iterable[key]
    elif isinstance(iterable, list):
        for item in iter(iterable[start:end]):
            yield item
    else:
        index = 0
        for item in iter(iterable):
            index += 1
            if index > end:
                break
            elif index > start:
                yield item

@task(files=list, returns=list)
def task_read_files(file_paths):
    """

    :param file_paths:
    :return:
    """
    ret = list()
    for file_path in file_paths:
        content = open(file_path).read()
        ret.append((file_path, content))

    return ret

@task(returns=1)
def task_map_partition(f, partition_loader, *args, **kwargs):
    """
    Apply a function to a partition in a new task. The function should take an
    iterable as a parameter and return a list.
    :param f: A function that takes an iterable as a parameter
    :param partition_loader: Data generator
    :type partition_loader: BaseDataGenerator
    :return: future object of the list containing results
    """
    data = partition_loader.retrieve_data()
    res = f(data, *args, **kwargs)
    del data
    return res

@task(first=INOUT)
def reduce_dicts(first, *args):
    dicts = iter(args)

    for _dict in dicts:
        for k in _dict:
            first[k] += _dict[k]

@task(returns=list, iterator=IN)
def task_dict_to_list(iterator, total_parts, partition_num):
    """
    Convert a dictionary to a list to be used as partitions later.
    :param iterator:
    :param total_parts:
    :param partition_num:
    :return:
    """
    ret = list()
    sorted_keys = sorted(iterator.keys())
    total = len(sorted_keys)
    chunk_size = max(1, total / total_parts)
    start = chunk_size * partition_num
    is_last = (total_parts == partition_num + 1)

    if is_last:
        for i in sorted_keys[start:]:
            ret.append((i, iterator[i]))
    else:
        for i in sorted_keys[start:start+chunk_size]:
            ret.append((i, iterator[i]))

    return ret

@task(returns=1, first=INOUT)
def merge_dicts(first, second, merger_function):
    """
    """
    for key in second:
        val = second[key]
        first[key] = merger_function(first[key], val) if key in first else val

@task(returns=1)
def map_partition(f, loader):
    """
    """
    res = f(loader)
    return res

@task(returns=1)
def reduce_partition(f, partition):
    """
    TODO: Should it be a Tree Reducer as well?
    """
    iterator = iter(partition)
    try:
        res = next(iterator)
    except StopIteration:
        return

    for el in partition:
        res = f(res, el)

    return res

@task(returns=1)
def reduce_multiple(f, *args):
    """
    """
    partitions = iter(args)
    try:
        res = next(partitions)[0]
    except StopIteration:
        return

    for part in partitions:
        if part:
            res = f(res, part[0])

    return [res]
<commit_msg>Separate 'load and map' and 'regular map' task functions.<commit_after>#
#  Copyright 2018 Barcelona Supercomputing Center (www.bsc.es)
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
from pycompss.api.api import compss_wait_on as cwo
from pycompss.api.parameter import INOUT, IN
from pycompss.api.task import task

marker = "COMPSS_DEFAULT_VALUE_TO_BE_USED_AS_A_MARKER"

@task(iterator=IN, returns=list)
def get_next_partition(iterable, start, end):
    """
    Divide and retrieve the next partition.
    :return:
    """

    # If it's a dict
    if isinstance(iterable, dict):
        sorted_keys = sorted(iterable.keys())
        for key in sorted_keys[start:end]:
            yield key, iterable[key]
    elif isinstance(iterable, list):
        for item in iter(iterable[start:end]):
            yield item
    else:
        index = 0
        for item in iter(iterable):
            index += 1
            if index > end:
                break
            elif index > start:
                yield item

@task(files=list, returns=list)
def task_read_files(file_paths):
    """

    :param file_paths:
    :return:
    """
    ret = list()
    for file_path in file_paths:
        content = open(file_path).read()
        ret.append((file_path, content))

    return ret

@task(returns=1)
def load_and_map_partition(f, partition_loader):
    """
    Apply a function to a partition in a new task. The function should take an
    iterable as a parameter and return a list.
    :param f: A function that takes an iterable as a parameter
    :param partition_loader: Data generator
    :type partition_loader: BaseDataGenerator
    :return: future object of the list containing results
    """
    data = partition_loader.retrieve_data()
    res = f(data)
    del data
    return res

@task(returns=1)
def map_partition(f, partition):
    """
    """
    res = f(partition)
    del partition
    return res

@task(first=INOUT)
def reduce_dicts(first, *args):
    dicts = iter(args)

    for _dict in dicts:
        for k in _dict:
            first[k] += _dict[k]

@task(returns=list, iterator=IN)
def task_dict_to_list(iterator, total_parts, partition_num):
    """
    Convert a dictionary to a list to be used as partitions later.
    :param iterator:
    :param total_parts:
    :param partition_num:
    :return:
    """
    ret = list()
    sorted_keys = sorted(iterator.keys())
    total = len(sorted_keys)
    chunk_size = max(1, total / total_parts)
    start = chunk_size * partition_num
    is_last = (total_parts == partition_num + 1)

    if is_last:
        for i in sorted_keys[start:]:
            ret.append((i, iterator[i]))
    else:
        for i in sorted_keys[start:start+chunk_size]:
            ret.append((i, iterator[i]))

    return ret

@task(returns=1, first=INOUT)
def merge_dicts(first, second, merger_function):
    """
    """
    for key in second:
        val = second[key]
        first[key] = merger_function(first[key], val) if key in first else val

@task(returns=1)
def reduce_partition(f, partition):
    """
    TODO: Should it be a Tree Reducer as well?
    """
    iterator = iter(partition)
    try:
        res = next(iterator)
    except StopIteration:
        return

    for el in partition:
        res = f(res, el)

    return res

@task(returns=1)
def reduce_multiple(f, *args):
    """
    """
    partitions = iter(args)
    try:
        res = next(partitions)[0]
    except StopIteration:
        return

    for part in partitions:
        if part:
            res = f(res, part[0])

    return [res]
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError
from datetime import date
from fec_alerts.models import new_filing
from formdata.models import SkedE
from summary_data.models import Candidate_Overlay
from reconciliation.fec_reconciler import match_by_name

cycle_start = date(2013,1,1)
THIS_CYCLE = '2014'

def set_data_from_self(skedeline):
    name = None
    if skedeline.candidate_middle_name:
        name = "%s, %s %s" % (skedeline.candidate_last_name, skedeline.candidate_first_name,  skedeline.candidate_middle_name)
    else:
        name = "%s, %s" % (skedeline.candidate_last_name, skedeline.candidate_first_name)

    skedeline.candidate_district_checked = skedeline.candidate_district
    skedeline.candidate_office_checked = skedeline.candidate_office
    skedeline.candidate_state_checked = skedeline.candidate_state
    skedeline.candidate_name_checked = name
    skedeline.support_oppose_checked = skedeline.support_oppose_code
    skedeline.save()

def set_data_from_candidate_id(skedeline, candidate_id):

    try:
        this_candidate = Candidate_Overlay.objects.get(fec_id=candidate_id, cycle=(THIS_CYCLE))
        skedeline.candidate_id_checked = this_candidate.fec_id
        skedeline.candidate_checked  = this_candidate
        skedeline.candidate_district_checked = this_candidate.office_district
        skedeline.district_checked = this_candidate.district
        skedeline.candidate_office_checked = this_candidate.office
        skedeline.candidate_party_checked = this_candidate.party
        skedeline.candidate_state_checked = this_candidate.state
        skedeline.candidate_name_checked = this_candidate.name
        skedeline.support_oppose_checked = skedeline.support_oppose_code
        skedeline.save()
        return True

    except Candidate_Overlay.DoesNotExist:
        return False

def fuzzy_match_candidate(skedeline):
    state = skedeline.candidate_state
    name_to_check = "%s, %s" % (skedeline.candidate_last_name, skedeline.candidate_first_name)
    office = skedeline.candidate_office

    result = match_by_name(name_to_check, state=None, office=None, cycle=THIS_CYCLE, reverse_name_order=False)
    if result:
        if result[0]['match']:
            print "Fuzzy matching matched %s to %s with id %s" % (name_to_check, result[0]['name'], result[0]['id'])

            return set_data_from_candidate_id(skedeline, result[0]['id'])

    print "Fuzzy matching couldn't match %s" % name_to_check
    return False

def attach_ie_target(skedeline):
    candidate_id = skedeline.candidate_id_number

    # If there's a candidate id, enter the data from the overlay
    if skedeline.expenditure_date_formatted >= cycle_start:

        if candidate_id:
            result = set_data_from_candidate_id(skedeline, candidate_id)

            if result:
                return True

    else:
        # don't care about these.
        return False

    # if we're still here, try a fuzzy match

    fuzzy_match_result = fuzzy_match_candidate(skedeline)
    if fuzzy_match_result:
        return True

    # fall back on data that's already there.
    set_data_from_self(skedeline)
    return False

class Command(BaseCommand):
    help = "Set the name and details of the candidate targetted"
    requires_model_validation = False

    def handle(self, *args, **options):
        filings_to_process = new_filing.objects.filter(data_is_processed=True, body_rows_superceded=True).exclude(ie_rows_processed=True).order_by('filing_number')
        for this_filing in filings_to_process:
            lines_present = this_filing.lines_present
            has_sked_E = False
            try:
                lines_present['E']
                if int(lines_present['E']) > 0:
                    has_sked_E = True
            except KeyError:
                continue

            if has_sked_E:
                #print "processing %s " % (this_filing.filing_number)
                #print lines_present, lines_present['E']

                skedelines = SkedE.objects.filter(filing_number=this_filing.filing_number)

                for skede in skedelines:
                    attach_ie_target(skede)
            # mark that we've been processed.
            this_filing.ie_rows_processed=True
            this_filing.save()<commit_msg>ignore out of cycle ies<commit_after>from django.core.management.base import BaseCommand, CommandError
from datetime import date
from fec_alerts.models import new_filing
from formdata.models import SkedE
from summary_data.models import Candidate_Overlay
from reconciliation.fec_reconciler import match_by_name

cycle_start = date(2013,1,1)
THIS_CYCLE = '2014'

def set_data_from_self(skedeline):
    name = None
    if skedeline.candidate_middle_name:
        name = "%s, %s %s" % (skedeline.candidate_last_name, skedeline.candidate_first_name,  skedeline.candidate_middle_name)
    else:
        name = "%s, %s" % (skedeline.candidate_last_name, skedeline.candidate_first_name)

    skedeline.candidate_district_checked = skedeline.candidate_district
    skedeline.candidate_office_checked = skedeline.candidate_office
    skedeline.candidate_state_checked = skedeline.candidate_state
    skedeline.candidate_name_checked = name
    skedeline.support_oppose_checked = skedeline.support_oppose_code
    skedeline.save()

def set_data_from_candidate_id(skedeline, candidate_id):

    try:
        this_candidate = Candidate_Overlay.objects.get(fec_id=candidate_id, cycle=(THIS_CYCLE))
        skedeline.candidate_id_checked = this_candidate.fec_id
        skedeline.candidate_checked  = this_candidate
        skedeline.candidate_district_checked = this_candidate.office_district
        skedeline.district_checked = this_candidate.district
        skedeline.candidate_office_checked = this_candidate.office
        skedeline.candidate_party_checked = this_candidate.party
        skedeline.candidate_state_checked = this_candidate.state
        skedeline.candidate_name_checked = this_candidate.name
        skedeline.support_oppose_checked = skedeline.support_oppose_code
        skedeline.save()
        return True

    except Candidate_Overlay.DoesNotExist:
        return False

def fuzzy_match_candidate(skedeline):
    state = skedeline.candidate_state
    name_to_check = "%s, %s" % (skedeline.candidate_last_name, skedeline.candidate_first_name)
    office = skedeline.candidate_office

    result = match_by_name(name_to_check, state=None, office=None, cycle=THIS_CYCLE, reverse_name_order=False)
    if result:
        if result[0]['match']:
            print "Fuzzy matching matched %s to %s with id %s" % (name_to_check, result[0]['name'], result[0]['id'])

            return set_data_from_candidate_id(skedeline, result[0]['id'])

    print "Fuzzy matching couldn't match %s" % name_to_check
    return False

def attach_ie_target(skedeline):
    candidate_id = skedeline.candidate_id_number

    # If there's a candidate id, enter the data from the overlay
    if skedeline.expenditure_date_formatted >= cycle_start:

        if candidate_id:
            result = set_data_from_candidate_id(skedeline, candidate_id)

            if result:
                return True

        else:

            # if we're still here, try a fuzzy match

            fuzzy_match_result = fuzzy_match_candidate(skedeline)
            if fuzzy_match_result:
                return True

            # fall back on data that's already there.
            set_data_from_self(skedeline)
            return False

    else:
        # don't care about the out of cycle stuff.
        return False

class Command(BaseCommand):
    help = "Set the name and details of the candidate targetted"
    requires_model_validation = False

    def handle(self, *args, **options):
        filings_to_process = new_filing.objects.filter(data_is_processed=True, body_rows_superceded=True).exclude(ie_rows_processed=True).order_by('filing_number')
        for this_filing in filings_to_process:
            lines_present = this_filing.lines_present
            has_sked_E = False
            try:
                lines_present['E']
                if int(lines_present['E']) > 0:
                    has_sked_E = True
            except KeyError:
                continue

            if has_sked_E:
                #print "processing %s " % (this_filing.filing_number)
                #print lines_present, lines_present['E']

                skedelines = SkedE.objects.filter(filing_number=this_filing.filing_number)

                for skede in skedelines:
                    attach_ie_target(skede)
            # mark that we've been processed.
            this_filing.ie_rows_processed=True
            this_filing.save()<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Script for updating a directory of repositories."""
import logging
import os

import click
from git import InvalidGitRepositoryError, Repo
from git.exc import GitCommandError

def check_changes(current, fetch_info_list, branch_list):
    """Check for changes in local branches and remote."""
    log = logging.getLogger(__name__)

    for fetch_info in fetch_info_list:
        log.debug('Checking for change in %s', fetch_info.name)

        if current[fetch_info.ref] != fetch_info.commit:
            log.info('%s has updates, %s..%s', fetch_info.name,
                     current[fetch_info.ref], fetch_info.commit)

    for branch in branch_list:
        log.debug('Checking for change in %s', branch.name)

        if current[branch] != branch.commit:
            log.info('%s updated, %s..%s', branch.name, current[branch],
                     branch.commit)

    return True

def update_repo(directory):
    """Update a repository.

    Returns:
        False if bad repository.
        True if everything worked.
    """
    log = logging.getLogger(__name__)

    try:
        repo = Repo(directory)
        current = {ref: ref.commit for ref in repo.refs}
        log.info('Updating %s', repo)
    except InvalidGitRepositoryError:
        log.warning('%s is not a valid repository.', directory)
        return False

    try:
        remote = repo.remote()
    except ValueError:
        log.warning('Check remotes for %s: %s', directory, repo.remotes)
        return False

    try:
        fetch_info_list = remote.pull()
    except GitCommandError as error:
        log.fatal('Pull failed. %s', error)
        return False

    check_changes(current, fetch_info_list, repo.branches)

    return True

@click.command()
@click.option('-d', '--debug', help='Set DEBUG level logging.', is_flag=True)
@click.argument('dir', default='.')
def main(**kwargs):
    """Update repositories in a directory.

    By default, the current working directory list is used for finding valid
    repositories.
    """
    logging.basicConfig(level=logging.INFO)

    log = logging.getLogger(__name__)

    if kwargs['debug']:
        logging.root.setLevel(logging.DEBUG)

    main_dir = kwargs['dir']
    log.info('Finding directories in %s', main_dir)

    dir_list = os.listdir(main_dir)
    log.debug('List of directories: %s', dir_list)

    # Git directory was passed in, not a directory of Git directories
    if '.git' in dir_list:
        dir_list = kwargs['dir']

    for directory in dir_list:
        update_repo(os.path.join(main_dir, directory))

if __name__ == '__main__':
    main()
<commit_msg>check_changes: Add argument detail in docstring<commit_after>#!/usr/bin/env python
"""Script for updating a directory of repositories."""
import logging
import os

import click
from git import InvalidGitRepositoryError, Repo
from git.exc import GitCommandError

def check_changes(current, fetch_info_list, branch_list):
    """Check for changes in local branches and remote.

    Args:
        current: Dict(reference: commit) from before `git pull` operation.
        fetch_info_list: List of remote references from `git pull`.
        branch_list: List of branches in repository.
    """
    log = logging.getLogger(__name__)

    for fetch_info in fetch_info_list:
        log.debug('Checking for change in %s', fetch_info.name)

        if current[fetch_info.ref] != fetch_info.commit:
            log.info('%s has updates, %s..%s', fetch_info.name,
                     current[fetch_info.ref], fetch_info.commit)

    for branch in branch_list:
        log.debug('Checking for change in %s', branch.name)

        if current[branch] != branch.commit:
            log.info('%s updated, %s..%s', branch.name, current[branch],
                     branch.commit)

    return True

def update_repo(directory):
    """Update a repository.

    Returns:
        False if bad repository.
        True if everything worked.
    """
    log = logging.getLogger(__name__)

    try:
        repo = Repo(directory)
        current = {ref: ref.commit for ref in repo.refs}
        log.info('Updating %s', repo)
    except InvalidGitRepositoryError:
        log.warning('%s is not a valid repository.', directory)
        return False

    try:
        remote = repo.remote()
    except ValueError:
        log.warning('Check remotes for %s: %s', directory, repo.remotes)
        return False

    try:
        fetch_info_list = remote.pull()
    except GitCommandError as error:
        log.fatal('Pull failed. %s', error)
        return False

    check_changes(current, fetch_info_list, repo.branches)

    return True

@click.command()
@click.option('-d', '--debug', help='Set DEBUG level logging.', is_flag=True)
@click.argument('dir', default='.')
def main(**kwargs):
    """Update repositories in a directory.

    By default, the current working directory list is used for finding valid
    repositories.
    """
    logging.basicConfig(level=logging.INFO)

    log = logging.getLogger(__name__)

    if kwargs['debug']:
        logging.root.setLevel(logging.DEBUG)

    main_dir = kwargs['dir']
    log.info('Finding directories in %s', main_dir)

    dir_list = os.listdir(main_dir)
    log.debug('List of directories: %s', dir_list)

    # Git directory was passed in, not a directory of Git directories
    if '.git' in dir_list:
        dir_list = kwargs['dir']

    for directory in dir_list:
        update_repo(os.path.join(main_dir, directory))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import subprocess
import sys

from gitgoggles.asciitable import AsciiTable, AsciiCell
from gitgoggles.git import Repository, TrackingBranch, LocalBranch, PublishedBranch, TrackedBranch
from gitgoggles.utils import colored

TAG_PREFIX = 'codereview--'

def get_status():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    refs = repo.branches(LocalBranch, TrackingBranch, PublishedBranch)
    tags = repo.tags()

    table = AsciiTable([u'Status', u'Branch', u'Review', u'Ahead', u'Behind', u'Pull', u'Push', u'Modified'])

    if repo.configs.get('gitgoggles.colors.disabled', 'False') == 'True':
        colored.disabled = True

    icons = {
        'unknown': repo.configs.get('gitgoggles.icons.unknown', u'\u203D'),
        'success': repo.configs.get('gitgoggles.icons.success', u'\u2714'),
        'failure': repo.configs.get('gitgoggles.icons.failure', u'\u2718'),
        }
    colors = {
        'local': repo.configs.get('gitgoggles.colors.local', 'cyan'),
        'new': repo.configs.get('gitgoggles.colors.new', 'red'),
        'review': repo.configs.get('gitgoggles.colors.review', 'red'),
        'merge': repo.configs.get('gitgoggles.colors.merge', 'yellow'),
        'done': repo.configs.get('gitgoggles.colors.done', 'green'),
        }

    for ref in refs:
        parent = ref.name in ('staging', 'master',) and 'master' or 'staging'
        codereview_tag = "%s%s" % (TAG_PREFIX, ref.shortname)

        color = 'red'
        status = u'?'
        review_commits = 0
        ahead_commits = ref.ahead
        behind_commits = ref.behind
        pull = ref.pull
        push = ref.push

        if ref.__class__ in (TrackingBranch, LocalBranch, TrackedBranch):
            if codereview_tag not in [ x.name for x in tags ]:
                review_commits = len(repo.git('log', '--pretty=format:%H', '%s..%s' % (ref.merge_refspec or '', ref.refspec), split=True))
            else:
                review_commits = len(repo.git('log', '--pretty=format:%H', '%s..%s' % (codereview_tag, ref.refspec), split=True))
        else:
            review_commits = None

        if ref.__class__ == LocalBranch:
            color, status = colors['local'], u'local'
        elif codereview_tag not in [ x.name for x in tags ]:
            color, status = colors['new'], u'new'
        else:
            if review_commits:
                color, status = colors['review'], u'review'
            else:
                if ahead_commits:
                    color, status = colors['merge'], u'merge'
                else:
                    color, status = colors['done'], u'done'

        review = bool(review_commits) or None
        ahead = bool(ahead_commits) or None
        behind = bool(behind_commits) or None
        tracked = ref.__class__ in (TrackingBranch, LocalBranch, TrackedBranch)

        review_text, review_color = review_commits is not None and (u'%s unreviewed' % review_commits, review and color) or (icons['unknown'], 'yellow',)
        ahead_text, ahead_color = ahead_commits is not None and (u'%s ahead' % ahead_commits, ahead and color) or (icons['unknown'], 'yellow',)
        behind_text, behind_color = behind_commits is not None and (u'%s behind' % behind_commits, behind and color) or (icons['unknown'], 'yellow',)

        pull_text, pull_color = not tracked and (icons['unknown'], 'yellow',) or (pull and (icons['failure'], 'red',) or (icons['success'], 'green',))
        push_text, push_color = not tracked and (icons['unknown'], 'yellow',) or (push and (icons['failure'], 'red',) or (icons['success'], 'green',))

        table.add_row([
            AsciiCell(status.upper(), color),
            AsciiCell(ref.name),
            AsciiCell(review_text, review_color, reverse=review),
            AsciiCell(ahead_text, ahead_color, reverse=ahead),
            AsciiCell(behind_text, behind_color, reverse=behind),
            AsciiCell(pull_text, pull_color),
            AsciiCell(push_text, push_color),
            AsciiCell(ref.modified),
            ])

    table.render()

def complete_review():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    branch = repo.branch()
    repo.git('tag', '-a', '%s%s' % (TAG_PREFIX, branch), '-f', '-m', 'creating code review for branch %s' % branch)
    repo.git('push', '--tags')
    print 'Created tag %s%s' % (TAG_PREFIX, branch)
    repo.git('checkout', 'staging')
    print 'Switched back to staging branch.'

def start_review():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    branch = repo.branch()
    tags = repo.tags()

    parent = branch in ('staging', 'master',) and 'master' or 'staging'

    cr_tag = '%s%s' % (TAG_PREFIX, branch)

    if cr_tag in [ x.name for x in tags ]:
        repo.git('diff', '-w', '%s..%s' % (cr_tag, branch), join=True)
    else:
        repo.git('diff', '-w', '%s..%s' % (parent, branch), join=True)

def update_branches():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    branch = repo.branch()
    refs = repo.branches(TrackingBranch)
    for branch in refs:
        if branch.pull:
            repo.git('checkout', branch.name)
            repo.git('pull')

<commit_msg>Altered config value for color disabling.<commit_after>import subprocess
import sys

from gitgoggles.asciitable import AsciiTable, AsciiCell
from gitgoggles.git import Repository, TrackingBranch, LocalBranch, PublishedBranch, TrackedBranch
from gitgoggles.utils import colored

TAG_PREFIX = 'codereview--'

def get_status():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    refs = repo.branches(LocalBranch, TrackingBranch, PublishedBranch)
    tags = repo.tags()

    table = AsciiTable([u'Status', u'Branch', u'Review', u'Ahead', u'Behind', u'Pull', u'Push', u'Modified'])

    if repo.configs.get('gitgoggles.colors', 'True') == 'False':
        colored.disabled = True

    icons = {
        'unknown': repo.configs.get('gitgoggles.icons.unknown', u'\u203D'),
        'success': repo.configs.get('gitgoggles.icons.success', u'\u2714'),
        'failure': repo.configs.get('gitgoggles.icons.failure', u'\u2718'),
        }
    colors = {
        'local': repo.configs.get('gitgoggles.colors.local', 'cyan'),
        'new': repo.configs.get('gitgoggles.colors.new', 'red'),
        'review': repo.configs.get('gitgoggles.colors.review', 'red'),
        'merge': repo.configs.get('gitgoggles.colors.merge', 'yellow'),
        'done': repo.configs.get('gitgoggles.colors.done', 'green'),
        }

    for ref in refs:
        parent = ref.name in ('staging', 'master',) and 'master' or 'staging'
        codereview_tag = "%s%s" % (TAG_PREFIX, ref.shortname)

        color = 'red'
        status = u'?'
        review_commits = 0
        ahead_commits = ref.ahead
        behind_commits = ref.behind
        pull = ref.pull
        push = ref.push

        if ref.__class__ in (TrackingBranch, LocalBranch, TrackedBranch):
            if codereview_tag not in [ x.name for x in tags ]:
                review_commits = len(repo.git('log', '--pretty=format:%H', '%s..%s' % (ref.merge_refspec or '', ref.refspec), split=True))
            else:
                review_commits = len(repo.git('log', '--pretty=format:%H', '%s..%s' % (codereview_tag, ref.refspec), split=True))
        else:
            review_commits = None

        if ref.__class__ == LocalBranch:
            color, status = colors['local'], u'local'
        elif codereview_tag not in [ x.name for x in tags ]:
            color, status = colors['new'], u'new'
        else:
            if review_commits:
                color, status = colors['review'], u'review'
            else:
                if ahead_commits:
                    color, status = colors['merge'], u'merge'
                else:
                    color, status = colors['done'], u'done'

        review = bool(review_commits) or None
        ahead = bool(ahead_commits) or None
        behind = bool(behind_commits) or None
        tracked = ref.__class__ in (TrackingBranch, LocalBranch, TrackedBranch)

        review_text, review_color = review_commits is not None and (u'%s unreviewed' % review_commits, review and color) or (icons['unknown'], 'yellow',)
        ahead_text, ahead_color = ahead_commits is not None and (u'%s ahead' % ahead_commits, ahead and color) or (icons['unknown'], 'yellow',)
        behind_text, behind_color = behind_commits is not None and (u'%s behind' % behind_commits, behind and color) or (icons['unknown'], 'yellow',)

        pull_text, pull_color = not tracked and (icons['unknown'], 'yellow',) or (pull and (icons['failure'], 'red',) or (icons['success'], 'green',))
        push_text, push_color = not tracked and (icons['unknown'], 'yellow',) or (push and (icons['failure'], 'red',) or (icons['success'], 'green',))

        table.add_row([
            AsciiCell(status.upper(), color),
            AsciiCell(ref.name),
            AsciiCell(review_text, review_color, reverse=review),
            AsciiCell(ahead_text, ahead_color, reverse=ahead),
            AsciiCell(behind_text, behind_color, reverse=behind),
            AsciiCell(pull_text, pull_color),
            AsciiCell(push_text, push_color),
            AsciiCell(ref.modified),
            ])

    table.render()

def complete_review():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    branch = repo.branch()
    repo.git('tag', '-a', '%s%s' % (TAG_PREFIX, branch), '-f', '-m', 'creating code review for branch %s' % branch)
    repo.git('push', '--tags')
    print 'Created tag %s%s' % (TAG_PREFIX, branch)
    repo.git('checkout', 'staging')
    print 'Switched back to staging branch.'

def start_review():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    branch = repo.branch()
    tags = repo.tags()

    parent = branch in ('staging', 'master',) and 'master' or 'staging'

    cr_tag = '%s%s' % (TAG_PREFIX, branch)

    if cr_tag in [ x.name for x in tags ]:
        repo.git('diff', '-w', '%s..%s' % (cr_tag, branch), join=True)
    else:
        repo.git('diff', '-w', '%s..%s' % (parent, branch), join=True)

def update_branches():
    repo = Repository()

    if repo.configs.get('gitgoggles.fetch', 'True') != 'False':
        repo.fetch()

    branch = repo.branch()
    refs = repo.branches(TrackingBranch)
    for branch in refs:
        if branch.pull:
            repo.git('checkout', branch.name)
            repo.git('pull')

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2015-2017 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import re
from shlex import quote

from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_clusters
from paasta_tools.cli.utils import list_instances
from paasta_tools.cli.utils import run_on_master
from paasta_tools.utils import DEFAULT_SOA_DIR
from paasta_tools.utils import list_services
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import paasta_print
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import PaastaNotConfiguredError
from paasta_tools.utils import SystemPaastaConfig

ARG_DEFAULTS = dict(
    common=dict(
        service=None,
        instance=None,
        cluster=None,  # load from system paasta config later
        verbose=False,
    ),
    start=dict(
        cmd=None,
        detach=False,
        staging_timeout=240.0,
        instances=1,
        docker_image=None,
        dry_run=False,
        constraint=[],
    ),
    stop=dict(run_id=None, framework_id=None),
    list=dict(),
)

def get_system_paasta_config():
    try:
        return load_system_paasta_config()
    except PaastaNotConfiguredError:
        paasta_print(
            PaastaColors.yellow(
                "Warning: Couldn't load config files from '/etc/paasta'. This "
                "indicates PaaSTA is not configured locally on this host, and "
                "remote-run may not behave the same way it would behave on a "
                "server configured for PaaSTA.",
            ),
            sep='\n',
        )
        return SystemPaastaConfig(
            {"volumes": []},
            '/etc/paasta',
)

def add_common_args_to_parser(parser):
    parser.add_argument(
        '-s', '--service',
        help='The name of the service you wish to inspect. Required.',
        required=True,
    ).completer = lazy_choices_completer(list_services)
    parser.add_argument(
        '-i', '--instance',
        help=(
            "Simulate a docker run for a particular instance of the "
            "service, like 'main' or 'canary'. Required."
        ),
        required=True,
    ).completer = lazy_choices_completer(list_instances)
    parser.add_argument(
        '-c', '--cluster',
        help=(
            'The name of the cluster you wish to run your task on. '
            'If omitted, uses the default cluster defined in the paasta '
            f'remote-run configs.'
        ),
        default=ARG_DEFAULTS['common']['cluster'],
    ).completer = lazy_choices_completer(list_clusters)
    parser.add_argument(
        '-v', '--verbose',
        help='Show more output',
        action='store_true',
        default=ARG_DEFAULTS['common']['verbose'],
    )

def add_start_parser(subparser):
    parser = subparser.add_parser(
        'start',
        help="Start task subcommand",
    )
    add_common_args_to_parser(parser)
    parser.add_argument(
        '-C', '--cmd',
        help=(
            'Run Docker container with particular command, for example: '
            '"bash". By default will use the command or args specified by the '
            'soa-configs or what was specified in the Dockerfile'
        ),
        default=ARG_DEFAULTS['start']['cmd'],
    ),
    parser.add_argument(
        '-D', '--detach',
        help='Launch in background',
        action='store_true',
        default=ARG_DEFAULTS['start']['detach'],
    )
    parser.add_argument(
        '-t', '--staging-timeout',
        help='A timeout for the task to be launching before killed',
        default=ARG_DEFAULTS['start']['staging_timeout'],
        type=float,
    )
    parser.add_argument(
        '-j', '--instances',
        help='Number of copies of the task to launch',
        default=ARG_DEFAULTS['start']['instances'],
        type=int,
    )
    parser.add_argument(
        '--docker-image',
        help=(
            'URL of docker image to use. '
            'Defaults to using the deployed docker image.'
        ),
        default=ARG_DEFAULTS['start']['docker_image'],
    )
    parser.add_argument(
        '-R', '--run-id',
        help='ID of task to stop',
        default=ARG_DEFAULTS['stop']['run_id'],
    )
    parser.add_argument(
        '-d', '--dry-run',
        help=(
            'Don\'t launch the task. '
            'Instead output task that would have been launched'
        ),
        action='store_true',
        default=ARG_DEFAULTS['start']['dry_run'],
    )
    parser.add_argument(
        '-X', '--constraint',
        help='Constraint option, format: <attr>,OP[,<value>], OP can be one '
        'of the following: EQUALS matches attribute value exactly, LIKE and '
        'UNLIKE match on regular expression, MAX_PER constrains number of '
        'tasks per attribute value, UNIQUE is the same as MAX_PER,1',
        action='append',
        default=ARG_DEFAULTS['start']['constraint'],
    )
    return parser

def add_stop_parser(subparser):
    parser = subparser.add_parser(
        'stop',
        help="Stop task subcommand",
    )
    add_common_args_to_parser(parser)
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        '-R', '--run-id',
        help='ID of task to stop',
        default=ARG_DEFAULTS['stop']['run_id'],
    )
    group.add_argument(
        '-F', '--framework-id',
        help=(
            'ID of framework to stop. Must belong to remote-run of selected '
            'service instance.'
        ),
        default=ARG_DEFAULTS['stop']['framework_id'],
    )
    return parser

def add_list_parser(subparser):
    parser = subparser.add_parser(
        'list',
        help="List tasks subcommand",
    )
    add_common_args_to_parser(parser)
    return parser

def add_subparser(subparsers):
    main_parser = subparsers.add_parser(
        'remote-run',
        help="Schedule Mesos to run adhoc command in context of a service",
        description=(
            "`paasta remote-run` is useful for running adhoc commands in "
            "context of a service's Docker image. The command will be "
            "scheduled on a Mesos cluster and stdout/stderr printed after "
            "execution is finished."
        ),
        epilog=(
            "Note: `paasta remote-run` uses Mesos API that may require "
            "authentication."
        ),
    )
    main_subs = main_parser.add_subparsers(
        dest='action',
        help='Subcommands of remote-run',
    )
    add_start_parser(main_subs)
    add_stop_parser(main_subs)
    add_list_parser(main_subs)
    main_parser.set_defaults(command=paasta_remote_run)

def split_constraints(constraints):
    return [c.split(',', 2) for c in constraints]

def create_remote_run_command(args):
    cmd_parts = ['/usr/bin/paasta_remote_run', args.action]
    arg_vars = vars(args)
    arg_defaults = dict(ARG_DEFAULTS[args.action])  # copy dict
    arg_defaults.update(ARG_DEFAULTS['common'])
    arg_defaults.pop('constraint')  # needs conversion to json

    for k in arg_vars.keys():
        if k not in arg_defaults:  # skip keys we don't know about
            continue
        v = arg_vars[k]
        if v == arg_defaults[k]:  # skip values that have default value
            continue
        k = re.sub(r'_', '-', k)
        if isinstance(v, bool) and v:
            cmd_parts.append(f'--{k}')
        else:
            cmd_parts.extend([f'--{k}', quote(str(v))])

    # constraint, convert to json
    if len(arg_vars['constraint']) > 0:
        constraints = split_constraints(arg_vars['constraint'])
        cmd_parts.extend(['--constraints-json', quote(json.dumps(constraints))])

    return cmd_parts

def paasta_remote_run(args):
    system_paasta_config = get_system_paasta_config()

    if not args.cluster:
        default_cluster = system_paasta_config.get_remote_run_config().get('default_cluster')
        if not default_cluster:
            paasta_print(PaastaColors.red(
                "Error: no cluster specified and no default cluster available"
            ))
            return 1
        args.cluster = default_cluster

    cmd_parts = create_remote_run_command(args)
    graceful_exit = (args.action == 'start' and not args.detach)
    return_code, status = run_on_master(
        cluster=args.cluster,
        system_paasta_config=system_paasta_config,
        cmd_parts=cmd_parts,
        graceful_exit=graceful_exit,
    )

    # Status results are streamed. This print is for possible error messages.
    if status is not None:
        for line in status.rstrip().split('\n'):
            paasta_print('    %s' % line)

    return return_code
<commit_msg>pre-commit changes<commit_after>#!/usr/bin/env python
# Copyright 2015-2017 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import re
from shlex import quote

from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_clusters
from paasta_tools.cli.utils import list_instances
from paasta_tools.cli.utils import run_on_master
from paasta_tools.utils import list_services
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import paasta_print
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import PaastaNotConfiguredError
from paasta_tools.utils import SystemPaastaConfig

ARG_DEFAULTS = dict(
    common=dict(
        service=None,
        instance=None,
        cluster=None,  # load from system paasta config later
        verbose=False,
    ),
    start=dict(
        cmd=None,
        detach=False,
        staging_timeout=240.0,
        instances=1,
        docker_image=None,
        dry_run=False,
        constraint=[],
    ),
    stop=dict(run_id=None, framework_id=None),
    list=dict(),
)

def get_system_paasta_config():
    try:
        return load_system_paasta_config()
    except PaastaNotConfiguredError:
        paasta_print(
            PaastaColors.yellow(
                "Warning: Couldn't load config files from '/etc/paasta'. This "
                "indicates PaaSTA is not configured locally on this host, and "
                "remote-run may not behave the same way it would behave on a "
                "server configured for PaaSTA.",
            ),
            sep='\n',
        )
        return SystemPaastaConfig(
            {"volumes": []},
            '/etc/paasta',
        )

def add_common_args_to_parser(parser):
    parser.add_argument(
        '-s', '--service',
        help='The name of the service you wish to inspect. Required.',
        required=True,
    ).completer = lazy_choices_completer(list_services)
    parser.add_argument(
        '-i', '--instance',
        help=(
            "Simulate a docker run for a particular instance of the "
            "service, like 'main' or 'canary'. Required."
        ),
        required=True,
    ).completer = lazy_choices_completer(list_instances)
    parser.add_argument(
        '-c', '--cluster',
        help=(
            'The name of the cluster you wish to run your task on. '
            'If omitted, uses the default cluster defined in the paasta '
            f'remote-run configs.'
        ),
        default=ARG_DEFAULTS['common']['cluster'],
    ).completer = lazy_choices_completer(list_clusters)
    parser.add_argument(
        '-v', '--verbose',
        help='Show more output',
        action='store_true',
        default=ARG_DEFAULTS['common']['verbose'],
    )

def add_start_parser(subparser):
    parser = subparser.add_parser(
        'start',
        help="Start task subcommand",
    )
    add_common_args_to_parser(parser)
    parser.add_argument(
        '-C', '--cmd',
        help=(
            'Run Docker container with particular command, for example: '
            '"bash". By default will use the command or args specified by the '
            'soa-configs or what was specified in the Dockerfile'
        ),
        default=ARG_DEFAULTS['start']['cmd'],
    ),
    parser.add_argument(
        '-D', '--detach',
        help='Launch in background',
        action='store_true',
        default=ARG_DEFAULTS['start']['detach'],
    )
    parser.add_argument(
        '-t', '--staging-timeout',
        help='A timeout for the task to be launching before killed',
        default=ARG_DEFAULTS['start']['staging_timeout'],
        type=float,
    )
    parser.add_argument(
        '-j', '--instances',
        help='Number of copies of the task to launch',
        default=ARG_DEFAULTS['start']['instances'],
        type=int,
    )
    parser.add_argument(
        '--docker-image',
        help=(
            'URL of docker image to use. '
            'Defaults to using the deployed docker image.'
        ),
        default=ARG_DEFAULTS['start']['docker_image'],
    )
    parser.add_argument(
        '-R', '--run-id',
        help='ID of task to stop',
        default=ARG_DEFAULTS['stop']['run_id'],
    )
    parser.add_argument(
        '-d', '--dry-run',
        help=(
            'Don\'t launch the task. '
            'Instead output task that would have been launched'
        ),
        action='store_true',
        default=ARG_DEFAULTS['start']['dry_run'],
    )
    parser.add_argument(
        '-X', '--constraint',
        help='Constraint option, format: <attr>,OP[,<value>], OP can be one '
        'of the following: EQUALS matches attribute value exactly, LIKE and '
        'UNLIKE match on regular expression, MAX_PER constrains number of '
        'tasks per attribute value, UNIQUE is the same as MAX_PER,1',
        action='append',
        default=ARG_DEFAULTS['start']['constraint'],
    )
    return parser

def add_stop_parser(subparser):
    parser = subparser.add_parser(
        'stop',
        help="Stop task subcommand",
    )
    add_common_args_to_parser(parser)
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        '-R', '--run-id',
        help='ID of task to stop',
        default=ARG_DEFAULTS['stop']['run_id'],
    )
    group.add_argument(
        '-F', '--framework-id',
        help=(
            'ID of framework to stop. Must belong to remote-run of selected '
            'service instance.'
        ),
        default=ARG_DEFAULTS['stop']['framework_id'],
    )
    return parser

def add_list_parser(subparser):
    parser = subparser.add_parser(
        'list',
        help="List tasks subcommand",
    )
    add_common_args_to_parser(parser)
    return parser

def add_subparser(subparsers):
    main_parser = subparsers.add_parser(
        'remote-run',
        help="Schedule Mesos to run adhoc command in context of a service",
        description=(
            "`paasta remote-run` is useful for running adhoc commands in "
            "context of a service's Docker image. The command will be "
            "scheduled on a Mesos cluster and stdout/stderr printed after "
            "execution is finished."
        ),
        epilog=(
            "Note: `paasta remote-run` uses Mesos API that may require "
            "authentication."
        ),
    )
    main_subs = main_parser.add_subparsers(
        dest='action',
        help='Subcommands of remote-run',
    )
    add_start_parser(main_subs)
    add_stop_parser(main_subs)
    add_list_parser(main_subs)
    main_parser.set_defaults(command=paasta_remote_run)

def split_constraints(constraints):
    return [c.split(',', 2) for c in constraints]

def create_remote_run_command(args):
    cmd_parts = ['/usr/bin/paasta_remote_run', args.action]
    arg_vars = vars(args)
    arg_defaults = dict(ARG_DEFAULTS[args.action])  # copy dict
    arg_defaults.update(ARG_DEFAULTS['common'])
    arg_defaults.pop('constraint')  # needs conversion to json

    for k in arg_vars.keys():
        if k not in arg_defaults:  # skip keys we don't know about
            continue
        v = arg_vars[k]
        if v == arg_defaults[k]:  # skip values that have default value
            continue
        k = re.sub(r'_', '-', k)
        if isinstance(v, bool) and v:
            cmd_parts.append(f'--{k}')
        else:
            cmd_parts.extend([f'--{k}', quote(str(v))])

    # constraint, convert to json
    if len(arg_vars['constraint']) > 0:
        constraints = split_constraints(arg_vars['constraint'])
        cmd_parts.extend(['--constraints-json', quote(json.dumps(constraints))])

    return cmd_parts

def paasta_remote_run(args):
    system_paasta_config = get_system_paasta_config()

    if not args.cluster:
        default_cluster = system_paasta_config.get_remote_run_config().get('default_cluster')
        if not default_cluster:
            paasta_print(PaastaColors.red(
                "Error: no cluster specified and no default cluster available",
            ))
            return 1
        args.cluster = default_cluster

    cmd_parts = create_remote_run_command(args)
    graceful_exit = (args.action == 'start' and not args.detach)
    return_code, status = run_on_master(
        cluster=args.cluster,
        system_paasta_config=system_paasta_config,
        cmd_parts=cmd_parts,
        graceful_exit=graceful_exit,
    )

    # Status results are streamed. This print is for possible error messages.
    if status is not None:
        for line in status.rstrip().split('\n'):
            paasta_print('    %s' % line)

    return return_code
<|endoftext|>


<commit_before># map from numerical coordinates to letters used by SGF
SGF_POS = " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

class Collection:
    def __init__(self, parser=None):
        self.parser = parser
        if parser:
            self.setup()
        self.children = []

    def setup(self):
        self.parser.start_gametree = self.my_start_gametree

    def my_start_gametree(self):
        self.children.append(GameTree(self, self.parser))

    def output(self, f):
        for child in self.children:
            child.output(f)

class GameTree:
    def __init__(self, parent, parser=None):
        self.parent = parent
        self.parser = parser
        if parser:
            self.setup()
        self.nodes = []
        self.children = []

    def setup(self):
        self.parser.start_gametree = self.my_start_gametree
        self.parser.end_gametree = self.my_end_gametree
        self.parser.start_node = self.my_start_node

    def my_start_node(self):
        if len(self.nodes) > 0:
            previous = self.nodes[-1]
        elif self.parent.__class__ == GameTree:
            previous = self.parent.nodes[-1]
        else:
            previous = None
        node = Node(self, previous, self.parser)
        if len(self.nodes) == 0:
            node.first = True
            if self.parent.__class__ == GameTree:
                if len(previous.variations) > 0:
                    previous.variations[-1].next_variation = node
                    node.previous_variation = previous.variations[-1]
                previous.variations.append(node)
            else:
                if len(self.parent.children) > 1:
                    node.previous_variation = self.parent.children[-2].nodes[0]
                    self.parent.children[-2].nodes[0].next_variation = node

        self.nodes.append(node)

    def my_start_gametree(self):
        self.children.append(GameTree(self, self.parser))

    def my_end_gametree(self):
        self.parent.setup()

    def output(self, f):
        f.write("(")
        for node in self.nodes:
            node.output(f)
        for child in self.children:
            child.output(f)
        f.write(")")

class Node:
    def __init__(self, parent, previous, parser=None):
        self.parent = parent
        self.previous = previous
        self.parser = parser
        if parser:
            self.setup()
        self.properties = {}
        self.next = None
        self.previous_variation = None
        self.next_variation = None
        self.first = False
        self.variations = []
        if previous and not previous.next:
            previous.next = self

    def setup(self):
        self.parser.start_property = self.my_start_property
        self.parser.add_prop_value = self.my_add_prop_value
        self.parser.end_property = self.my_end_property
        self.parser.end_node = self.my_end_node

    def my_start_property(self, identifier):
        # @@@ check for duplicates
        self.current_property = identifier
        self.current_prop_value = []

    def my_add_prop_value(self, value):
        self.current_prop_value.append(value)

    def my_end_property(self):
        self.properties[self.current_property] = self.current_prop_value

    def my_end_node(self):
        self.parent.setup()

    def output(self, f):
        f.write(";")
        for property in self.properties.keys():
            f.write(property)
            for value in self.properties[property]:
                if "\\" in value:
                    value = "\\\\".join(value.split("\\"))
                if "]" in value:
                    value = "\\]".join(value.split("]"))
                f.write("[%s]" % value)

class ParseException(Exception):
    pass

class Parser:

    def parse(self, sgf_string):

        def whitespace(ch):
            return ch in " \t\r\n"

        def ucletter(ch):
            return ch in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

        def lcletter(ch):
            return ch in "abcdefghijklmnopqrstuvwxyz"

        state = 0

        for ch in sgf_string:
            if state == 0:
                if whitespace(ch):
                    state = 0
                elif ch == '(':
                    self.start_gametree()
                    state = 1
                else:
                    state = 0  # ignore everything up to first (
                    # raise ParseException(ch, state)
            elif state == 1:
                if whitespace(ch):
                    state = 1
                elif ch == ";":
                    self.start_node()
                    state = 2
                else:
                    raise ParseException(ch, state)
            elif state == 2:
                if whitespace(ch):
                    state = 2
                elif ucletter(ch):
                    prop_ident = ch
                    state = 3
                elif ch == ";":
                    self.end_node()
                    self.start_node()
                    state = 2
                elif ch == "(":
                    self.end_node()
                    self.start_gametree()
                    state = 1
                elif ch == ")":
                    self.end_node()
                    self.end_gametree()
                    state = 4
                else:
                    raise ParseException(ch, state)
            elif state == 3:
                if ucletter(ch):
                    prop_ident = prop_ident + ch
                    state = 3
                elif lcletter(ch):  # @@@ currently ignoring lowercase
                    state = 3
                elif ch == "[":
                    self.start_property(prop_ident)
                    prop_value = ""
                    state = 5
                else:
                    raise ParseException(ch, state)
            elif state == 4:
                if ch == ")":
                    self.end_gametree()
                    state = 4
                elif whitespace(ch):
                    state = 4
                elif ch == "(":
                    self.start_gametree()
                    state = 1
                else:
                    raise ParseException(ch, state)
            elif state == 5:
                if ch == "\\":
                    state = 6
                elif ch == "]":
                    self.add_prop_value(prop_value)
                    state = 7
                else:
                    prop_value = prop_value + ch
            elif state == 6:
                prop_value = prop_value + ch
                state = 5
            elif state == 7:
                if whitespace(ch):
                    state = 7
                elif ch == "[":
                    prop_value = ""
                    state = 5
                elif ch == ";":
                    self.end_property()
                    self.end_node()
                    self.start_node()
                    state = 2
                elif ucletter(ch):
                    self.end_property()
                    prop_ident = ch
                    state = 3
                elif ch == ")":
                    self.end_property()
                    self.end_node()
                    self.end_gametree()
                    state = 4
                elif ch == "(":
                    self.end_property()
                    self.end_node()
                    self.start_gametree()
                    state = 1
                else:
                    raise ParseException(ch, state)
            else:
                raise ParseException(ch, state)

        if state != 4:
            raise ParseException(ch, state)
<commit_msg>made collections indexable and iterable<commit_after># map from numerical coordinates to letters used by SGF
SGF_POS = " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

class Collection:
    def __init__(self, parser=None):
        self.parser = parser
        if parser:
            self.setup()
        self.children = []

    def setup(self):
        self.parser.start_gametree = self.my_start_gametree

    def my_start_gametree(self):
        self.children.append(GameTree(self, self.parser))

    def __getitem__(self, k):
        return self.children[k]

    def __iter__(self):
        return iter(self.children)

    def output(self, f):
        for child in self.children:
            child.output(f)

class GameTree:
    def __init__(self, parent, parser=None):
        self.parent = parent
        self.parser = parser
        if parser:
            self.setup()
        self.nodes = []
        self.children = []

    def setup(self):
        self.parser.start_gametree = self.my_start_gametree
        self.parser.end_gametree = self.my_end_gametree
        self.parser.start_node = self.my_start_node

    def my_start_node(self):
        if len(self.nodes) > 0:
            previous = self.nodes[-1]
        elif self.parent.__class__ == GameTree:
            previous = self.parent.nodes[-1]
        else:
            previous = None
        node = Node(self, previous, self.parser)
        if len(self.nodes) == 0:
            node.first = True
            if self.parent.__class__ == GameTree:
                if len(previous.variations) > 0:
                    previous.variations[-1].next_variation = node
                    node.previous_variation = previous.variations[-1]
                previous.variations.append(node)
            else:
                if len(self.parent.children) > 1:
                    node.previous_variation = self.parent.children[-2].nodes[0]
                    self.parent.children[-2].nodes[0].next_variation = node

        self.nodes.append(node)

    def my_start_gametree(self):
        self.children.append(GameTree(self, self.parser))

    def my_end_gametree(self):
        self.parent.setup()

    def output(self, f):
        f.write("(")
        for node in self.nodes:
            node.output(f)
        for child in self.children:
            child.output(f)
        f.write(")")

class Node:
    def __init__(self, parent, previous, parser=None):
        self.parent = parent
        self.previous = previous
        self.parser = parser
        if parser:
            self.setup()
        self.properties = {}
        self.next = None
        self.previous_variation = None
        self.next_variation = None
        self.first = False
        self.variations = []
        if previous and not previous.next:
            previous.next = self

    def setup(self):
        self.parser.start_property = self.my_start_property
        self.parser.add_prop_value = self.my_add_prop_value
        self.parser.end_property = self.my_end_property
        self.parser.end_node = self.my_end_node

    def my_start_property(self, identifier):
        # @@@ check for duplicates
        self.current_property = identifier
        self.current_prop_value = []

    def my_add_prop_value(self, value):
        self.current_prop_value.append(value)

    def my_end_property(self):
        self.properties[self.current_property] = self.current_prop_value

    def my_end_node(self):
        self.parent.setup()

    def output(self, f):
        f.write(";")
        for property in self.properties.keys():
            f.write(property)
            for value in self.properties[property]:
                if "\\" in value:
                    value = "\\\\".join(value.split("\\"))
                if "]" in value:
                    value = "\\]".join(value.split("]"))
                f.write("[%s]" % value)

class ParseException(Exception):
    pass

class Parser:

    def parse(self, sgf_string):

        def whitespace(ch):
            return ch in " \t\r\n"

        def ucletter(ch):
            return ch in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

        def lcletter(ch):
            return ch in "abcdefghijklmnopqrstuvwxyz"

        state = 0

        for ch in sgf_string:
            if state == 0:
                if whitespace(ch):
                    state = 0
                elif ch == '(':
                    self.start_gametree()
                    state = 1
                else:
                    state = 0  # ignore everything up to first (
                    # raise ParseException(ch, state)
            elif state == 1:
                if whitespace(ch):
                    state = 1
                elif ch == ";":
                    self.start_node()
                    state = 2
                else:
                    raise ParseException(ch, state)
            elif state == 2:
                if whitespace(ch):
                    state = 2
                elif ucletter(ch):
                    prop_ident = ch
                    state = 3
                elif ch == ";":
                    self.end_node()
                    self.start_node()
                    state = 2
                elif ch == "(":
                    self.end_node()
                    self.start_gametree()
                    state = 1
                elif ch == ")":
                    self.end_node()
                    self.end_gametree()
                    state = 4
                else:
                    raise ParseException(ch, state)
            elif state == 3:
                if ucletter(ch):
                    prop_ident = prop_ident + ch
                    state = 3
                elif lcletter(ch):  # @@@ currently ignoring lowercase
                    state = 3
                elif ch == "[":
                    self.start_property(prop_ident)
                    prop_value = ""
                    state = 5
                else:
                    raise ParseException(ch, state)
            elif state == 4:
                if ch == ")":
                    self.end_gametree()
                    state = 4
                elif whitespace(ch):
                    state = 4
                elif ch == "(":
                    self.start_gametree()
                    state = 1
                else:
                    raise ParseException(ch, state)
            elif state == 5:
                if ch == "\\":
                    state = 6
                elif ch == "]":
                    self.add_prop_value(prop_value)
                    state = 7
                else:
                    prop_value = prop_value + ch
            elif state == 6:
                prop_value = prop_value + ch
                state = 5
            elif state == 7:
                if whitespace(ch):
                    state = 7
                elif ch == "[":
                    prop_value = ""
                    state = 5
                elif ch == ";":
                    self.end_property()
                    self.end_node()
                    self.start_node()
                    state = 2
                elif ucletter(ch):
                    self.end_property()
                    prop_ident = ch
                    state = 3
                elif ch == ")":
                    self.end_property()
                    self.end_node()
                    self.end_gametree()
                    state = 4
                elif ch == "(":
                    self.end_property()
                    self.end_node()
                    self.start_gametree()
                    state = 1
                else:
                    raise ParseException(ch, state)
            else:
                raise ParseException(ch, state)

        if state != 4:
            raise ParseException(ch, state)
<|endoftext|>