<commit_before>import tensorflow as tf

from .flag import FLAGS, FlagAdder
from .estimator import def_estimator
from .inputs import def_def_train_input_fn, def_def_eval_input_fn

def def_def_experiment_fn(batch_inputs=True,
                          prepare_filename_queues=True,
                          distributed=False):
    adder = FlagAdder()

    for mode in [tf.contrib.learn.ModeKeys.TRAIN,
                 tf.contrib.learn.ModeKeys.EVAL]:
        adder.add_flag("{}_steps".format(mode), type=int,
                       help="Maximum number of {} steps".format(mode))

    adder.add_flag("min_eval_frequency", type=int, default=1,
                   help="Minimum evaluation frequency in number of model "
                        "savings")

    estimator = def_estimator(distributed)
    def_train_input_fn = def_def_train_input_fn(batch_inputs,
                                                prepare_filename_queues)
    def_eval_input_fn = def_def_eval_input_fn(batch_inputs,
                                              prepare_filename_queues)

    def def_experiment_fn(model_fn, train_input_fn, eval_input_fn=None):
        def experiment_fn(output_dir):
            return tf.contrib.learn.Experiment(
                estimator(model_fn, output_dir),
                def_train_input_fn(train_input_fn),
                def_eval_input_fn(eval_input_fn or train_input_fn),
                **adder.flags)

        return experiment_fn

    return def_experiment_fn
<commit_msg>Fix help message of --min_eval_frequency flag<commit_after>import tensorflow as tf

from .flag import FLAGS, FlagAdder
from .estimator import def_estimator
from .inputs import def_def_train_input_fn, def_def_eval_input_fn

def def_def_experiment_fn(batch_inputs=True,
                          prepare_filename_queues=True,
                          distributed=False):
    adder = FlagAdder()

    for mode in [tf.contrib.learn.ModeKeys.TRAIN,
                 tf.contrib.learn.ModeKeys.EVAL]:
        adder.add_flag("{}_steps".format(mode), type=int,
                       help="Maximum number of {} steps".format(mode))

    adder.add_flag("min_eval_frequency", type=int, default=1,
                   help="Minimum evaluation frequency in number of train steps")

    estimator = def_estimator(distributed)
    def_train_input_fn = def_def_train_input_fn(batch_inputs,
                                                prepare_filename_queues)
    def_eval_input_fn = def_def_eval_input_fn(batch_inputs,
                                              prepare_filename_queues)

    def def_experiment_fn(model_fn, train_input_fn, eval_input_fn=None):
        def experiment_fn(output_dir):
            return tf.contrib.learn.Experiment(
                estimator(model_fn, output_dir),
                def_train_input_fn(train_input_fn),
                def_eval_input_fn(eval_input_fn or train_input_fn),
                **adder.flags)

        return experiment_fn

    return def_experiment_fn
<|endoftext|>


<commit_before>"""Qopen: Seperation of intrinsic and scattering Q by envelope inversion"""

from qopen.core import run

__version__ = '1.6'
<commit_msg>raise dev version<commit_after>"""Qopen: Seperation of intrinsic and scattering Q by envelope inversion"""

from qopen.core import run

__version__ = '1.7-dev'
<|endoftext|>


<commit_before>#pylint:disable=too-many-instance-attributes
#pylint:disable=too-few-public-methods
#pylint:disable=attribute-defined-outside-init
#pylint:disable=bare-except
#pylint:disable=no-value-for-parameter

""" Class Research and auxiliary classes for multiple experiments. """

import os
from copy import copy
from collections import OrderedDict
from math import ceil
import json
import dill

from ..config import Config
from .distributor import Tasks, Distributor
from .workers import PipelineWorker, SavingWorker
from .grid import Grid

class Research:
    """ Class Research for multiple experiments with pipelines. """
    def __init__(self):
        self.pipelines = OrderedDict()

    def add_pipeline(self, pipeline, variables, preproc=None, config=None, name=None,
                     execute_for=None, run=False, **kwargs):
        """ Add new pipeline to research.

        Parameters
        ----------
        pipeline : dataset.Pipeline
            if preproc is None, pipeline must have run action with lazy=True. All parameters that are
            defined in grid should be defined as C('parameter_name'). Corresponding parameter in grid
            must have the same 'parameter_name'
        variables : str or list of str
            names of pipeline variables to remember at each repetition. All of them must be defined in pipeline,
            not in preproc.
        preproc : dataset.Pipeline or None
            if preproc is not None it must have run action with lazy=True. For resulting batch
            pipeline.execute_for(batch) will be called.
        config : Config or dict (default None)
            pipeline config with parameters that doesn't change between experiments.
        name : str (default None)
            pipeline name. If name is None, pipeline will have name 'ppl_{index}'
        execute_for : int, list of ints or None
            If -1, pipeline will be executed just at last iteration.
            If positive int, pipeline will be excuted for iterations with that step
            If list of ints, pipeline will be excuted for that iterations
            If None, pipeline will executed at each iteration.
        kwargs :
            parameters in pipeline config that depends on the names of the other pipeline. For example,
            if test pipeline imports model from the other pipeline with name 'train' in Researcn,
            corresponding parameter in import_model must be C('import_from') and add_pipeline
            must be called with parameter import_from='train'.
        """
        name = name or 'ppl_' + str(len(self.pipelines))
        config = config or Config()
        variables = variables or []

        if not isinstance(variables, list):
            variables = [variables]
        if name in self.pipelines:
            raise ValueError('Pipeline with name {} was alredy existed'.format(name))

        self.pipelines[name] = {
            'ppl': pipeline,
            'cfg': config,
            'var': variables,
            'execute_for': execute_for,
            'preproc': preproc,
            'kwargs': kwargs,
            'run': run
        }

        return self

    def add_grid_config(self, grid_config):
        """ Add grid of pipeline parameters.

        Parameters
        ----------
        grid_config : dict, Grid or Option
            if dict it should have items parameter_name: list of values.
        """
        self.grid_config = Grid(grid_config)
        return self

    def _create_tasks(self, n_reps, n_iters, n_groups, name):
        if isinstance(n_groups, int):
            n_models = n_groups
        elif n_groups is None:
            n_models = 1
        else:
            n_models = len(n_groups)

        configs_with_repetitions = [(idx, configs)
                                    for idx in range(n_reps)
                                    for configs in self.grid_config.gen_configs()]

        configs_chunks = self._chunks(configs_with_repetitions, n_models)

        self.tasks = (
            {'pipelines': self.pipelines,
             'n_iters': n_iters,
             'configs': list(zip(*chunk))[1],
             'repetition': list(zip(*chunk))[0],
             'n_groups': n_groups,
             'name': name
            }
            for chunk in configs_chunks
        )
        self.n_tasks = ceil(len(configs_with_repetitions) / n_models)
        self.tasks = Tasks(self.tasks)

    def _chunks(self, array, size):
        """ Divide array into chunks of the fixed size. """
        for i in range(0, len(array), size):
            yield array[i:i + size]

    def run(self, n_reps, n_iters, n_workers=1, n_groups=1, name=None, save_model=False):
        """ Run research.

        Parameters
        ----------
        n_reps : int
            number of repetitions with each combination of parameters
        n_iters: int
            number of iterations for each configurations of each pipeline.
        n_workers : int (default 1) or list of Workers
            If int - number of workers to run pipelines or workers that will run them. By default,
            PipelineWorker will be used.
            If list - instances of Worker class.
        n_groups: int or list of dicts
            If int - number of pipelines with different configs that will use the same prepared batch
            from preproc. If n_groups - list of dicts with additional configs to each pipeline.
            For example, if there are 2 GPUs, we can define parameter 'device' in model config as C('device')
            and define n_groups as [{'device': 0}, {'device': 1}].
        name : str or None
            name folder to save research. By default is 'research'.
        save_model : bool
            save or not the model 'model' at the first repetition from 'train' pipeline.
            If n_workers is not int there is no difference between True and False.

        At each iteration all add pipelines will be runned with some config from grid.
        """
        self.n_reps = n_reps
        self.n_iters = n_iters
        self.n_workers = n_workers
        self.n_groups = n_groups

        self.name = self._does_exist(name)

        # dump information about research
        self.save()

        self._create_tasks(n_reps, n_iters, n_groups, self.name)

        if isinstance(n_workers, int) or isinstance(n_workers[0], (dict, Config)):
            if save_model:
                worker = SavingWorker # worker that saves model at first repetition
            else:
                worker = PipelineWorker
        else:
            worker = None
        distr = Distributor(n_workers, worker)
        distr.run(self.tasks, dirname=self.name, n_tasks=self.n_tasks)
        return self

    def _does_exist(self, name):
        name = name or 'research'
        if not os.path.exists(name):
            dirname = name
        else:
            i = 1
            while os.path.exists(name + '_' + str(i)):
                i += 1
            dirname = name + '_' + str(i)
        os.makedirs(dirname)
        return dirname

    def save(self):
        """ Save description of the research to folder name/description. """
        with open(os.path.join(self.name, 'description'), 'wb') as file:
            dill.dump(self, file)
        with open(os.path.join(self.name, 'description_research.json'), 'w') as file:
            file.write(json.dumps(self._json(), default=self._set_default_json))
        with open(os.path.join(self.name, 'description_alias.json'), 'w') as file:
            file.write(json.dumps(self.grid_config.description(), default=self._set_default_json))

    def _set_default_json(self, obj):
        try:
            json.dumps(obj)
        except TypeError:
            return str(obj)

    def _json(self):
        description = copy(self.__dict__)
        description['grid_config'] = self.grid_config.value()
        _pipelines = dict()
        for name, pipeline in self.pipelines.items():
            _pipelines[name] = copy(pipeline)
            del _pipelines[name]['ppl']
            _pipelines[name]['cfg'] = _pipelines[name]['cfg'].flatten()
        description['pipelines'] = _pipelines
        return description

    @classmethod
    def load(cls, name):
        """ Load description of the research from name/description. """
        with open(os.path.join(name, 'description'), 'rb') as file:
            return dill.load(file)
<commit_msg>add doc-strings<commit_after>#pylint:disable=too-many-instance-attributes
#pylint:disable=too-few-public-methods
#pylint:disable=attribute-defined-outside-init
#pylint:disable=bare-except
#pylint:disable=no-value-for-parameter

""" Class Research and auxiliary classes for multiple experiments. """

import os
from copy import copy
from collections import OrderedDict
from math import ceil
import json
import dill

from ..config import Config
from .distributor import Tasks, Distributor
from .workers import PipelineWorker, SavingWorker
from .grid import Grid

class Research:
    """ Class Research for multiple experiments with pipelines. """
    def __init__(self):
        self.pipelines = OrderedDict()

    def add_pipeline(self, pipeline, variables, preproc=None, config=None, name=None,
                     execute_for=None, run=False, **kwargs):
        """ Add new pipeline to research.

        Parameters
        ----------
        pipeline : dataset.Pipeline
            if preproc is None, pipeline must have run action with lazy=True and n_epochs=None. All parameters that are
            defined in grid should be defined as C('parameter_name'). Corresponding parameter in grid
            must have the same 'parameter_name'.
        variables : str or list of str
            names of pipeline variables to save after each repetition. All of them must be defined in pipeline,
            not in preproc.
        preproc : dataset.Pipeline or None
            if preproc is not None it must have run action with lazy=True and n_epochs=None. For resulting batch
            pipeline.execute_for(batch) will be called.
        config : Config or dict (default None)
            pipeline config with parameters that doesn't change between experiments.
        name : str (default None)
            pipeline name. If name is None, pipeline will have name 'ppl_{index}'
        execute_for : int, list of ints or None
            If -1, pipeline will be executed just at last iteration.
            If positive int, pipeline will be excuted for iterations with that step
            If list of ints, pipeline will be excuted for that iterations
            If None, pipeline will executed at each iteration.
        run : bool (default False)
            if False then .next_batch() will be applied to pipeline, else .run().
        kwargs :
            parameters in pipeline config that depends on the names of the other pipeline. For example,
            if test pipeline imports model from the other pipeline with name 'train' in Researcn,
            corresponding parameter in import_model must be C('import_from') and add_pipeline
            must be called with parameter import_from='train'.
        """
        name = name or 'ppl_' + str(len(self.pipelines))
        config = config or Config()
        variables = variables or []

        if not isinstance(variables, list):
            variables = [variables]
        if name in self.pipelines:
            raise ValueError('Pipeline with name {} was alredy existed'.format(name))

        self.pipelines[name] = {
            'ppl': pipeline,
            'cfg': config,
            'var': variables,
            'execute_for': execute_for,
            'preproc': preproc,
            'kwargs': kwargs,
            'run': run
        }

        return self

    def add_grid_config(self, grid_config):
        """ Add grid of pipeline parameters.

        Parameters
        ----------
        grid_config : dict, Grid or Option
            if dict it should have items parameter_name: list of values.
        """
        self.grid_config = Grid(grid_config)
        return self

    def _create_tasks(self, n_reps, n_iters, n_groups, name):
        """ Create Tasks instance with tasks to run. Each task is one repetition for each config
        from grid_config.

        Parameters
        ----------
        n_reps : int

        n_iters : int

        n_groups : int

        name : str
            name of research.
        """
        if isinstance(n_groups, int):
            n_models = n_groups
        elif n_groups is None:
            n_models = 1
        else:
            n_models = len(n_groups)

        configs_with_repetitions = [(idx, configs)
                                    for idx in range(n_reps)
                                    for configs in self.grid_config.gen_configs()]

        configs_chunks = self._chunks(configs_with_repetitions, n_models)

        self.tasks = (
            {'pipelines': self.pipelines,
             'n_iters': n_iters,
             'configs': list(zip(*chunk))[1],
             'repetition': list(zip(*chunk))[0],
             'n_groups': n_groups,
             'name': name
            }
            for chunk in configs_chunks
        )
        self.n_tasks = ceil(len(configs_with_repetitions) / n_models)
        self.tasks = Tasks(self.tasks)

    def _chunks(self, array, size):
        """ Divide array into chunks of the fixed size.

        Parameters
        ----------
        array : list or np.ndarray

        size : int
            chunk size.
        """
        for i in range(0, len(array), size):
            yield array[i:i + size]

    def run(self, n_reps, n_iters, n_workers=1, n_groups=1, name=None, save_model=False):
        """ Run research.

        Parameters
        ----------
        n_reps : int
            number of repetitions with each combination of parameters from grid_config.
        n_iters: int
            number of iterations for each configurations of each pipeline.
        n_workers : int (default 1), list of instances of Worker or list of dicts (Configs).
            Workers (processes) to run tasks in parallel.
            If int - number of workers to run pipelines or workers that will run them. By default,
                PipelineWorker will be used.
            If list of instances of Worker - workers to run tasks.
            If list of dicts (Configs) - list of additional configs which will be appended to configs from tasks.
                Each element corresponds to worker.
        n_groups: int or list of dicts (Configs)
            Is using if preproc is not None. Number of different configs which will use the same batch from preproc pipeline.
            Pipelines will be executed in different threads.
            If int - number of pipelines with different configs that will use the same prepared batch
                from preproc.
            If list of dicts (Configs) - list of dicts with additional configs to each pipeline.

            For example, if there are 2 GPUs, we can define parameter 'device' in model config as C('device')
            and define n_groups as [{'device': 0}, {'device': 1}].
        name : str or None
            name folder to save research. By default is 'research'.
        save_model : bool
            save or not the model with name 'model' at the first repetition from 'train' pipeline.
            If n_workers is not int there is no difference between True and False.

        At each iteration all add pipelines will be runned with some config from grid.
        """
        self.n_reps = n_reps
        self.n_iters = n_iters
        self.n_workers = n_workers
        self.n_groups = n_groups

        self.name = self._does_exist(name)

        # dump information about research
        self.save()

        self._create_tasks(n_reps, n_iters, n_groups, self.name)

        if isinstance(n_workers, int) or isinstance(n_workers[0], (dict, Config)):
            if save_model:
                worker = SavingWorker # worker that saves model at first repetition
            else:
                worker = PipelineWorker
        else:
            worker = None
        distr = Distributor(n_workers, worker)
        distr.run(self.tasks, dirname=self.name, n_tasks=self.n_tasks)
        return self

    def _does_exist(self, name):
        name = name or 'research'
        if not os.path.exists(name):
            dirname = name
        else:
            i = 1
            while os.path.exists(name + '_' + str(i)):
                i += 1
            dirname = name + '_' + str(i)
        os.makedirs(dirname)
        return dirname

    def save(self):
        """ Save description of the research to folder name/description. """
        with open(os.path.join(self.name, 'description'), 'wb') as file:
            dill.dump(self, file)
        with open(os.path.join(self.name, 'description_research.json'), 'w') as file:
            file.write(json.dumps(self._json(), default=self._set_default_json))
        with open(os.path.join(self.name, 'description_alias.json'), 'w') as file:
            file.write(json.dumps(self.grid_config.description(), default=self._set_default_json))

    def _set_default_json(self, obj):
        try:
            json.dumps(obj)
        except TypeError:
            return str(obj)

    def _json(self):
        description = copy(self.__dict__)
        description['grid_config'] = self.grid_config.value()
        _pipelines = dict()
        for name, pipeline in self.pipelines.items():
            _pipelines[name] = copy(pipeline)
            del _pipelines[name]['ppl']
            _pipelines[name]['cfg'] = _pipelines[name]['cfg'].flatten()
        description['pipelines'] = _pipelines
        return description

    @classmethod
    def load(cls, name):
        """ Load description of the research from name/description. """
        with open(os.path.join(name, 'description'), 'rb') as file:
            return dill.load(file)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import logging
from datetime import datetime
from django.db import models
from django.contrib.auth.models import User, Group
from django.utils.translation import ugettext_lazy as _
from django.utils import simplejson
from logical.models import Database

from util.models import BaseModel

LOG = logging.getLogger(__name__)

class TaskHistory(BaseModel):

    class Meta:
      verbose_name_plural = "Task histories"

    STATUS_PENDING = 'PENDING'
    STATUS_RUNNING = 'RUNNING'
    STATUS_SUCCESS = 'SUCCESS'
    STATUS_ERROR = 'ERROR'
    STATUS_WARNING = 'WARNING'
    STATUS_WAITING = 'WAITING'

    _STATUS = [STATUS_PENDING, STATUS_RUNNING, STATUS_SUCCESS, STATUS_ERROR, STATUS_WARNING, STATUS_WAITING]

    task_id = models.CharField(_('Task ID'), max_length=200, null=True, blank=True, editable=False)
    task_name = models.CharField(_('Task Name'), max_length=200, null=True, blank=True)
    user = models.CharField(max_length=255, null=True, blank=True)
    ended_at = models.DateTimeField(verbose_name=_("Ended at"), null=True, blank=True, editable=False)
    task_status = models.CharField(_('Task Status'), max_length=100, default=STATUS_PENDING)
    context = models.TextField(null=True, blank=True)
    details = models.TextField(verbose_name=_("Details"), null=True, blank=True)
    arguments = models.TextField(verbose_name=_("Arguments"), null=True, blank=True)
    db_id = models.ForeignKey(Database, related_name="database", null=True, blank=True, on_delete=models.SET_NULL)

    def __unicode__(self):
        return u"%s" % self.task_id

    def load_context_data(self):
        if self.context == '':
            self.context = '{}'
        self.context_data = simplejson.loads(self.context)
        return self.context_data

    def update_details(self, details, persist=False):
        """
        Method to update the details of a task history.
        TODO: should we put a timestamp in details? should we append the details?
        """

        if self.details:
            self.details = "\n%s%s" % (self.details, details)
        else:
            print "None"
            self.details = details

        if persist:
            self.save()

    def update_status_for(self, status, details=None):
        if status not in TaskHistory._STATUS:
            raise RuntimeError("Invalid task status")

        self.task_status = status
        self.details = (self.details or " ") + "\n" + str(details)
        if status in [TaskHistory.STATUS_SUCCESS, TaskHistory.STATUS_ERROR, TaskHistory.STATUS_WARNING]:
            self.update_ended_at()
        else:
            self.save()

    def update_dbid(self, db):
        self.db_id = db
        self.save()

    def update_ended_at(self):
        self.ended_at = datetime.now()
        self.save()

    @classmethod
    def register(cls, request=None, user=None, task_history=None):

        LOG.info("task id: %s | task name: %s | " % (request.id,
                                                     request.task))
        print request.args

        if not task_history:
            task_history = TaskHistory()

        if task_history.task_id:
            task_history.task_id += ';' + request.id
        else:
            task_history.task_id = request.id

        task_history.task_name = request.task
        task_history.task_status = TaskHistory.STATUS_RUNNING

        if request.task == 'notification.tasks.create_database':
            task_history.arguments = "Database name: {0},\nEnvironment: {1},\
            \nProject: {2},\nPlan: {3}".format(
                request.kwargs['name'], request.kwargs['environment'],
                request.kwargs['project'], request.kwargs['plan'])

        elif request.task == 'notification.tasks.resize_database':
            task_history.arguments = "Database name: {0},\nNew Offering: {1}".format(
                request.kwargs['database'].name, request.kwargs['cloudstackpack'])

        elif request.task=='notification.tasks.destroy_database':
            task_history.arguments = "Database name: {0},\nUser: {1}".format(
                request.kwargs['database'].name, request.kwargs['user'])

        elif request.task=='notification.tasks.clone_database':
            task_history.arguments = "Database name: {0},\nClone: {1},\nPlan: {2},\
            \nEnvironment: {3}".format(
                request.kwargs['origin_database'].name, str(request.kwargs['clone_name']),
                str(request.kwargs['plan']), str(request.kwargs['environment']))

        else:
            task_history.arguments = request.args

        if user:
            task_history.user = str(user.username)

        task_history.save()

        return task_history
<commit_msg>Add worker_name to task_history record context<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import logging
from datetime import datetime
from django.db import models
from django.contrib.auth.models import User, Group
from django.utils.translation import ugettext_lazy as _
from django.utils import simplejson
from logical.models import Database

from util.models import BaseModel

LOG = logging.getLogger(__name__)

class TaskHistory(BaseModel):

    class Meta:
      verbose_name_plural = "Task histories"

    STATUS_PENDING = 'PENDING'
    STATUS_RUNNING = 'RUNNING'
    STATUS_SUCCESS = 'SUCCESS'
    STATUS_ERROR = 'ERROR'
    STATUS_WARNING = 'WARNING'
    STATUS_WAITING = 'WAITING'

    _STATUS = [STATUS_PENDING, STATUS_RUNNING, STATUS_SUCCESS, STATUS_ERROR, STATUS_WARNING, STATUS_WAITING]

    task_id = models.CharField(_('Task ID'), max_length=200, null=True, blank=True, editable=False)
    task_name = models.CharField(_('Task Name'), max_length=200, null=True, blank=True)
    user = models.CharField(max_length=255, null=True, blank=True)
    ended_at = models.DateTimeField(verbose_name=_("Ended at"), null=True, blank=True, editable=False)
    task_status = models.CharField(_('Task Status'), max_length=100, default=STATUS_PENDING)
    context = models.TextField(null=True, blank=True)
    details = models.TextField(verbose_name=_("Details"), null=True, blank=True)
    arguments = models.TextField(verbose_name=_("Arguments"), null=True, blank=True)
    db_id = models.ForeignKey(Database, related_name="database", null=True, blank=True, on_delete=models.SET_NULL)

    def __unicode__(self):
        return u"%s" % self.task_id

    def load_context_data(self):
        if self.context == '':
            self.context = '{}'
        self.context_data = simplejson.loads(self.context)
        return self.context_data

    def update_details(self, details, persist=False):
        """
        Method to update the details of a task history.
        TODO: should we put a timestamp in details? should we append the details?
        """

        if self.details:
            self.details = "\n%s%s" % (self.details, details)
        else:
            print "None"
            self.details = details

        if persist:
            self.save()

    def update_status_for(self, status, details=None):
        if status not in TaskHistory._STATUS:
            raise RuntimeError("Invalid task status")

        self.task_status = status
        self.details = (self.details or " ") + "\n" + str(details)
        if status in [TaskHistory.STATUS_SUCCESS, TaskHistory.STATUS_ERROR, TaskHistory.STATUS_WARNING]:
            self.update_ended_at()
        else:
            self.save()

    def update_dbid(self, db):
        self.db_id = db
        self.save()

    def update_ended_at(self):
        self.ended_at = datetime.now()
        self.save()

    @classmethod
    def register(cls, request=None, user=None, task_history=None, worker_name=None):

        LOG.info("task id: %s | task name: %s | " % (request.id,
                                                     request.task))

        if not task_history:
            task_history = TaskHistory()

        if task_history.task_id:
            task_history.task_id += ';' + request.id
        else:
            task_history.task_id = request.id

        task_history.task_name = request.task
        task_history.task_status = TaskHistory.STATUS_RUNNING

        if task_history.context:
            task_history.context.update({"worker_name": worker_name})
        else:
            task_history.context = {"worker_name": worker_name}

        if request.task == 'notification.tasks.create_database':
            task_history.arguments = "Database name: {0},\nEnvironment: {1},\
            \nProject: {2},\nPlan: {3}".format(
                request.kwargs['name'], request.kwargs['environment'],
                request.kwargs['project'], request.kwargs['plan'])

        elif request.task == 'notification.tasks.resize_database':
            task_history.arguments = "Database name: {0},\nNew Offering: {1}".format(
                request.kwargs['database'].name, request.kwargs['cloudstackpack'])

        elif request.task=='notification.tasks.destroy_database':
            task_history.arguments = "Database name: {0},\nUser: {1}".format(
                request.kwargs['database'].name, request.kwargs['user'])

        elif request.task=='notification.tasks.clone_database':
            task_history.arguments = "Database name: {0},\nClone: {1},\nPlan: {2},\
            \nEnvironment: {3}".format(
                request.kwargs['origin_database'].name, str(request.kwargs['clone_name']),
                str(request.kwargs['plan']), str(request.kwargs['environment']))

        else:
            task_history.arguments = request.args

        if user:
            task_history.user = str(user.username)

        task_history.save()

        return task_history
<|endoftext|>


<commit_before>"""Evaluation metrics."""

import numpy as np
import warnings
from deepchem.utils.save import log
import sklearn.metrics
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import recall_score
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import precision_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import auc
from sklearn.metrics import jaccard_similarity_score
from sklearn.metrics import f1_score
from scipy.stats import pearsonr

def to_one_hot(y, n_classes=2):
  """Transforms label vector into one-hot encoding.

    Turns y into vector of shape [n_samples, 2] (assuming binary labels).

    y: np.ndarray
      A vector of shape [n_samples, 1]
    """
  n_samples = np.shape(y)[0]
  y_hot = np.zeros((n_samples, n_classes))
  y_hot[np.arange(n_samples), y.astype(np.int64)] = 1
  return y_hot

def from_one_hot(y, axis=1):
  """Transorms label vector from one-hot encoding.

    y: np.ndarray
      A vector of shape [n_samples, num_classes]
    """
  return np.argmax(y, axis=axis)

def _ensure_one_hot(y):
  """If neceessary, convert class labels to one-hot encoding."""
  if len(y.shape) == 1:
    return to_one_hot(y)
  return y

def _ensure_class_labels(y):
  """If necessary, convert one-hot encoding to class labels."""
  if len(y.shape) == 2:
    return from_one_hot(y)
  return y

def roc_auc_score(y, y_pred):
  """Area under the receiver operating characteristic curve."""
  if y.shape != y_pred.shape:
    y = _ensure_one_hot(y)
  return sklearn.metrics.roc_auc_score(y, y_pred)

def accuracy_score(y, y_pred):
  y = _ensure_class_labels(y)
  y_pred = _ensure_class_labels(y_pred)
  return sklearn.metrics.accuracy_score(y, y_pred)

def balanced_accuracy_score(y, y_pred):
  """Computes balanced accuracy score."""
  num_positive = float(np.count_nonzero(y))
  num_negative = float(len(y) - num_positive)
  pos_weight = num_negative / num_positive
  weights = np.ones_like(y)
  weights[y != 0] = pos_weight
  return accuracy_score(y, y_pred, sample_weight=weights)

def pearson_r2_score(y, y_pred):
  """Computes Pearson R^2 (square of Pearson correlation)."""
  return pearsonr(y, y_pred)[0]**2

def jaccard_index(y, y_pred):
  """Computes Jaccard Index which is the Intersection Over Union metric
       which is commonly used in image segmentation tasks

      Parameters
      ----------
      y: ground truth array
      y_pred: predicted array
    """
  return jaccard_similarity_score(y, y_pred)

def pixel_error(y, y_pred):
  """defined as 1 - the maximal F-score of pixel similarity,
       or squared Euclidean distance between the original and the result labels.

      Parameters
      ----------
      y: ground truth array
      y_pred: predicted array
    """
  return 1 - f1_score(y, y_pred)

def prc_auc_score(y, y_pred):
  """Compute area under precision-recall curve"""
  if y.shape != y_pred.shape:
    y = _ensure_one_hot(y)
  assert y_pred.shape == y.shape
  assert y_pred.shape[1] == 2
  precision, recall, _ = precision_recall_curve(y[:, 1], y_pred[:, 1])
  return auc(recall, precision)

def rms_score(y_true, y_pred):
  """Computes RMS error."""
  return np.sqrt(mean_squared_error(y_true, y_pred))

def mae_score(y_true, y_pred):
  """Computes MAE."""
  return mean_absolute_error(y_true, y_pred)

def kappa_score(y_true, y_pred):
  """Calculate Cohen's kappa for classification tasks.

    See https://en.wikipedia.org/wiki/Cohen%27s_kappa

    Note that this implementation of Cohen's kappa expects binary labels.

    Args:
      y_true: Numpy array containing true values.
      y_pred: Numpy array containing predicted values.

    Returns:
      kappa: Numpy array containing kappa for each classification task.

    Raises:
      AssertionError: If y_true and y_pred are not the same size, or if class
        labels are not in [0, 1].
    """
  assert len(y_true) == len(y_pred), 'Number of examples does not match.'
  yt = np.asarray(y_true, dtype=int)
  yp = np.asarray(y_pred, dtype=int)
  assert np.array_equal(
      np.unique(yt),
      [0, 1]), ('Class labels must be binary: %s' % np.unique(yt))
  observed_agreement = np.true_divide(
      np.count_nonzero(np.equal(yt, yp)), len(yt))
  expected_agreement = np.true_divide(
      np.count_nonzero(yt == 1) * np.count_nonzero(yp == 1) +
      np.count_nonzero(yt == 0) * np.count_nonzero(yp == 0),
      len(yt)**2)
  kappa = np.true_divide(observed_agreement - expected_agreement,
                         1.0 - expected_agreement)
  return kappa

def bedroc_score(y_true, y_pred, alpha=20.0):
  """BEDROC metric implemented according to Truchon and Bayley that modifies
  the ROC score by allowing for a factor of early recognition

    References:
      The original paper by Truchon et al. is located at
      https://pubs.acs.org/doi/pdf/10.1021/ci600426e

    Args:
      y_true (array_like):
        Binary class labels. 1 for positive class, 0 otherwise
      y_pred (array_like):
        Predicted labels
      alpha (float), default 20.0:
        Early recognition parameter

    Returns:
      float: Value in [0, 1] that indicates the degree of early recognition

  """

  assert len(y_true) == len(y_pred), 'Number of examples do not match'

  assert np.array_equal(
      np.unique(y_true).astype(int),
      [0, 1]), ('Class labels must be binary: %s' % np.unique(y_true))

  from rdkit.ML.Scoring.Scoring import CalcBEDROC

  yt = np.asarray(y_true)
  yp = np.asarray(y_pred)

  yt = yt.flatten()
  yp = yp[:, 1].flatten()  # Index 1 because one_hot predictions

  scores = list(zip(yt, yp))
  scores = sorted(scores, key=lambda pair: pair[1], reverse=True)

  return CalcBEDROC(scores, 0, alpha)

class Metric(object):
  """Wrapper class for computing user-defined metrics."""

  def __init__(self,
               metric,
               task_averager=None,
               name=None,
               threshold=None,
               verbose=True,
               mode=None,
               compute_energy_metric=False):
    """
        Args:
          metric: function that takes args y_true, y_pred (in that order) and
                  computes desired score.
          task_averager: If not None, should be a function that averages metrics
                  across tasks. For example, task_averager=np.mean. If task_averager
                  is provided, this task will be inherited as a multitask metric.
        """
    self.metric = metric
    self.task_averager = task_averager
    self.is_multitask = (self.task_averager is not None)
    if name is None:
      if not self.is_multitask:
        self.name = self.metric.__name__
      else:
        self.name = self.task_averager.__name__ + "-" + self.metric.__name__
    else:
      self.name = name
    self.verbose = verbose
    self.threshold = threshold
    if mode is None:
      if self.metric.__name__ in [
          "roc_auc_score", "matthews_corrcoef", "recall_score",
          "accuracy_score", "kappa_score", "precision_score",
          "balanced_accuracy_score", "prc_auc_score", "f1_score"
      ]:
        mode = "classification"
      elif self.metric.__name__ in [
          "pearson_r2_score", "r2_score", "mean_squared_error",
          "mean_absolute_error", "rms_score", "mae_score", "pearsonr"
      ]:
        mode = "regression"
      else:
        raise ValueError("Must specify mode for new metric.")
    assert mode in ["classification", "regression"]
    if self.metric.__name__ in [
        "accuracy_score", "balanced_accuracy_score", "recall_score",
        "matthews_corrcoef", "precision_score", "f1_score"
    ] and threshold is None:
      self.threshold = 0.5
    self.mode = mode
    # The convention used is that the first task is the metric.
    # TODO(rbharath, joegomes): This doesn't seem like it should be hard-coded as
    # an option in the Metric class. Instead, this should be possible to move into
    # user-space as a custom task_averager function.
    self.compute_energy_metric = compute_energy_metric

  def compute_metric(self,
                     y_true,
                     y_pred,
                     w=None,
                     n_classes=2,
                     filter_nans=True,
                     per_task_metrics=False):
    """Compute a performance metric for each task.

        Parameters
        ----------
        y_true: np.ndarray
          An np.ndarray containing true values for each task.
        y_pred: np.ndarray
          An np.ndarray containing predicted values for each task.
        w: np.ndarray, optional
          An np.ndarray containing weights for each datapoint.
        n_classes: int, optional
          Number of classes in data for classification tasks.
        filter_nans: bool, optional
          Remove NaN values in computed metrics
        per_task_metrics: bool, optional
          If true, return computed metric for each task on multitask dataset.

        Returns
        -------
        A numpy nd.array containing metric values for each task.
        """
    n_samples = y_true.shape[0]
    expected_dims = (3 if self.mode == "classification" else 2)
    if len(y_pred.shape) < expected_dims:
      n_tasks = 1
      y_true = np.expand_dims(y_true, 1)
      y_pred = np.expand_dims(y_pred, 1)
    else:
      n_tasks = y_pred.shape[1]
    if w is None or len(w) == 0:
      w = np.ones((n_samples, n_tasks))
    computed_metrics = []
    for task in range(n_tasks):
      y_task = y_true[:, task]
      y_pred_task = y_pred[:, task]
      if len(w.shape) == 1:
        w_task = w
      elif w.shape[1] == 1:
        w_task = w[:, 0]
      else:
        w_task = w[:, task]

      metric_value = self.compute_singletask_metric(y_task, y_pred_task, w_task)
      computed_metrics.append(metric_value)
    log("computed_metrics: %s" % str(computed_metrics), self.verbose)
    if n_tasks == 1:
      computed_metrics = computed_metrics[0]
    if not self.is_multitask:
      return computed_metrics
    else:
      if filter_nans:
        computed_metrics = np.array(computed_metrics)
        computed_metrics = computed_metrics[~np.isnan(computed_metrics)]
      if self.compute_energy_metric:
        # TODO(rbharath, joegomes): What is this magic number?
        force_error = self.task_averager(computed_metrics[1:]) * 4961.47596096
        print("Force error (metric: np.mean(%s)): %f kJ/mol/A" % (self.name,
                                                                  force_error))
        return computed_metrics[0]
      elif not per_task_metrics:
        return self.task_averager(computed_metrics)
      else:
        return self.task_averager(computed_metrics), computed_metrics

  def compute_singletask_metric(self, y_true, y_pred, w):
    """Compute a metric value.

    Args:
      y_true: A list of arrays containing true values for each task.
      y_pred: A list of arrays containing predicted values for each task.

    Returns:
      Float metric value.

    Raises:
      NotImplementedError: If metric_str is not in METRICS.
    """

    y_true = np.array(np.squeeze(y_true[w != 0]))
    y_pred = np.array(np.squeeze(y_pred[w != 0]))

    if len(y_true.shape) == 0:
      n_samples = 1
    else:
      n_samples = y_true.shape[0]
    # If there are no nonzero examples, metric is ill-defined.
    if not y_true.size:
      return np.nan
    if self.threshold is not None:
      y_pred = y_pred[:, 1]
      y_pred = np.greater(y_pred, self.threshold)
    if len(y_true.shape) == 0:
      y_true = np.expand_dims(y_true, 0)
    if len(y_pred.shape) == 0:
      y_pred = np.expand_dims(y_pred, 0)
    try:
      metric_value = self.metric(y_true, y_pred)
    except (AssertionError, ValueError) as e:
      warnings.warn("Error calculating metric %s: %s" % (self.name, e))
      metric_value = np.nan
    return metric_value
<commit_msg>sklearn fix<commit_after>"""Evaluation metrics."""

import numpy as np
import warnings
from deepchem.utils.save import log
import sklearn.metrics
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import recall_score
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import precision_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import auc
from sklearn.metrics import jaccard_score
from sklearn.metrics import f1_score
from scipy.stats import pearsonr

def to_one_hot(y, n_classes=2):
  """Transforms label vector into one-hot encoding.

    Turns y into vector of shape [n_samples, 2] (assuming binary labels).

    y: np.ndarray
      A vector of shape [n_samples, 1]
    """
  n_samples = np.shape(y)[0]
  y_hot = np.zeros((n_samples, n_classes))
  y_hot[np.arange(n_samples), y.astype(np.int64)] = 1
  return y_hot

def from_one_hot(y, axis=1):
  """Transorms label vector from one-hot encoding.

    y: np.ndarray
      A vector of shape [n_samples, num_classes]
    """
  return np.argmax(y, axis=axis)

def _ensure_one_hot(y):
  """If neceessary, convert class labels to one-hot encoding."""
  if len(y.shape) == 1:
    return to_one_hot(y)
  return y

def _ensure_class_labels(y):
  """If necessary, convert one-hot encoding to class labels."""
  if len(y.shape) == 2:
    return from_one_hot(y)
  return y

def roc_auc_score(y, y_pred):
  """Area under the receiver operating characteristic curve."""
  if y.shape != y_pred.shape:
    y = _ensure_one_hot(y)
  return sklearn.metrics.roc_auc_score(y, y_pred)

def accuracy_score(y, y_pred):
  y = _ensure_class_labels(y)
  y_pred = _ensure_class_labels(y_pred)
  return sklearn.metrics.accuracy_score(y, y_pred)

def balanced_accuracy_score(y, y_pred):
  """Computes balanced accuracy score."""
  num_positive = float(np.count_nonzero(y))
  num_negative = float(len(y) - num_positive)
  pos_weight = num_negative / num_positive
  weights = np.ones_like(y)
  weights[y != 0] = pos_weight
  return accuracy_score(y, y_pred, sample_weight=weights)

def pearson_r2_score(y, y_pred):
  """Computes Pearson R^2 (square of Pearson correlation)."""
  return pearsonr(y, y_pred)[0]**2

def jaccard_index(y, y_pred):
  """Computes Jaccard Index which is the Intersection Over Union metric
       which is commonly used in image segmentation tasks

      Parameters
      ----------
      y: ground truth array
      y_pred: predicted array
    """
  return jaccard_score(y, y_pred)

def pixel_error(y, y_pred):
  """defined as 1 - the maximal F-score of pixel similarity,
       or squared Euclidean distance between the original and the result labels.

      Parameters
      ----------
      y: ground truth array
      y_pred: predicted array
    """
  return 1 - f1_score(y, y_pred)

def prc_auc_score(y, y_pred):
  """Compute area under precision-recall curve"""
  if y.shape != y_pred.shape:
    y = _ensure_one_hot(y)
  assert y_pred.shape == y.shape
  assert y_pred.shape[1] == 2
  precision, recall, _ = precision_recall_curve(y[:, 1], y_pred[:, 1])
  return auc(recall, precision)

def rms_score(y_true, y_pred):
  """Computes RMS error."""
  return np.sqrt(mean_squared_error(y_true, y_pred))

def mae_score(y_true, y_pred):
  """Computes MAE."""
  return mean_absolute_error(y_true, y_pred)

def kappa_score(y_true, y_pred):
  """Calculate Cohen's kappa for classification tasks.

    See https://en.wikipedia.org/wiki/Cohen%27s_kappa

    Note that this implementation of Cohen's kappa expects binary labels.

    Args:
      y_true: Numpy array containing true values.
      y_pred: Numpy array containing predicted values.

    Returns:
      kappa: Numpy array containing kappa for each classification task.

    Raises:
      AssertionError: If y_true and y_pred are not the same size, or if class
        labels are not in [0, 1].
    """
  assert len(y_true) == len(y_pred), 'Number of examples does not match.'
  yt = np.asarray(y_true, dtype=int)
  yp = np.asarray(y_pred, dtype=int)
  assert np.array_equal(
      np.unique(yt),
      [0, 1]), ('Class labels must be binary: %s' % np.unique(yt))
  observed_agreement = np.true_divide(
      np.count_nonzero(np.equal(yt, yp)), len(yt))
  expected_agreement = np.true_divide(
      np.count_nonzero(yt == 1) * np.count_nonzero(yp == 1) +
      np.count_nonzero(yt == 0) * np.count_nonzero(yp == 0),
      len(yt)**2)
  kappa = np.true_divide(observed_agreement - expected_agreement,
                         1.0 - expected_agreement)
  return kappa

def bedroc_score(y_true, y_pred, alpha=20.0):
  """BEDROC metric implemented according to Truchon and Bayley that modifies
  the ROC score by allowing for a factor of early recognition

    References:
      The original paper by Truchon et al. is located at
      https://pubs.acs.org/doi/pdf/10.1021/ci600426e

    Args:
      y_true (array_like):
        Binary class labels. 1 for positive class, 0 otherwise
      y_pred (array_like):
        Predicted labels
      alpha (float), default 20.0:
        Early recognition parameter

    Returns:
      float: Value in [0, 1] that indicates the degree of early recognition

  """

  assert len(y_true) == len(y_pred), 'Number of examples do not match'

  assert np.array_equal(
      np.unique(y_true).astype(int),
      [0, 1]), ('Class labels must be binary: %s' % np.unique(y_true))

  from rdkit.ML.Scoring.Scoring import CalcBEDROC

  yt = np.asarray(y_true)
  yp = np.asarray(y_pred)

  yt = yt.flatten()
  yp = yp[:, 1].flatten()  # Index 1 because one_hot predictions

  scores = list(zip(yt, yp))
  scores = sorted(scores, key=lambda pair: pair[1], reverse=True)

  return CalcBEDROC(scores, 0, alpha)

class Metric(object):
  """Wrapper class for computing user-defined metrics."""

  def __init__(self,
               metric,
               task_averager=None,
               name=None,
               threshold=None,
               verbose=True,
               mode=None,
               compute_energy_metric=False):
    """
        Args:
          metric: function that takes args y_true, y_pred (in that order) and
                  computes desired score.
          task_averager: If not None, should be a function that averages metrics
                  across tasks. For example, task_averager=np.mean. If task_averager
                  is provided, this task will be inherited as a multitask metric.
        """
    self.metric = metric
    self.task_averager = task_averager
    self.is_multitask = (self.task_averager is not None)
    if name is None:
      if not self.is_multitask:
        self.name = self.metric.__name__
      else:
        self.name = self.task_averager.__name__ + "-" + self.metric.__name__
    else:
      self.name = name
    self.verbose = verbose
    self.threshold = threshold
    if mode is None:
      if self.metric.__name__ in [
          "roc_auc_score", "matthews_corrcoef", "recall_score",
          "accuracy_score", "kappa_score", "precision_score",
          "balanced_accuracy_score", "prc_auc_score", "f1_score"
      ]:
        mode = "classification"
      elif self.metric.__name__ in [
          "pearson_r2_score", "r2_score", "mean_squared_error",
          "mean_absolute_error", "rms_score", "mae_score", "pearsonr"
      ]:
        mode = "regression"
      else:
        raise ValueError("Must specify mode for new metric.")
    assert mode in ["classification", "regression"]
    if self.metric.__name__ in [
        "accuracy_score", "balanced_accuracy_score", "recall_score",
        "matthews_corrcoef", "precision_score", "f1_score"
    ] and threshold is None:
      self.threshold = 0.5
    self.mode = mode
    # The convention used is that the first task is the metric.
    # TODO(rbharath, joegomes): This doesn't seem like it should be hard-coded as
    # an option in the Metric class. Instead, this should be possible to move into
    # user-space as a custom task_averager function.
    self.compute_energy_metric = compute_energy_metric

  def compute_metric(self,
                     y_true,
                     y_pred,
                     w=None,
                     n_classes=2,
                     filter_nans=True,
                     per_task_metrics=False):
    """Compute a performance metric for each task.

        Parameters
        ----------
        y_true: np.ndarray
          An np.ndarray containing true values for each task.
        y_pred: np.ndarray
          An np.ndarray containing predicted values for each task.
        w: np.ndarray, optional
          An np.ndarray containing weights for each datapoint.
        n_classes: int, optional
          Number of classes in data for classification tasks.
        filter_nans: bool, optional
          Remove NaN values in computed metrics
        per_task_metrics: bool, optional
          If true, return computed metric for each task on multitask dataset.

        Returns
        -------
        A numpy nd.array containing metric values for each task.
        """
    n_samples = y_true.shape[0]
    expected_dims = (3 if self.mode == "classification" else 2)
    if len(y_pred.shape) < expected_dims:
      n_tasks = 1
      y_true = np.expand_dims(y_true, 1)
      y_pred = np.expand_dims(y_pred, 1)
    else:
      n_tasks = y_pred.shape[1]
    if w is None or len(w) == 0:
      w = np.ones((n_samples, n_tasks))
    computed_metrics = []
    for task in range(n_tasks):
      y_task = y_true[:, task]
      y_pred_task = y_pred[:, task]
      if len(w.shape) == 1:
        w_task = w
      elif w.shape[1] == 1:
        w_task = w[:, 0]
      else:
        w_task = w[:, task]

      metric_value = self.compute_singletask_metric(y_task, y_pred_task, w_task)
      computed_metrics.append(metric_value)
    log("computed_metrics: %s" % str(computed_metrics), self.verbose)
    if n_tasks == 1:
      computed_metrics = computed_metrics[0]
    if not self.is_multitask:
      return computed_metrics
    else:
      if filter_nans:
        computed_metrics = np.array(computed_metrics)
        computed_metrics = computed_metrics[~np.isnan(computed_metrics)]
      if self.compute_energy_metric:
        # TODO(rbharath, joegomes): What is this magic number?
        force_error = self.task_averager(computed_metrics[1:]) * 4961.47596096
        print("Force error (metric: np.mean(%s)): %f kJ/mol/A" % (self.name,
                                                                  force_error))
        return computed_metrics[0]
      elif not per_task_metrics:
        return self.task_averager(computed_metrics)
      else:
        return self.task_averager(computed_metrics), computed_metrics

  def compute_singletask_metric(self, y_true, y_pred, w):
    """Compute a metric value.

    Args:
      y_true: A list of arrays containing true values for each task.
      y_pred: A list of arrays containing predicted values for each task.

    Returns:
      Float metric value.

    Raises:
      NotImplementedError: If metric_str is not in METRICS.
    """

    y_true = np.array(np.squeeze(y_true[w != 0]))
    y_pred = np.array(np.squeeze(y_pred[w != 0]))

    if len(y_true.shape) == 0:
      n_samples = 1
    else:
      n_samples = y_true.shape[0]
    # If there are no nonzero examples, metric is ill-defined.
    if not y_true.size:
      return np.nan
    if self.threshold is not None:
      y_pred = y_pred[:, 1]
      y_pred = np.greater(y_pred, self.threshold)
    if len(y_true.shape) == 0:
      y_true = np.expand_dims(y_true, 0)
    if len(y_pred.shape) == 0:
      y_pred = np.expand_dims(y_pred, 0)
    try:
      metric_value = self.metric(y_true, y_pred)
    except (AssertionError, ValueError) as e:
      warnings.warn("Error calculating metric %s: %s" % (self.name, e))
      metric_value = np.nan
    return metric_value
<|endoftext|>


<commit_before>import sys, os
import re
import networkx as nx
import pyparsing
import cPickle as pickle
import matplotlib.pyplot as plt

amr_nx_graph = None

# def post_order(G):
# 	root = [G.graph['root']]
# 	edge_list = []
# 	visited = []
# 	for start in root:
# 		if start in visited:
# 			continue
#
# 		start_children = list(G[root[0]])
# 		# sort children by span order
# 		start_children_span = sorted([ (G.node[x]['span'][0],x) for x in start_children],reverse=True)
# 		start_children = [p[1] for p in start_children_span]
# 		stack = [(start, start_children)]
# 		#visited.append(start)
# 		while stack:
# 			#print stack
# 			parent, children = stack[-1]
# 			# Visit the parent
# 			# Then visit all children
# 			for i in range(len(children)):
# 				child = children[i]
# 				#print child
# 				if child not in visited:
# 					edge_list.append((parent,child))
# 					#visited.append(child)
# 					new_children = list(G[child])
# 					children_span = sorted([(G.node[x]['span'][0], x) for x in new_children],reverse=True)
# 					#print children_span
# 					new_children = [p[1] for p in children_span]
# 					#stack = [(start, start_children)]
# 					stack.append((child, new_children))
# 			visited.append(parent)

def pre_order(G):
	root = [G.graph['root']]
	edge_list = []
	visited = []
	for start in root:
		if start in visited:
			continue

		start_children = list(G[root[0]])
		# sort children by span order
		start_children_span = sorted([ (G.node[x]['span'][0],x) for x in start_children],reverse=True)
		start_children = [p[1] for p in start_children_span]
		stack = [(start, start_children)]
		#visited.append(start)
		while stack:
			#print stack
			parent, children = stack.pop()
			# Visit the parent
			if parent in visited:
				continue
			visited.append(parent)
			# Then visit all children
			for i in range(len(children)):
				child = children[i]
				#print child
				if child not in visited:
					edge_list.append((parent,child))
					#visited.append(child)
					new_children = list(G[child])
					children_span = sorted([(G.node[x]['span'][0], x) for x in new_children],reverse=True)
					#print children_span
					new_children = [p[1] for p in children_span]
					#stack = [(start, start_children)]
					stack.append((child, new_children))

	return visited

def count_swaps(original,ordered):
	one_swap = 0
	more_swaps = 0
	for i in range(len(original)):
		idx = ordered.index(original[i])
		if idx == i - 1:
			one_swap += 1
		elif idx > i:
			more_swaps += 1

	return one_swap, more_swaps

def is_projective(G,original):

	edge_dict = {x:1 for x in G.edges()}
	#print edge_dict
	crossing_edges = 0

	for each_edge in edge_dict:
		#print G.node[each_edge[0]]
		h = G.node[each_edge[0]]['span'][0]
		t = G.node[each_edge[1]]['span'][0]
		#print h,t
		for each_edge2 in edge_dict:

			h2 = G.node[each_edge2[0]]['span'][0]
			t2 = G.node[each_edge2[1]]['span'][0]

			if t < h:

				h,t=t,h

			if h < h2 < t:
				if t2 < h or t2 > t:
					crossing_edges += 1
			elif h < t2 < t:
				if h2 < h or h2 > t:
					crossing_edges += 1

	#print crossing_edges

def main(argv):

	all_concept = {"NULL":1}
	all_relations = {"ROOT":1,"NOEDGE":0}
	concept_count = 2
	relation_count = 1
	#amr_aligned = open("amr-small")
	#amr_aligned = open("amr-release-1.0-training-full.aligned")
	amr_aligned = open(argv[0])
	pos_file = open(argv[1])
	line = amr_aligned.readline()
	current_amr_graph = nx.MultiDiGraph()
	prev_node = ""
	prev_span = []
	while (line != ""):
		if line.startswith("# ::id"):
			current_id = line.split()[2]
			current_amr_graph.graph['id'] =  current_id
		elif line.startswith("# ::tok"):
			current_sent = line.strip("# ::tok").strip()
			current_amr_graph.graph['sent'] = current_sent
		elif line.startswith("# ::node"):
			x = line.strip().split('\t')
			curr_node_id = x[1]
			curr_node_label = x[2]
			try:
				current_node_span = [int(p) for p in x[3].split('-')]
				final_span = range(current_node_span[0], current_node_span[1])
				if final_span == prev_span:
					if curr_node_label[0] != '"':
						current_amr_graph.node[prev_node]['node_label'] += "_{}".format(curr_node_label)
				else:
					prev_span = final_span
					prev_node = curr_node_id
					current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
					#current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
				if not final_span:
					final_span = [-1]
					prev_span = [-1]
					prev_node = curr_node_id
					current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
			except IndexError:
				final_span = [-1]
				prev_span = [-1]
				prev_node = curr_node_id
				current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
			#Add node to graph
			# if curr_node_id not in nodes:
			# 	nodes[curr_node]

		elif line.startswith("# ::root"):
			x = line.strip().split('\t')
			current_amr_graph.graph['root'] = x[1]
		elif line.startswith("# ::edge"):
			x = line.strip().split('\t')
			head = x[4]
			tail = x[5]
			if head not in current_amr_graph.node or tail not in current_amr_graph.node:
				line = amr_aligned.readline()
				continue
			label  = x[2]
			current_amr_graph.add_edge(head,tail,key=label,edge_label=label)
		elif line.strip() == "":
			span_sorted = sorted([(current_amr_graph.node[x]['span'][0],current_amr_graph.node[x]['span'][-1],
			x,current_amr_graph.node[x]['span']) for x in current_amr_graph.node.keys() if current_amr_graph.node[x]['span'][0] != -1 ])
			for each_edge in current_amr_graph.edges():
				lhs = each_edge[0]
				rhs = each_edge[1]
				lhs_name = current_amr_graph.node[lhs]['node_label']
				rhs_name = current_amr_graph.node[rhs]['node_label']
			#print current_amr_graph.node
			#print span_sorted
			sent_tok = current_amr_graph.graph['sent'].split()
			i = 0
			j = 0
			count = 1
			pos = pos_file.readline().strip().split()
			pos_t = [x.split('_')[1] for x in pos]
			extra = 0
			while (i <len(sent_tok)):
				if j == len(span_sorted):
					break
				if span_sorted[j][0] == i:
					node = span_sorted[j][2]
					node_label = current_amr_graph.node[node]['node_label']
					if node_label not in all_concept:
						all_concept[node_label] = concept_count
						concept_count += 1
					c_lab = all_concept[node_label]
					parent_info = current_amr_graph.pred[node]
					#If it is a root
					if len(parent_info) == 0:
						e_lab = all_relations["ROOT"]
						sys.stdout.write("{} {} {} {}:{}".format(0,e_lab,c_lab,0,"ROOT"))
						#print count,"\t",' '.join(sent_tok[i:span_sorted[j][1]+1]),"\t",POS,"\t",0,"\t","ROOT","\t",node_label
					else:
						# Write parent info
						last_col = ""
						for parent in parent_info.keys():
							#parent = parent_info.keys()[0]
							label = parent_info[parent].keys()[0]
							if label not in all_relations:
								all_relations[label] = relation_count
								relation_count += 1
							e_lab = all_relations[label]
							parent_pos = current_amr_graph.node[parent]['span'][0]
							last_col += "{}:{}::".format(current_amr_graph.node[parent]['node_label'
																						''],label)
							sys.stdout.write("{} {} {} ".format(parent_pos+1,e_lab,c_lab))
						sys.stdout.write(last_col)
						# Word
					sys.stdout.write("|w {} ".format(' '.join(sent_tok[i:span_sorted[j][1]+1])))
					sys.stdout.write("|p {}\n".format('_'.join(pos_t[i:span_sorted[j][1]+1])))

						#print p
					#print count,"\t",' '.join(sent_tok[i:span_sorted[j][1]+1]),"\t",POS,"\t",parent_pos+1+extra,"\t",label,"\t",node_label

					new_i = span_sorted[j][1] + 1
					j += 1
					if j == len(span_sorted):
						break
					if span_sorted[j][0] != i:
						i = new_i
					else:
						extra += 1
				else:
					e_lab = all_relations["NOEDGE"]
					c_lab = all_concept["NULL"]
					sys.stdout.write("{} {} {} {}:{}".format(0,e_lab,c_lab,0,"NOEDGE"))
					sys.stdout.write("|w {} ".format(sent_tok[i]))
					sys.stdout.write("|p {}\n".format(pos_t[i]))
					#print count,"\t",sent_tok[i],"\t",POS,"\t",0,"\t","NOEDGE","\t","NULL"
					i += 1
				count += 1
			print
			print "Done"
			current_amr_graph = nx.MultiDiGraph()
			prev_node = ""
			prev_span = []

		line = amr_aligned.readline()

	amr_aligned.close()
	with open("nodes_dict", 'w') as f:
		nodes_list = sorted([ (key,all_concept[key])for key in all_concept],key=lambda x:x[1])
		for each_node in nodes_list:
			f.write("{0}\t{1}\n".format(each_node[0],each_node[1]))

	with open("edges_dict", 'w') as f:
		edges_list = sorted([ (key,all_relations[key])for key in all_relations],key=lambda x:x[1])
		for each_node in edges_list:
			f.write("{0}\t{1}\n".format(each_node[0],each_node[1]))

if __name__ == "__main__":
	main(sys.argv[1:])
<commit_msg>Updated to fix multiple roots bug<commit_after>import sys, os
import re
import networkx as nx
import pyparsing
import cPickle as pickle
import matplotlib.pyplot as plt

amr_nx_graph = None

# def post_order(G):
# 	root = [G.graph['root']]
# 	edge_list = []
# 	visited = []
# 	for start in root:
# 		if start in visited:
# 			continue
#
# 		start_children = list(G[root[0]])
# 		# sort children by span order
# 		start_children_span = sorted([ (G.node[x]['span'][0],x) for x in start_children],reverse=True)
# 		start_children = [p[1] for p in start_children_span]
# 		stack = [(start, start_children)]
# 		#visited.append(start)
# 		while stack:
# 			#print stack
# 			parent, children = stack[-1]
# 			# Visit the parent
# 			# Then visit all children
# 			for i in range(len(children)):
# 				child = children[i]
# 				#print child
# 				if child not in visited:
# 					edge_list.append((parent,child))
# 					#visited.append(child)
# 					new_children = list(G[child])
# 					children_span = sorted([(G.node[x]['span'][0], x) for x in new_children],reverse=True)
# 					#print children_span
# 					new_children = [p[1] for p in children_span]
# 					#stack = [(start, start_children)]
# 					stack.append((child, new_children))
# 			visited.append(parent)

def pre_order(G):
	root = [G.graph['root']]
	edge_list = []
	visited = []
	for start in root:
		if start in visited:
			continue

		start_children = list(G[root[0]])
		# sort children by span order
		start_children_span = sorted([ (G.node[x]['span'][0],x) for x in start_children],reverse=True)
		start_children = [p[1] for p in start_children_span]
		stack = [(start, start_children)]
		#visited.append(start)
		while stack:
			#print stack
			parent, children = stack.pop()
			# Visit the parent
			if parent in visited:
				continue
			visited.append(parent)
			# Then visit all children
			for i in range(len(children)):
				child = children[i]
				#print child
				if child not in visited:
					edge_list.append((parent,child))
					#visited.append(child)
					new_children = list(G[child])
					children_span = sorted([(G.node[x]['span'][0], x) for x in new_children],reverse=True)
					#print children_span
					new_children = [p[1] for p in children_span]
					#stack = [(start, start_children)]
					stack.append((child, new_children))

	return visited

def count_swaps(original,ordered):
	one_swap = 0
	more_swaps = 0
	for i in range(len(original)):
		idx = ordered.index(original[i])
		if idx == i - 1:
			one_swap += 1
		elif idx > i:
			more_swaps += 1

	return one_swap, more_swaps

def is_projective(G,original):

	edge_dict = {x:1 for x in G.edges()}
	#print edge_dict
	crossing_edges = 0

	for each_edge in edge_dict:
		#print G.node[each_edge[0]]
		h = G.node[each_edge[0]]['span'][0]
		t = G.node[each_edge[1]]['span'][0]
		#print h,t
		for each_edge2 in edge_dict:

			h2 = G.node[each_edge2[0]]['span'][0]
			t2 = G.node[each_edge2[1]]['span'][0]

			if t < h:

				h,t=t,h

			if h < h2 < t:
				if t2 < h or t2 > t:
					crossing_edges += 1
			elif h < t2 < t:
				if h2 < h or h2 > t:
					crossing_edges += 1

	#print crossing_edges

def main(argv):

	all_concept = {"NULL":1}
	all_relations = {"ROOT":1,"NOEDGE":0}
	concept_count = 2
	relation_count = 1
	#amr_aligned = open("amr-small")
	#pos_file = open("amr-small.pos")
	#amr_aligned = open("amr-release-1.0-training-full.aligned")
	#pos_file = open("amr-release-1.0-training-full.pos")
	amr_aligned = open(argv[0])
	pos_file = open(argv[1])
	line = amr_aligned.readline()
	current_amr_graph = nx.MultiDiGraph()
	prev_node = ""
	prev_span = []
	main_count = 0
	folded = {}
	while (line != ""):
		if line.startswith("# ::id"):
			current_id = line.split()[2]
			current_amr_graph.graph['id'] =  current_id
		elif line.startswith("# ::tok"):
			current_sent = line.strip("# ::tok").strip()
			current_amr_graph.graph['sent'] = current_sent
		elif line.startswith("# ::node"):
			x = line.strip().split('\t')
			curr_node_id = x[1]
			curr_node_label = x[2]
			try:
				current_node_span = [int(p) for p in x[3].split('-')]
				final_span = range(current_node_span[0], current_node_span[1])
				if final_span == prev_span:
					if curr_node_label[0] != '"':
						folded[curr_node_id] = prev_node
						current_amr_graph.node[prev_node]['node_label'] += "_{}".format(curr_node_label)
				else:
					prev_span = final_span
					prev_node = curr_node_id
					current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
					#current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
				if not final_span:
					final_span = [-1]
					prev_span = [-1]
					prev_node = curr_node_id
					current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
			except IndexError:
				final_span = [-1]
				prev_span = [-1]
				prev_node = curr_node_id
				current_amr_graph.add_node(curr_node_id, node_label=curr_node_label, span=final_span)
			#Add node to graph
			# if curr_node_id not in nodes:
			# 	nodes[curr_node]

		elif line.startswith("# ::root"):
			x = line.strip().split('\t')
			current_amr_graph.graph['root'] = x[1]
		elif line.startswith("# ::edge"):
			x = line.strip().split('\t')
			head = x[4]
			tail = x[5]
			if head in folded:
				head = folded[head]
			if tail in folded:
				tail = folded[tail]
			if head not in current_amr_graph.node or tail not in current_amr_graph.node:
				line = amr_aligned.readline()
				continue
			label  = x[2]
			current_amr_graph.add_edge(head,tail,key=label,edge_label=label)
		elif line.strip() == "":
			span_sorted = sorted([(current_amr_graph.node[x]['span'][0],current_amr_graph.node[x]['span'][-1],
			x,current_amr_graph.node[x]['span']) for x in current_amr_graph.node.keys() if current_amr_graph.node[x]['span'][0] != -1 ])
			for each_edge in current_amr_graph.edges():
				lhs = each_edge[0]
				rhs = each_edge[1]
				lhs_name = current_amr_graph.node[lhs]['node_label']
				rhs_name = current_amr_graph.node[rhs]['node_label']
			#print current_amr_graph.node
			#print span_sorted
			sent_tok = current_amr_graph.graph['sent'].split()
			i = 0
			j = 0
			count = 1
			pos = pos_file.readline().strip().split()
			pos_t = [x.split('_')[1] for x in pos]
			extra = 0
			while (i <len(sent_tok)):
				if j == len(span_sorted):
					break
				if span_sorted[j][0] == i:
					node = span_sorted[j][2]
					node_label = current_amr_graph.node[node]['node_label']
					if node_label not in all_concept:
						all_concept[node_label] = concept_count
						concept_count += 1
					c_lab = all_concept[node_label]
					parent_info = current_amr_graph.pred[node]
					#If it is a root
					if len(parent_info) == 0:
						e_lab = all_relations["ROOT"]
						sys.stdout.write("{} {} {} {}:{}".format(0,e_lab,c_lab,0,"ROOT"))
						#print count,"\t",' '.join(sent_tok[i:span_sorted[j][1]+1]),"\t",POS,"\t",0,"\t","ROOT","\t",node_label
					else:
						# Write parent info
						last_col = ""
						for parent in parent_info.keys():
							#parent = parent_info.keys()[0]
							label = parent_info[parent].keys()[0]
							if label not in all_relations:
								all_relations[label] = relation_count
								relation_count += 1
							e_lab = all_relations[label]
							parent_pos = current_amr_graph.node[parent]['span'][0]
							last_col += "{}:{}::".format(current_amr_graph.node[parent]['node_label'
																						''],label)
							sys.stdout.write("{} {} {} ".format(parent_pos+1,e_lab,c_lab))
						sys.stdout.write(last_col)
						# Word
					sys.stdout.write("|w {} ".format(' '.join(sent_tok[i:span_sorted[j][1]+1])))
					sys.stdout.write("|p {}\n".format('_'.join(pos_t[i:span_sorted[j][1]+1])))

						#print p
					#print count,"\t",' '.join(sent_tok[i:span_sorted[j][1]+1]),"\t",POS,"\t",parent_pos+1+extra,"\t",label,"\t",node_label

					new_i = span_sorted[j][1] + 1
					j += 1
					if j == len(span_sorted):
						break
					if span_sorted[j][0] != i:
						i = new_i
					else:
						extra += 1
				else:
					e_lab = all_relations["NOEDGE"]
					c_lab = all_concept["NULL"]
					sys.stdout.write("{} {} {} {}:{}".format(0,e_lab,c_lab,0,"NOEDGE"))
					sys.stdout.write("|w {} ".format(sent_tok[i]))
					sys.stdout.write("|p {}\n".format(pos_t[i]))
					#print count,"\t",sent_tok[i],"\t",POS,"\t",0,"\t","NOEDGE","\t","NULL"
					i += 1
				count += 1
			print
			current_amr_graph = nx.MultiDiGraph()
			main_count += 1
			prev_node = ""
			prev_span = []
			folded = {}

		line = amr_aligned.readline()

	amr_aligned.close()
	with open("nodes_dict", 'w') as f:
		nodes_list = sorted([ (key,all_concept[key])for key in all_concept],key=lambda x:x[1])
		for each_node in nodes_list:
			f.write("{0}\t{1}\n".format(each_node[0],each_node[1]))

	with open("edges_dict", 'w') as f:
		edges_list = sorted([ (key,all_relations[key])for key in all_relations],key=lambda x:x[1])
		for each_node in edges_list:
			f.write("{0}\t{1}\n".format(each_node[0],each_node[1]))

if __name__ == "__main__":
	main(sys.argv[1:])
<|endoftext|>


<commit_before>import subprocess
import os
import json

class ExifTool(object):

    sentinel = "{ready}\n"

    def __init__(self, executable = os.chdir(os.path.abspath(__file__)[0:os.path.abspath(__file__).rfind('\\')]) + "exiftool.exe"):
        self.executable = executable

    def __enter__(self):
        self.process = subprocess.Popen(
            [self.executable,  "-@", "-"],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        return self

    def  __exit__(self, exc_type, exc_value, traceback):
        self.process.stdin.write("-stay_open\nFalse\n")
        self.process.stdin.flush()

    def execute(self, *args):
        args = args + ("-execute\n",)
        self.process.stdin.write(str.join("\n", args))
        self.process.stdin.flush()
        output = ""
        fd = self.process.stdout.fileno()
        while not output.endswith(self.sentinel):
            output += os.read(fd, 4096)
        return output[:-len(self.sentinel)]

    def get_metadata(self, *filenames):
        return json.loads(self.execute("-G", "-j", "-n", *filenames))

def getEXIFTags(file_name):
    #NOT WORKING!!!! call fuctions to get json Tags
    with ExifTool() as e:
        metadata = e.get_metadata(file_name)
    return metadata

def setEXIFTag(file_name, tag='comment', info='GIMP Python Script Created with '):
    #NOT WORKING!!! set tag and return true if successful
    from subprocess import call
    import os
    os.chdir(os.path.abspath(__file__)[0:os.path.abspath(__file__).rfind('\\')])
    return call('exiftool.exe -set' + file_name + ' ' + tag + ' ' + info)
<commit_msg>removed unneeded space<commit_after>import subprocess
import os
import json

class ExifTool(object):

    sentinel = "{ready}\n"

    def __init__(self, executable = os.chdir(os.path.abspath(__file__)[0:os.path.abspath(__file__).rfind('\\')]) + "exiftool.exe"):
        self.executable = executable

    def __enter__(self):
        self.process = subprocess.Popen(
            [self.executable,  "-@", "-"],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.process.stdin.write("-stay_open\nFalse\n")
        self.process.stdin.flush()

    def execute(self, *args):
        args = args + ("-execute\n",)
        self.process.stdin.write(str.join("\n", args))
        self.process.stdin.flush()
        output = ""
        fd = self.process.stdout.fileno()
        while not output.endswith(self.sentinel):
            output += os.read(fd, 4096)
        return output[:-len(self.sentinel)]

    def get_metadata(self, *filenames):
        return json.loads(self.execute("-G", "-j", "-n", *filenames))

def getEXIFTags(file_name):
    #NOT WORKING!!!! call fuctions to get json Tags
    with ExifTool() as e:
        metadata = e.get_metadata(file_name)
    return metadata

def setEXIFTag(file_name, tag='comment', info='GIMP Python Script Created with '):
    #NOT WORKING!!! set tag and return true if successful
    from subprocess import call
    import os
    os.chdir(os.path.abspath(__file__)[0:os.path.abspath(__file__).rfind('\\')])
    return call('exiftool.exe -set' + file_name + ' ' + tag + ' ' + info)
<|endoftext|>


<commit_before>from symbolic import SymbolicReference
import os
from git.objects import Object
from git.util import (
					LazyMixin,
					Iterable,
					)

from gitdb.util import (
							isfile,
							hex_to_bin
						)

__all__ = ["Reference"]

#{ Utilities
def require_remote_ref_path(func):
	"""A decorator raising a TypeError if we are not a valid remote, based on the path"""
	def wrapper(self, *args):
		if not self.path.startswith(self._remote_common_path_default + "/"):
			raise ValueError("ref path does not point to a remote reference: %s" % path)
		return func(self, *args)
	#END wrapper
	wrapper.__name__ = func.__name__
	return wrapper
#}END utilites

class Reference(SymbolicReference, LazyMixin, Iterable):
	"""Represents a named reference to any object. Subclasses may apply restrictions though,
	i.e. Heads can only point to commits."""
	__slots__ = tuple()
	_points_to_commits_only = False
	_resolve_ref_on_create = True
	_common_path_default = "refs"

	def __init__(self, repo, path, check_path = True):
		"""Initialize this instance
		:param repo: Our parent repository

		:param path:
			Path relative to the .git/ directory pointing to the ref in question, i.e.
			refs/heads/master
		:param check_path: if False, you can provide any path. Otherwise the path must start with the
			default path prefix of this type."""
		if check_path and not path.startswith(self._common_path_default+'/'):
			raise ValueError("Cannot instantiate %r from path %s" % (self.__class__.__name__, path))
		super(Reference, self).__init__(repo, path)

	def __str__(self):
		return self.name

	#{ Interface

	def set_object(self, object, logmsg = None):
		"""Special version which checks if the head-log needs an update as well"""
		oldbinsha = None
		if logmsg is not None:
			head = self.repo.head
			if not head.is_detached and head.ref == self:
				oldbinsha = self.commit.binsha
			#END handle commit retrieval
		#END handle message is set

		super(Reference, self).set_object(object, logmsg)

		if oldbinsha is not None:
			# /* from refs.c in git-source
			# * Special hack: If a branch is updated directly and HEAD
			# * points to it (may happen on the remote side of a push
			# * for example) then logically the HEAD reflog should be
			# * updated too.
			# * A generic solution implies reverse symref information,
			# * but finding all symrefs pointing to the given branch
			# * would be rather costly for this rare event (the direct
			# * update of a branch) to be worth it.  So let's cheat and
			# * check with HEAD only which should cover 99% of all usage
			# * scenarios (even 100% of the default ones).
			# */
			self.repo.head.log_append(oldbinsha, logmsg)
		#END check if the head

	# NOTE: Don't have to overwrite properties as the will only work without a the log

	@property
	def name(self):
		""":return: (shortest) Name of this reference - it may contain path components"""
		# first two path tokens are can be removed as they are
		# refs/heads or refs/tags or refs/remotes
		tokens = self.path.split('/')
		if len(tokens) < 3:
			return self.path		   # could be refs/HEAD
		return '/'.join(tokens[2:])

	@classmethod
	def iter_items(cls, repo, common_path = None):
		"""Equivalent to SymbolicReference.iter_items, but will return non-detached
		references as well."""
		return cls._iter_items(repo, common_path)

	#}END interface

	#{ Remote Interface

	@property
	@require_remote_ref_path
	def remote_name(self):
		"""
		:return:
			Name of the remote we are a reference of, such as 'origin' for a reference
			named 'origin/master'"""
		tokens = self.path.split('/')
		# /refs/remotes/<remote name>/<branch_name>
		return tokens[2]

	@property
	@require_remote_ref_path
	def remote_head(self):
		""":return: Name of the remote head itself, i.e. master.
		:note: The returned name is usually not qualified enough to uniquely identify
			a branch"""
		tokens = self.path.split('/')
		return '/'.join(tokens[3:])

	#} END remote interface
<commit_msg>Fixed NameError<commit_after>from symbolic import SymbolicReference
import os
from git.objects import Object
from git.util import (
					LazyMixin,
					Iterable,
					)

from gitdb.util import (
							isfile,
							hex_to_bin
						)

__all__ = ["Reference"]

#{ Utilities
def require_remote_ref_path(func):
	"""A decorator raising a TypeError if we are not a valid remote, based on the path"""
	def wrapper(self, *args):
		if not self.path.startswith(self._remote_common_path_default + "/"):
			raise ValueError("ref path does not point to a remote reference: %s" % self.path)
		return func(self, *args)
	#END wrapper
	wrapper.__name__ = func.__name__
	return wrapper
#}END utilites

class Reference(SymbolicReference, LazyMixin, Iterable):
	"""Represents a named reference to any object. Subclasses may apply restrictions though,
	i.e. Heads can only point to commits."""
	__slots__ = tuple()
	_points_to_commits_only = False
	_resolve_ref_on_create = True
	_common_path_default = "refs"

	def __init__(self, repo, path, check_path = True):
		"""Initialize this instance
		:param repo: Our parent repository

		:param path:
			Path relative to the .git/ directory pointing to the ref in question, i.e.
			refs/heads/master
		:param check_path: if False, you can provide any path. Otherwise the path must start with the
			default path prefix of this type."""
		if check_path and not path.startswith(self._common_path_default+'/'):
			raise ValueError("Cannot instantiate %r from path %s" % (self.__class__.__name__, path))
		super(Reference, self).__init__(repo, path)

	def __str__(self):
		return self.name

	#{ Interface

	def set_object(self, object, logmsg = None):
		"""Special version which checks if the head-log needs an update as well"""
		oldbinsha = None
		if logmsg is not None:
			head = self.repo.head
			if not head.is_detached and head.ref == self:
				oldbinsha = self.commit.binsha
			#END handle commit retrieval
		#END handle message is set

		super(Reference, self).set_object(object, logmsg)

		if oldbinsha is not None:
			# /* from refs.c in git-source
			# * Special hack: If a branch is updated directly and HEAD
			# * points to it (may happen on the remote side of a push
			# * for example) then logically the HEAD reflog should be
			# * updated too.
			# * A generic solution implies reverse symref information,
			# * but finding all symrefs pointing to the given branch
			# * would be rather costly for this rare event (the direct
			# * update of a branch) to be worth it.  So let's cheat and
			# * check with HEAD only which should cover 99% of all usage
			# * scenarios (even 100% of the default ones).
			# */
			self.repo.head.log_append(oldbinsha, logmsg)
		#END check if the head

	# NOTE: Don't have to overwrite properties as the will only work without a the log

	@property
	def name(self):
		""":return: (shortest) Name of this reference - it may contain path components"""
		# first two path tokens are can be removed as they are
		# refs/heads or refs/tags or refs/remotes
		tokens = self.path.split('/')
		if len(tokens) < 3:
			return self.path		   # could be refs/HEAD
		return '/'.join(tokens[2:])

	@classmethod
	def iter_items(cls, repo, common_path = None):
		"""Equivalent to SymbolicReference.iter_items, but will return non-detached
		references as well."""
		return cls._iter_items(repo, common_path)

	#}END interface

	#{ Remote Interface

	@property
	@require_remote_ref_path
	def remote_name(self):
		"""
		:return:
			Name of the remote we are a reference of, such as 'origin' for a reference
			named 'origin/master'"""
		tokens = self.path.split('/')
		# /refs/remotes/<remote name>/<branch_name>
		return tokens[2]

	@property
	@require_remote_ref_path
	def remote_head(self):
		""":return: Name of the remote head itself, i.e. master.
		:note: The returned name is usually not qualified enough to uniquely identify
			a branch"""
		tokens = self.path.split('/')
		return '/'.join(tokens[3:])

	#} END remote interface
<|endoftext|>