<commit_before>import os
import requests
import json

import sys
PY2 = sys.version_info[0] == 2

from django.core.management.base import BaseCommand

# URL = 'https://django.datamodeling.online'
URL = 'http://localhost:5001'

class Command(BaseCommand):
    help = 'Export as a diagram to django.datamodeling.online.'

    def add_arguments(self, parser):
        parser.add_argument('app_label', type=str)
        parser.add_argument(
            '--name', dest='diagram_name', default=None,
            help="Name of the imported diagram.",
        )

    def handle(self, *args, **options):
        app_label = options['app_label']
        diagram = make_diagram(app_label)
        diagram = json.dumps(diagram)
        if os.environ.get('DEBUG'):
            with open('dia.json', 'w') as f:
                f.write(diagram)
        import uuid
        path = uuid.uuid4().hex
        # upload it
        upload_url = f'{URL}/import/upload/{path}'
        r = requests.post(upload_url, data=diagram)
        if r.status_code != 200:
            self.stderr.write(f'{r.status_code} from {upload_url}')
            return
        diagram_name = options['diagram_name'] or app_label
        link = f'{URL}/import/{diagram_name}/{path}'
        self.stdout.write(f'\nTo finish import please open {link} in your browser.')

OPTIONS = (
    'verbose_name', 'verbose_name_plural', 'db_table', 'ordering',
    'unique_together', 'get_latest_by', 'order_with_respect_to',
    'app_label', 'db_tablespace', 'abstract', 'managed',
    'auto_created', 'index_together', 'default_permissions',
    'default_related_name', 'required_db_features',
    'required_db_vendor', 'base_manager_name', 'default_manager_name',
)

from math import ceil, sqrt

class Placing:

    SPACE_HEIGHT = 30
    SPACE_WIDTH = 20

    def __init__(self, entities):
        self.entities = entities

    def apply(self):
        size = int(ceil(sqrt(len(self.entities))))
        top = self.SPACE_HEIGHT
        height = 0
        for row in range(size):
            left = self.SPACE_WIDTH
            for col in range(size):
                index = row * size + col
                try:
                    entity = self.entities[index]
                    h = self.get_height(entity)
                    if h > height:
                        height = h
                except IndexError:
                    return
                w = self.get_width(entity)
                entity.update({
                    'left': left, 'top': top, 'width': round(w), 'height': round(h),
                })
                left += w + self.SPACE_WIDTH
            top += height + self.SPACE_HEIGHT

    def get_height(self, entity):
        rows_count = len(entity['attributes'])
        return 68 + 22.2 * rows_count

    def get_width(self, entity):
        return 250

def make_diagram(app_label):
    from django.apps import apps
    app_config = apps.get_app_config(app_label)
    entities = []
    for M in app_config.get_models():
        dic = to_dict(M)
        dic['name'] = M.__name__
        options = {}
        for name in OPTIONS:
            value = getattr(M._meta, name)
            if value is None:
                continue
            if hasattr(value, '_proxy____cast'):
                value = value._proxy____cast()
            if name == 'verbose_name':
                # This restriction is on the frontend
                value = M.__name__
            elif name == 'verbose_name_plural':
                value = value.capitalize()
            options[name] = value
            options['indexes'] = []
            for index in M._meta.indexes:
                index_dic = {
                    'fields': index.fields,
                }
                if index.name:
                    index_dic['name'] = index.name
                options['indexes'].append(index_dic)
        dic.update(options)
        entities.append(dic)

    placing = Placing(entities)
    placing.apply()
    dic = {
        'entities': entities,
        'state': {
            'scrollTop': 0,
            'scrollLeft': 0,
        },
    }
    if entities:
        dic['state'].update({
            'scrollTop': entities[0]['top'],
            'scrollLeft': entities[0]['left'],
        })
    return dic

def to_dict(model):
    def keyfunc(f):
        if hasattr(f, 'field'):
            f = f.field
        return f.creation_counter
    fields = sorted(model._meta.get_fields(), key=keyfunc)
    attrs = []

    parent = None
    parent_fields = []
    for parent in model._meta.parents:
        parent_fields = parent._meta.get_fields()
    for field in fields:
        if not hasattr(field, 'deconstruct'):
            continue
        if field in parent_fields:
            continue
        attr = Field.to_dict(field)
        if attr:
            attrs.append(attr)

    return {
        'name': model.__name__,
        'attributes': attrs,
        'inheritsFrom': parent and parent.__name__,
    }

import inspect

class Field:

    @classmethod
    def to_dict(cls, field):
        name, callable, args, kwargs = field.deconstruct()
        kwargs['name'] = name
        kwargs['type'] = callable.split('.')[-1]
        assert not args
        if kwargs['type'] == 'AutoField' and name == 'id':
            return None
        kwargs = cls.transform(kwargs, field)
        if kwargs is None:
            return None
        for k, v in tuple(kwargs.items()):
            if inspect.isfunction(v) and v.__name__.isupper():
                kwargs[k] = v.__name__
        kwargs.setdefault('objectType', 'attribute')
        return kwargs

    @classmethod
    def transform(cls, dic, field):
        if 'to' in dic:
            dic['to'] = dic['to'].split('.')[-1]
            dic['related_name'] = field.rel.name
            dic['objectType'] = 'attributeRelationship'
            to_app = field.rel.to._meta.app_label
            if to_app != field.model._meta.app_label:
                dic['to'] = f"{to_app}.{dic['to']}"
        if dic.get('parent_link'):
            return None
        return dic
<commit_msg>replace localhost<commit_after>import os
import requests
import json

import sys
PY2 = sys.version_info[0] == 2

from django.core.management.base import BaseCommand

URL = 'https://django.datamodeling.online'
# URL = 'http://localhost:5001'

class Command(BaseCommand):
    help = 'Export as a diagram to django.datamodeling.online.'

    def add_arguments(self, parser):
        parser.add_argument('app_label', type=str)
        parser.add_argument(
            '--name', dest='diagram_name', default=None,
            help="Name of the imported diagram.",
        )

    def handle(self, *args, **options):
        app_label = options['app_label']
        diagram = make_diagram(app_label)
        diagram = json.dumps(diagram)
        if os.environ.get('DEBUG'):
            with open('dia.json', 'w') as f:
                f.write(diagram)
        import uuid
        path = uuid.uuid4().hex
        # upload it
        upload_url = f'{URL}/import/upload/{path}'
        r = requests.post(upload_url, data=diagram)
        if r.status_code != 200:
            self.stderr.write(f'{r.status_code} from {upload_url}')
            return
        diagram_name = options['diagram_name'] or app_label
        link = f'{URL}/import/{diagram_name}/{path}'
        self.stdout.write(f'\nTo finish import please open {link} in your browser.')

OPTIONS = (
    'verbose_name', 'verbose_name_plural', 'db_table', 'ordering',
    'unique_together', 'get_latest_by', 'order_with_respect_to',
    'app_label', 'db_tablespace', 'abstract', 'managed',
    'auto_created', 'index_together', 'default_permissions',
    'default_related_name', 'required_db_features',
    'required_db_vendor', 'base_manager_name', 'default_manager_name',
)

from math import ceil, sqrt

class Placing:

    SPACE_HEIGHT = 30
    SPACE_WIDTH = 20

    def __init__(self, entities):
        self.entities = entities

    def apply(self):
        size = int(ceil(sqrt(len(self.entities))))
        top = self.SPACE_HEIGHT
        height = 0
        for row in range(size):
            left = self.SPACE_WIDTH
            for col in range(size):
                index = row * size + col
                try:
                    entity = self.entities[index]
                    h = self.get_height(entity)
                    if h > height:
                        height = h
                except IndexError:
                    return
                w = self.get_width(entity)
                entity.update({
                    'left': left, 'top': top, 'width': round(w), 'height': round(h),
                })
                left += w + self.SPACE_WIDTH
            top += height + self.SPACE_HEIGHT

    def get_height(self, entity):
        rows_count = len(entity['attributes'])
        return 68 + 22.2 * rows_count

    def get_width(self, entity):
        return 250

def make_diagram(app_label):
    from django.apps import apps
    app_config = apps.get_app_config(app_label)
    entities = []
    for M in app_config.get_models():
        dic = to_dict(M)
        dic['name'] = M.__name__
        options = {}
        for name in OPTIONS:
            value = getattr(M._meta, name)
            if value is None:
                continue
            if hasattr(value, '_proxy____cast'):
                value = value._proxy____cast()
            if name == 'verbose_name':
                # This restriction is on the frontend
                value = M.__name__
            elif name == 'verbose_name_plural':
                value = value.capitalize()
            options[name] = value
            options['indexes'] = []
            for index in M._meta.indexes:
                index_dic = {
                    'fields': index.fields,
                }
                if index.name:
                    index_dic['name'] = index.name
                options['indexes'].append(index_dic)
        dic.update(options)
        entities.append(dic)

    placing = Placing(entities)
    placing.apply()
    dic = {
        'entities': entities,
        'state': {
            'scrollTop': 0,
            'scrollLeft': 0,
        },
    }
    if entities:
        dic['state'].update({
            'scrollTop': entities[0]['top'],
            'scrollLeft': entities[0]['left'],
        })
    return dic

def to_dict(model):
    def keyfunc(f):
        if hasattr(f, 'field'):
            f = f.field
        return f.creation_counter
    fields = sorted(model._meta.get_fields(), key=keyfunc)
    attrs = []

    parent = None
    parent_fields = []
    for parent in model._meta.parents:
        parent_fields = parent._meta.get_fields()
    for field in fields:
        if not hasattr(field, 'deconstruct'):
            continue
        if field in parent_fields:
            continue
        attr = Field.to_dict(field)
        if attr:
            attrs.append(attr)

    return {
        'name': model.__name__,
        'attributes': attrs,
        'inheritsFrom': parent and parent.__name__,
    }

import inspect

class Field:

    @classmethod
    def to_dict(cls, field):
        name, callable, args, kwargs = field.deconstruct()
        kwargs['name'] = name
        kwargs['type'] = callable.split('.')[-1]
        assert not args
        if kwargs['type'] == 'AutoField' and name == 'id':
            return None
        kwargs = cls.transform(kwargs, field)
        if kwargs is None:
            return None
        for k, v in tuple(kwargs.items()):
            if inspect.isfunction(v) and v.__name__.isupper():
                kwargs[k] = v.__name__
        kwargs.setdefault('objectType', 'attribute')
        return kwargs

    @classmethod
    def transform(cls, dic, field):
        if 'to' in dic:
            dic['to'] = dic['to'].split('.')[-1]
            dic['related_name'] = field.rel.name
            dic['objectType'] = 'attributeRelationship'
            to_app = field.rel.to._meta.app_label
            if to_app != field.model._meta.app_label:
                dic['to'] = f"{to_app}.{dic['to']}"
        if dic.get('parent_link'):
            return None
        return dic
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from models import area
import core.command
from core.localisation import _

class move(core.command.command):
	"""
	Move command
	"""

	def run(self):
		"""
		c.run()

		Move the player in the desired direction.
		"""
		if len(self._args) == 0:
			raise core.command.exception(_('ERROR_MOVE_NO_DIRECTION_GIVEN'))

		direction = self._args[0]
		if direction not in area.area.getDirections():
			raise core.command.exception(_('ERROR_MOVE_INVALID_DIRECTION_%s') % direction)
		curAreaId = self._player.getAreaId()
		curArea = area.model.loadById(curAreaId)

		a = area.area.getNeighbourFromDirection(curAreaId, direction)

		if area.area.canGoTo(curArea['directions'], direction) is False or a is None:
			raise core.command.exception(_('ERROR_MOVE_DIRECTION_NOT_AVAILABLE'))
		else:
			self._player.goTo(a._model['id_area'])

		return {'direction': direction}

	def render(self, data):
		return _('MOVE_CONFIRMATION_%s') % data['direction']
<commit_msg>After a player moves in an area, get a random enemy to fight<commit_after># -*- coding: utf-8 -*-

from models import area, creature
import core.command
from core.localisation import _

class move(core.command.command):
	"""
	Move command
	"""

	def run(self):
		"""
		c.run()

		Move the player in the desired direction.
		"""
		if len(self._args) == 0:
			raise core.command.exception(_('ERROR_MOVE_NO_DIRECTION_GIVEN'))

		direction = self._args[0]
		if direction not in area.area.getDirections():
			raise core.command.exception(_('ERROR_MOVE_INVALID_DIRECTION_%s') % direction)
		curAreaId = self._player.getAreaId()
		curArea = area.model.loadById(curAreaId)

		a = area.area.getNeighbourFromDirection(curAreaId, direction)

		if area.area.canGoTo(curArea['directions'], direction) is False or a is None:
			raise core.command.exception(_('ERROR_MOVE_DIRECTION_NOT_AVAILABLE'))
		else:
			self._player.goTo(a._model['id_area'])
			hasCreature = area.area.hasCreature(a)
			if hasCreature:
				enemy = creature.creature.getFromAreaType(
					a._model['id_area_type']
				)

		return {'direction': direction}

	def render(self, data):
		return _('MOVE_CONFIRMATION_%s') % data['direction']
<|endoftext|>


<commit_before>import packages.project as project
import packages.primitive as primitive
import packages.utils as utils
import packages.processing
import packages.io
import argparse
from matplotlib import pyplot as plt
import matplotlib.mlab as mlab
import numpy as np
from scipy.stats import norm,kstest,skewtest,kurtosistest,normaltest
import os.path
import unitTests.relations as test_relations
import packages.relationGraph as relgraph
import packages.polarPlot as polarplot
import math

compareToGt = False

itmin=0
itmax=20

## Load a mesh as GT (must have correct normals), the input point cloud,
## and the reconstruction output: primitives + assignment.
##
##   1. Compute the cloud to GT assignment
##      We now have two assigments per points: GT and estimated
##   2. For each point, compute a distance between GT and estimated normal
##   3. Analyse the resulting distance distribution:
##        - variance and mean
##
##
##
################################################################################
## Command line parsing
parser = argparse.ArgumentParser(description='Compute reconstruction error in normal space.')
parser.add_argument('gtmesh')
parser.add_argument('cloud')
parser.add_argument('primitives')
parser.add_argument('assignment')

args = parser.parse_args()

<commit_msg>Add some stat computations<commit_after>import packages.project as project
import packages.primitive as primitive
import packages.utils as utils
import packages.processing
import packages.io
import argparse
from matplotlib import pyplot as plt
import matplotlib.mlab as mlab
import numpy as np
from scipy.stats import norm,kstest,skewtest,kurtosistest,normaltest
import os.path
import unitTests.relations as test_relations
import packages.relationGraph as relgraph
import packages.polarPlot as polarplot
import math

compareToGt = False

itmin=0
itmax=20

## Load a mesh as GT (must have correct normals), the input point cloud,
## and the reconstruction output: primitives + assignment.
##
##   1. Compute the cloud to GT assignment
##      We now have two assigments per points: GT and estimated
##   2. For each point, compute a distance between GT and estimated normal
##   3. Analyse the resulting distance distribution:
##        - variance and mean
##
##
##
################################################################################
## Command line parsing
parser = argparse.ArgumentParser(description='Compute reconstruction error in normal space.')
parser.add_argument('angles')

args = parser.parse_args()

anglesFile  = args.angles

angles = np.array( [np.float32(line.strip()) for line in open(anglesFile)] )

print angles
print "mean   = ", np.mean(angles)
print "var    = ", np.var(angles)
print "median = ", np.median(angles)

<|endoftext|>


<commit_before>import random

# Make a list of allowed parameter values.
def linearRange(start, stop, step):
  pval = start
  plist = [pval]
  while pval < stop:
    pval = pval + step
    plist.append(pval)
  return plist

class Parameters(object):
  '''
  This class provides methods for searching ranges of parameters to see how
  they affect performance.
  '''
  def __init__(self):
    '''
    Have to keep track of the names and valid values of each parameter
    defined by the user.
    '''
    # list of parameter names
    self._names = []

    # list of allowed parameter values
    self._allowedValues = []

    # list of past and present parameter values
    self._values = []

    # list of past and present results that correspond to each set of parameter
    # values
    self._results = []

    # the number of possible combinations of parameter values for all parameters
    self.numCombinations = 1

  def define(self, name, allowedValues):
    '''
    This method allows users to define a parameter by providing its name and
    a list of values for the parameter.
    '''
    if name not in self._names:
      self._names.append(name)
      self._allowedValues.append(allowedValues)
      self.numCombinations = self.numCombinations * len(allowedValues)
    else:
      print "Parameter: ", name, " is already defined!"

  def getNames(self):
    '''
    This method returns the names of all defined parameters.
    '''
    return self._names

  def getValue(self, name):
    '''
    This method returns the current value of the parameter specified by name.
    '''
    assert name in self._names
    i = self._names.index(name)
    assert len(self._values[-1]) > i
    return self._values[-1][i]

  def getAllValues(self):
    '''
    This method returns the current values of all defined parameters.
    '''
    return self._values[-1]

  def appendResults(self,item):
    '''
    This method adds an item to the results list.
    '''
    print "Just completed parameter Combination: ", self.getAllValues()
    self._results.append(item)
    print
    print "Parameter combinations completed: ",
    print len(self._results), "/", self.numCombinations
    print

  def getNumResults(self):
    '''
    This method returns the number of items in the results list.
    '''
    return len(self._results)

  def printResults(self, resultNames, formatStrings):
    '''
    This method prints a summary of all the results.
    '''
    print
    print "Summary of Results"
    print
    headerList = self.getNames()
    headerList.extend(resultNames)
    headerString = ", ".join(headerList)
    print headerString
    for i, result in enumerate(self._results):
      valueString = str(self._values[i])[1:-1]
      for j,formatString in enumerate(formatStrings):
        valueString += formatString % result[j]
      print valueString

  def nextRandomCombination(self):
    '''
    This method randomly selects a value for each parameter from its list of
    allowed parameter values.  If the resulting combination has already been
    used then it tries again.
    '''
    random_combination = [random.choice(self._allowedValues[i])
      for i in range(len(self._names))]
    if random_combination in self._values:
      self.nextRandomCombination()
    else:
      self._values.append(random_combination)
      print "Parameter Combination: ", self.getAllValues()
      print

  def nextCombination(self):
    '''
    This method finds the next combination of parameters values using the
    allowed value lists for each parameter.
    '''
    if len(self._values) == 0:
      # list of values is empty so this is the first combination,
      # each parameter gets the first value in its list of allowed values
      self._values.append([self._allowedValues[i][0]
        for i in range(len(self._names))])
    else:
      self._values.append(self._values[-1])
      i = 0
      while i < len(self._names):
        # if current value is not the last in the list
        if self._values[-1][i] != self._allowedValues[i][-1]:
          # change parameter to next value in allowed value list and return
          index = self._allowedValues[i].index(self._values[-1][i])
          self._values[-1][i] = self._allowedValues[i][index + 1]
          i = len(self._names)
        else:
          # change parameter to first value in allowed value list
          self._values[-1][i] = self._allowedValues[i][0]
          # move next parameter to next value in its allowed value list
          i = i + 1

    print "Parameter Combination: ", self.getAllValues()
    print

<commit_msg>Changed this module to store historical value indexes for each parameter instead of the actual values.  This is more efficient, but more importantly it fixed a bug where copying the last set of values when appending to the list resulted in pass by reference instead of pass by value which destroyed the value history of some parameters.<commit_after>import random

# Make a list of allowed parameter values.
def linearRange(start, stop, step):
  pval = start
  plist = [pval]
  while pval < stop:
    pval = pval + step
    plist.append(pval)
  return plist

class Parameters(object):
  '''
  This class provides methods for searching ranges of parameters to see how
  they affect performance.
  '''
  def __init__(self):
    '''
    Have to keep track of the names and valid values of each parameter
    defined by the user.
    '''
    # list of parameter names
    self._names = []

    # list of allowed parameter values
    self._allowedValues = []

    # list of past and present parameter value indexes
    self._valueIndexes = []

    # list of past and present results that correspond to each set of parameter
    # values
    self._results = []

    # the number of possible combinations of parameter values for all parameters
    self.numCombinations = 1

  def define(self, name, allowedValues):
    '''
    This method allows users to define a parameter by providing its name and
    a list of values for the parameter.
    '''
    if name not in self._names:
      self._names.append(name)
      self._allowedValues.append(allowedValues)
      self.numCombinations = self.numCombinations * len(allowedValues)
    else:
      print "Parameter: ", name, " is already defined!"

  def getNames(self):
    '''
    This method returns the names of all defined parameters.
    '''
    return self._names

  def getValue(self, name):
    '''
    This method returns the current value of the parameter specified by name.
    '''
    assert name in self._names
    i = self._names.index(name)
    assert len(self._valueIndexes[-1]) > i
    return self._allowedValues[i][self._valueIndexes[-1][i]]

  def getAllValues(self):
    '''
    This method returns the current values of all defined parameters.
    '''
    return [self._allowedValues[i][j] for i,j in
      enumerate(self._valueIndexes[-1])]

  def appendResults(self,item):
    '''
    This method adds an item to the results list.
    '''
    print "Just completed parameter Combination: ", self.getAllValues()
    self._results.append(item)
    print
    print "Parameter combinations completed: ",
    print len(self._results), "/", self.numCombinations
    print

  def getNumResults(self):
    '''
    This method returns the number of items in the results list.
    '''
    return len(self._results)

  def printResults(self, resultNames, formatStrings):
    '''
    This method prints a summary of all the results.
    '''
    print
    print "Summary of Results"
    print
    headerList = self.getNames()
    headerList.extend(resultNames)
    headerString = ", ".join(headerList)
    print headerString
    for i, result in enumerate(self._results):
      valueString = str([self._allowedValues[j][k] for j,k in
        enumerate(self._valueIndexes[i])])[1:-1]
      for j,formatString in enumerate(formatStrings):
        valueString += formatString % result[j]
      print valueString

  def nextRandomCombination(self):
    '''
    This method randomly selects a value for each parameter from its list of
    allowed parameter values.  If the resulting combination has already been
    used then it tries again.
    '''
    random_combination = [random.choice(self._allowedValues[i])
      for i in range(len(self._names))]
    if random_combination in self._values:
      self.nextRandomCombination()
    else:
      self._values.append(random_combination)
      print "Parameter Combination: ", self.getAllValues()
      print

  def nextCombination(self):
    '''
    This method finds the next combination of parameter values using the
    allowed value lists for each parameter.
    '''
    if len(self._valueIndexes) == 0:
      # list of value indexes is empty so this is the first combination,
      # each parameter gets the first value in its list of allowed values
      self._valueIndexes.append([0 for i in range(len(self._names))])
    else:
      newValueIndexes = self._valueIndexes[-1][:]
      i = 0
      while i < len(self._names):
        # if current value is not the last in the list
        if self._valueIndexes[-1][i] != len(self._allowedValues[i]) - 1:
          # change parameter to next value in allowed value list and return
          newValueIndexes[i] += 1
          break
        else:
          # change parameter to first value in allowed value list
          newValueIndexes[i] = 0
          # move next parameter to next value in its allowed value list
          i = i + 1
      self._valueIndexes.append(newValueIndexes)

    print "Parameter Combination: ", self.getAllValues()
    print

<|endoftext|>


<commit_before>random_seed = 3 # fix random seed for reproducibility
number_of_experiment_words = 8

meaningless_token = 'VOID'
meaningless_token_frequency = 0.005 # frequency before the replacement procedure

experiment_word_occurrence_min = 10000

word_freq_experiment_ratio = 1./2
word_freq_experiment_power_max = 20

coocc_noise_experiment_max_value = 7
coocc_noise_experiment_freq_reduction = 2. / (coocc_noise_experiment_max_value + 1)
<commit_msg>fix parameters to give full span of coocc noise proportions<commit_after>random_seed = 3 # fix random seed for reproducibility
number_of_experiment_words = 8

meaningless_token = 'VOID'
meaningless_token_frequency = 0.005 # frequency before the replacement procedure

experiment_word_occurrence_min = 10000

word_freq_experiment_ratio = 1./2
word_freq_experiment_power_max = 20

coocc_noise_experiment_max_value = 7
coocc_noise_experiment_freq_reduction = 2. / coocc_noise_experiment_max_value
<|endoftext|>


<commit_before># -*-coding:Utf-8 -*

# Copyright (c) 2010 LE GOFF Vincent
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# * Neither the name of the copyright holder nor the names of its contributors
#   may be used to endorse or promote products derived from this software
#   without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Ce fichier définit la classe BaseObj définie plus bas."""

import time

class BaseObj:
    """Cette classe définit la base d'un objet destiné à être enregistré,
    directement ou indirectement dans un fichier.

    Rappelons que :
    *   Les objets destinés à être DIRECTEMENT enregistrés dans des fichiers
        doivent être hérités de 'ObjetID' (voir abstraits/id/__init__.py)
        La classe ObjetID hérite elle-même de BaseObj.
    *   Les objets destinés à être INDIRECTEMENT enregistrés dans des fichiers
        doivent être hérités de BaseObj.
        Ces objets sont ceux destinés à être enregistrés dans des fichiers
        sous la forme d'attributs d'autres objets par exemple.

    On se base sur un dictionnaire représentant les attributs, lui-même étant
    un attribut de classe :
    -   les clés sont constituées des noms des attributs
    -   les valeurs sont les valeurs par défaut des attributs correspondants

    Exemple d'utilisation :
    >>> from abstraits.obase import BaseObj
    >>> dic_attributs = {
    ...     "nom": "inconnu",
    ...     "race": RACE_INCONNU,
    ...     "talents": [],
    ... }
    >>> class Joueur(BaseObj):
    ...     attributs = dic_attributs

    Ce dictionnaire est utilisé :
    *   A la création du personnage : on va tout simplement copier
        chaque attribut et valeur par défaut dans l'objet à créer
    *   A la récupération de l'objet (méthode '__setstate__') :
        On écrit dans ce cas les valeurs par défaut des attributs
        sans écraser ceux déjà présents dans l'objet
        Ainsi, on peut rajouter des attributs dans des objets d'une session du
        module à l'autre.

    """
    attributs = {} # dictionnaire des attributs

    # Dictionnaires à NE PAS redéfinir dans les sous-classes :
    trace_p_ids = {}

    def __init__(self):
        """Constructeur. On copie tous les attributs dans self"""
        self.__dict__.update(type(self).attributs)
        self.p_id = id(self)

    def __getstate__(self):
        """Au moment de l'enregistrement, on met à jour le timestamp"""
        self._ts = time.time()
        return self.__dict__

    def __setstate__(self, dico_attrs):
        """Méthode appelée lors de la désérialisation de l'objet"""
        attributs = dict(type(self).attributs)
        attributs.update(dico_attrs)
        self.__dict__.update(attributs)
        # Si l'objet est déjà tracé dans trace_p_ids, on réc upère son __dict__
        # Sinon, on ajoute son __dict__ dans trace_p_ids
        # Note importante: à chaque fois qu'un objet est sauvegardé, il
        # enregistre le timestamp de cette sauvegarde.
        # Si on trouve une référence vers le même objet mais dont le timestamp
        # de son enregistrement est supérieur à celui enregistré dans
        # trace_p_ids, c'est le plus récent qui est conservé.
        # Pour résumer, on considère que c'est le dernier objet enregistré le
        # plus à jour.
        if self.p_id in type(self).trace_p_ids.keys():
            if self._ts > type(self).trace_p_ids[self.p_id]._ts:
                # self est plus récent que celui de trace_p_ids
                type(self).trace_p_ids[self.p_id].__dict__.update( \
                        self.__dict__)
            self.__dict__ = type(self).trace_p_ids[self.p_id].__dict__
        else:
            type(self).trace_p_ids[self.p_id] = self
            self.p_id = id(self)
<commit_msg>Ajout de code pour utiliser 'BaseObj' dans des cas d'héritage multiple :     Si on définit une classe A, héritée de BaseObj et une classe B, héritée de     A, le dictionnaire d'attribut de B sera un regroupement de son propre     dictionnaire d'attribut et du dictionnaire d'attribut de A<commit_after># -*-coding:Utf-8 -*

# Copyright (c) 2010 LE GOFF Vincent
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# * Neither the name of the copyright holder nor the names of its contributors
#   may be used to endorse or promote products derived from this software
#   without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Ce fichier définit la classe BaseObj définie plus bas."""

import time

class BaseObj:
    """Cette classe définit la base d'un objet destiné à être enregistré,
    directement ou indirectement dans un fichier.

    Rappelons que :
    *   Les objets destinés à être DIRECTEMENT enregistrés dans des fichiers
        doivent être hérités de 'ObjetID' (voir abstraits/id/__init__.py)
        La classe ObjetID hérite elle-même de BaseObj.
    *   Les objets destinés à être INDIRECTEMENT enregistrés dans des fichiers
        doivent être hérités de BaseObj.
        Ces objets sont ceux destinés à être enregistrés dans des fichiers
        sous la forme d'attributs d'autres objets par exemple.

    On se base sur un dictionnaire représentant les attributs, lui-même étant
    un attribut de classe :
    -   les clés sont constituées des noms des attributs
    -   les valeurs sont les valeurs par défaut des attributs correspondants

    Exemple d'utilisation :
    >>> from abstraits.obase import BaseObj
    >>> dic_attributs = {
    ...     "nom": "inconnu",
    ...     "race": RACE_INCONNU,
    ...     "talents": [],
    ... }
    >>> class Joueur(BaseObj):
    ...     attributs = dic_attributs

    Ce dictionnaire est utilisé :
    *   A la création du personnage : on va tout simplement copier
        chaque attribut et valeur par défaut dans l'objet à créer
    *   A la récupération de l'objet (méthode '__setstate__') :
        On écrit dans ce cas les valeurs par défaut des attributs
        sans écraser ceux déjà présents dans l'objet
        Ainsi, on peut rajouter des attributs dans des objets d'une session du
        module à l'autre.

    """
    attributs = {} # dictionnaire des attributs

    # Dictionnaires à NE PAS redéfinir dans les sous-classes :
    trace_p_ids = {}

    def __init__(self):
        """Constructeur. On copie tous les attributs dans self"""
        attributs = dict(type(self).attributs)
        # On parcourt les descendants éventuels de la classe
        # Ainsi, on ajoute au dictionnaire des attributs les attributs des
        # objets-parents
        for classe in type(self).__bases:
            if hasattr(self, "attributs"):
                n_attributs = dict(classe.attributs)
                attributs = n_attributs.update(attributs)
        self.__dict__.update(attributs)
        self.p_id = id(self)

    def __getstate__(self):
        """Au moment de l'enregistrement, on met à jour le timestamp"""
        self._ts = time.time()
        return self.__dict__

    def __setstate__(self, dico_attrs):
        """Méthode appelée lors de la désérialisation de l'objet"""
        attributs = dict(type(self).attributs)
        attributs.update(dico_attrs)
        self.__dict__.update(attributs)
        # Si l'objet est déjà tracé dans trace_p_ids, on réc upère son __dict__
        # Sinon, on ajoute son __dict__ dans trace_p_ids
        # Note importante: à chaque fois qu'un objet est sauvegardé, il
        # enregistre le timestamp de cette sauvegarde.
        # Si on trouve une référence vers le même objet mais dont le timestamp
        # de son enregistrement est supérieur à celui enregistré dans
        # trace_p_ids, c'est le plus récent qui est conservé.
        # Pour résumer, on considère que c'est le dernier objet enregistré le
        # plus à jour.
        if self.p_id in type(self).trace_p_ids.keys():
            if self._ts > type(self).trace_p_ids[self.p_id]._ts:
                # self est plus récent que celui de trace_p_ids
                type(self).trace_p_ids[self.p_id].__dict__.update( \
                        self.__dict__)
            self.__dict__ = type(self).trace_p_ids[self.p_id].__dict__
        else:
            type(self).trace_p_ids[self.p_id] = self
            self.p_id = id(self)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""paved.util -- helper functions.
"""
import re
from paver.easy import info, options, path, dry, sh

try:
    if not options.virtualenv.activate_cmd:
        raise AttributeError
except AttributeError:
    options.setdotted('virtualenv.activate_cmd', 'source ' + options.paved.cwd / 'source' / 'bin' / 'activate')

from . import paved

__all__ = ['rmFilePatterns', 'rmDirPatterns', 'shv', 'update']

def _walkWithAction(*patterns, **kwargs):
    use_path = path(kwargs.get('use_path', options.paved.cwd))
    action = kwargs.pop('action')
    assert type(action) is str
    assert hasattr(use_path, action)

    use_regex = kwargs.get('use_regex')
    errors = kwargs.get('errors', 'warn')
    walk_method = kwargs.get('walk_method', 'walkfiles')

    for p in patterns:
        info("Looking for %s" % p)
        if use_regex:
            cp = re.compile(p)
            p = None
        for f in getattr(use_path, walk_method)(pattern=p, errors=errors):
            if use_regex and not cp.search(str(f)):
                continue
            if f.exists():
                msg = "%s %s..." % (action, f)
                dry(msg, getattr(f, action))

def rmFilePatterns(*patterns, **kwargs):
    """Remove all files under the given path with the given patterns.
    """
    kwargs['action'] = 'remove'
    kwargs['walk_method'] = 'walkfiles'
    return _walkWithAction(*patterns, **kwargs)

def rmDirPatterns(*patterns, **kwargs):
    """Remove all directories under the current path with the given patterns.
    """
    kwargs['action'] = 'rmtree'
    kwargs['walk_method'] = 'walkdirs'
    return _walkWithAction(*patterns, **kwargs)

def shv(command, capture=False, ignore_error=False, cwd=None):
    """Run the given command inside the virtual environment, if available:
    """
    try:
        command = "%s; %s" % (options.virtualenv.activate_cmd, command)
    except AttributeError:
        pass
    return sh(command, capture=capture, ignore_error=ignore_error, cwd=cwd)

def update(dst, src):
    """Recursively update the destination dict-like object with the source dict-like object.

    Useful for merging options and Bunches together!

    Based on:
    http://code.activestate.com/recipes/499335-recursively-update-a-dictionary-without-hitting-py/#c1
    """
    stack = [(dst, src)]

    def isdict(o):
        return hasattr(o, '__setitem__')

    while stack:
        current_dst, current_src = stack.pop()
        for key in current_src:
            if key not in current_dst:
                current_dst[key] = current_src[key]
            else:
                if isdict(current_src[key]) and isdict(current_dst[key]):
                    stack.append((current_dst[key], current_src[key]))
                else:
                    current_dst[key] = current_src[key]
    return dst
<commit_msg>Fixed dict-like detection.<commit_after># -*- coding: utf-8 -*-
"""paved.util -- helper functions.
"""
import re
from paver.easy import info, options, path, dry, sh

try:
    if not options.virtualenv.activate_cmd:
        raise AttributeError
except AttributeError:
    options.setdotted('virtualenv.activate_cmd', 'source ' + options.paved.cwd / 'source' / 'bin' / 'activate')

from . import paved

__all__ = ['rmFilePatterns', 'rmDirPatterns', 'shv', 'update']

def _walkWithAction(*patterns, **kwargs):
    use_path = path(kwargs.get('use_path', options.paved.cwd))
    action = kwargs.pop('action')
    assert type(action) is str
    assert hasattr(use_path, action)

    use_regex = kwargs.get('use_regex')
    errors = kwargs.get('errors', 'warn')
    walk_method = kwargs.get('walk_method', 'walkfiles')

    for p in patterns:
        info("Looking for %s" % p)
        if use_regex:
            cp = re.compile(p)
            p = None
        for f in getattr(use_path, walk_method)(pattern=p, errors=errors):
            if use_regex and not cp.search(str(f)):
                continue
            if f.exists():
                msg = "%s %s..." % (action, f)
                dry(msg, getattr(f, action))

def rmFilePatterns(*patterns, **kwargs):
    """Remove all files under the given path with the given patterns.
    """
    kwargs['action'] = 'remove'
    kwargs['walk_method'] = 'walkfiles'
    return _walkWithAction(*patterns, **kwargs)

def rmDirPatterns(*patterns, **kwargs):
    """Remove all directories under the current path with the given patterns.
    """
    kwargs['action'] = 'rmtree'
    kwargs['walk_method'] = 'walkdirs'
    return _walkWithAction(*patterns, **kwargs)

def shv(command, capture=False, ignore_error=False, cwd=None):
    """Run the given command inside the virtual environment, if available:
    """
    try:
        command = "%s; %s" % (options.virtualenv.activate_cmd, command)
    except AttributeError:
        pass
    return sh(command, capture=capture, ignore_error=ignore_error, cwd=cwd)

def update(dst, src):
    """Recursively update the destination dict-like object with the source dict-like object.

    Useful for merging options and Bunches together!

    Based on:
    http://code.activestate.com/recipes/499335-recursively-update-a-dictionary-without-hitting-py/#c1
    """
    stack = [(dst, src)]

    def isdict(o):
        return hasattr(o, 'keys')

    while stack:
        current_dst, current_src = stack.pop()
        for key in current_src:
            if key not in current_dst:
                current_dst[key] = current_src[key]
            else:
                if isdict(current_src[key]) and isdict(current_dst[key]):
                    stack.append((current_dst[key], current_src[key]))
                else:
                    current_dst[key] = current_src[key]
    return dst
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.test import TestCase

from ansible.models import Playbook

class PlaybookListViewTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        Playbook.query_set.create(
            username='lozadaomr',
            repository='ansi-dst',
            inventory='hosts',
            user='ubuntu'
        )

    def test_view_url_exists_at_desired_location(self):
        resp = self.client.get('/playbooks/')
        self.assertEqual(resp.status_code, 200)

    def test_view_playbook_list_url_accessible_by_name(self):
        resp = self.client.get(reverse('ansible:playbook-list'))
        self.assertEqual(resp.status_code, 200)

    def test_view_playbook_detail_url_accessible_by_name(self):
        resp = self.client.get(
            reverse('ansible:playbook-detail', kwargs={'pk':1})
        )
        self.assertEqual(resp.status_code, 200)

    def test_view_uses_correct_template(self):
        resp = self.client.get(reverse('ansible:playbook-list'))
        self.assertEqual(resp.status_code, 200)
        self.assertTemplateUsed(resp, 'ansible/playbook_list.html')
<commit_msg>Try limiting query_set on unit tests<commit_after>from django.core.urlresolvers import reverse
from django.test import TestCase

from ansible.models import Playbook

class PlaybookListViewTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        Playbook.query_set.create(
            username='lozadaomr',
            repository='ansi-dst',
            inventory='hosts',
            user='ubuntu'
        )

    def test_view_url_exists_at_desired_location(self):
        resp = self.client.get('/playbooks/')
        self.assertEqual(resp.status_code, 200)

    def test_view_playbook_list_url_accessible_by_name(self):
        resp = self.client.get(reverse('ansible:playbook-list'))
        self.assertEqual(resp.status_code, 200)

    def test_view_playbook_detail_url_accessible_by_name(self):
        # TODO: Testing to make sure this is the same TestData
        pk = Playbook.query_set.filter(pk=1).values('pk')[:1]
        print pk
        resp = self.client.get(
            reverse('ansible:playbook-detail', kwargs={'pk': pk[0]['pk']})
        )
        self.assertEqual(resp.status_code, 200)

    def test_view_uses_correct_template(self):
        resp = self.client.get(reverse('ansible:playbook-list'))
        self.assertEqual(resp.status_code, 200)
        self.assertTemplateUsed(resp, 'ansible/playbook_list.html')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Filesystem functionality."""

import logging
import os

import magic

from pic2map.gps import filter_files_with_gps_metadata

logger = logging.getLogger(__name__)

class TreeExplorer(object):

    """Look for image files in a tree and return them.

    :param directory: Base directory for the tree to be explored.
    :type directory: str

    """

    def __init__(self, directory):
        """Initialize tree explorer."""
        self.directory = directory

    def paths(self):
        """Return paths to picture files found under directory.

        :return: Paths to picture files
        :rtype: list(str)

        """
        paths = self._explore()
        logger.info(
            '%d picture files found under %s',
            len(paths),
            self.directory)

        valid_paths = filter_files_with_gps_metadata(paths)
        logger.info(
            '%d picture files with GPS metadata found under %s',
            len(valid_paths),
            self.directory)

        return valid_paths

    def _explore(self):
        """Walk from base directory and return files that match pattern.

        :returns: Image files found under directory
        :rtype: list(str)

        """
        paths = []
        for (dirpath, _dirnames, filenames) in os.walk(self.directory):
            logger.debug('Exploring %s...', dirpath)

            # Check if any filename is a picture file
            for filename in filenames:
                path = os.path.join(dirpath, filename)

                # Skip missing files like broken symbolic links
                if not os.path.isfile(path):
                    logger.warning('Unable to access file: %r', path)
                    continue

                if 'JPEG image data' in magic.from_file(path):
                    paths.append(path)

        return paths
<commit_msg>Do not filter files in TreeExplorer<commit_after># -*- coding: utf-8 -*-
"""Filesystem functionality."""

import logging
import os

import magic

logger = logging.getLogger(__name__)

class TreeExplorer(object):

    """Look for image files in a tree and return them.

    :param directory: Base directory for the tree to be explored.
    :type directory: str

    """

    def __init__(self, directory):
        """Initialize tree explorer."""
        self.directory = directory

    def paths(self):
        """Return paths to picture files found under directory.

        :return: Paths to picture files
        :rtype: list(str)

        """
        paths = self._explore()
        logger.info(
            '%d picture files found under %s',
            len(paths),
            self.directory)

        return paths

    def _explore(self):
        """Walk from base directory and return files that match pattern.

        :returns: Image files found under directory
        :rtype: list(str)

        """
        paths = []
        for (dirpath, _dirnames, filenames) in os.walk(self.directory):
            logger.debug('Exploring %s...', dirpath)

            # Check if any filename is a picture file
            for filename in filenames:
                path = os.path.join(dirpath, filename)

                # Skip missing files like broken symbolic links
                if not os.path.isfile(path):
                    logger.warning('Unable to access file: %r', path)
                    continue

                if 'JPEG image data' in magic.from_file(path):
                    paths.append(path)

        return paths
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import logging
import xml.etree.ElementTree as ET
from datetime import datetime

from indra.statements import *
from indra.util import UnicodeXMLTreeBuilder as UTB

logger = logging.getLogger(__name__)

class CWMSError(Exception):
    pass

POLARITY_DICT = {'CC': {'ONT::CAUSE': 1,
                        'ONT::INFLUENCE': 1},
                 'EVENT': {'ONT::INCREASE': 1,
                           'ONT::MODULATE': None,
                           'ONT::DECREASE': -1,
                           'ONT::INHIBIT': -1,
                           'ONT::TRANSFORM': None,
                           'ONT::STIMULATE': 1}}

class CWMSProcessor(object):
    """The CWMSProcessor currently extracts causal relationships between
    terms (nouns) in EKB. In the future, this processor can be extended to
    extract other types of relations, or to extract relations involving
    events.

    For more details on the TRIPS EKB XML format, see
    http://trips.ihmc.us/parser/cgi/drum

    Parameters
    ----------
    xml_string : str
        A TRIPS extraction knowledge base (EKB) in XML format as a string.

    Attributes
    ----------
    tree : xml.etree.ElementTree.Element
        An ElementTree object representation of the TRIPS EKB XML.
    doc_id: str
        Document ID
    statements : list[indra.statements.Statement]
        A list of INDRA Statements that were extracted from the EKB.
    sentences : dict[str: str]
        The list of all sentences in the EKB with their IDs
    paragraphs : dict[str: str]
        The list of all paragraphs in the EKB with their IDs
    par_to_sec : dict[str: str]
        A map from paragraph IDs to their associated section types
    """

    def __init__(self, xml_string):
        self.statements = []
        # Parse XML
        try:
            self.tree = ET.XML(xml_string, parser=UTB())
        except ET.ParseError:
            logger.error('Could not parse XML string')
            self.tree = None
            return

        # Get the document ID from the EKB tag.
        self.doc_id = self.tree.attrib.get('id')

        # Store all paragraphs and store all sentences in a data structure
        paragraph_tags = self.tree.findall('input/paragraphs/paragraph')
        sentence_tags = self.tree.findall('input/sentences/sentence')
        self.paragraphs = {p.attrib['id']: p.text for p in paragraph_tags}
        self.sentences = {s.attrib['id']: s.text for s in sentence_tags}
        self.par_to_sec = {p.attrib['id']: p.attrib.get('sec-type')
                           for p in paragraph_tags}

        # Keep a list of unhandled events for development purposes
        self._unhandled_events = []

    def extract_causal_relations(self):
        events = self.tree.findall("CC/[type]")
        for event in events:
            ev_type = event.find('type').text
            if ev_type not in POLARITY_DICT['CC']:
                self._unhandled_events.append(ev_type)
                continue
            subj = self._get_event(event, "arg/[@role=':FACTOR']")
            obj = self._get_event(event, "arg/[@role=':OUTCOME']")
            if subj is None or obj is None:
                continue
            obj.delta['polarity'] = POLARITY_DICT['CC'][ev_type]
            time, location = self._extract_time_loc(event)
            if time or location:
                context = WorldContext(time=time, geo_location=location)
            else:
                context = None
            ev = self._get_evidence(event, context)
            st = Influence(subj, obj, evidence=[ev])
            self.statements.append(st)

        events = self.tree.findall("EVENT/[type]")
        for event in events:
            ev_type = event.find('type').text
            if ev_type not in POLARITY_DICT['EVENT']:
                self._unhandled_events.append(ev_type)
                continue
            subj = self._get_event(event, "*[@role=':AGENT']")
            obj = self._get_event(event, "*[@role=':AFFECTED']")
            if subj is None or obj is None:
                continue
            obj.delta['polarity'] = POLARITY_DICT['EVENT'][ev_type]
            time, location = self._extract_time_loc(event)
            if time or location:
                context = WorldContext(time=time, geo_location=location)
            else:
                context = None
            ev = self._get_evidence(event, context)
            st = Influence(subj, obj, evidence=[ev])
            self.statements.append(st)
        # In some EKBs we get two redundant relations over the same arguments,
        # we eliminate these
        self._remove_multi_extraction_artifacts()

        # Print unhandled event types
        logger.debug('Unhandled event types: %s' %
                     (', '.join(sorted(list(set(self._unhandled_events))))))

    def extract_events(self):
        events = self.tree.findall("EVENT/[type='ONT::INCREASE']") + (
            self.tree.findall("EVENT/[type='ONT::DECREASE']"))
        for event_entry in events:
            evid = self._get_evidence(event_entry, context=None)
            event = self._get_event(event_entry, "*[@role=':AFFECTED']",
                                    evidence=[evid])
            if event is None:
                continue
            self.statements.append(event)

        self._remove_multi_extraction_artifacts()

    def _get_event(self, event, find_str, evidence=None):
        """Get a concept referred from the event by the given string."""
        # Get the term with the given element id
        element = event.find(find_str)
        if element is None:
            return None
        element_id = element.attrib.get('id')
        element_term = self.tree.find("*[@id='%s']" % element_id)
        if element_term is None:
            return None
        time, location = self._extract_time_loc(element_term)

        # Now see if there is a modifier like assoc-with connected
        # to the main concept
        assoc_with = self._get_assoc_with(element_term)

        # Get the element's text and use it to construct a Concept
        element_text_element = element_term.find('text')
        if element_text_element is None:
            return None
        element_text = element_text_element.text
        element_db_refs = {'TEXT': element_text}
        element_name = sanitize_name(element_text)

        element_type_element = element_term.find('type')
        if element_type_element is not None:
            element_db_refs['CWMS'] = element_type_element.text
            # If there's an assoc-with, we tack it on as extra grounding
            if assoc_with is not None:
                element_db_refs['CWMS'] += ('|%s' % assoc_with)

        concept = Concept(element_name, db_refs=element_db_refs)
        if time or location:
            context = WorldContext(time=time, geo_location=location)
        else:
            context = None
        event_obj = Event(concept, context=context, evidence=evidence)
        return event_obj

    def _extract_time_loc(self, term):
        """Get the location from a term (CC or TERM)"""
        loc = term.find('location')
        if loc is None:
            loc_context = None
        else:
            loc_id = loc.attrib.get('id')
            loc_term = self.tree.find("*[@id='%s']" % loc_id)
            text = loc_term.findtext('text')
            name = loc_term.findtext('name')
            loc_context = RefContext(name=text)
        time = term.find('time')
        if time is None:
            time_context = None
        else:
            time_id = time.attrib.get('id')
            time_term = self.tree.find("*[@id='%s']" % time_id)
            if time_term is not None:
                text = time_term.findtext('text')
                timex = time_term.find('timex')
                if timex is not None:
                    year = timex.findtext('year')
                    try:
                        year = int(year)
                    except Exception:
                        year = None
                    month = timex.findtext('month')
                    day = timex.findtext('day')
                    if year and (month or day):
                        try:
                            month = int(month)
                        except Exception:
                            month = 1
                        try:
                            day = int(day)
                        except Exception:
                            day = 1
                        start = datetime(year, month, day)
                        time_context = TimeContext(text=text, start=start)
                    else:
                        time_context = TimeContext(text=text)
                else:
                    time_context = TimeContext(text=text)
            else:
                time_context = None
        return time_context, loc_context

    def _get_assoc_with(self, element_term):
        # NOTE: there could be multiple assoc-withs here that we may
        # want to handle
        assoc_with = element_term.find('assoc-with')
        if assoc_with is not None:
            # We first identify the ID of the assoc-with argument
            assoc_with_id = assoc_with.attrib.get('id')
            # In some cases the assoc-with has no ID but has a type
            # defined in place that we can get
            if assoc_with_id is None:
                assoc_with_grounding = assoc_with.find('type').text
                return assoc_with_grounding
            # If the assoc-with has an ID then find the TERM
            # corresponding to it
            assoc_with_term = self.tree.find("*[@id='%s']" % assoc_with_id)
            if assoc_with_term is not None:
                # We then get the grounding for the term
                assoc_with_grounding = assoc_with_term.find('type').text
                return assoc_with_grounding
        return None

    def _get_evidence(self, event_tag, context):
        text = self._get_evidence_text(event_tag)
        sec = self._get_section(event_tag)
        epi = {'direct': False}
        if sec:
            epi['section_type'] = sec
        ev = Evidence(source_api='cwms', text=text, pmid=self.doc_id,
                      epistemics=epi, context=context)
        return ev

    def _get_evidence_text(self, event_tag):
        """Extract the evidence for an event.

        Pieces of text linked to an EVENT are fragments of a sentence. The
        EVENT refers to the paragraph ID and the "uttnum", which corresponds
        to a sentence ID. Here we find and return the full sentence from which
        the event was taken.
        """
        par_id = event_tag.attrib.get('paragraph')
        uttnum = event_tag.attrib.get('uttnum')
        event_text = event_tag.find('text')
        if self.sentences is not None and uttnum is not None:
            sentence = self.sentences[uttnum]
        elif event_text is not None:
            sentence = event_text.text
        else:
            sentence = None
        return sentence

    def _get_section(self, event_tag):
        par_id = event_tag.attrib.get('paragraph')
        sec = self.par_to_sec.get(par_id)
        return sec

    def _remove_multi_extraction_artifacts(self):
        # Build up a dict of evidence matches keys with statement UUIDs
        evmks = {}
        logger.info('Starting with %d Statements.' % len(self.statements))
        for stmt in self.statements:
            # Standalone Events will be handled later, so we do not add them
            if isinstance(stmt, Event):
                continue
            evmk = (stmt.evidence[0].matches_key() +
                    stmt.subj.matches_key() + stmt.obj.matches_key())
            if evmk not in evmks:
                evmks[evmk] = [stmt.uuid]
            else:
                evmks[evmk].append(stmt.uuid)
        # This is a list of groups of statement UUIDs that are redundant
        multi_evmks = [v for k, v in evmks.items() if len(v) > 1]
        # We now figure out if anything needs to be removed
        to_remove = []
        # Influence statements to be removed
        for uuids in multi_evmks:
            stmts = [s for s in self.statements if (s.uuid in uuids and
                                                    isinstance(s, Influence))]
            stmts = sorted(stmts, key=lambda x: x.polarity_count(),
                           reverse=True)
            to_remove += [s.uuid for s in stmts[1:]]
        # Standalone events to be removed
        matches_keys = ''.join([k for k in evmks.keys()])
        for stmt in self.statements:
            if isinstance(stmt, Event):
                if stmt.matches_key() in matches_keys:
                    logger.info('Found an Event that is part of '
                                'Causal Relationship')
                    to_remove.append(stmt.uuid)
        # Remove all reduncdant statements
        if to_remove:
            logger.info('Found %d Statements to remove' % len(to_remove))
        self.statements = [s for s in self.statements
                           if s.uuid not in to_remove]

def sanitize_name(txt):
    name = txt.replace('\n', '')
    return name
<commit_msg>Remove redundant events and rel_objects in CWMS<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import logging
import xml.etree.ElementTree as ET
from datetime import datetime

from indra.statements import *
from indra.util import UnicodeXMLTreeBuilder as UTB

logger = logging.getLogger(__name__)

class CWMSError(Exception):
    pass

POLARITY_DICT = {'CC': {'ONT::CAUSE': 1,
                        'ONT::INFLUENCE': 1},
                 'EVENT': {'ONT::INCREASE': 1,
                           'ONT::MODULATE': None,
                           'ONT::DECREASE': -1,
                           'ONT::INHIBIT': -1,
                           'ONT::TRANSFORM': None,
                           'ONT::STIMULATE': 1}}

class CWMSProcessor(object):
    """The CWMSProcessor currently extracts causal relationships between
    terms (nouns) in EKB. In the future, this processor can be extended to
    extract other types of relations, or to extract relations involving
    events.

    For more details on the TRIPS EKB XML format, see
    http://trips.ihmc.us/parser/cgi/drum

    Parameters
    ----------
    xml_string : str
        A TRIPS extraction knowledge base (EKB) in XML format as a string.

    Attributes
    ----------
    tree : xml.etree.ElementTree.Element
        An ElementTree object representation of the TRIPS EKB XML.
    doc_id: str
        Document ID
    statements : list[indra.statements.Statement]
        A list of INDRA Statements that were extracted from the EKB.
    sentences : dict[str: str]
        The list of all sentences in the EKB with their IDs
    paragraphs : dict[str: str]
        The list of all paragraphs in the EKB with their IDs
    par_to_sec : dict[str: str]
        A map from paragraph IDs to their associated section types
    """

    def __init__(self, xml_string):
        self.statements = []
        # Parse XML
        try:
            self.tree = ET.XML(xml_string, parser=UTB())
        except ET.ParseError:
            logger.error('Could not parse XML string')
            self.tree = None
            return

        # Get the document ID from the EKB tag.
        self.doc_id = self.tree.attrib.get('id')

        # Store all paragraphs and store all sentences in a data structure
        paragraph_tags = self.tree.findall('input/paragraphs/paragraph')
        sentence_tags = self.tree.findall('input/sentences/sentence')
        self.paragraphs = {p.attrib['id']: p.text for p in paragraph_tags}
        self.sentences = {s.attrib['id']: s.text for s in sentence_tags}
        self.par_to_sec = {p.attrib['id']: p.attrib.get('sec-type')
                           for p in paragraph_tags}

        # Keep a list of relation's object ids
        self.relation_obj_ids = []

        # Keep a list of unhandled events for development purposes
        self._unhandled_events = []

    def extract_causal_relations(self):
        events = self.tree.findall("CC/[type]")
        for event in events:
            ev_type = event.find('type').text
            if ev_type not in POLARITY_DICT['CC']:
                self._unhandled_events.append(ev_type)
                continue
            subj = self._get_event(event, "arg/[@role=':FACTOR']")
            obj = self._get_event(event, "arg/[@role=':OUTCOME']")
            if subj is None or obj is None:
                continue
            obj.delta['polarity'] = POLARITY_DICT['CC'][ev_type]
            time, location = self._extract_time_loc(event)
            if time or location:
                context = WorldContext(time=time, geo_location=location)
            else:
                context = None
            ev = self._get_evidence(event, context)
            st = Influence(subj, obj, evidence=[ev])
            self.statements.append(st)

        events = self.tree.findall("EVENT/[type]")
        for event in events:
            ev_type = event.find('type').text
            if ev_type not in POLARITY_DICT['EVENT']:
                self._unhandled_events.append(ev_type)
                continue
            subj = self._get_event(event, "*[@role=':AGENT']")
            obj = self._get_event(event, "*[@role=':AFFECTED']")
            if subj is None or obj is None:
                continue
            self.relation_obj_ids.append(self._get_event_id(
                event, "*[@role=':AFFECTED']"))
            obj.delta['polarity'] = POLARITY_DICT['EVENT'][ev_type]
            time, location = self._extract_time_loc(event)
            if time or location:
                context = WorldContext(time=time, geo_location=location)
            else:
                context = None
            ev = self._get_evidence(event, context)
            st = Influence(subj, obj, evidence=[ev])
            self.statements.append(st)
        # In some EKBs we get two redundant relations over the same arguments,
        # we eliminate these
        self._remove_multi_extraction_artifacts()

        # Print unhandled event types
        logger.debug('Unhandled event types: %s' %
                     (', '.join(sorted(list(set(self._unhandled_events))))))

    def extract_events(self):
        if not self.relation_obj_ids:
            self.extract_causal_relations()
        events = self.tree.findall("EVENT/[type='ONT::INCREASE']") + (
            self.tree.findall("EVENT/[type='ONT::DECREASE']"))
        for event_entry in events:
            # Check if event is part of a causal relation
            event_id = self._get_event_id(event_entry, "*[@role=':AFFECTED']")
            if event_id in self.relation_obj_ids:
                continue
            # Make an Event statement if it is a standalone event
            evidence = self._get_evidence(event_entry, context=None)
            event = self._get_event(event_entry, "*[@role=':AFFECTED']",
                                    evidence=[evidence])
            if event is None:
                continue
            self.statements.append(event)

        self._remove_multi_extraction_artifacts()

    def _get_event(self, event, find_str, evidence=None):
        """Get a concept referred from the event by the given string."""
        # Get the term with the given element id
        element_id = self._get_event_id(event, find_str)
        if element_id is None:
            return None
        element_term = self.tree.find("*[@id='%s']" % element_id)
        if element_term is None:
            return None
        time, location = self._extract_time_loc(element_term)

        # Now see if there is a modifier like assoc-with connected
        # to the main concept
        assoc_with = self._get_assoc_with(element_term)

        # Get the element's text and use it to construct a Concept
        element_text_element = element_term.find('text')
        if element_text_element is None:
            return None
        element_text = element_text_element.text
        element_db_refs = {'TEXT': element_text}
        element_name = sanitize_name(element_text)

        element_type_element = element_term.find('type')
        if element_type_element is not None:
            element_db_refs['CWMS'] = element_type_element.text
            # If there's an assoc-with, we tack it on as extra grounding
            if assoc_with is not None:
                element_db_refs['CWMS'] += ('|%s' % assoc_with)

        concept = Concept(element_name, db_refs=element_db_refs)
        if time or location:
            context = WorldContext(time=time, geo_location=location)
        else:
            context = None
        event_obj = Event(concept, context=context, evidence=evidence)
        return event_obj

    def _get_event_id(self, event, find_str):
        element = event.find(find_str)
        if element is None:
            return None
        element_id = element.attrib.get('id')
        return element_id

    def _extract_time_loc(self, term):
        """Get the location from a term (CC or TERM)"""
        loc = term.find('location')
        if loc is None:
            loc_context = None
        else:
            loc_id = loc.attrib.get('id')
            loc_term = self.tree.find("*[@id='%s']" % loc_id)
            text = loc_term.findtext('text')
            name = loc_term.findtext('name')
            loc_context = RefContext(name=text)
        time = term.find('time')
        if time is None:
            time_context = None
        else:
            time_id = time.attrib.get('id')
            time_term = self.tree.find("*[@id='%s']" % time_id)
            if time_term is not None:
                text = time_term.findtext('text')
                timex = time_term.find('timex')
                if timex is not None:
                    year = timex.findtext('year')
                    try:
                        year = int(year)
                    except Exception:
                        year = None
                    month = timex.findtext('month')
                    day = timex.findtext('day')
                    if year and (month or day):
                        try:
                            month = int(month)
                        except Exception:
                            month = 1
                        try:
                            day = int(day)
                        except Exception:
                            day = 1
                        start = datetime(year, month, day)
                        time_context = TimeContext(text=text, start=start)
                    else:
                        time_context = TimeContext(text=text)
                else:
                    time_context = TimeContext(text=text)
            else:
                time_context = None
        return time_context, loc_context

    def _get_assoc_with(self, element_term):
        # NOTE: there could be multiple assoc-withs here that we may
        # want to handle
        assoc_with = element_term.find('assoc-with')
        if assoc_with is not None:
            # We first identify the ID of the assoc-with argument
            assoc_with_id = assoc_with.attrib.get('id')
            # In some cases the assoc-with has no ID but has a type
            # defined in place that we can get
            if assoc_with_id is None:
                assoc_with_grounding = assoc_with.find('type').text
                return assoc_with_grounding
            # If the assoc-with has an ID then find the TERM
            # corresponding to it
            assoc_with_term = self.tree.find("*[@id='%s']" % assoc_with_id)
            if assoc_with_term is not None:
                # We then get the grounding for the term
                assoc_with_grounding = assoc_with_term.find('type').text
                return assoc_with_grounding
        return None

    def _get_evidence(self, event_tag, context):
        text = self._get_evidence_text(event_tag)
        sec = self._get_section(event_tag)
        epi = {'direct': False}
        if sec:
            epi['section_type'] = sec
        ev = Evidence(source_api='cwms', text=text, pmid=self.doc_id,
                      epistemics=epi, context=context)
        return ev

    def _get_evidence_text(self, event_tag):
        """Extract the evidence for an event.

        Pieces of text linked to an EVENT are fragments of a sentence. The
        EVENT refers to the paragraph ID and the "uttnum", which corresponds
        to a sentence ID. Here we find and return the full sentence from which
        the event was taken.
        """
        par_id = event_tag.attrib.get('paragraph')
        uttnum = event_tag.attrib.get('uttnum')
        event_text = event_tag.find('text')
        if self.sentences is not None and uttnum is not None:
            sentence = self.sentences[uttnum]
        elif event_text is not None:
            sentence = event_text.text
        else:
            sentence = None
        return sentence

    def _get_section(self, event_tag):
        par_id = event_tag.attrib.get('paragraph')
        sec = self.par_to_sec.get(par_id)
        return sec

    def _remove_multi_extraction_artifacts(self):
        # Build up a dict of evidence matches keys with statement UUIDs
        evmks = {}
        logger.info('Starting with %d Statements.' % len(self.statements))
        for stmt in self.statements:
            # Standalone Events will be handled later, so we do not add them
            if isinstance(stmt, Event):
                continue
            evmk = (stmt.evidence[0].matches_key() +
                    stmt.subj.matches_key() + stmt.obj.matches_key())
            if evmk not in evmks:
                evmks[evmk] = [stmt.uuid]
            else:
                evmks[evmk].append(stmt.uuid)
        # This is a list of groups of statement UUIDs that are redundant
        multi_evmks = [v for k, v in evmks.items() if len(v) > 1]
        # We now figure out if anything needs to be removed
        to_remove = []
        # Remove redundant statements
        for uuids in multi_evmks:
            # Influence statements to be removed
            infl_stmts = [s for s in self.statements if (
                            s.uuid in uuids and isinstance(s, Influence))]
            infl_stmts = sorted(infl_stmts, key=lambda x: x.polarity_count(),
                                reverse=True)
            to_remove += [s.uuid for s in infl_stmts[1:]]
            # Standalone events to be removed
            events = [s for s in self.statements if (
                        s.uuid in uuids and isinstance(s, Event))]
            events = sorted(events, key=lambda x: event_delta_score(x),
                            reverse=True)
            to_remove += [e.uuid for e in events[1:]]

        # Remove all redundant statements
        if to_remove:
            logger.info('Found %d Statements to remove' % len(to_remove))
        self.statements = [s for s in self.statements
                           if s.uuid not in to_remove]

def sanitize_name(txt):
    name = txt.replace('\n', '')
    return name

def event_delta_score(stmt):
    pol_score = 1 if stmt.delta['polarity'] is not None else 0
    adj_score = len(stmt.delta['adjectives'])
    return (pol_score + adj_score)
<|endoftext|>


<commit_before>__all__ = ['process_annotations', 'get_annotations']

import requests
from indra.config import get_config
from .processor import HypothesisProcessor
from .annotator import statement_to_annotations

base_url = 'https://api.hypothes.is/api/'
api_key = get_config('HYPOTHESIS_API_KEY')
headers = {'Authorization': 'Bearer %s' % api_key,
           'Accept': 'application/vnd.hypothesis.v1+json',
           'content-type': 'application/json'}
indra_group = get_config('HYPOTHESIS_GROUP')

def send_get_request(endpoint, **params):
    """Send a request to the hypothes.is web service and return JSON response.

    Note that it is assumed that `HYPOTHESIS_API_KEY` is set either as a
    configuration entry or as an environmental variable.

    Parameters
    ----------
    endpoint : str
        The endpoint to call, e.g., `search`.
    params : kwargs
        A set of keyword arguments that are passed to the `requests.get` call
        as `params`.
    """
    if api_key is None:
        return ValueError('No API key set in HYPOTHESIS_API_KEY')
    res = requests.get(base_url + endpoint, headers=headers,
                       params=params)
    res.raise_for_status()
    return res.json()

def send_post_request(endpoint, **params):
    """Send a request to the hypothes.is web service and return JSON response.

    Note that it is assumed that `HYPOTHESIS_API_KEY` is set either as a
    configuration entry or as an environmental variable.

    Parameters
    ----------
    endpoint : str
        The endpoint to call, e.g., `search`.
    params : kwargs
        A set of keyword arguments that are passed to the `requests.post` call
        as `json`.
    """
    if api_key is None:
        return ValueError('No API key set in HYPOTHESIS_API_KEY')
    res = requests.post(base_url + endpoint, headers=headers,
                        json=params)
    res.raise_for_status()
    return res.json()

def upload_annotation(url, annotation, target_text=None, tags=None,
                      group=None):
    if group is None:
        if indra_group:
            group = indra_group
        else:
            raise ValueError('No group provided and HYPOTHESIS_GROUP '
                             'is not set.')
    params = {
        'uri': url,
        'group': group,
        'text': annotation,
    }
    if target_text:
        params['target'] = [{
            'source': [url],
            'selector': [
                {'type': 'TextQuoteSelector',
                 'exact': target_text}
            ]
        }]
    if tags:
        params['tags'] = tags
    res = send_post_request('annotations', **params)
    return res

def upload_statement_annotation(stmt):
    annotations = statement_to_annotations(stmt, annotate_agents=True)
    for annotation in annotations:
        annotation['tags'].append('indra_upload')
        upload_annotation(**annotation)

def get_annotations(group=None):
    """Return annotations in hypothes.is in a given group.

    Parameters
    ----------
    group : Optional[str]
        The hypothesi.is key of the group (not its name). If not given, the
        HYPOTHESIS_GROUP configuration in the config file or an environmental
        variable is used.
    """
    if group is None:
        if indra_group:
            group = indra_group
        else:
            raise ValueError('No group provided and HYPOTHESIS_GROUP '
                             'is not set.')
    res = send_get_request('search', group=group, limit=200)
    annotations = res.get('rows', [])
    return annotations

def process_annotations(group=None, reader=None, grounder=None):
    """Process annotations in hypothes.is in a given group.

    Parameters
    ----------
    group : Optional[str]
        The hypothesi.is key of the group (not its name). If not given, the
        HYPOTHESIS_GROUP configuration in the config file or an environmental
        variable is used.
    reader : Optiona[function]
        A handle for a function which takes a single str argument
        (text to process) and returns a processor object with a statements
        attribute containing INDRA Statements. By default, the REACH reader's
        process_text function is used with default parameters. Note that
        if the function requires extra parameters other than the input text,
        functools.partial can be used to set those.
    grounder : Optional[function]
        A handle for a function which takes a positional str argument (entity
        text to ground) and an optional context key word argument and returns
        a list of objects matching the structure of gilda.grounder.ScoredMatch.
        By default, Gilda's ground function is used for grounding.

    Returns
    -------
    HypothesisProcessor
        A HypothesisProcessor object which contains a list of extracted
        INDRA Statements in its statements attribute, and a list of extracted
        grounding curations in its groundings attribute.
    """
    annotations = get_annotations(group=group)
    hp = HypothesisProcessor(annotations, reader=reader, grounder=grounder)
    hp.extract_statements()
    hp.extract_groundings()
    return hp
<commit_msg>Update docstrings for new functions<commit_after>__all__ = ['process_annotations', 'get_annotations']

import requests
from indra.config import get_config
from .processor import HypothesisProcessor
from .annotator import statement_to_annotations

base_url = 'https://api.hypothes.is/api/'
api_key = get_config('HYPOTHESIS_API_KEY')
headers = {'Authorization': 'Bearer %s' % api_key,
           'Accept': 'application/vnd.hypothesis.v1+json',
           'content-type': 'application/json'}
indra_group = get_config('HYPOTHESIS_GROUP')

def send_get_request(endpoint, **params):
    """Send a request to the hypothes.is web service and return JSON response.

    Note that it is assumed that `HYPOTHESIS_API_KEY` is set either as a
    configuration entry or as an environmental variable.

    Parameters
    ----------
    endpoint : str
        The endpoint to call, e.g., `search`.
    params : kwargs
        A set of keyword arguments that are passed to the `requests.get` call
        as `params`.
    """
    if api_key is None:
        return ValueError('No API key set in HYPOTHESIS_API_KEY')
    res = requests.get(base_url + endpoint, headers=headers,
                       params=params)
    res.raise_for_status()
    return res.json()

def send_post_request(endpoint, **params):
    """Send a post request to the hypothes.is web service and return JSON
    response.

    Note that it is assumed that `HYPOTHESIS_API_KEY` is set either as a
    configuration entry or as an environmental variable.

    Parameters
    ----------
    endpoint : str
        The endpoint to call, e.g., `search`.
    params : kwargs
        A set of keyword arguments that are passed to the `requests.post` call
        as `json`.
    """
    if api_key is None:
        return ValueError('No API key set in HYPOTHESIS_API_KEY')
    res = requests.post(base_url + endpoint, headers=headers,
                        json=params)
    res.raise_for_status()
    return res.json()

def upload_annotation(url, annotation, target_text=None, tags=None,
                      group=None):
    """Upload an annotation to hypothes.is.

    Parameters
    ----------
    url : str
        The URL of the resource being annotated.
    annotation : str
        The text content of the annotation itself.
    target_text : Optional[str]
        The specific span of text that the annotation applies to.
    tags : list[str]
        A list of tags to apply to the annotation.
    group : Optional[str]
        The hypothesi.is key of the group (not its name). If not given, the
        HYPOTHESIS_GROUP configuration in the config file or an environmental
        variable is used.

    Returns
    -------
    json
        The full response JSON from the web service.
    """
    if group is None:
        if indra_group:
            group = indra_group
        else:
            raise ValueError('No group provided and HYPOTHESIS_GROUP '
                             'is not set.')
    params = {
        'uri': url,
        'group': group,
        'text': annotation,
    }
    if target_text:
        params['target'] = [{
            'source': [url],
            'selector': [
                {'type': 'TextQuoteSelector',
                 'exact': target_text}
            ]
        }]
    if tags:
        params['tags'] = tags
    res = send_post_request('annotations', **params)
    return res

def upload_statement_annotation(stmt, annotate_agents=True):
    """Construct and upload all annotations for a given INDRA Statement.

    Parameters
    ----------
    stmt : indra.statements.Statement
        An INDRA Statement.
    annotate_agents : Optional[bool]
        If True, the agents in the annotation text are linked to outside
        databases based on their grounding. Default: True
    """
    annotations = statement_to_annotations(stmt,
                                           annotate_agents=annotate_agents)
    for annotation in annotations:
        annotation['tags'].append('indra_upload')
        upload_annotation(**annotation)

def get_annotations(group=None):
    """Return annotations in hypothes.is in a given group.

    Parameters
    ----------
    group : Optional[str]
        The hypothesi.is key of the group (not its name). If not given, the
        HYPOTHESIS_GROUP configuration in the config file or an environmental
        variable is used.
    """
    if group is None:
        if indra_group:
            group = indra_group
        else:
            raise ValueError('No group provided and HYPOTHESIS_GROUP '
                             'is not set.')
    res = send_get_request('search', group=group, limit=200)
    annotations = res.get('rows', [])
    return annotations

def process_annotations(group=None, reader=None, grounder=None):
    """Process annotations in hypothes.is in a given group.

    Parameters
    ----------
    group : Optional[str]
        The hypothesi.is key of the group (not its name). If not given, the
        HYPOTHESIS_GROUP configuration in the config file or an environmental
        variable is used.
    reader : Optiona[function]
        A handle for a function which takes a single str argument
        (text to process) and returns a processor object with a statements
        attribute containing INDRA Statements. By default, the REACH reader's
        process_text function is used with default parameters. Note that
        if the function requires extra parameters other than the input text,
        functools.partial can be used to set those.
    grounder : Optional[function]
        A handle for a function which takes a positional str argument (entity
        text to ground) and an optional context key word argument and returns
        a list of objects matching the structure of gilda.grounder.ScoredMatch.
        By default, Gilda's ground function is used for grounding.

    Returns
    -------
    HypothesisProcessor
        A HypothesisProcessor object which contains a list of extracted
        INDRA Statements in its statements attribute, and a list of extracted
        grounding curations in its groundings attribute.
    """
    annotations = get_annotations(group=group)
    hp = HypothesisProcessor(annotations, reader=reader, grounder=grounder)
    hp.extract_statements()
    hp.extract_groundings()
    return hp
<|endoftext|>


<commit_before>'''
Parse CLI options
'''
# Import python libs
import os
import copy
import argparse
# Import pkgcmp libs
import pkgcmp.scan
# Import third party libs
import yaml

DEFAULTS = {'cachedir': '/var/cache/pkgcmp'}

def parse():
    '''
    Parse!!
    '''
    parser = argparse.ArgumentParser(description='The pkgcmp map generator')
    parser.add_argument(
            '--cachedir',
            dest='cachedir',
            default=None,
            help='The location to store all the files while working')
    parser.add_argument(
            '--config',
            dest='config',
            default='/etc/pkgcmp/pkgcmp',
            help='The location of the pkgcmp config file')
    opts = parser.parse_args().__dict__
    conf = config(opts['config'])
    for key in opts:
        if opts[key] is not None:
            conf[key] = opts[key]
    return conf

def config(cfn):
    '''
    Read in the config file
    '''
    ret = copy.copy(DEFAULTS)
    if os.path.isfile(cfn):
        with open(cfn, 'r') as cfp:
            conf = yaml.safe_load(cfp)
            print(conf)
            ret.update(conf)
    return ret

class PkgCmp:
    '''
    Build and run the application
    '''
    def __init__(self):
        self.opts = parse()
        self.scan = pkgcmp.scan.Scanner(self.opts)

    def run(self):
        self.scan.run()
<commit_msg>watch for empty configs<commit_after>'''
Parse CLI options
'''
# Import python libs
import os
import copy
import argparse
# Import pkgcmp libs
import pkgcmp.scan
# Import third party libs
import yaml

DEFAULTS = {'cachedir': '/var/cache/pkgcmp'}

def parse():
    '''
    Parse!!
    '''
    parser = argparse.ArgumentParser(description='The pkgcmp map generator')
    parser.add_argument(
            '--cachedir',
            dest='cachedir',
            default=None,
            help='The location to store all the files while working')
    parser.add_argument(
            '--config',
            dest='config',
            default='/etc/pkgcmp/pkgcmp',
            help='The location of the pkgcmp config file')
    opts = parser.parse_args().__dict__
    conf = config(opts['config'])
    for key in opts:
        if opts[key] is not None:
            conf[key] = opts[key]
    return conf

def config(cfn):
    '''
    Read in the config file
    '''
    ret = copy.copy(DEFAULTS)
    if os.path.isfile(cfn):
        with open(cfn, 'r') as cfp:
            conf = yaml.safe_load(cfp)
            if isinstance(conf, dict):
                ret.update(conf)
    return ret

class PkgCmp:
    '''
    Build and run the application
    '''
    def __init__(self):
        self.opts = parse()
        self.scan = pkgcmp.scan.Scanner(self.opts)

    def run(self):
        self.scan.run()
<|endoftext|>