<commit_before>import re
import sys
import os
import time
import shutil
import subprocess
from lingcod.common.utils import get_logger
log = get_logger()

DJANGO = True
try:
    from django.conf import settings
except ImportError:
    # not in a django env
    DJANGO = False

'''
Sets up a grass environment in the current running process.  Allows grass commands
to be run in a shell environment.  Optimally grass python bindings would be used in the
future once they become more stable and full-featured

Grass is not thread-safe, see http://grass.osgeo.org/wiki/Parallel_GRASS_jobs
Because of this there can be issues with race conditions while working within
the same mapset with more than one process.  The solution is to create a new
mapset for each analysis run, copy the necessary maps into the new mapset,
run the analysis, and then remove the mapset when you're done.

@author: Tim Welch, Ecotrust 2009
@author: Matthew Perry, Ecotrust 2011
Concepts taken from the PyWPS project.

Example usage (django):
    g = Grass('mlpa_sc')
    result = g.run('r.info soilmoisture')

    # Django Settings:
    #    GRASS_GISBASE = '/usr/local/grass-6.4.1RC2'
    #    GRASS_GISDBASE = '/home/grass'

Example usage (outside django):
    g = Grass('mlpa_sc', gisdbase="/home/grass", gisbase="/usr/local/grass-6.4.1RC2")
    result = g.run('r.info soilmoisture')

'''

class Grass:
    '''
    If being used to load the source mapset then tmpMapset should be None or False.
    If you're running any commands with outputs, use tmpMapset=True (autogenerates
      tmpMapset name) or specify a unique name.
    autoclean=True/False determines if tmpMapset gets deleted at Grass.__del__()
    gisdbase and gisbase should be set in django settings or overriden with kwargs
    '''
    def __init__(self, location, srcMapset='PERMANENT', tmpMapset=True,
                 gisdbase=None, gisbase=None, autoclean=True):
        self.autoclean = False
        self.verbose = False

        self.location = location
        if tmpMapset == True:
            # unique name not specified, autogenerate it
            self.tmpMapset = 'tmpmapset_%s_%s' % (srcMapset, str(time.time()))
        elif tmpMapset:
            self.tmpMapset = tmpMapset
        self.srcMapset = srcMapset
        self.templateMapset = 'mm_template_mapset'

        if DJANGO and not gisdbase:
            self.gisdbase = settings.GRASS_GISDBASE
        else:
            self.gisdbase = gisdbase
        if DJANGO and not gisbase:
            self.gisbase = settings.GRASS_GISBASE
        else:
            self.gisbase = gisbase

        self.locationPath = os.path.join(self.gisdbase, self.location) #mlpa_nc
        self.tmpMapsetPath = os.path.join(self.locationPath, self.tmpMapset)
        self.srcMapsetPath = os.path.join(self.locationPath, self.srcMapset) #PERMANENT
        self.templatePath = os.path.join(self.locationPath, self.templateMapset) #mm_template_mapset

        if not tmpMapset:
            self.curMapset = srcMapset
            self.grassRcFile = os.path.join(gisdbase, "mm_grass6rc")
            self.setupTmpGrassEnv(mktmp=False)
            self.autoclean = autoclean
        else:
            self.curMapset = self.tmpMapset
            self.grassRcFile = os.path.join(self.tmpMapsetPath, "mm_grass6rc")
            self.setupTmpGrassEnv()

        #Turn on verbose debug output
        #self.runCmd('g.gisenv set=DEBUG=3')

    def __del__(self):
        if self.autoclean:
            self.cleanup()

    def __repr__(self):
        if self.tmpMapset:
            return u'Grass instance at temporary mapset %s (%s)' % (self.tmpMapset, self.gisdbase)
        else:
            return u'Grass instance at permanent mapset %s' % self.srcMapsetPath

    @property
    def grass_tmp(self):
        try:
            return settings.GRASS_TMP
        except:
            return '/tmp'

    @property
    def grass_path(self):
        try:
            return settings.GRASS_PATH
        except:
            return '%(GRASS)s/bin:%(GRASS)s/scripts:/usr/local/sbin:/usr/local/bin:' \
                   '/usr/sbin:/usr/bin:/sbin:/bin' % {'GRASS': self.gisbase}

    @property
    def grass_version(self):
        try:
            return settings.GRASS_VERSION
        except:
            try:
                return self.gisbase.split('-')[1].replace('/','')
            except:
                return "6.4.1"

    @property
    def grass_lib_path(self):
        try:
            return settings.GRASS_LIB_PATH
        except:
            return '%s/lib' % self.gisbase

    '''
    Setup grass environment to use a temporary mapset as the current mapset.
    Base maps can be used from the src mapset as it will be in your
    mapset path, but results will go to the temporary mapset.
    grassenv settings are system specific. To get these values start a
    grass shell and echo the environment variables to get the necessary
    values.  eg.  >grass64 ; >echo $GISBASE
    '''
    def setupTmpGrassEnv(self, mktmp=True):
        grassenv = {
            'HOME' : self.grass_tmp,
            'PATH' : self.grass_path,
            'GRASS_VERSION' : self.grass_version,
            'GISBASE': self.gisbase,
            'LD_LIBRARY_PATH' : self.grass_lib_path,
            'LOCATION_NAME' : self.location,
            'MAPSET' : self.srcMapset,
            'GISDBASE' : self.gisdbase,
            'GRASS_GUI' : 'text',
            'GIS_LOCK' : str(os.getpid())
        }

        for key in grassenv.keys():
            self.setEnv(key, grassenv[key])

        #Create temporary space for current analysis run
        if mktmp:
            self.createTempMapset()
        #Change to the directory with the RC file
        os.chdir(self.locationPath)
        #Output a new rc file, overwriting any old ones
        self.writeGrassRc()

    '''
    Create a temporary mapset
    '''
    def createTempMapset(self):
        if not os.path.exists(self.tmpMapsetPath):
            if os.path.exists(self.templatePath):
                # use template mapset directory if available
                shutil.copytree(self.templatePath, self.tmpMapsetPath)
            else:
                # .. otherwise create the bare minimum file structure
                os.mkdir(self.tmpMapsetPath)
                shutil.copyfile(
                        os.path.join(self.srcMapsetPath, 'DEFAULT_WIND'),
                        os.path.join(self.tmpMapsetPath, 'DEFAULT_WIND'))
                shutil.copyfile(
                        os.path.join(self.srcMapsetPath, 'WIND'),
                        os.path.join(self.tmpMapsetPath, 'WIND'))
                myname = open(os.path.join(self.tmpMapsetPath, 'MYNAME'),'w')
                myname.write(self.tmpMapset)
                myname.close()

    '''
    Cleanup disk such as temporary mapset
    '''
    def cleanup(self):
        if self.tmpMapset:
            shutil.rmtree(self.tmpMapsetPath)

    '''
    Output grassrc file used by Grass.  Set to use the current mapset
    '''
    def writeGrassRc(self):
        gisrc = open(self.grassRcFile,"w")
        gisrc.write("LOCATION_NAME: %s\n" % (self.location))
        gisrc.write("MAPSET: %s\n" % (self.curMapset))
        gisrc.write("DIGITIZER: none\n")
        gisrc.write("GISDBASE: %s\n" % (self.gisdbase))
        gisrc.write("OVERWRITE: 1\n")
        gisrc.write("GRASS_GUI: text\n")
        gisrc.close()
        os.environ['GISRC'] = self.grassRcFile

    '''
    Set a single environment variable to the given value
    '''
    def setEnv(self, key,value):
        origValue = os.getenv(key)
        #if origValue:
        #    value  += ":"+origValue
        os.putenv(key,value)
        os.environ[key] = value
        return

    '''
    Returns the raster path for the current mapset
    '''
    def getRastPath(self):
        return self.gisdbase+self.location+'/'+self.curMapset+'/cell/'

    def run(self, cmd, nice=None):
        if nice:
            cmd = 'nice -n %d %s' % (nice,cmd)
        if self.verbose:
            log.debug(cmd)
        proc = subprocess.Popen(cmd, shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,)
        out, err = proc.communicate()
        returncode = proc.returncode

        if returncode != 0:
            raise Exception("\nCommand failed with return code %s: \n %s \n %s" % (returncode, cmd, err))
        elif err and self.verbose:
            log.debug(err)
        return out

    ############ Shortcut Commands ############

    '''
    Copy a vector map from the base mapset to the temporary mapset
    '''
    def copyMap(self, type, mapName):
        command = "g.copy %s=%s@%s,%s" % (type, mapName, self.srcMapset, mapName)
        self.run(command)

    '''
    Input a vector ogr datasource (input) and save as a grass vector map (output)
    '''
    def v_in_ogr(self, input, output):
        command = 'v.in.ogr -o dsn=%s output=%s' % (input, output)
        self.run(command)

    '''
    Input a raster ogr datasource (input) and save as a grass raster map (output)
    '''
    def r_in_gdal(self, input, output):
        command = 'r.in.gdal -o input=%s output=%s' % (input, output)
        self.run(command)

    '''
    Calculates the sum of all cell values where input is the map name
    '''
    def r_sum(self, input):
        value = 0.0
        valueResult = self.run('r.sum '+input)
        valueList = re.split('SUM = ', valueResult)
        if len(valueList) > 0:
            value = float(valueList[1])
            return value
        else:
            return None

    '''
    Calculate the area of a given raster (positive value cells) where input is the name
    of the map layer and cell_size is the width of each square cell in the raster
    '''
    def r_area(self, input, cell_size):
        #Change all value cells to 1 for counting
        maskCmd = 'r.mapcalc "maskMap=if(%s, 1)"' % (input)
        self.run(maskCmd, nice=1)

        #Calculate total fishing area
        area = 0.0
        cells = 0
        areaResult = self.run('r.sum maskMap')
        areaList = re.split('SUM = ', areaResult)
        if len(areaList) > 0:
            cells = float(areaList[1])
            area = self.__areaOfCells(cells, cell_size)
            return (cells, area)
        else:
            return None

    '''
    Intersect raster map m1 with m2, storing the cell value of m2 in the resulting raster
    '''
    def r_intersect(self, result, m1, m2):
        intersectCmd = 'r.mapcalc "%s=if(%s,%s)"' % (result, m1, m2)
        self.run(intersectCmd, nice=1)

    '''
    Converts vMap a vector map, to rMap a raster map.  Where vectors overlap, raster cells
    are given the value of val
    '''
    def v_to_r(self, vMap, rMap, val):
        command = 'v.to.rast use=val value=%s in=%s out=%s' % (val, vMap, rMap)
        self.run(command, nice=1)

    '''
    Calculates area where numCells is the number of square cells composing the area
    and cell_size is the width of the square cell
    '''
    def __areaOfCells(self, numCells, cell_size):
        return numCells * (cell_size * cell_size)

    '''
    Removes a grass map layer where type is the layer type 'vect' or 'rast' and layers
    is the name of the map layer
    '''
    def g_remove(self, type, layer):
        command = 'g.remove %s=%s' % (type,layer)
        self.run(command)

    '''
    returns a dict of available raster and vector datasets as a python list
    '''
    def list(self):
        cmd = 'g.mlist type=rast'
        rasts = self.run(cmd)
        cmd = 'g.mlist type=vect'
        vects = self.run(cmd)
        return {'rast': rasts.split(), 'vect': vects.split()}

<commit_msg>Set autoclean properly in grass object<commit_after>import re
import sys
import os
import time
import shutil
import subprocess
from lingcod.common.utils import get_logger
log = get_logger()

DJANGO = True
try:
    from django.conf import settings
except ImportError:
    # not in a django env
    DJANGO = False

'''
Sets up a grass environment in the current running process.  Allows grass commands
to be run in a shell environment.  Optimally grass python bindings would be used in the
future once they become more stable and full-featured

Grass is not thread-safe, see http://grass.osgeo.org/wiki/Parallel_GRASS_jobs
Because of this there can be issues with race conditions while working within
the same mapset with more than one process.  The solution is to create a new
mapset for each analysis run, copy the necessary maps into the new mapset,
run the analysis, and then remove the mapset when you're done.

@author: Tim Welch, Ecotrust 2009
@author: Matthew Perry, Ecotrust 2011
Concepts taken from the PyWPS project.

Example usage (django):
    g = Grass('mlpa_sc')
    result = g.run('r.info soilmoisture')

    # Django Settings:
    #    GRASS_GISBASE = '/usr/local/grass-6.4.1RC2'
    #    GRASS_GISDBASE = '/home/grass'

Example usage (outside django):
    g = Grass('mlpa_sc', gisdbase="/home/grass", gisbase="/usr/local/grass-6.4.1RC2")
    result = g.run('r.info soilmoisture')

'''

class Grass:
    '''
    If being used to load the source mapset then tmpMapset should be None or False.
    If you're running any commands with outputs, use tmpMapset=True (autogenerates
      tmpMapset name) or specify a unique name.
    autoclean=True/False determines if tmpMapset gets deleted at Grass.__del__()
    gisdbase and gisbase should be set in django settings or overriden with kwargs
    '''
    def __init__(self, location, srcMapset='PERMANENT', tmpMapset=True,
                 gisdbase=None, gisbase=None, autoclean=True):
        self.autoclean = autoclean
        self.verbose = False

        self.location = location
        if tmpMapset == True:
            # unique name not specified, autogenerate it
            self.tmpMapset = 'tmpmapset_%s_%s' % (srcMapset, str(time.time()))
        elif tmpMapset:
            self.tmpMapset = tmpMapset
        self.srcMapset = srcMapset
        self.templateMapset = 'mm_template_mapset'

        if DJANGO and not gisdbase:
            self.gisdbase = settings.GRASS_GISDBASE
        else:
            self.gisdbase = gisdbase
        if DJANGO and not gisbase:
            self.gisbase = settings.GRASS_GISBASE
        else:
            self.gisbase = gisbase

        self.locationPath = os.path.join(self.gisdbase, self.location) #mlpa_nc
        self.tmpMapsetPath = os.path.join(self.locationPath, self.tmpMapset)
        self.srcMapsetPath = os.path.join(self.locationPath, self.srcMapset) #PERMANENT
        self.templatePath = os.path.join(self.locationPath, self.templateMapset) #mm_template_mapset

        if not tmpMapset:
            self.curMapset = srcMapset
            self.grassRcFile = os.path.join(gisdbase, "mm_grass6rc")
            self.setupTmpGrassEnv(mktmp=False)
        else:
            self.curMapset = self.tmpMapset
            self.grassRcFile = os.path.join(self.tmpMapsetPath, "mm_grass6rc")
            self.setupTmpGrassEnv()

        #Turn on verbose debug output
        #self.runCmd('g.gisenv set=DEBUG=3')

    def __del__(self):
        if self.autoclean:
            self.cleanup()

    def __repr__(self):
        if self.tmpMapset:
            return u'Grass instance at temporary mapset %s (%s)' % (self.tmpMapset, self.gisdbase)
        else:
            return u'Grass instance at permanent mapset %s' % self.srcMapsetPath

    @property
    def grass_tmp(self):
        try:
            return settings.GRASS_TMP
        except:
            return '/tmp'

    @property
    def grass_path(self):
        try:
            return settings.GRASS_PATH
        except:
            return '%(GRASS)s/bin:%(GRASS)s/scripts:/usr/local/sbin:/usr/local/bin:' \
                   '/usr/sbin:/usr/bin:/sbin:/bin' % {'GRASS': self.gisbase}

    @property
    def grass_version(self):
        try:
            return settings.GRASS_VERSION
        except:
            try:
                return self.gisbase.split('-')[1].replace('/','')
            except:
                return "6.4.1"

    @property
    def grass_lib_path(self):
        try:
            return settings.GRASS_LIB_PATH
        except:
            return '%s/lib' % self.gisbase

    '''
    Setup grass environment to use a temporary mapset as the current mapset.
    Base maps can be used from the src mapset as it will be in your
    mapset path, but results will go to the temporary mapset.
    grassenv settings are system specific. To get these values start a
    grass shell and echo the environment variables to get the necessary
    values.  eg.  >grass64 ; >echo $GISBASE
    '''
    def setupTmpGrassEnv(self, mktmp=True):
        grassenv = {
            'HOME' : self.grass_tmp,
            'PATH' : self.grass_path,
            'GRASS_VERSION' : self.grass_version,
            'GISBASE': self.gisbase,
            'LD_LIBRARY_PATH' : self.grass_lib_path,
            'LOCATION_NAME' : self.location,
            'MAPSET' : self.srcMapset,
            'GISDBASE' : self.gisdbase,
            'GRASS_GUI' : 'text',
            'GIS_LOCK' : str(os.getpid())
        }

        for key in grassenv.keys():
            self.setEnv(key, grassenv[key])

        #Create temporary space for current analysis run
        if mktmp:
            self.createTempMapset()
        #Change to the directory with the RC file
        os.chdir(self.locationPath)
        #Output a new rc file, overwriting any old ones
        self.writeGrassRc()

    '''
    Create a temporary mapset
    '''
    def createTempMapset(self):
        if not os.path.exists(self.tmpMapsetPath):
            if os.path.exists(self.templatePath):
                # use template mapset directory if available
                shutil.copytree(self.templatePath, self.tmpMapsetPath)
            else:
                # .. otherwise create the bare minimum file structure
                os.mkdir(self.tmpMapsetPath)
                shutil.copyfile(
                        os.path.join(self.srcMapsetPath, 'DEFAULT_WIND'),
                        os.path.join(self.tmpMapsetPath, 'DEFAULT_WIND'))
                shutil.copyfile(
                        os.path.join(self.srcMapsetPath, 'WIND'),
                        os.path.join(self.tmpMapsetPath, 'WIND'))
                myname = open(os.path.join(self.tmpMapsetPath, 'MYNAME'),'w')
                myname.write(self.tmpMapset)
                myname.close()

    '''
    Cleanup disk such as temporary mapset
    '''
    def cleanup(self):
        if self.tmpMapset:
            shutil.rmtree(self.tmpMapsetPath)

    '''
    Output grassrc file used by Grass.  Set to use the current mapset
    '''
    def writeGrassRc(self):
        gisrc = open(self.grassRcFile,"w")
        gisrc.write("LOCATION_NAME: %s\n" % (self.location))
        gisrc.write("MAPSET: %s\n" % (self.curMapset))
        gisrc.write("DIGITIZER: none\n")
        gisrc.write("GISDBASE: %s\n" % (self.gisdbase))
        gisrc.write("OVERWRITE: 1\n")
        gisrc.write("GRASS_GUI: text\n")
        gisrc.close()
        os.environ['GISRC'] = self.grassRcFile

    '''
    Set a single environment variable to the given value
    '''
    def setEnv(self, key,value):
        origValue = os.getenv(key)
        #if origValue:
        #    value  += ":"+origValue
        os.putenv(key,value)
        os.environ[key] = value
        return

    '''
    Returns the raster path for the current mapset
    '''
    def getRastPath(self):
        return self.gisdbase+self.location+'/'+self.curMapset+'/cell/'

    def run(self, cmd, nice=None):
        if nice:
            cmd = 'nice -n %d %s' % (nice,cmd)
        if self.verbose:
            log.debug(cmd)
        proc = subprocess.Popen(cmd, shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,)
        out, err = proc.communicate()
        returncode = proc.returncode

        if returncode != 0:
            raise Exception("\nCommand failed with return code %s: \n %s \n %s" % (returncode, cmd, err))
        elif err and self.verbose:
            log.debug(err)
        return out

    ############ Shortcut Commands ############

    '''
    Copy a vector map from the base mapset to the temporary mapset
    '''
    def copyMap(self, type, mapName):
        command = "g.copy %s=%s@%s,%s" % (type, mapName, self.srcMapset, mapName)
        self.run(command)

    '''
    Input a vector ogr datasource (input) and save as a grass vector map (output)
    '''
    def v_in_ogr(self, input, output):
        command = 'v.in.ogr -o dsn=%s output=%s' % (input, output)
        self.run(command)

    '''
    Input a raster ogr datasource (input) and save as a grass raster map (output)
    '''
    def r_in_gdal(self, input, output):
        command = 'r.in.gdal -o input=%s output=%s' % (input, output)
        self.run(command)

    '''
    Calculates the sum of all cell values where input is the map name
    '''
    def r_sum(self, input):
        value = 0.0
        valueResult = self.run('r.sum '+input)
        valueList = re.split('SUM = ', valueResult)
        if len(valueList) > 0:
            value = float(valueList[1])
            return value
        else:
            return None

    '''
    Calculate the area of a given raster (positive value cells) where input is the name
    of the map layer and cell_size is the width of each square cell in the raster
    '''
    def r_area(self, input, cell_size):
        #Change all value cells to 1 for counting
        maskCmd = 'r.mapcalc "maskMap=if(%s, 1)"' % (input)
        self.run(maskCmd, nice=1)

        #Calculate total fishing area
        area = 0.0
        cells = 0
        areaResult = self.run('r.sum maskMap')
        areaList = re.split('SUM = ', areaResult)
        if len(areaList) > 0:
            cells = float(areaList[1])
            area = self.__areaOfCells(cells, cell_size)
            return (cells, area)
        else:
            return None

    '''
    Intersect raster map m1 with m2, storing the cell value of m2 in the resulting raster
    '''
    def r_intersect(self, result, m1, m2):
        intersectCmd = 'r.mapcalc "%s=if(%s,%s)"' % (result, m1, m2)
        self.run(intersectCmd, nice=1)

    '''
    Converts vMap a vector map, to rMap a raster map.  Where vectors overlap, raster cells
    are given the value of val
    '''
    def v_to_r(self, vMap, rMap, val):
        command = 'v.to.rast use=val value=%s in=%s out=%s' % (val, vMap, rMap)
        self.run(command, nice=1)

    '''
    Calculates area where numCells is the number of square cells composing the area
    and cell_size is the width of the square cell
    '''
    def __areaOfCells(self, numCells, cell_size):
        return numCells * (cell_size * cell_size)

    '''
    Removes a grass map layer where type is the layer type 'vect' or 'rast' and layers
    is the name of the map layer
    '''
    def g_remove(self, type, layer):
        command = 'g.remove %s=%s' % (type,layer)
        self.run(command)

    '''
    returns a dict of available raster and vector datasets as a python list
    '''
    def list(self):
        cmd = 'g.mlist type=rast'
        rasts = self.run(cmd)
        cmd = 'g.mlist type=vect'
        vects = self.run(cmd)
        return {'rast': rasts.split(), 'vect': vects.split()}

<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2011-2017 Luc Saffre
# This file is part of Lino Cosi.
#
# Lino Cosi is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Lino Cosi is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with Lino Cosi.  If not, see
# <http://www.gnu.org/licenses/>.

"""Default settings module for a :ref:`cosi` project. This is being
inherited by the other applications below :mod:`lino_cosi.projects`.

"""

from __future__ import unicode_literals

from lino.projects.std.settings import *

from django.utils.translation import ugettext_lazy as _

import lino_cosi

class Site(Site):
    """Base class for a :ref:`cosi` application.

    """

    verbose_name = "Lino Cosi"
    version = lino_cosi.SETUP_INFO['version']
    url = lino_cosi.SETUP_INFO['url']

    demo_fixtures = 'std few_countries minimal_ledger euvatrates \
    furniture demo demo_bookings payments demo2'.split()

    # languages = 'en de fr'
    languages = 'en'

    user_types_module = 'lino_cosi.lib.cosi.user_types'
    custom_layouts_module = 'lino_cosi.lib.cosi.layouts'

    default_build_method = 'wkhtmltopdf'

    # textfield_format = 'html'

    def get_installed_apps(self):
        yield super(Site, self).get_installed_apps()
        yield 'lino.modlib.gfks'
        # yield 'lino.modlib.system'
        yield 'lino.modlib.users'
        yield 'lino_xl.lib.countries'
        yield 'lino_cosi.lib.contacts'
        #~ yield 'lino_xl.lib.households'
        yield 'lino_xl.lib.products'
        yield 'lino_xl.lib.accounts'
        yield 'lino_xl.lib.sepa'

        yield 'lino_xl.lib.excerpts'

        # yield 'lino_xl.lib.outbox'
        # yield 'lino.modlib.uploads'
        yield 'lino_xl.lib.appypod'
        yield 'lino.modlib.export_excel'
        yield 'lino.modlib.tinymce'
        yield 'lino.modlib.wkhtmltopdf'

        # ledger must come before sales because its demo fixture
        # creates journals

        # yield 'lino.modlib.ledger'
        yield 'lino_xl.lib.vat'
        yield 'lino_xl.lib.finan'
        yield 'lino_xl.lib.sales'
        yield 'lino_xl.lib.invoicing'
        yield 'lino_xl.lib.declarations'
        #~ 'lino.modlib.journals',
        #~ 'lino_xl.lib.projects',
        #~ yield 'lino_xl.lib.blogs'
        #~ yield 'lino.modlib.tickets'
        #~ 'lino.modlib.links',
        #~ 'lino_xl.lib.thirds',
        #~ yield 'lino_xl.lib.postings'
        # yield 'lino_xl.lib.pages'
        # yield 'lino_cosi.lib.cosi'

    def setup_plugins(self):
        """
        Change the default value of certain plugin settings.

        """
        super(Site, self).setup_plugins()
        self.plugins.countries.configure(hide_region=True)
        self.plugins.ledger.configure(use_pcmn=True)
        self.plugins.countries.configure(country_code='BE')

    def setup_actions(self):
        super(Site, self).setup_actions()
        partners = self.modules.contacts
        from lino.core.merge import MergeAction
        for m in (partners.Person, partners.Organisation):
            m.define_action(merge_row=MergeAction(m))

    def setup_layouts(self):
        super(Site, self).setup_layouts()

        self.modules.system.SiteConfigs.set_detail_layout("""
        site_company next_partner_id:10
        default_build_method
        clients_account   sales_account
        suppliers_account purchases_account
        """)

        self.modules.accounts.Accounts.set_detail_layout("""
        ref:10 name id:5
        seqno group type clearable
        ledger.MovementsByAccount
        """)

# class DocsSite(Site):
#     """A special variant used to build the docs.
#     """
#     def get_installed_apps(self):
#         yield super(DocsSite, self).get_installed_apps()
#         yield 'lino_xl.lib.courses'  # needed for Sphinx autosummary
#         yield 'lino_xl.lib.vatless'

    # def get_apps_modifiers(self, **kw):
    #     kw = super(DocsSite, self).get_apps_modifiers(**kw)
    #     kw.update(sales='lino_cosi.lib.auto.sales')
    #     return kw
<commit_msg>http://luc.lino-framework.org/blog/2017/0712.html<commit_after># -*- coding: UTF-8 -*-
# Copyright 2011-2017 Luc Saffre
# This file is part of Lino Cosi.
#
# Lino Cosi is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Lino Cosi is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with Lino Cosi.  If not, see
# <http://www.gnu.org/licenses/>.

"""Default settings module for a :ref:`cosi` project. This is being
inherited by the other applications below :mod:`lino_cosi.projects`.

"""

from __future__ import unicode_literals

from lino.projects.std.settings import *

from django.utils.translation import ugettext_lazy as _

import lino_cosi

class Site(Site):
    """Base class for a :ref:`cosi` application.

    """

    verbose_name = "Lino Cosi"
    version = lino_cosi.SETUP_INFO['version']
    url = lino_cosi.SETUP_INFO['url']

    demo_fixtures = 'std few_countries minimal_ledger euvatrates \
    furniture demo demo_bookings payments demo2'.split()

    # languages = 'en de fr'
    languages = 'en'

    user_types_module = 'lino_cosi.lib.cosi.user_types'
    custom_layouts_module = 'lino_cosi.lib.cosi.layouts'

    default_build_method = 'wkhtmltopdf'

    # textfield_format = 'html'

    def get_installed_apps(self):
        yield super(Site, self).get_installed_apps()
        yield 'lino.modlib.gfks'
        # yield 'lino.modlib.system'
        yield 'lino.modlib.users'
        yield 'lino_xl.lib.countries'
        yield 'lino_cosi.lib.contacts'
        #~ yield 'lino_xl.lib.households'
        yield 'lino_xl.lib.products'
        yield 'lino_xl.lib.accounts'
        yield 'lino_xl.lib.sepa'

        yield 'lino_xl.lib.excerpts'

        # yield 'lino_xl.lib.outbox'
        # yield 'lino.modlib.uploads'
        yield 'lino_xl.lib.appypod'
        yield 'lino.modlib.export_excel'
        yield 'lino.modlib.tinymce'
        yield 'lino.modlib.wkhtmltopdf'

        # ledger must come before sales because its demo fixture
        # creates journals

        # yield 'lino.modlib.ledger'
        yield 'lino_xl.lib.vat'
        yield 'lino_xl.lib.finan'
        yield 'lino_xl.lib.sales'
        yield 'lino_xl.lib.invoicing'
        yield 'lino_xl.lib.bevat'
        #~ 'lino.modlib.journals',
        #~ 'lino_xl.lib.projects',
        #~ yield 'lino_xl.lib.blogs'
        #~ yield 'lino.modlib.tickets'
        #~ 'lino.modlib.links',
        #~ 'lino_xl.lib.thirds',
        #~ yield 'lino_xl.lib.postings'
        # yield 'lino_xl.lib.pages'
        # yield 'lino_cosi.lib.cosi'

    def setup_plugins(self):
        """
        Change the default value of certain plugin settings.

        """
        super(Site, self).setup_plugins()
        self.plugins.countries.configure(hide_region=True)
        self.plugins.ledger.configure(use_pcmn=True)
        self.plugins.countries.configure(country_code='BE')

    def setup_actions(self):
        super(Site, self).setup_actions()
        partners = self.modules.contacts
        from lino.core.merge import MergeAction
        for m in (partners.Person, partners.Organisation):
            m.define_action(merge_row=MergeAction(m))

    def setup_layouts(self):
        super(Site, self).setup_layouts()

        self.modules.system.SiteConfigs.set_detail_layout("""
        site_company next_partner_id:10
        default_build_method
        clients_account   sales_account
        suppliers_account purchases_account
        """)

        self.modules.accounts.Accounts.set_detail_layout("""
        ref:10 name id:5
        seqno group type clearable
        ledger.MovementsByAccount
        """)

# class DocsSite(Site):
#     """A special variant used to build the docs.
#     """
#     def get_installed_apps(self):
#         yield super(DocsSite, self).get_installed_apps()
#         yield 'lino_xl.lib.courses'  # needed for Sphinx autosummary
#         yield 'lino_xl.lib.vatless'

    # def get_apps_modifiers(self, **kw):
    #     kw = super(DocsSite, self).get_apps_modifiers(**kw)
    #     kw.update(sales='lino_cosi.lib.auto.sales')
    #     return kw
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
import datetime
import json
from datetime import timedelta
from django.contrib.auth.models import User
from django.http import HttpResponse, Http404
from django.shortcuts import get_object_or_404
from django.utils.timezone import UTC
from social.apps.django_app.default.models import UserSocialAuth
from courseware import grades
from courseware.models import StudentModule , OfflineComputedGrade
from courseware.courses import get_course_by_id
from models import Teacher
from tasks import offline_calc
import time
import logging

log = logging.getLogger(__name__)

class JsonResponse(HttpResponse):
    """
    Wrapper for HttpResponse with the right content type and the dump to json.
    """
    def __init__(self, content={}, mimetype=None, status=None,
                 content_type='application/json'):
        super(JsonResponse, self).__init__(json.dumps(content), mimetype=mimetype,
                                           status=status, content_type=content_type)

def heartbeat(request):
    now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

    risposta = {
        "alive_at": now
    }

    return JsonResponse(risposta)

def teacher_view(request, id_teacher):
    try:
        teacher = get_object_or_404(Teacher, id_teacher=id_teacher)
    except Http404:
        emptyresponse = {}
        return JsonResponse(emptyresponse)
    name = u'%s %s' % (teacher.first_name, teacher.last_name)
    if teacher.image:
        imageurl = teacher.image
    else:
        imageurl = ''  # TODO: mettiamo un placeholder?

    descriptions = []
    for d in teacher.teacherdescription_set.all():
        descriptions.append(
            {"language": d.language,
             "label": d.label}
        )
    risposta = {
        "name": name,
        "imageUrl": imageurl,
        "desc": descriptions
    }
    return JsonResponse(risposta)

def user_courses(request, eco_user_id):
    risposta = []
    try:
        usa = get_object_or_404(UserSocialAuth, uid=eco_user_id)
    except Http404:
        return JsonResponse(risposta)
    # The pre-fetching of groups is done to make auth checks not require an
    # additional DB lookup (this kills the Progress page in particular).
    student = User.objects.prefetch_related("groups").get(id=usa.user.id)

    course_enrollements = student.courseenrollment_set.all()
    now = datetime.datetime.now(UTC())
    current_milli_time = lambda: int(round(time.time() * 1000))
    start = current_milli_time()
    for ce in course_enrollements:
        course_start_elaboration = current_milli_time();
        course_key = ce.course_id
        course_key_str = u'%s' % course_key
        course = get_course_by_id(course_key)
        log.info( "Get course for course_key"+str(course_key_str)+" take "+str (current_milli_time() - course_start_elaboration)+" milliseconds")
        grade_summary = optimized_grade(student,request,course)  #grades.grade(student, request, course)
        log.info( "Grade elabortion for course "+str(course_key_str)+" take "+str (current_milli_time() - course_start_elaboration)+" milliseconds")

        modules = StudentModule.objects.filter(student=student, course_id=course_key)
        viewCount = modules.count()
        if viewCount > 0:
            firstViewDate = modules.order_by('created')[0].created.strftime("%Y-%m-%dT%H:%M:%S")
            lastViewDate = modules.order_by('-modified')[0].modified.strftime("%Y-%m-%dT%H:%M:%S")
        else:
            firstViewDate = ""
            lastViewDate = ""

        # If cutoff is reached, and today > course.end_date -> course.end_date else ""
        nonzero_cutoffs = [cutoff for cutoff in course.grade_cutoffs.values() if cutoff > 0]
        success_cutoff = min(nonzero_cutoffs) if nonzero_cutoffs else None
        completedDate = ""
        if success_cutoff and grade_summary['percent'] > success_cutoff:
            if course.end < now:
                completedDate = course.end.strftime("%Y-%m-%dT%H:%M:%S")

        # Sum of difference between created and modified StudentModule for this course
        spentTime = timedelta()
        for m in modules:
            spentTime += m.modified - m.created
        spentTime = str(spentTime.seconds * 1000)  # Total time the user spent in this course in milliseconds

        risposta.append(
            {
                "id": course_key_str,  # it is a representation like u'edX/DemoX/Demo_Course'
                "viewCount": viewCount,
                "progressPercentage": grade_summary['percent'],
                # "currentPill": 3,  NOT USED because we provide progressPercentage
                "firstViewDate": firstViewDate,  # First StudentModule created
                "lastViewDate": lastViewDate,  # Last StudentModule modified
                "completedDate": completedDate,
                "spentTime": spentTime
            }
        )
    log.info("Full api elabortaion takes "+str(current_milli_time() -  start)+" milliseconds")
    return JsonResponse(risposta)

def optimized_grade(student, request, course):
    '''
    Similar to instructor offline_gradecal.student_grades the offline_gradecalc but we need
    to set a periodic task to update those data with a day(?) retention.
    Update need the django command compute_grades in background
    '''

    now = datetime.datetime.now(UTC())
    needRecalculation = False
    try:
        ocg = OfflineComputedGrade.objects.get(user=student, course_id=course.id)
        if (ocg + datetime.timedelta(days=1)) > now :
            offline_calc.delay(course)
        return json.loads(ocg.gradeset)
    except OfflineComputedGrade.DoesNotExist:
        log.info("OfflineComputedGrade DoesNotExist for "+str(student)+" in "+str(course.id))
        grade_summary = dict(percent = 0 )  # assume this and run task for calculate
        offline_calc.delay(course)
        return grade_summary

<commit_msg>Cleaning log debug<commit_after>#-*- coding: utf-8 -*-
import datetime
import json
from datetime import timedelta
from django.contrib.auth.models import User
from django.http import HttpResponse, Http404
from django.shortcuts import get_object_or_404
from django.utils.timezone import UTC
from social.apps.django_app.default.models import UserSocialAuth
from courseware import grades
from courseware.models import StudentModule , OfflineComputedGrade
from courseware.courses import get_course_by_id
from models import Teacher
from tasks import offline_calc
import time

class JsonResponse(HttpResponse):
    """
    Wrapper for HttpResponse with the right content type and the dump to json.
    """
    def __init__(self, content={}, mimetype=None, status=None,
                 content_type='application/json'):
        super(JsonResponse, self).__init__(json.dumps(content), mimetype=mimetype,
                                           status=status, content_type=content_type)

def heartbeat(request):
    now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

    risposta = {
        "alive_at": now
    }

    return JsonResponse(risposta)

def teacher_view(request, id_teacher):
    try:
        teacher = get_object_or_404(Teacher, id_teacher=id_teacher)
    except Http404:
        emptyresponse = {}
        return JsonResponse(emptyresponse)
    name = u'%s %s' % (teacher.first_name, teacher.last_name)
    if teacher.image:
        imageurl = teacher.image
    else:
        imageurl = ''  # TODO: mettiamo un placeholder?

    descriptions = []
    for d in teacher.teacherdescription_set.all():
        descriptions.append(
            {"language": d.language,
             "label": d.label}
        )
    risposta = {
        "name": name,
        "imageUrl": imageurl,
        "desc": descriptions
    }
    return JsonResponse(risposta)

def user_courses(request, eco_user_id):
    risposta = []
    try:
        usa = get_object_or_404(UserSocialAuth, uid=eco_user_id)
    except Http404:
        return JsonResponse(risposta)
    # The pre-fetching of groups is done to make auth checks not require an
    # additional DB lookup (this kills the Progress page in particular).
    student = User.objects.prefetch_related("groups").get(id=usa.user.id)

    course_enrollements = student.courseenrollment_set.all()
    now = datetime.datetime.now(UTC())
    for ce in course_enrollements:
        course_key = ce.course_id
        course_key_str = u'%s' % course_key
        try:
            course = get_course_by_id(course_key)
        except Http404:
            # This souldn't be happen if course is delete correctly (deleting also enrollments)
            continue
        grade_summary = optimized_grade(student,request,course)  #grades.grade(student, request, course)

        modules = StudentModule.objects.filter(student=student, course_id=course_key)
        viewCount = modules.count()
        if viewCount > 0:
            firstViewDate = modules.order_by('created')[0].created.strftime("%Y-%m-%dT%H:%M:%S")
            lastViewDate = modules.order_by('-modified')[0].modified.strftime("%Y-%m-%dT%H:%M:%S")
        else:
            firstViewDate = ""
            lastViewDate = ""

        # If cutoff is reached, and today > course.end_date -> course.end_date else ""
        nonzero_cutoffs = [cutoff for cutoff in course.grade_cutoffs.values() if cutoff > 0]
        success_cutoff = min(nonzero_cutoffs) if nonzero_cutoffs else None
        completedDate = ""
        if success_cutoff and grade_summary['percent'] > success_cutoff:
            if course.end < now:
                completedDate = course.end.strftime("%Y-%m-%dT%H:%M:%S")

        # Sum of difference between created and modified StudentModule for this course
        spentTime = timedelta()
        for m in modules:
            spentTime += m.modified - m.created
        spentTime = str(spentTime.seconds * 1000)  # Total time the user spent in this course in milliseconds

        risposta.append(
            {
                "id": course_key_str,  # it is a representation like u'edX/DemoX/Demo_Course'
                "viewCount": viewCount,
                "progressPercentage": grade_summary['percent'],
                # "currentPill": 3,  NOT USED because we provide progressPercentage
                "firstViewDate": firstViewDate,  # First StudentModule created
                "lastViewDate": lastViewDate,  # Last StudentModule modified
                "completedDate": completedDate,
                "spentTime": spentTime
            }
        )
    return JsonResponse(risposta)

def optimized_grade(student, request, course):
    '''
    Similar to instructor offline_gradecal.student_grades the offline_gradecalc but we need
    to set a periodic task to update those data with a day(?) retention.
    Update need the django command compute_grades in background
    '''

    now = datetime.datetime.now(UTC())
    needRecalculation = False
    try:
        ocg = OfflineComputedGrade.objects.get(user=student, course_id=course.id)
        if (ocg.updated + datetime.timedelta(days=1)) < now :
            offline_calc.delay(course)
        return json.loads(ocg.gradeset)
    except OfflineComputedGrade.DoesNotExist:
        grade_summary = dict(percent = 0 )  # assume this and run task for calculate
        offline_calc.delay(course)
        return grade_summary

<|endoftext|>


<commit_before>#!/usr/bin/env python

from PyQt4.QtGui import *
from PyQt4.QtCore import *

from roomlistwidget import RoomListWidget

class RoomReqListWidget(RoomListWidget):

    def getIconSize(self):
        return QSize(100, 100)

    def setRowSelected(self, id_item):
        if self.item is not None:
            item = self.table.findItems(id_item, Qt.MatchExactly)
            if item:
                row = item[0].row()
                self.table.selectRow(row)
<commit_msg>change function for select initial element in room's list; now search in all event's requirements and find the correct room to select<commit_after>#!/usr/bin/env python

from PyQt4.QtGui import *
from PyQt4.QtCore import *

from roomlistwidget import RoomListWidget

class RoomReqListWidget(RoomListWidget):

    def getIconSize(self):
        return QSize(100, 100)

    def setRowSelected(self, id_room):
        if self.item is not None:
            for requirement in self.event.requirements:
                if requirement.tag_name == "var_req" and\
                   requirement.value == id_room:
                    item = self.table.findItems(id_room, Qt.MatchExactly)
                    if item:
                        row = item[0].row()
                        self.table.selectRow(row)
                        break
<|endoftext|>


<commit_before># coding: utf-8

"""
デコレータについてのサンプルです。
"""
from trypython.common.commoncls import SampleBase
from trypython.common.commonfunc import pr

def log_it(func):
    """
    デコレータ関数

    :param func: 元の関数
    :return: デコレータ関数
    """

    def decorator_function(*args, **kwargs):
        pr('decorator', func.__name__, 'func-name')
        pr('decorator', args, '*args')
        pr('decorator', kwargs, '**kwargs')

        result = func(*args, **kwargs)
        pr('decorator', result, 'result')
        pr('----------------------------------', '')

        return result

    return decorator_function

def sum_it(func):
    """
    デコレータ関数

    :param func: 元の関数
    :return: デコレータ関数
    """

    def decorator_function(*args, **kwargs):
        func_result = func(*args, **kwargs)

        result = sum(range(1, func_result + 1))
        pr('sum_it', result)

        return result

    return decorator_function

class Sample(SampleBase):
    def exec(self):
        #
        # デコレータは、入力として関数を取り
        # 別の関数を返す関数のこと
        #
        # 元の関数をラップして、付加操作を追加する
        #

        #
        # そのまま利用
        #
        decorator_function = log_it(self.test_func)
        decorator_function(1, 2)

        #
        # 元の関数に @log_it を付与した版
        #
        self.test_func2(name='test value')

        #
        # @log_it を付与せずに
        # 関数をデコレータ関数で上書きした版
        #
        self.test_func3()

        #
        # @xxxxx を複数付与することも可能
        # 関数に最も近いデコレータから先に実行される
        #
        self.test_func4(1, 9)

        #
        # 関数の名前がどのようになるか
        #
        pr('test_func.__name__', self.test_func.__name__)
        pr('test_func2.__name__', self.test_func2.__name__)
        pr('test_func3.__name__', self.test_func3.__name__)
        pr('test_func4.__name__', self.test_func4.__name__)

    def test_func(self, x, y):
        pr('test-func', f'hello world {x} {y}')

    @log_it
    def test_func2(self, **kwargs):
        pr('test-func2', f'hello world {kwargs}')

    def test_func3(self):
        pr('test-func3', 'test-func3')

    test_func3 = log_it(test_func3)

    @sum_it
    @log_it
    def test_func4(self, *args):
        return sum(args)

    # 以下と同じことになる
    # test_func4 = sum_it(log_it(test_func4))

def go():
    obj = Sample()
    obj.exec()

if __name__ == '__main__':
    go()
<commit_msg>update decorator sample<commit_after># coding: utf-8

"""
デコレータについてのサンプルです。
"""
import functools

from trypython.common.commoncls import SampleBase
from trypython.common.commonfunc import pr

def log_it(func):
    """
    デコレータ関数

    :param func: 元の関数
    :return: デコレータ関数
    """

    def decorator_function(*args, **kwargs):
        pr('decorator', func.__name__, 'func-name')
        pr('decorator', args, '*args')
        pr('decorator', kwargs, '**kwargs')

        result = func(*args, **kwargs)
        pr('decorator', result, 'result')
        pr('----------------------------------', '')

        return result

    return decorator_function

def sum_it(func):
    """
    デコレータ関数

    :param func: 元の関数
    :return: デコレータ関数
    """

    def decorator_function(*args, **kwargs):
        func_result = func(*args, **kwargs)

        result = sum(range(1, func_result + 1))
        pr('sum_it', result)

        return result

    return decorator_function

def upper(func):
    # functools.wraps() を利用するとオリジナル関数のメタデータが上書きされない
    @functools.wraps(func)
    def decorator_function(*args, **kwargs):
        result: str = func(*args, **kwargs)
        return result.upper()

    return decorator_function

class Sample(SampleBase):
    def exec(self):
        #
        # デコレータは、入力として関数を取り
        # 別の関数を返す関数のこと
        #
        # 元の関数をラップして、付加操作を追加する
        #

        #
        # そのまま利用
        #
        decorator_function = log_it(self.test_func)
        decorator_function(1, 2)

        #
        # 元の関数に @log_it を付与した版
        #
        self.test_func2(name='test value')

        #
        # @log_it を付与せずに
        # 関数をデコレータ関数で上書きした版
        #
        self.test_func3()

        #
        # @xxxxx を複数付与することも可能
        # 関数に最も近いデコレータから先に実行される
        #
        self.test_func4(1, 9)

        #
        # functools.wrap()を利用した版
        #
        pr('test_func5', self.test_func5('hello decorator'))

        #
        # 関数の名前がどのようになるか
        #
        pr('test_func.__name__', self.test_func.__name__)
        pr('test_func2.__name__', self.test_func2.__name__)
        pr('test_func3.__name__', self.test_func3.__name__)
        pr('test_func4.__name__', self.test_func4.__name__)
        pr('test_func5.__name__', self.test_func5.__name__)

    def test_func(self, x, y):
        pr('test-func', f'hello world {x} {y}')

    @log_it
    def test_func2(self, **kwargs):
        pr('test-func2', f'hello world {kwargs}')

    def test_func3(self):
        pr('test-func3', 'test-func3')

    test_func3 = log_it(test_func3)

    @sum_it
    @log_it
    def test_func4(self, *args):
        return sum(args)

    # 以下と同じことになる
    # test_func4 = sum_it(log_it(test_func4))

    @upper
    def test_func5(self, message: str):
        return message

def go():
    obj = Sample()
    obj.exec()

if __name__ == '__main__':
    go()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding:utf-8 -*-
from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function

class Node(object):
    def __init__(self, value):
        self.left = None
        self.right = None
        self.value = value

# recursive
def preOrderRecur(root):
    if not root:
        return
    print(root.value)

    if root.left:
        preOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)

def inOrderRecur(root):
    if not root:
        return

    if root.left:
        inOrderRecur(root.left)

    print(root.value)

    if root.right:
        inOrderRecur(root.right)

def postOrderRecur(root):
    if not root:
        return

    if root.left:
        postOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)
    print(root.value)

# iterative
def preOrder(root):
    if root is None:
        return []

    stack = [root]
    preorder = []
    while stack:
        node = stack.pop()
        preorder.append(node.val)
        if node.right:
            stack.append(node.right)
        if node.left:
            stack.append(node.left)

    return preorder

def inOrder(root):
    if not root:
        return

    stack = []
    inorder = []
    cur = root
    while stack or cur:
        if cur:
            stack.append(cur)
            cur = cur.left
        else:
            cur = stack.pop()
            inorder.append(cur.value)
            cur = cur.right

    return inorder
<commit_msg>add iterative postorder<commit_after>#!/usr/bin/env python
# -*- coding:utf-8 -*-
from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function

class Node(object):
    def __init__(self, value):
        self.left = None
        self.right = None
        self.value = value

# recursive
def preOrderRecur(root):
    if not root:
        return
    print(root.value)

    if root.left:
        preOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)

def inOrderRecur(root):
    if not root:
        return

    if root.left:
        inOrderRecur(root.left)

    print(root.value)

    if root.right:
        inOrderRecur(root.right)

def postOrderRecur(root):
    if not root:
        return

    if root.left:
        postOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)
    print(root.value)

# iterative
def preOrder(root):
    if root is None:
        return []

    stack = [root]
    preorder = []
    while stack:
        node = stack.pop()
        preorder.append(node.val)
        if node.right:
            stack.append(node.right)
        if node.left:
            stack.append(node.left)

    return preorder

def inOrder(root):
    if not root:
        return

    stack = []
    inorder = []
    cur = root
    while stack or cur:
        if cur:
            stack.append(cur)
            cur = cur.left
        else:
            cur = stack.pop()
            inorder.append(cur.value)
            cur = cur.right

    return inorder

def postOrder(root):
    if not root:
        return

    stack = []
    postorder = []
    cur = root
    while stack or cur:
        while cur:
            stack.append(cur)
            cur = cur.left

        if stack:
            cur = stack.pop()
            postorder.append(cur.value)
            cur = cur.right

    return postorder
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding:utf-8 -*-
from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function

class Node(object):
    def __init__(self, value):
        self.left = None
        self.right = None
        self.value = value

# recursive
def preOrderRecur(root):
    if not root:
        return
    print(root.value)

    if root.left:
        preOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)

def inOrderRecur(root):
    if not root:
        return

    if root.left:
        inOrderRecur(root.left)

    print(root.value)

    if root.right:
        inOrderRecur(root.right)

def postOrderRecur(root):
    if not root:
        return

    if root.left:
        postOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)
    print(root.value)

# iterative
def preOrder(root):
    if root is None:
        return []

    stack = [root]
    preorder = []
    while stack:
        node = stack.pop()
        preorder.append(node.val)
        if node.right:
            stack.append(node.right)
        if node.left:
            stack.append(node.left)

    return preorder

def inOrder(root):
    if not root:
        return

    stack = []
    inorder = []
    cur = root
    while stack or cur:
        if cur:
            stack.append(cur)
            cur = cur.left
        else:
            cur = stack.pop()
            inorder.append(cur.value)
            cur = cur.right

    return inorder

def postOrder(root):
    if not root:
        return

    stack = []
    postorder = []
    cur = root
    while stack or cur:
        while cur:
            stack.append(cur)
            cur = cur.left

        if stack:
            cur = stack.pop()
            postorder.append(cur.value)
            cur = cur.right

    return postorder

def levelOrder(root):
    if not root:
        return

    queue = []
    queue.append(root)
    while queue:
        cur = queue.pop(0)
        # do something to cur

        if cur.left:
            queue.append(cur.left)

        if cur.right:
            queue.append(cur.right)
<commit_msg>adjust the level order return value<commit_after>#!/usr/bin/env python
# -*- coding:utf-8 -*-
from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function

class Node(object):
    def __init__(self, value):
        self.left = None
        self.right = None
        self.value = value

# recursive
def preOrderRecur(root):
    if not root:
        return
    print(root.value)

    if root.left:
        preOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)

def inOrderRecur(root):
    if not root:
        return

    if root.left:
        inOrderRecur(root.left)

    print(root.value)

    if root.right:
        inOrderRecur(root.right)

def postOrderRecur(root):
    if not root:
        return

    if root.left:
        postOrderRecur(root.left)
    if root.right:
        postOrderRecur(root.right)
    print(root.value)

# iterative
def preOrder(root):
    if root is None:
        return []

    stack = [root]
    preorder = []
    while stack:
        node = stack.pop()
        preorder.append(node.val)
        if node.right:
            stack.append(node.right)
        if node.left:
            stack.append(node.left)

    return preorder

def inOrder(root):
    if not root:
        return

    stack = []
    inorder = []
    cur = root
    while stack or cur:
        if cur:
            stack.append(cur)
            cur = cur.left
        else:
            cur = stack.pop()
            inorder.append(cur.value)
            cur = cur.right

    return inorder

def postOrder(root):
    if not root:
        return

    stack = []
    postorder = []
    cur = root
    while stack or cur:
        while cur:
            stack.append(cur)
            cur = cur.left

        if stack:
            cur = stack.pop()
            postorder.append(cur.value)
            cur = cur.right

    return postorder

def levelOrder(root):
    if not root:
        return

    queue = []
    levelorder = []
    queue.append(root)
    while queue:
        cur = queue.pop(0)
        levelorder.append(cur)
        # do something to cur

        if cur.left:
            queue.append(cur.left)

        if cur.right:
            queue.append(cur.right)

    return levelorder
<|endoftext|>


<commit_before>import Adafruit_Thermal

maxCol = 32
printer = Adafruit_Thermal("/dev/ttyO2", 19200, timeout=5)
printer.begin()
printer.upsideDownOn()

def parseTest(text):
    r = text.split(' ')
    # r.reverse()
    curLine = []
    fin = []
    for w in r:
    	if len(w)+len(curLine) > maxCol:
    		fin.append(curLine)
    		curLine = []
    		curLine.append(w)
    	else:
    		curLine.append(w)
    print fin
    return
<commit_msg>adding text method to prevent linebreaks from cutting words in half<commit_after>import Adafruit_Thermal

maxCol = 32
printer = Adafruit_Thermal("/dev/ttyO2", 19200, timeout=5)
printer.begin()
printer.upsideDownOn()

def parseTest(text):
    r = text.split(' ')
    # r.reverse()
    curLine = []
    fin = []
    for w in r:
    	if len(w)+len(curLine) > maxCol:
    		fin.append(curLine)
    		curLine = []
    		curLine.append(w)
    	else:
    		curLine.append(w)
    print fin
<|endoftext|>


<commit_before>from nalaf.learning.taggers import RelationExtractor
from nalaf.learning.taggers import StubSameSentenceRelationExtractor
from nalaf.learning.svmlight import SVMLightTreeKernels
from nalaf.preprocessing.tokenizers import TmVarTokenizer, NLTK_TOKENIZER
from nalaf.structures.relation_pipelines import RelationExtractionPipeline
from loctext.features.specific import LocationWordFeatureGenerator
from loctext.features.specific import ProteinWordFeatureGenerator
from nalaf.features.relations import TokenFeatureGenerator
from nalaf.features.relations.context import LinearDistanceFeatureGenerator
from nalaf.features.relations.context import EntityOrderFeatureGenerator
from nalaf.features.relations.context import IntermediateTokensFeatureGenerator
from nalaf.features.relations.path import PathFeatureGenerator
from nalaf.features.relations.sentence import NamedEntityCountFeatureGenerator, BagOfWordsFeatureGenerator, StemmedBagOfWordsFeatureGenerator
from nalaf.features.relations.new.sentence import SentenceFeatureGenerator
from nalaf.features.relations.entityhead import EntityHeadTokenFeatureGenerator, EntityHeadTokenUpperCaseFeatureGenerator, EntityHeadTokenDigitsFeatureGenerator, EntityHeadTokenPunctuationFeatureGenerator, EntityHeadTokenChainFeatureGenerator
from nalaf.preprocessing.edges import SentenceDistanceEdgeGenerator
from nalaf import print_verbose, print_debug

class LocTextSSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=0)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id):

        return [
            SentenceFeatureGenerator(
                f_counts=1,
                f_counts_in_between=2,
                f_order=None,  # 3
                f_bow=None,  # 5
                f_pos=None,  # 6
                f_tokens_count=7

            ),
        ]

    # @staticmethod
    # def default_feature_generators(prot_e_id, loc_e_id, graphs=None):
    #
    #     GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs
    #
    #     return [
    #         LocationWordFeatureGenerator(
    #             loc_e_id,
    #             prefix1=2
    #         ),
    #
    #         ProteinWordFeatureGenerator(
    #             GRAPHS_CLOSURE_VARIABLE,
    #             prefix_PWPE_bow=7,
    #             prefix_PWPE_bow_masked=9,
    #             prefix_PWPE_dep=10,
    #             prefix_protein_word_found=13,
    #             prefix_protein_not_word_found=14
    #         ),
    #
    #         IntermediateTokensFeatureGenerator(
    #             prefix_fwd_pos_intermediate=34,
    #
    #             prefix_bkd_bow_intermediate=35,
    #             prefix_bkd_bow_intermediate_masked=36,
    #             prefix_bkd_stem_intermediate=37,
    #             prefix_bkd_pos_intermediate=38,
    #
    #             prefix_bow_intermediate=39,
    #             prefix_bow_intermediate_masked=40,
    #             prefix_stem_intermediate=41,
    #             prefix_pos_intermediate=42,
    #         ),
    #
    #         LinearDistanceFeatureGenerator(
    #             distance=5,
    #             prefix_entity_linear_distance_greater_than=43,
    #             prefix_entity_linear_distance_lesser_than=44,
    #             # prefix_entity_linear_distance=45
    #         ),
    #
    #         EntityOrderFeatureGenerator(
    #             prefix_order_entity1_entity2=46,
    #             prefix_order_entity2_entity1=47,
    #         ),
    #
    #         PathFeatureGenerator(
    #             GRAPHS_CLOSURE_VARIABLE,
    #
    #             token_feature_generator=TokenFeatureGenerator(
    #                 # prefix_txt=78,  # 73 in relna
    #                 prefix_pos=79,  # 74
    #                 prefix_masked_txt=77,  # 75
    #                 prefix_stem_masked_txt=81,  # 76
    #                 prefix_ann_type=80,  # 77
    #             ),
    #
    #             prefix_45_len_tokens=73,
    #             prefix_46_len=None,  # None
    #             prefix_47_word_in_path=None,  # None
    #             prefix_48_dep_forward=65,
    #             prefix_49_dep_reverse=66,
    #             prefix_50_internal_pos=67,
    #             prefix_51_internal_masked_txt=68,
    #             prefix_52_internal_txt=69,
    #             prefix_53_internal_stem=70,
    #             prefix_54_internal_dep_forward=71,
    #             prefix_55_internal_dep_reverse=72,
    #             prefix_56_token_path=64,
    #             prefix_57_dep_style_gram=60,
    #             prefix_58_edge_gram=None,  # None
    #             prefix_59_ann_edge_gram=None,  # None
    #             prefix_60_edge_directions=63,
    #             prefix_61_dep_1=49,
    #             prefix_62_masked_txt_dep_0=50,
    #             prefix_63_pos_dep_0=51,
    #             prefix_64_ann_type_1=52,
    #             prefix_65_dep_to_1=None,
    #             prefix_66_masked_txt_dep_to_0=53,
    #             prefix_67_pos_to=54,
    #             prefix_68_ann_type_2=55,
    #             prefix_69_gov_g_text=56,
    #             prefix_70_gov_g_pos=57,
    #             prefix_71_gov_anns=58,
    #             prefix_72_triple=59,
    #         ),
    #
    #         EntityHeadTokenUpperCaseFeatureGenerator(
    #             prefix_entity1_upper_case_middle=87.1,
    #             prefix_entity2_upper_case_middle=87.2,
    #         ),
    #
    #         EntityHeadTokenDigitsFeatureGenerator(
    #             prefix_entity1_has_hyphenated_digits=89.1,
    #             prefix_entity2_has_hyphenated_digits=89.2,
    #         ),
    #
    #         EntityHeadTokenPunctuationFeatureGenerator(
    #             prefix_entity1_has_hyphen=90.1,
    #             prefix_entity1_has_fslash=91.1,
    #             prefix_entity2_has_hyphen=90.2,
    #             prefix_entity2_has_fslash=91.2,
    #         ),
    #
    #         BagOfWordsFeatureGenerator(
    #             prefix_bow_text=2,
    #             prefix_ne_bow_count=3,
    #         ),
    #
    #         StemmedBagOfWordsFeatureGenerator(
    #             prefix_bow_stem=4
    #         ),
    #
    #         NamedEntityCountFeatureGenerator(
    #             prot_e_id,
    #             prefix=107
    #         ),
    #
    #         NamedEntityCountFeatureGenerator(
    #             loc_e_id,
    #             prefix=108
    #         )
    #     ]

class LocTextDSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=1)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id, graphs=None):
        from loctext.features import DS

        GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs

        return [
            # Comment from Shrikant:
            # TODO ...Commenting follwing two function calls increases the overall Fscore by 0.3 points...

            # Trigram
            DS.AnyNGramFeatureGenerator(
                n_gram=3,
                #
                prefix_pos=3
            ),

            # Bigram
            DS.AnyNGramFeatureGenerator(
                n_gram=2,
                #
                prefix_bow=5,
                prefix_bow_masked=6,
                prefix_pos=7,
                prefix_stem=8
            ),

            DS.PatternFeatureGenerator(
                e1_class=prot_e_id,
                e2_class=loc_e_id,
                #
                prefix_ProtVerbWord=39,
                prefix_LocVerbWord=41,
                prefix_ProtVerbWordLocVerbWord=43,
                prefix_WordVerbProtLocVerbWord=45,
            ),

            DS.SameWordFeatureGenerator(
                prefix_sameWordsSamePOS=48,
                prefix_sameStemSamePOS=50,
            ),

            DS.LocEntityFeatureGenerator(
                localization_class_id=loc_e_id,
                prefix_localizationVerb=51
            ),

            DS.IndividualSentencesFeatureGenerator(
                prefix_sentence_1_POS=54,
                #
                prefix_sentence_2_POS=57
            ),

            DS.IntermediateTokenFeatureGenerator(
                prefix_fwdPOSIntermeditate=61,
                prefix_bkwdPOSIntermeditate=65,
                prefix_unorderedPOSIntermeditate=69,
            ),

            DS.LinearDistanceFeatureGenerator(
                # TODO such a small distance doesn't seem to make sense
                distance_threshold=5,
                #
                prefix_entityLinearDistGreaterThan=70,
                prefix_entityLinearDistLessThanOrEqual=71,
                prefix_entityLinearDist=72,
                prefix_entityLinearDistOffsets=72.1,
            ),

            #####

            EntityHeadTokenFeatureGenerator(

            ),

            EntityHeadTokenUpperCaseFeatureGenerator(
                prefix_entity1_upper_case_start=111.1,
                prefix_entity2_upper_case_start=111.2,
                prefix_entity1_upper_case_middle=112.1,
                prefix_entity2_upper_case_middle=112.1,
            ),

            EntityHeadTokenDigitsFeatureGenerator(
                prefix_entity1_has_digits=113.1,
                prefix_entity2_has_digits=113.2,
                prefix_entity1_has_hyphenated_digits=114.1,
                prefix_entity2_has_hyphenated_digits=114.2,
            ),

            EntityHeadTokenPunctuationFeatureGenerator(
                prefix_entity1_has_hyphen=115.1,
                prefix_entity2_has_hyphen=115.2,
                prefix_entity1_has_fslash=116.1,
                prefix_entity2_has_fslash=116.2,
            ),

            EntityHeadTokenChainFeatureGenerator(

            ),

            DS.BowFeatureGenerator(
                prefix_bow_of_tokens_and_entities=129,
            ),

            NamedEntityCountFeatureGenerator(
                prot_e_id,
                prefix=130
            ),

            NamedEntityCountFeatureGenerator(
                loc_e_id,
                prefix=131
            ),
        ]

class LocTextCombinedModelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            ss_model,
            ds_model):

        super().__init__(entity1_class, entity2_class, rel_type)

        self.ss_model = ss_model
        self.ds_model = ds_model
        self.submodels = [self.ss_model, self.ds_model]

    def annotate(self, target_corpus):

        for model in self.submodels:
            model.annotate(target_corpus)

        return target_corpus
<commit_msg>do not use<commit_after>from nalaf.learning.taggers import RelationExtractor
from nalaf.learning.taggers import StubSameSentenceRelationExtractor
from nalaf.learning.svmlight import SVMLightTreeKernels
from nalaf.preprocessing.tokenizers import TmVarTokenizer, NLTK_TOKENIZER
from nalaf.structures.relation_pipelines import RelationExtractionPipeline
from loctext.features.specific import LocationWordFeatureGenerator
from loctext.features.specific import ProteinWordFeatureGenerator
from nalaf.features.relations import TokenFeatureGenerator
from nalaf.features.relations.context import LinearDistanceFeatureGenerator
from nalaf.features.relations.context import EntityOrderFeatureGenerator
from nalaf.features.relations.context import IntermediateTokensFeatureGenerator
from nalaf.features.relations.path import PathFeatureGenerator
from nalaf.features.relations.sentence import NamedEntityCountFeatureGenerator, BagOfWordsFeatureGenerator, StemmedBagOfWordsFeatureGenerator
from nalaf.features.relations.new.sentence import SentenceFeatureGenerator
from nalaf.features.relations.entityhead import EntityHeadTokenFeatureGenerator, EntityHeadTokenUpperCaseFeatureGenerator, EntityHeadTokenDigitsFeatureGenerator, EntityHeadTokenPunctuationFeatureGenerator, EntityHeadTokenChainFeatureGenerator
from nalaf.preprocessing.edges import SentenceDistanceEdgeGenerator
from nalaf import print_verbose, print_debug

class LocTextSSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=0)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id):

        return [
            SentenceFeatureGenerator(
                f_counts=1,
                f_counts_in_between=2,
                f_order=None,  # 3
                f_bow=None,  # 5
                f_pos=None,  # 6
                f_tokens_count=None

            ),
        ]

    # @staticmethod
    # def default_feature_generators(prot_e_id, loc_e_id, graphs=None):
    #
    #     GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs
    #
    #     return [
    #         LocationWordFeatureGenerator(
    #             loc_e_id,
    #             prefix1=2
    #         ),
    #
    #         ProteinWordFeatureGenerator(
    #             GRAPHS_CLOSURE_VARIABLE,
    #             prefix_PWPE_bow=7,
    #             prefix_PWPE_bow_masked=9,
    #             prefix_PWPE_dep=10,
    #             prefix_protein_word_found=13,
    #             prefix_protein_not_word_found=14
    #         ),
    #
    #         IntermediateTokensFeatureGenerator(
    #             prefix_fwd_pos_intermediate=34,
    #
    #             prefix_bkd_bow_intermediate=35,
    #             prefix_bkd_bow_intermediate_masked=36,
    #             prefix_bkd_stem_intermediate=37,
    #             prefix_bkd_pos_intermediate=38,
    #
    #             prefix_bow_intermediate=39,
    #             prefix_bow_intermediate_masked=40,
    #             prefix_stem_intermediate=41,
    #             prefix_pos_intermediate=42,
    #         ),
    #
    #         LinearDistanceFeatureGenerator(
    #             distance=5,
    #             prefix_entity_linear_distance_greater_than=43,
    #             prefix_entity_linear_distance_lesser_than=44,
    #             # prefix_entity_linear_distance=45
    #         ),
    #
    #         EntityOrderFeatureGenerator(
    #             prefix_order_entity1_entity2=46,
    #             prefix_order_entity2_entity1=47,
    #         ),
    #
    #         PathFeatureGenerator(
    #             GRAPHS_CLOSURE_VARIABLE,
    #
    #             token_feature_generator=TokenFeatureGenerator(
    #                 # prefix_txt=78,  # 73 in relna
    #                 prefix_pos=79,  # 74
    #                 prefix_masked_txt=77,  # 75
    #                 prefix_stem_masked_txt=81,  # 76
    #                 prefix_ann_type=80,  # 77
    #             ),
    #
    #             prefix_45_len_tokens=73,
    #             prefix_46_len=None,  # None
    #             prefix_47_word_in_path=None,  # None
    #             prefix_48_dep_forward=65,
    #             prefix_49_dep_reverse=66,
    #             prefix_50_internal_pos=67,
    #             prefix_51_internal_masked_txt=68,
    #             prefix_52_internal_txt=69,
    #             prefix_53_internal_stem=70,
    #             prefix_54_internal_dep_forward=71,
    #             prefix_55_internal_dep_reverse=72,
    #             prefix_56_token_path=64,
    #             prefix_57_dep_style_gram=60,
    #             prefix_58_edge_gram=None,  # None
    #             prefix_59_ann_edge_gram=None,  # None
    #             prefix_60_edge_directions=63,
    #             prefix_61_dep_1=49,
    #             prefix_62_masked_txt_dep_0=50,
    #             prefix_63_pos_dep_0=51,
    #             prefix_64_ann_type_1=52,
    #             prefix_65_dep_to_1=None,
    #             prefix_66_masked_txt_dep_to_0=53,
    #             prefix_67_pos_to=54,
    #             prefix_68_ann_type_2=55,
    #             prefix_69_gov_g_text=56,
    #             prefix_70_gov_g_pos=57,
    #             prefix_71_gov_anns=58,
    #             prefix_72_triple=59,
    #         ),
    #
    #         EntityHeadTokenUpperCaseFeatureGenerator(
    #             prefix_entity1_upper_case_middle=87.1,
    #             prefix_entity2_upper_case_middle=87.2,
    #         ),
    #
    #         EntityHeadTokenDigitsFeatureGenerator(
    #             prefix_entity1_has_hyphenated_digits=89.1,
    #             prefix_entity2_has_hyphenated_digits=89.2,
    #         ),
    #
    #         EntityHeadTokenPunctuationFeatureGenerator(
    #             prefix_entity1_has_hyphen=90.1,
    #             prefix_entity1_has_fslash=91.1,
    #             prefix_entity2_has_hyphen=90.2,
    #             prefix_entity2_has_fslash=91.2,
    #         ),
    #
    #         BagOfWordsFeatureGenerator(
    #             prefix_bow_text=2,
    #             prefix_ne_bow_count=3,
    #         ),
    #
    #         StemmedBagOfWordsFeatureGenerator(
    #             prefix_bow_stem=4
    #         ),
    #
    #         NamedEntityCountFeatureGenerator(
    #             prot_e_id,
    #             prefix=107
    #         ),
    #
    #         NamedEntityCountFeatureGenerator(
    #             loc_e_id,
    #             prefix=108
    #         )
    #     ]

class LocTextDSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=1)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id, graphs=None):
        from loctext.features import DS

        GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs

        return [
            # Comment from Shrikant:
            # TODO ...Commenting follwing two function calls increases the overall Fscore by 0.3 points...

            # Trigram
            DS.AnyNGramFeatureGenerator(
                n_gram=3,
                #
                prefix_pos=3
            ),

            # Bigram
            DS.AnyNGramFeatureGenerator(
                n_gram=2,
                #
                prefix_bow=5,
                prefix_bow_masked=6,
                prefix_pos=7,
                prefix_stem=8
            ),

            DS.PatternFeatureGenerator(
                e1_class=prot_e_id,
                e2_class=loc_e_id,
                #
                prefix_ProtVerbWord=39,
                prefix_LocVerbWord=41,
                prefix_ProtVerbWordLocVerbWord=43,
                prefix_WordVerbProtLocVerbWord=45,
            ),

            DS.SameWordFeatureGenerator(
                prefix_sameWordsSamePOS=48,
                prefix_sameStemSamePOS=50,
            ),

            DS.LocEntityFeatureGenerator(
                localization_class_id=loc_e_id,
                prefix_localizationVerb=51
            ),

            DS.IndividualSentencesFeatureGenerator(
                prefix_sentence_1_POS=54,
                #
                prefix_sentence_2_POS=57
            ),

            DS.IntermediateTokenFeatureGenerator(
                prefix_fwdPOSIntermeditate=61,
                prefix_bkwdPOSIntermeditate=65,
                prefix_unorderedPOSIntermeditate=69,
            ),

            DS.LinearDistanceFeatureGenerator(
                # TODO such a small distance doesn't seem to make sense
                distance_threshold=5,
                #
                prefix_entityLinearDistGreaterThan=70,
                prefix_entityLinearDistLessThanOrEqual=71,
                prefix_entityLinearDist=72,
                prefix_entityLinearDistOffsets=72.1,
            ),

            #####

            EntityHeadTokenFeatureGenerator(

            ),

            EntityHeadTokenUpperCaseFeatureGenerator(
                prefix_entity1_upper_case_start=111.1,
                prefix_entity2_upper_case_start=111.2,
                prefix_entity1_upper_case_middle=112.1,
                prefix_entity2_upper_case_middle=112.1,
            ),

            EntityHeadTokenDigitsFeatureGenerator(
                prefix_entity1_has_digits=113.1,
                prefix_entity2_has_digits=113.2,
                prefix_entity1_has_hyphenated_digits=114.1,
                prefix_entity2_has_hyphenated_digits=114.2,
            ),

            EntityHeadTokenPunctuationFeatureGenerator(
                prefix_entity1_has_hyphen=115.1,
                prefix_entity2_has_hyphen=115.2,
                prefix_entity1_has_fslash=116.1,
                prefix_entity2_has_fslash=116.2,
            ),

            EntityHeadTokenChainFeatureGenerator(

            ),

            DS.BowFeatureGenerator(
                prefix_bow_of_tokens_and_entities=129,
            ),

            NamedEntityCountFeatureGenerator(
                prot_e_id,
                prefix=130
            ),

            NamedEntityCountFeatureGenerator(
                loc_e_id,
                prefix=131
            ),
        ]

class LocTextCombinedModelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            ss_model,
            ds_model):

        super().__init__(entity1_class, entity2_class, rel_type)

        self.ss_model = ss_model
        self.ds_model = ds_model
        self.submodels = [self.ss_model, self.ds_model]

    def annotate(self, target_corpus):

        for model in self.submodels:
            model.annotate(target_corpus)

        return target_corpus
<|endoftext|>


<commit_before>"""
MIT License

Copyright (c) 2018 Chad Rosenquist

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

Created on April 4, 2018

@author: Chad Rosenquist
"""

import logging
from functools import wraps
from enum import Enum

class DisplayLogs(Enum):
    """Determines when to display logs.

    :Values:
        * NEVER: Never display the logs.  The logs will always be discarded.
        * FAILURE: Display the logs only if the test case fails.
        * ALWAYS: Always displays the logs - pass or fail.
    """
    NEVER = 1
    FAILURE = 2
    ALWAYS = 3

def capturelogs(logger=None, level=None, display_logs=DisplayLogs.FAILURE):
    """Very similar to self.assertLogs() except can be used a function decorator, reducing clutter in test functions.

    :param logger: Name of logger, or an actual logger.  Defaults to root logger.
    :param level: Log level as a text string.  Defaults to 'INFO'.
    :param DisplayLogs display_logs:  Determines when to display logs.
        * DisplayLogs.NEVER: Never display the logs.  The logs will always be discarded.
        * DisplayLogs.FAILURE: Display the logs only if the test case fails.
        * DisplayLogs.ALWAYS: Always displays the logs - pass or fail.

    Example::

        import logging
        from loggingtestcase import capturelogs

        @capturelogs('foo', level='INFO')
        def test_capture_logs(self, logs):
            logging.getLogger('foo').info('first message')
            logging.getLogger('foo.bar').error('second message')

            self.assertEqual(logs.output, ['INFO:foo:first message',
                                           'ERROR:foo.bar:second message'])
            self.assertEqual(logs.records[0].message, 'first message')
            self.assertEqual(logs.records[1].message, 'second message')

    Always display logs example::

        import logging
        from loggingtestcase import capturelogs, DisplayLogs

        @capturelogs('foo', level='INFO', display_logs=DisplayLogs.ALWAYS)
        def test_always_display_logs(self, logs):
            logging.getLogger('foo').info('first message')
            self.assertTrue(False)
            self.assertEqual(logs.output, ['INFO:foo:first message'])

    """
    def decorate(func):
        log = _set_the_logger(logger)
        log_level = _set_the_level(level)

        @wraps(func)
        def wrapper(*args, **kwargs):
            assertion_error_raised = False

            # Capture the logs.
            formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s")
            handler = _CaptureLogsHandler()
            handler.setFormatter(formatter)
            old_handlers = log.handlers[:]
            old_level = log.level
            old_propagate = log.propagate
            log.handlers = [handler]
            log.setLevel(log_level)
            log.propagate = False

            try:
                # Call the function, adding the captured_logs as an argument.
                return_value = func(*args, handler.captured_logs, **kwargs)
                return return_value

            except Exception:
                assertion_error_raised = True
                raise

            finally:
                # Restore the logs.
                log.handlers = old_handlers
                log.level = old_level
                log.propagate = old_propagate

                # Print logs to the original handler.
                if display_logs is DisplayLogs.ALWAYS \
                        or (display_logs is DisplayLogs.FAILURE and assertion_error_raised):
                    for record in handler.captured_logs.records:
                        log.log(record.levelno, record.msg)
        return wrapper
    return decorate

def _set_the_logger(logger):
    if isinstance(logger, logging.Logger):
        return logger
    else:
        return logging.getLogger(logger)

def _set_the_level(level):
    if level:
        # noinspection PyProtectedMember
        return logging._nameToLevel.get(level, level)
    else:
        return logging.INFO

class _TheCapturedLogs(object):
    """Stores the captured logs."""
    def __init__(self):
        self.records = []
        self.output = []

    def __repr__(self):
        return 'records = {0} output = {1}'.format(self.records, self.output)

class _CaptureLogsHandler(logging.Handler):
    """Captures the logs and stores them so they can be inspected later."""
    def __init__(self):
        logging.Handler.__init__(self)
        self.captured_logs = _TheCapturedLogs()

    def flush(self):
        pass

    def emit(self, record):
        self.captured_logs.records.append(record)
        message = self.format(record)
        self.captured_logs.output.append(message)
<commit_msg>Fix Python 3.5+ only syntax<commit_after>"""
MIT License

Copyright (c) 2018 Chad Rosenquist

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

Created on April 4, 2018

@author: Chad Rosenquist
"""

import logging
from functools import wraps
from enum import Enum

class DisplayLogs(Enum):
    """Determines when to display logs.

    :Values:
        * NEVER: Never display the logs.  The logs will always be discarded.
        * FAILURE: Display the logs only if the test case fails.
        * ALWAYS: Always displays the logs - pass or fail.
    """
    NEVER = 1
    FAILURE = 2
    ALWAYS = 3

def capturelogs(logger=None, level=None, display_logs=DisplayLogs.FAILURE):
    """Very similar to self.assertLogs() except can be used a function decorator, reducing clutter in test functions.

    :param logger: Name of logger, or an actual logger.  Defaults to root logger.
    :param level: Log level as a text string.  Defaults to 'INFO'.
    :param DisplayLogs display_logs:  Determines when to display logs.
        * DisplayLogs.NEVER: Never display the logs.  The logs will always be discarded.
        * DisplayLogs.FAILURE: Display the logs only if the test case fails.
        * DisplayLogs.ALWAYS: Always displays the logs - pass or fail.

    Example::

        import logging
        from loggingtestcase import capturelogs

        @capturelogs('foo', level='INFO')
        def test_capture_logs(self, logs):
            logging.getLogger('foo').info('first message')
            logging.getLogger('foo.bar').error('second message')

            self.assertEqual(logs.output, ['INFO:foo:first message',
                                           'ERROR:foo.bar:second message'])
            self.assertEqual(logs.records[0].message, 'first message')
            self.assertEqual(logs.records[1].message, 'second message')

    Always display logs example::

        import logging
        from loggingtestcase import capturelogs, DisplayLogs

        @capturelogs('foo', level='INFO', display_logs=DisplayLogs.ALWAYS)
        def test_always_display_logs(self, logs):
            logging.getLogger('foo').info('first message')
            self.assertTrue(False)
            self.assertEqual(logs.output, ['INFO:foo:first message'])

    """
    def decorate(func):
        log = _set_the_logger(logger)
        log_level = _set_the_level(level)

        @wraps(func)
        def wrapper(*args, **kwargs):
            assertion_error_raised = False

            # Capture the logs.
            formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s")
            handler = _CaptureLogsHandler()
            handler.setFormatter(formatter)
            old_handlers = log.handlers[:]
            old_level = log.level
            old_propagate = log.propagate
            log.handlers = [handler]
            log.setLevel(log_level)
            log.propagate = False

            try:
                # Call the function, adding the captured_logs as an argument.
                args = list(args) + [handler.captured_logs]
                return_value = func(*args, **kwargs)
                return return_value

            except Exception:
                assertion_error_raised = True
                raise

            finally:
                # Restore the logs.
                log.handlers = old_handlers
                log.level = old_level
                log.propagate = old_propagate

                # Print logs to the original handler.
                if display_logs is DisplayLogs.ALWAYS \
                        or (display_logs is DisplayLogs.FAILURE and assertion_error_raised):
                    for record in handler.captured_logs.records:
                        log.log(record.levelno, record.msg)
        return wrapper
    return decorate

def _set_the_logger(logger):
    if isinstance(logger, logging.Logger):
        return logger
    else:
        return logging.getLogger(logger)

def _set_the_level(level):
    if level:
        # noinspection PyProtectedMember
        return logging._nameToLevel.get(level, level)
    else:
        return logging.INFO

class _TheCapturedLogs(object):
    """Stores the captured logs."""
    def __init__(self):
        self.records = []
        self.output = []

    def __repr__(self):
        return 'records = {0} output = {1}'.format(self.records, self.output)

class _CaptureLogsHandler(logging.Handler):
    """Captures the logs and stores them so they can be inspected later."""
    def __init__(self):
        logging.Handler.__init__(self)
        self.captured_logs = _TheCapturedLogs()

    def flush(self):
        pass

    def emit(self, record):
        self.captured_logs.records.append(record)
        message = self.format(record)
        self.captured_logs.output.append(message)
<|endoftext|>