<commit_before>#!/usr/bin/env python
# =====================================================================

# Globally useful modules:

import numpy
import sys,getopt
from PIL import Image
import humvi
import os

# =====================================================================

def compose(argv):
    """
    NAME
        compose.py

    PURPOSE
      Make a color composite PNG image from 3 FITS images, using the Lupton
      algorithm. Part of the HumVI package.

    COMMENTS
      Reads filter name and zero point from header and tries to set scales
      automatically - unless scales are set on command line. Note
      telescope/survey has to be recognised for this to work...

    USAGE
      compose.py [flags] red.fits green.fits blue.fits

    FLAGS
      -h        Print this message
      -v        Verbose operation
      -b --subtract-background

    INPUTS
      red.fits etc  Names of FITS files containing image data

    OPTIONAL INPUTS
      -s --scales   3,2,4     Comma-separated scales for R,G,B channels [None]
      -p --parameters  5,0.05 Non-linearity parameters Q,alpha
      -o --output   outfile   Name of output filename [guessed]
      -x --saturate-to style  Saturation method, color or white [white]
      -z --offset 0.1       Offset image level (+ve = gray background)
      -m --mask -1.0        Mask images below level [don't]

    OUTPUTS
      stdout      Useful information
      outfile     Output plot in png format

    EXAMPLES

        Note the different alpha required, to cope with survey depth!

        CFHTLS:
          compose.py  -v -s 0.4,0.6,1.7 -z 0.0 -p 1.7,0.09 -m -1.0 \
             -o examples/CFHTLS-test_gri.png \
                examples/CFHTLS-test_i.fits \
                examples/CFHTLS-test_r.fits \
                examples/CFHTLS-test_g.fits

        PS1 (Needs optimizing on a larger image.):
          compose.py  -v -s 0.6,0.6,1.7 -z 0.0 -p 1.7,0.00006 -m -1.0 \
             -o examples/PS1-test_riz.png \
                examples/PS1-test_z.fits \
                examples/PS1-test_i.fits \
                examples/PS1-test_r.fits

        DES (Very experimental. No images checked in.):
          compose.py -v -s 2.0,1.5,2.5 -z 0.0 -p 2.0,0.04 -m -1.0 \
             -o examples/DES-test_gri.png \
                examples/DES-test_i.fits \
                examples/DES-test_r.fits \
                examples/DES-test_g.fits

        VICS82:
          compose.py -v -s 1.0,1.4,2.0 -z 0.0 -p 1.5,0.4 -m -1.0 \
             -o examples/VICS82-test_iJKs.png \
                examples/VICS82-test_Ks.fits \
                examples/VICS82-test_J.fits \
                examples/VICS82-test_i.fits
        KiDS:
          compose.py -v -s 0.4,1.0,1.7 -z 0.0 -p 1.0,0.02 -m -1.0 \
             -o examples/KiDS-test_gri.png \
                examples/KiDS-test_i.fits \
                examples/KiDS-test_r.fits \
                examples/KiDS-test_g.fits

    BUGS

    - Renormalized scales will not be appropriate if one image is in very
      different units to another, or if images are in counts, not counts per
      second or AB maggies.

    - Code currently assumes one file per channel, whereas we might want to
      use N>3 images in combination.  The image to channel transformation
      would be performed after scaling to flux units but before
      stretching. Masking would need to be done at this point too.

    - Should cope with just two, or even one, input image file(s).

    HISTORY
      2012-05-11    started Marshall (Oxford)
      2012-07-??    integrated with wherry.py Sandford (NYU)
      2012-12-19    defaults set for CFHTLS images Marshall (Oxford)
      2013-02-21    defaults set for PS1 images Marshall (Oxford)
      2013-02-26    experimenting with DES images Hirsch (UCL)
    """

    # -------------------------------------------------------------------

    try:
        opts, args = getopt.getopt(argv, "hvp:s:n:o:x:z:blwm:",\
        ["help","verbose","scales","pars","output","saturate-to","offset","subtract-background","lupton","wherry","mask"])
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        print compose.__doc__
        return

    vb = False

    outfile = "color.png"

    # Defaults optimized for CFHTLS...
    pars = '1.7,0.09'
    scales = '0.4,0.6,1.7'

    # More general sensible choices:
    backsub = False
    saturation = 'white'
    offset = 0.0
    mask = False
    masklevel = -1.0

    for o,a in opts:
        if o in ("-h", "--help"):
            print compose.__doc__
            return
        elif o in ("-v", "--verbose"):
            vb = True
        elif o in ("-p","--parameters"):
            pars = a
        elif o in ("-s","--scales"):
            scales = a
        elif o in ("-x","--saturate-to"):
            saturation = a
        elif o in ("-z","--offset"):
            offset = float(a)
        elif o in ("-m","--mask"):
            mask = True
            masklevel = float(a)
        elif o in ("-o","--output"):
            outfile = a
        elif o in ("-b","--subtract-background"):
            backsub = True
        else:
            assert False, "Unhandled option"

    # Check for datafiles in array args:

    print len(args)

    if len(args) == 3:
        rfile = args[0]
        gfile = args[1]
        bfile = args[2]
        if vb:
            print "Making color composite image of data in following files:",rfile,gfile,bfile
            print "Output will be written to",outfile
            if mask: print "Masking stretched pixel values less than",masklevel

    else:
        print compose.__doc__
        return

    # Parse nonlinearity parameters:
    Qs,alphas = pars.split(',')
    Q = float(Qs)
    alpha = float(alphas)

    # Parse channel colour scales:
    x,y,z = scales.split(',')
    rscale,gscale,bscale = float(x),float(y),float(z)

    # -------------------------------------------------------------------
    # Read in images, calibrated into flux units:

    band3 = humvi.channel(rfile)
    band2 = humvi.channel(gfile)
    band1 = humvi.channel(bfile)

    # Check shapes are equal:
    humvi.check_image_shapes(band1.image,band2.image,band3.image)

    # Subtract backgrounds (median, optional):
    if backsub:
      band1.subtract_background()
      band2.subtract_background()
      band3.subtract_background()

    # -------------------------------------------------------------------

    # BUG: as it stands, this code assumes one file one channel, whereas
    # in practice we might like to be able to make composites based on
    # N bands. Need to overload + operator for channels? Calib etc will
    # need altering as well as image.

    red = band3
    green = band2
    blue = band1

    # -------------------------------------------------------------------
    # Set scales determining color balance in composite:

    rscale,gscale,bscale = humvi.normalize_scales(rscale,gscale,bscale)
    red.set_scale(manually=rscale)
    green.set_scale(manually=gscale)
    blue.set_scale(manually=bscale)
    if vb: print 'Scales normalized to:',red.scale,green.scale,blue.scale

    # Scale images - only do once:
    red.apply_scale()
    green.apply_scale()
    blue.apply_scale()

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Stretch images to cope with high dynamic range:

    if vb:
        print "Stretch parameters Q,alpha:",Q,alpha
        print "At low surface brightness levels, the channel images are further rescaled by alpha"
        print "Nonlinearity sets in at about 1/Q*alpha in the scaled intensity image:",1.0/(Q*alpha)

    # Compute total intensity image and the arcsinh of it:
    I = humvi.lupton_intensity(red.image,green.image,blue.image,type='sum')
    stretch = humvi.lupton_stretch(I,Q,alpha)

    # Apply stretch to channel images:
    r = stretch * red.image
    g = stretch * green.image
    b = stretch * blue.image

    if mask:
        # Mask problem areas - exact zeros or very negative patches should
        # be set to zero.

        # BUG: this should have been done after scaling but before conversion
        # to channels, as its the individual images that have problems...

        r,g,b = humvi.pjm_mask(r,g,b,masklevel)

    # Offset the stretched images to make zero level appear dark gray.
    # Negative offset makes background more black...
    r,g,b = humvi.pjm_offset(r,g,b,offset)

    if saturation == 'color':
        # Saturate to colour at some level - might as well be 1, since
        # Q redefines scale?:
        threshold = 1.0
        r,g,b = humvi.lupton_saturate(r,g,b,threshold)
    # Otherwise, saturate to white.

    # Package into a python Image, and write out to file:
    image = humvi.pack_up(r,g,b)
    image.save(outfile)

# ======================================================================

    if vb: print "Image saved to:",outfile
    return

# ======================================================================

if __name__ == '__main__':
    compose(sys.argv[1:])

# ======================================================================
<commit_msg>Tweaked scales for KiDS example<commit_after>#!/usr/bin/env python
# =====================================================================

# Globally useful modules:

import numpy
import sys,getopt
from PIL import Image
import humvi
import os

# =====================================================================

def compose(argv):
    """
    NAME
        compose.py

    PURPOSE
      Make a color composite PNG image from 3 FITS images, using the Lupton
      algorithm. Part of the HumVI package.

    COMMENTS
      Reads filter name and zero point from header and tries to set scales
      automatically - unless scales are set on command line. Note
      telescope/survey has to be recognised for this to work...

    USAGE
      compose.py [flags] red.fits green.fits blue.fits

    FLAGS
      -h        Print this message
      -v        Verbose operation
      -b --subtract-background

    INPUTS
      red.fits etc  Names of FITS files containing image data

    OPTIONAL INPUTS
      -s --scales   3,2,4     Comma-separated scales for R,G,B channels [None]
      -p --parameters  5,0.05 Non-linearity parameters Q,alpha
      -o --output   outfile   Name of output filename [guessed]
      -x --saturate-to style  Saturation method, color or white [white]
      -z --offset 0.1       Offset image level (+ve = gray background)
      -m --mask -1.0        Mask images below level [don't]

    OUTPUTS
      stdout      Useful information
      outfile     Output plot in png format

    EXAMPLES

        Note the different alpha required, to cope with survey depth!

        CFHTLS:
          compose.py  -v -s 0.4,0.6,1.7 -z 0.0 -p 1.7,0.09 -m -1.0 \
             -o examples/CFHTLS-test_gri.png \
                examples/CFHTLS-test_i.fits \
                examples/CFHTLS-test_r.fits \
                examples/CFHTLS-test_g.fits

        PS1 (Needs optimizing on a larger image.):
          compose.py  -v -s 0.6,0.6,1.7 -z 0.0 -p 1.7,0.00006 -m -1.0 \
             -o examples/PS1-test_riz.png \
                examples/PS1-test_z.fits \
                examples/PS1-test_i.fits \
                examples/PS1-test_r.fits

        DES (Very experimental. No images checked in.):
          compose.py -v -s 2.0,1.5,2.5 -z 0.0 -p 2.0,0.04 -m -1.0 \
             -o examples/DES-test_gri.png \
                examples/DES-test_i.fits \
                examples/DES-test_r.fits \
                examples/DES-test_g.fits

        VICS82:
          compose.py -v -s 1.0,1.4,2.0 -z 0.0 -p 1.5,0.4 -m -1.0 \
             -o examples/VICS82-test_iJKs.png \
                examples/VICS82-test_Ks.fits \
                examples/VICS82-test_J.fits \
                examples/VICS82-test_i.fits
        KiDS:
          compose.py -v -s 0.4,0.6,1.7 -z 0.0 -p 1.5,0.02 -m -1.0 \
             -o examples/KiDS-test_gri.png \
                examples/KiDS-test_i.fits \
                examples/KiDS-test_r.fits \
                examples/KiDS-test_g.fits

    BUGS

    - Renormalized scales will not be appropriate if one image is in very
      different units to another, or if images are in counts, not counts per
      second or AB maggies.

    - Code currently assumes one file per channel, whereas we might want to
      use N>3 images in combination.  The image to channel transformation
      would be performed after scaling to flux units but before
      stretching. Masking would need to be done at this point too.

    - Should cope with just two, or even one, input image file(s).

    HISTORY
      2012-05-11    started Marshall (Oxford)
      2012-07-??    integrated with wherry.py Sandford (NYU)
      2012-12-19    defaults set for CFHTLS images Marshall (Oxford)
      2013-02-21    defaults set for PS1 images Marshall (Oxford)
      2013-02-26    experimenting with DES images Hirsch (UCL)
    """

    # -------------------------------------------------------------------

    try:
        opts, args = getopt.getopt(argv, "hvp:s:n:o:x:z:blwm:",\
        ["help","verbose","scales","pars","output","saturate-to","offset","subtract-background","lupton","wherry","mask"])
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        print compose.__doc__
        return

    vb = False

    outfile = "color.png"

    # Defaults optimized for CFHTLS...
    pars = '1.7,0.09'
    scales = '0.4,0.6,1.7'

    # More general sensible choices:
    backsub = False
    saturation = 'white'
    offset = 0.0
    mask = False
    masklevel = -1.0

    for o,a in opts:
        if o in ("-h", "--help"):
            print compose.__doc__
            return
        elif o in ("-v", "--verbose"):
            vb = True
        elif o in ("-p","--parameters"):
            pars = a
        elif o in ("-s","--scales"):
            scales = a
        elif o in ("-x","--saturate-to"):
            saturation = a
        elif o in ("-z","--offset"):
            offset = float(a)
        elif o in ("-m","--mask"):
            mask = True
            masklevel = float(a)
        elif o in ("-o","--output"):
            outfile = a
        elif o in ("-b","--subtract-background"):
            backsub = True
        else:
            assert False, "Unhandled option"

    # Check for datafiles in array args:

    print len(args)

    if len(args) == 3:
        rfile = args[0]
        gfile = args[1]
        bfile = args[2]
        if vb:
            print "Making color composite image of data in following files:",rfile,gfile,bfile
            print "Output will be written to",outfile
            if mask: print "Masking stretched pixel values less than",masklevel

    else:
        print compose.__doc__
        return

    # Parse nonlinearity parameters:
    Qs,alphas = pars.split(',')
    Q = float(Qs)
    alpha = float(alphas)

    # Parse channel colour scales:
    x,y,z = scales.split(',')
    rscale,gscale,bscale = float(x),float(y),float(z)

    # -------------------------------------------------------------------
    # Read in images, calibrated into flux units:

    band3 = humvi.channel(rfile)
    band2 = humvi.channel(gfile)
    band1 = humvi.channel(bfile)

    # Check shapes are equal:
    humvi.check_image_shapes(band1.image,band2.image,band3.image)

    # Subtract backgrounds (median, optional):
    if backsub:
      band1.subtract_background()
      band2.subtract_background()
      band3.subtract_background()

    # -------------------------------------------------------------------

    # BUG: as it stands, this code assumes one file one channel, whereas
    # in practice we might like to be able to make composites based on
    # N bands. Need to overload + operator for channels? Calib etc will
    # need altering as well as image.

    red = band3
    green = band2
    blue = band1

    # -------------------------------------------------------------------
    # Set scales determining color balance in composite:

    rscale,gscale,bscale = humvi.normalize_scales(rscale,gscale,bscale)
    red.set_scale(manually=rscale)
    green.set_scale(manually=gscale)
    blue.set_scale(manually=bscale)
    if vb: print 'Scales normalized to:',red.scale,green.scale,blue.scale

    # Scale images - only do once:
    red.apply_scale()
    green.apply_scale()
    blue.apply_scale()

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Stretch images to cope with high dynamic range:

    if vb:
        print "Stretch parameters Q,alpha:",Q,alpha
        print "At low surface brightness levels, the channel images are further rescaled by alpha"
        print "Nonlinearity sets in at about 1/Q*alpha in the scaled intensity image:",1.0/(Q*alpha)

    # Compute total intensity image and the arcsinh of it:
    I = humvi.lupton_intensity(red.image,green.image,blue.image,type='sum')
    stretch = humvi.lupton_stretch(I,Q,alpha)

    # Apply stretch to channel images:
    r = stretch * red.image
    g = stretch * green.image
    b = stretch * blue.image

    if mask:
        # Mask problem areas - exact zeros or very negative patches should
        # be set to zero.

        # BUG: this should have been done after scaling but before conversion
        # to channels, as its the individual images that have problems...

        r,g,b = humvi.pjm_mask(r,g,b,masklevel)

    # Offset the stretched images to make zero level appear dark gray.
    # Negative offset makes background more black...
    r,g,b = humvi.pjm_offset(r,g,b,offset)

    if saturation == 'color':
        # Saturate to colour at some level - might as well be 1, since
        # Q redefines scale?:
        threshold = 1.0
        r,g,b = humvi.lupton_saturate(r,g,b,threshold)
    # Otherwise, saturate to white.

    # Package into a python Image, and write out to file:
    image = humvi.pack_up(r,g,b)
    image.save(outfile)

# ======================================================================

    if vb: print "Image saved to:",outfile
    return

# ======================================================================

if __name__ == '__main__':
    compose(sys.argv[1:])

# ======================================================================
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

version = "HEAD"
<commit_msg>version: set version to 201203<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

version = "201203"
<|endoftext|>


<commit_before>from src.utils.downloads import Download
from data.data_loader import DataLoader
import re
import numpy as np

class Antibody:
    """
    TODO: write description
    """

    def __init__(self, sequence='', name='', numbering=None):
        self.raw_sequence = sequence
        self.sequence = sequence.replace('-', '').upper()
        self.name = name
        self.numbering = numbering
        self.hydrophobicity_matrix = np.array([])
        self.chain = ''
        self.mw = 0

    def apply_numbering(self, server='abysis', numbering_scheme='chothia'):

        available_numbering_schemes = ['chothia', 'chothia_ext', 'kabath']
        available_servers = ['abysis']

        assert numbering_scheme.lower() in available_numbering_schemes, \
            "Unknown Numbering scheme: {}. \
            Numbering schemes available: {}".format(numbering_scheme,
                                                    ', '.join(available_numbering_schemes))

        assert server in available_servers, "Unknown server: {}. \
            Available servers: {}".format(server, ' ,'.join(available_servers))

        self.numbering = get_ab_numbering(self.sequence, server, numbering_scheme)

        try:
            if self.numbering[0][0] == 'H':
                self.chain = 'Heavy'
            elif self.numbering[0][0] == 'L':
                self.chain = 'Light'
        except:
            self.chain = ''
            self.numbering = 'NA'
            print('Could not apply numbering scheme on provided sequence')

    def calculate_hydrophobicity(self, hydrophobicity_scores='ew'):

        # check if all the required parameters are in order
        available_hydrophobicity_scores = ['kd', 'ww', 'hh', 'mf', 'ew']

        if isinstance(hydrophobicity_scores, str):
            assert hydrophobicity_scores in available_hydrophobicity_scores, \
                "Chosen hydrophobicity scores ({}) not available. \
                Available hydrophobicity scores: {}".format(
                    hydrophobicity_scores, ' ,'.join(available_hydrophobicity_scores)
                )
        if self.chain == 'Light':
            with open('/Users/gilhoben/AbPyTools/data/NumberingSchemes/LightChothiaWithCDR.txt','r') as f:
                whole_sequence = [x.replace('\n', '') for x in f.readlines()]
        elif self.chain == 'Heavy':
            with open('/Users/gilhoben/AbPyTools/data/NumberingSchemes/HeavyChothiaWithCDR.txt','r') as f:
                whole_sequence = [x.replace('\n', '') for x in f.readlines()]
        else:
            self.hydrophobicity_matrix = 'NA'
            print('Could not calculate the hydrophobicity matrix of the \
                  the provided sequence')
            return

        # get the dictionary with the hydrophobicity scores
        aa_hydrophobicity_scores = get_aa_hydrophobicity_scores(hydrophobicity_scores)

        # instantiate numpy array
        self.hydrophobicity_matrix = np.zeros(len(whole_sequence))

        for i, position in enumerate(whole_sequence):

            if position not in self.numbering:
                self.hydrophobicity_matrix[i] = 0
    def ab_molecular_weight(self):

            else:
                position_in_data = self.numbering.index(position)
                self.hydrophobicity_matrix[i] = aa_hydrophobicity_scores[self.sequence[position_in_data]]
        data_loader = DataLoader(amino_acid_property=['MolecularWeight', 'average'])
        mw_dict = data_loader.get_data()

        return calculate_mw(self.sequence, mw_dict)

def get_ab_numbering(sequence, server, numbering_scheme):
    # check which server to use to get numbering
    if server.lower() == 'abysis':
        # find out which numbering scheme to use
        if numbering_scheme.lower() == 'chothia':
            scheme = '-c'
        elif numbering_scheme.lower() == 'chotia_ext':
            scheme = '-a'
        else:
            scheme = '-k'

        url = 'http://www.bioinf.org.uk/cgi-bin/abnum/abnum.pl?plain=1&aaseq={}&scheme={}'.format(sequence,
                                                                                scheme)
        numbering_table = Download(url, verbose=False)
        numbering_table.download()

        parsed_numbering_table = re.findall("[\S| ]+", numbering_table.html)

        numbering = [x[:-2] for x in parsed_numbering_table if x[-1]!= '-']

        # TODO: add more server options
    else:
        numbering = ['']

    return numbering

def get_aa_hydrophobicity_scores(hydrophobicity_scores='ew'):

    with open('/Users/gilhoben/AbPyTools/data/AminoAcidProperties.json') as f:
        hydrophobicity_data = json.load(f)
def calculate_mw(sequence, mw_dict):

    return hydrophobicity_data["hydrophobicity"][hydrophobicity_scores+'Hydrophobicity']
    return sum(mw_dict[x] for x in sequence) - (len(sequence) - 1) * mw_dict['water']
<commit_msg>added load method to calculate everything at once<commit_after>from src.utils.downloads import Download
from data.data_loader import DataLoader
import re
import numpy as np

class Antibody:
    """
    TODO: write description
    """

    def __init__(self, sequence='', name='', numbering=None):
        self.raw_sequence = sequence
        self.sequence = sequence.replace('-', '').upper()
        self.name = name
        self.numbering = numbering
        self.hydrophobicity_matrix = np.array([])
        self.chain = ''
        self.mw = 0

    def load(self):
        """
        Generates all the data available:
        - Antibody Numbering
        - Hydrophobicity matrix
        :return:

        """
        self.numbering, self.chain = self.ab_numbering()
        self.hydrophobicity_matrix = self.ab_hydrophobicity_matrix()
        self.mw = self.ab_molecular_weight()

    def ab_numbering(self, server='abysis', numbering_scheme='chothia'):
        # type: (str, str) -> object

        available_numbering_schemes = ['chothia', 'chothia_ext', 'kabath']
        available_servers = ['abysis']

        assert numbering_scheme.lower() in available_numbering_schemes, \
            "Unknown Numbering scheme: {}. \
            Numbering schemes available: {}".format(numbering_scheme,
                                                    ', '.join(available_numbering_schemes))

        assert server in available_servers, "Unknown server: {}. \
            Available servers: {}".format(server, ' ,'.join(available_servers))

        self.numbering = get_ab_numbering(self.sequence, server, numbering_scheme)

        try:
            if self.numbering[0][0] == 'H':
                self.chain = 'Heavy'
            elif self.numbering[0][0] == 'L':
                self.chain = 'Light'
        except:
            self.chain = ''
            self.numbering = 'NA'
            print('Could not apply numbering scheme on provided sequence')

    def calculate_hydrophobicity(self, hydrophobicity_scores='ew'):

        # check if all the required parameters are in order
        available_hydrophobicity_scores = ['kd', 'ww', 'hh', 'mf', 'ew']

        if isinstance(hydrophobicity_scores, str):
            assert hydrophobicity_scores in available_hydrophobicity_scores, \
                "Chosen hydrophobicity scores ({}) not available. \
                Available hydrophobicity scores: {}".format(
                    hydrophobicity_scores, ' ,'.join(available_hydrophobicity_scores)
                )
        if self.chain == 'Light':
            with open('/Users/gilhoben/AbPyTools/data/NumberingSchemes/LightChothiaWithCDR.txt','r') as f:
                whole_sequence = [x.replace('\n', '') for x in f.readlines()]
        elif self.chain == 'Heavy':
            with open('/Users/gilhoben/AbPyTools/data/NumberingSchemes/HeavyChothiaWithCDR.txt','r') as f:
                whole_sequence = [x.replace('\n', '') for x in f.readlines()]
        else:
            self.hydrophobicity_matrix = 'NA'
            print('Could not calculate the hydrophobicity matrix of the \
                  the provided sequence')
            return

        # get the dictionary with the hydrophobicity scores
        aa_hydrophobicity_scores = get_aa_hydrophobicity_scores(hydrophobicity_scores)

        # instantiate numpy array
        self.hydrophobicity_matrix = np.zeros(len(whole_sequence))

        for i, position in enumerate(whole_sequence):

            if position not in self.numbering:
                self.hydrophobicity_matrix[i] = 0
    def ab_molecular_weight(self):

            else:
                position_in_data = self.numbering.index(position)
                self.hydrophobicity_matrix[i] = aa_hydrophobicity_scores[self.sequence[position_in_data]]
        data_loader = DataLoader(amino_acid_property=['MolecularWeight', 'average'])
        mw_dict = data_loader.get_data()

        return calculate_mw(self.sequence, mw_dict)

def get_ab_numbering(sequence, server, numbering_scheme):
    # check which server to use to get numbering
    if server.lower() == 'abysis':
        # find out which numbering scheme to use
        if numbering_scheme.lower() == 'chothia':
            scheme = '-c'
        elif numbering_scheme.lower() == 'chotia_ext':
            scheme = '-a'
        else:
            scheme = '-k'

        url = 'http://www.bioinf.org.uk/cgi-bin/abnum/abnum.pl?plain=1&aaseq={}&scheme={}'.format(sequence,
                                                                                scheme)
        numbering_table = Download(url, verbose=False)
        numbering_table.download()

        parsed_numbering_table = re.findall("[\S| ]+", numbering_table.html)

        numbering = [x[:-2] for x in parsed_numbering_table if x[-1]!= '-']

        # TODO: add more server options
    else:
        numbering = ['']

    return numbering

def get_aa_hydrophobicity_scores(hydrophobicity_scores='ew'):

    with open('/Users/gilhoben/AbPyTools/data/AminoAcidProperties.json') as f:
        hydrophobicity_data = json.load(f)
def calculate_mw(sequence, mw_dict):

    return hydrophobicity_data["hydrophobicity"][hydrophobicity_scores+'Hydrophobicity']
    return sum(mw_dict[x] for x in sequence) - (len(sequence) - 1) * mw_dict['water']
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (C) 2008-2012 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Appier Framework. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2012 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import re
import copy
import base
import datetime

import util
import mongo
import exceptions

EMAIL_REGEX_VALUE = "^[\w\d\._%+-]+@[\w\d\.\-]+$"
""" The email regex value used to validate
if the provided value is in fact an email """

URL_REGEX_VALUE = "^\w+\:\/\/[^\:\/\?#]+(\:\d+)?(\/[^\?#]+)*\/?(\?[^#]*)?(#.*)?$"
""" The url regex value used to validate
if the provided value is in fact an URL/URI """

EMAIL_REGEX = re.compile(EMAIL_REGEX_VALUE)
""" The email regex used to validate
if the provided value is in fact an email """

URL_REGEX = re.compile(URL_REGEX_VALUE)
""" The url regex used to validate
if the provided value is in fact an URL/URI """

def validate(method = None, methods = [], object = None, ctx = None, build = True):
    # retrieves the base request object that is going to be used in
    # the construction of the object
    request = base.get_request()

    # uses the provided method to retrieves the complete
    # set of methods to be used for validation, this provides
    # an extra level of indirection
    methods = method and method() or methods
    errors = []

    # verifies if the provided object is valid in such case creates
    # a copy of it and uses it as the base object for validation
    # otherwise used an empty map (form validation)
    object = object and copy.copy(object) or {}

    # in case the build flag is set must process the received request
    # to correctly retrieve populate the object from it
    if build:
        # retrieves the current request data and tries to
        # "load" it as json data, in case it fails gracefully
        # handles the failure setting the value as an empty map
        data_j = util.request_json()

        # uses all the values referencing data in the request to try
        # to populate the object this way it may be constructed using
        # any of theses strategies (easier for the developer)
        for name, value in data_j.iteritems(): object[name] = value
        for name, value in request.files_s.iteritems(): object[name] = value
        for name, value in request.post_s.iteritems(): object[name] = value
        for name, value in request.params_s.iteritems(): object[name] = value

    for method in methods:
        try: method(object, ctx = ctx)
        except exceptions.ValidationInternalError, error:
            errors.append((error.name, error.message))

    errors_map = {}
    for name, message in errors:
        if not name in errors_map: errors_map[name] = []
        _errors = errors_map[name]
        _errors.append(message)

    return errors_map, object

def validate_b(method = None, methods = [], object = None, build = True):
    errors_map, object = validate(
        method = method,
        methods = methods,
        object = object,
        build = build
    )
    result = False if errors_map else True
    return result

def eq(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be equal to %s" % str(value_c)
        )
    return validation

def gt(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value > value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be greater than %s" % str(value_c)
        )
    return validation

def gte(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value >= value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be greater than or equal to %s" % str(value_c)
        )
    return validation

def lt(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value < value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be less than %s" % str(value_c)
        )
    return validation

def lte(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value <= value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be less than or equal to %s" % str(value_c)
        )
    return validation

def not_null(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if not value == None: return True
        raise exceptions.ValidationInternalError(name, "value is not set")
    return validation

def not_empty(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if len(value): return True
        raise exceptions.ValidationInternalError(name, "value is empty")
    return validation

def is_in(name, values):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value in values: return True
        raise exceptions.ValidationInternalError(name, "value is not in set")
    return validation

def is_email(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == "": return True
        if EMAIL_REGEX.match(value): return True
        raise exceptions.ValidationInternalError(name, "value is not a valid email")
    return validation

def is_url(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == "": return True
        if URL_REGEX.match(value): return True
        raise exceptions.ValidationInternalError(name, "value is not a valid url")
    return validation

def is_regex(name, regex):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == "": return True
        match = re.match(regex, value)
        if match: return True
        raise exceptions.ValidationInternalError(
            name, "value has incorrect format"
        )
    return validation

def field_eq(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v == field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be equal to %s" % field
        )
    return validation

def field_gt(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v > field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be greater than %s" % field
        )
    return validation

def field_lt(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v < field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be less than %s" % field
        )
    return validation

def string_gt(name, size):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if len(value) > size: return True
        raise exceptions.ValidationInternalError(
            name, "must be larger than %d characters" % size
        )
    return validation

def string_lt(name, size):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if len(value) < size: return True
        raise exceptions.ValidationInternalError(
            name, "must be smaller than %d characters" % size
        )
    return validation

def equals(first_name, second_name):
    def validation(object, ctx):
        first_value = object.get(first_name, None)
        second_value = object.get(second_name, None)
        if first_value == None: return True
        if second_value == None: return True
        if first_value == second_value: return True
        raise exceptions.ValidationInternalError(
            first_name, "value is not equals to %s" % second_name
        )
    return validation

def not_past(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value >= datetime.datetime.utcnow(): return True
        raise exceptions.ValidationInternalError(
            name, "date is in the past"
        )
    return validation

def not_duplicate(name, collection):
    def validation(object, ctx):
        _id = object.get("_id", None)
        value = object.get(name, None)
        if value == None: return True
        db = mongo.get_db()
        _collection = db[collection]
        item = _collection.find_one({name : value})
        if not item: return True
        if str(item["_id"]) == str(_id): return True
        raise exceptions.ValidationInternalError(
            name, "value is duplicate"
        )
    return validation

def all_different(name, name_ref = None):
    def validation(object, ctx):
        # uses the currently provided context to retrieve
        # the definition of the name to be validation and
        # in it's a valid relation type tries to retrieve
        # the underlying referenced name otherwise default
        # to the provided one or the id name
        cls = ctx.__class__
        definition = cls.definition_n(name)
        type = definition.get("type", unicode)
        _name_ref = name_ref or (hasattr(type, "_name") and type._name or "id")

        # tries to retrieve both the value for the identifier
        # in the current object and the values of the sequence
        # that is going to be used for all different matching in
        # case any of them does not exist returns valid
        value = object.get(name, None)
        if value == None: return True
        if len(value) == 0: return True

        # verifies if the sequence is in fact a proxy object and
        # contains the ids attribute in case that's the case the
        # ids attributes is retrieved as the sequence instead
        if hasattr(value, "ids"): values = value.ids

        # otherwise this is a normal sequence and the it must be
        # iterates to check if the reference name should be retrieve
        # or if the concrete values should be used instead
        else: values = [getattr(_value, _name_ref) if hasattr(_value, _name_ref) else _value\
            for _value in value]

        # creates a set structure from the the sequence of values
        # and in case the size of the sequence and the set are the
        # same the sequence is considered to not contain duplicates
        values_set = set(values)
        if len(value) == len(values_set): return True
        raise exceptions.ValidationInternalError(
            name, "has duplicates"
        )
    return validation

def no_self(name, name_ref = None):
    def validation(object, ctx):
        # uses the currently provided context to retrieve
        # the definition of the name to be validation and
        # in it's a valid relation type tries to retrieve
        # the underlying referenced name otherwise default
        # to the provided one or the id name
        cls = ctx.__class__
        definition = cls.definition_n(name)
        type = definition.get("type", unicode)
        _name_ref = name_ref or (hasattr(type, "_name") and type._name or "id")

        # tries to retrieve both the value for the identifier
        # in the current object and the values of the sequence
        # that is going to be used for existence matching in
        # case any of them does not exist returns valid
        _id = object.get(_name_ref, None)
        value = object.get(name, None)
        if _id == None: return True
        if value == None: return True

        # verifies if the sequence is in fact a proxy object and
        # contains the ids attribute in case that's the case the
        # ids attributes is retrieved as the sequence instead
        if hasattr(value, "ids"): values = value.ids

        # otherwise this is a normal sequence and the it must be
        # iterates to check if the reference name should be retrieve
        # or if the concrete values should be used instead
        else: values = [getattr(_value, _name_ref) if hasattr(_value, _name_ref) else _value\
            for _value in value]

        # verifies if the current identifier value exists in the
        # sequence and if that's the case raises the validation
        # exception indicating the validation problem
        exists = _id in values
        if not exists: return True
        raise exceptions.ValidationInternalError(
            name, "contains self"
        )
    return validation
<commit_msg>new validation methods<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (C) 2008-2012 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Appier Framework. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2012 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import re
import copy
import base
import datetime

import util
import mongo
import exceptions

EMAIL_REGEX_VALUE = "^[\w\d\._%+-]+@[\w\d\.\-]+$"
""" The email regex value used to validate
if the provided value is in fact an email """

URL_REGEX_VALUE = "^\w+\:\/\/[^\:\/\?#]+(\:\d+)?(\/[^\?#]+)*\/?(\?[^#]*)?(#.*)?$"
""" The url regex value used to validate
if the provided value is in fact an URL/URI """

EMAIL_REGEX = re.compile(EMAIL_REGEX_VALUE)
""" The email regex used to validate
if the provided value is in fact an email """

URL_REGEX = re.compile(URL_REGEX_VALUE)
""" The url regex used to validate
if the provided value is in fact an URL/URI """

def validate(method = None, methods = [], object = None, ctx = None, build = True):
    # retrieves the base request object that is going to be used in
    # the construction of the object
    request = base.get_request()

    # uses the provided method to retrieves the complete
    # set of methods to be used for validation, this provides
    # an extra level of indirection
    methods = method and method() or methods
    errors = []

    # verifies if the provided object is valid in such case creates
    # a copy of it and uses it as the base object for validation
    # otherwise used an empty map (form validation)
    object = object and copy.copy(object) or {}

    # in case the build flag is set must process the received request
    # to correctly retrieve populate the object from it
    if build:
        # retrieves the current request data and tries to
        # "load" it as json data, in case it fails gracefully
        # handles the failure setting the value as an empty map
        data_j = util.request_json()

        # uses all the values referencing data in the request to try
        # to populate the object this way it may be constructed using
        # any of theses strategies (easier for the developer)
        for name, value in data_j.iteritems(): object[name] = value
        for name, value in request.files_s.iteritems(): object[name] = value
        for name, value in request.post_s.iteritems(): object[name] = value
        for name, value in request.params_s.iteritems(): object[name] = value

    for method in methods:
        try: method(object, ctx = ctx)
        except exceptions.ValidationInternalError, error:
            errors.append((error.name, error.message))

    errors_map = {}
    for name, message in errors:
        if not name in errors_map: errors_map[name] = []
        _errors = errors_map[name]
        _errors.append(message)

    return errors_map, object

def validate_b(method = None, methods = [], object = None, build = True):
    errors_map, object = validate(
        method = method,
        methods = methods,
        object = object,
        build = build
    )
    result = False if errors_map else True
    return result

def eq(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be equal to %s" % str(value_c)
        )
    return validation

def gt(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value > value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be greater than %s" % str(value_c)
        )
    return validation

def gte(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value >= value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be greater than or equal to %s" % str(value_c)
        )
    return validation

def lt(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value < value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be less than %s" % str(value_c)
        )
    return validation

def lte(name, value_c):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value <= value_c: return True
        raise exceptions.ValidationInternalError(
            name, "must be less than or equal to %s" % str(value_c)
        )
    return validation

def not_null(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if not value == None: return True
        raise exceptions.ValidationInternalError(name, "value is not set")
    return validation

def not_empty(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if len(value): return True
        raise exceptions.ValidationInternalError(name, "value is empty")
    return validation

def is_in(name, values):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value in values: return True
        raise exceptions.ValidationInternalError(name, "value is not in set")
    return validation

def is_email(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == "": return True
        if EMAIL_REGEX.match(value): return True
        raise exceptions.ValidationInternalError(name, "value is not a valid email")
    return validation

def is_url(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == "": return True
        if URL_REGEX.match(value): return True
        raise exceptions.ValidationInternalError(name, "value is not a valid url")
    return validation

def is_regex(name, regex):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value == "": return True
        match = re.match(regex, value)
        if match: return True
        raise exceptions.ValidationInternalError(
            name, "value has incorrect format"
        )
    return validation

def field_eq(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v == field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be equal to %s" % field
        )
    return validation

def field_gt(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v > field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be greater than %s" % field
        )
    return validation

def field_gte(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v >= field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be greater or equal than %s" % field
        )
    return validation

def field_lt(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v < field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be less than %s" % field
        )
    return validation

def field_lte(name, field):
    def validation(object, ctx):
        name_v = object.get(name, None)
        field_v = object.get(field, None)
        if name_v == None: return True
        if field_v == None: return True
        if name_v <= field_v: return True
        raise exceptions.ValidationInternalError(
            name, "must be less or equal than %s" % field
        )
    return validation

def string_gt(name, size):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if len(value) > size: return True
        raise exceptions.ValidationInternalError(
            name, "must be larger than %d characters" % size
        )
    return validation

def string_lt(name, size):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if len(value) < size: return True
        raise exceptions.ValidationInternalError(
            name, "must be smaller than %d characters" % size
        )
    return validation

def equals(first_name, second_name):
    def validation(object, ctx):
        first_value = object.get(first_name, None)
        second_value = object.get(second_name, None)
        if first_value == None: return True
        if second_value == None: return True
        if first_value == second_value: return True
        raise exceptions.ValidationInternalError(
            first_name, "value is not equals to %s" % second_name
        )
    return validation

def not_past(name):
    def validation(object, ctx):
        value = object.get(name, None)
        if value == None: return True
        if value >= datetime.datetime.utcnow(): return True
        raise exceptions.ValidationInternalError(
            name, "date is in the past"
        )
    return validation

def not_duplicate(name, collection):
    def validation(object, ctx):
        _id = object.get("_id", None)
        value = object.get(name, None)
        if value == None: return True
        db = mongo.get_db()
        _collection = db[collection]
        item = _collection.find_one({name : value})
        if not item: return True
        if str(item["_id"]) == str(_id): return True
        raise exceptions.ValidationInternalError(
            name, "value is duplicate"
        )
    return validation

def all_different(name, name_ref = None):
    def validation(object, ctx):
        # uses the currently provided context to retrieve
        # the definition of the name to be validation and
        # in it's a valid relation type tries to retrieve
        # the underlying referenced name otherwise default
        # to the provided one or the id name
        cls = ctx.__class__
        definition = cls.definition_n(name)
        type = definition.get("type", unicode)
        _name_ref = name_ref or (hasattr(type, "_name") and type._name or "id")

        # tries to retrieve both the value for the identifier
        # in the current object and the values of the sequence
        # that is going to be used for all different matching in
        # case any of them does not exist returns valid
        value = object.get(name, None)
        if value == None: return True
        if len(value) == 0: return True

        # verifies if the sequence is in fact a proxy object and
        # contains the ids attribute in case that's the case the
        # ids attributes is retrieved as the sequence instead
        if hasattr(value, "ids"): values = value.ids

        # otherwise this is a normal sequence and the it must be
        # iterates to check if the reference name should be retrieve
        # or if the concrete values should be used instead
        else: values = [getattr(_value, _name_ref) if hasattr(_value, _name_ref) else _value\
            for _value in value]

        # creates a set structure from the the sequence of values
        # and in case the size of the sequence and the set are the
        # same the sequence is considered to not contain duplicates
        values_set = set(values)
        if len(value) == len(values_set): return True
        raise exceptions.ValidationInternalError(
            name, "has duplicates"
        )
    return validation

def no_self(name, name_ref = None):
    def validation(object, ctx):
        # uses the currently provided context to retrieve
        # the definition of the name to be validation and
        # in it's a valid relation type tries to retrieve
        # the underlying referenced name otherwise default
        # to the provided one or the id name
        cls = ctx.__class__
        definition = cls.definition_n(name)
        type = definition.get("type", unicode)
        _name_ref = name_ref or (hasattr(type, "_name") and type._name or "id")

        # tries to retrieve both the value for the identifier
        # in the current object and the values of the sequence
        # that is going to be used for existence matching in
        # case any of them does not exist returns valid
        _id = object.get(_name_ref, None)
        value = object.get(name, None)
        if _id == None: return True
        if value == None: return True

        # verifies if the sequence is in fact a proxy object and
        # contains the ids attribute in case that's the case the
        # ids attributes is retrieved as the sequence instead
        if hasattr(value, "ids"): values = value.ids

        # otherwise this is a normal sequence and the it must be
        # iterates to check if the reference name should be retrieve
        # or if the concrete values should be used instead
        else: values = [getattr(_value, _name_ref) if hasattr(_value, _name_ref) else _value\
            for _value in value]

        # verifies if the current identifier value exists in the
        # sequence and if that's the case raises the validation
        # exception indicating the validation problem
        exists = _id in values
        if not exists: return True
        raise exceptions.ValidationInternalError(
            name, "contains self"
        )
    return validation
<|endoftext|>


<commit_before># _*_ coding: utf-8 _*_

import pdb
import pytest
data = {('Roberts', 'John'): [20.00, 15.00],
        ('Edwards', 'Mary'): [5.00, 30.50],
        ('asdf', 'brice'): [2.35],
        }

NAME_CHECK_DATA = [('Bob Smith', 'Bob Smith'),
                   ('Bob Joe Smith', 'Bob Joe Smith'),
                   ('333', None)]

DONATION_CHECK_DATA = [('33', 33.0), ('45.52', 45.52), ('Buster', None)]

DONATION_DATA = [('2', 2.0), ('3.50', 3.5), ('4500', 4500.0)]

MENU_CHECK_DATA = [('1', '1'), ('2', '2'), ('q', 'q'),
                   ('quit', 'quit'), ('list', None)]

NAME_DATA = [('Mike Harrison', ('Harrison', 'Mike')),
             ('Dogulas Adams', ('Adams', 'Dogulas'))]

PROMPT_DATA = [('2', '2'), ('3', '3'), ('1', '1')]

@pytest.mark.parametrize('input, result', NAME_CHECK_DATA)
def test_check_data(input, result):
	"""Test name_check"""
    from mailroom import name_check
    assert name_check(input) == result

@pytest.mark.parametrize('input, result', DONATION_CHECK_DATA)
def test_check_data(input, result):
    """Test check_"""
    from mailroom import donation_check
    assert donation_check(input) == result

@pytest.mark.parametrize('input, result', MENU_CHECK_DATA)
def test_check_data(input, result):
    """Test menu_check function"""
    from mailroom import menu_check
    assert menu_check(input) == result

def test_sortbydonation():
    """Test Sort By donation function in mailroom"""
    from mailroom import sort_by_donation
    assert type(sort_by_donation(data)) == list
    assert len(sort_by_donation(data)) == 3
    data2 = sort_by_donation(data)
    for i in range(len(data2) - 1):
        assert data2[i] > data2[i + 1]  or len(data2) < 2

@pytest.mark.parametrize('xname, result', NAME_DATA)
def test_getname(xname, result):
    """Test Get Name function in mailroom"""
    from mailroom import get_name
    # name = 'Mike Kang Harrison'
    assert type(get_name(xname)) == tuple
    assert len(get_name(xname)) == 2
    assert get_name(xname) == result

@pytest.mark.parametrize('num, result', PROMPT_DATA)
def test_menuprompot(num, result):
    """ Test menu prompt"""
    from mailroom import menu_prompt
    # assert menu_prompt('2') == '2'
    assert menu_prompt(num) == result

@pytest.mark.parametrize('num, result', DONATION_DATA)
def test_getdonation(num, result):
    """Test Get Donation"""
    from mailroom import get_donation
    assert get_donation('2') == 2.0
    assert get_donation(num) == result
   <commit_msg>Fixed an indentation error<commit_after># _*_ coding: utf-8 _*_

import pdb
import pytest
data = {('Roberts', 'John'): [20.00, 15.00],
        ('Edwards', 'Mary'): [5.00, 30.50],
        ('asdf', 'brice'): [2.35],
        }

NAME_CHECK_DATA = [('Bob Smith', 'Bob Smith'),
                   ('Bob Joe Smith', 'Bob Joe Smith'),
                   ('333', None)]

DONATION_CHECK_DATA = [('33', 33.0), ('45.52', 45.52), ('Buster', None)]

DONATION_DATA = [('2', 2.0), ('3.50', 3.5), ('4500', 4500.0)]

MENU_CHECK_DATA = [('1', '1'), ('2', '2'), ('q', 'q'),
                   ('quit', 'quit'), ('list', None)]

NAME_DATA = [('Mike Harrison', ('Harrison', 'Mike')),
             ('Dogulas Adams', ('Adams', 'Dogulas'))]

PROMPT_DATA = [('2', '2'), ('3', '3'), ('1', '1')]

@pytest.mark.parametrize('input, result', NAME_CHECK_DATA)
def test_check_data(input, result):
    """Test name_check"""
    from mailroom import name_check
    assert name_check(input) == result

@pytest.mark.parametrize('input, result', DONATION_CHECK_DATA)
def test_check_data(input, result):
    """Test check_"""
    from mailroom import donation_check
    assert donation_check(input) == result

@pytest.mark.parametrize('input, result', MENU_CHECK_DATA)
def test_check_data(input, result):
    """Test menu_check function"""
    from mailroom import menu_check
    assert menu_check(input) == result

def test_sortbydonation():
    """Test Sort By donation function in mailroom"""
    from mailroom import sort_by_donation
    assert type(sort_by_donation(data)) == list
    assert len(sort_by_donation(data)) == 3
    data2 = sort_by_donation(data)
    for i in range(len(data2) - 1):
        assert data2[i] > data2[i + 1]  or len(data2) < 2

@pytest.mark.parametrize('xname, result', NAME_DATA)
def test_getname(xname, result):
    """Test Get Name function in mailroom"""
    from mailroom import get_name
    # name = 'Mike Kang Harrison'
    assert type(get_name(xname)) == tuple
    assert len(get_name(xname)) == 2
    assert get_name(xname) == result

@pytest.mark.parametrize('num, result', PROMPT_DATA)
def test_menuprompot(num, result):
    """ Test menu prompt"""
    from mailroom import menu_prompt
    # assert menu_prompt('2') == '2'
    assert menu_prompt(num) == result

@pytest.mark.parametrize('num, result', DONATION_DATA)
def test_getdonation(num, result):
    """Test Get Donation"""
    from mailroom import get_donation
    assert get_donation('2') == 2.0
    assert get_donation(num) == result
   <|endoftext|>


<commit_before>import logging
import sys
from xml.etree import ElementTree as ET
import re
from text.protein_entity import ProteinEntity

from token2 import Token2
from entity import Entities
from classification.re.relations import Pairs
from classification.re import ddi_kernels
from classification.re import relations
from text.chemical_entity import ChemdnerAnnotation
from text.mirna_entity import MirnaEntity

class Sentence(object):
    """Sentence from a document, to be annotated"""
    def __init__(self, text, offset=0, **kwargs):
        self.text = text
        self.sid = kwargs.get("sid")
        self.did = kwargs.get("did")
        self.entities = Entities(sid=self.sid, did=self.did)
        self.offset = offset
        self.pairs = Pairs()
        self.parsetree = None
        self.tokens = []

    def process_corenlp_sentence(self, corenlpres):
        """
        Process the results obtained with CoreNLP for this sentence
        :param corenlpres:
        :return:
        """
        # self.sentences = []
        if len(corenlpres['sentences']) > 1:
            sys.exit("Number of sentences from CoreNLP is not 1.")
        if len(corenlpres['sentences']) == 0:
            self.tokens = []
            self.create_newtoken("", {})
            logging.debug("no sentences")
            logging.debug(self.text)
            return
        sentence = corenlpres['sentences'][0]
        # print sentence
        self.parsetree = sentence['parsetree']
        for t in sentence['words']:
            # print t[0]
            if t[0]:
                # separate "-" when between words with more than 4 chars
                token_seq = re.split(r'-|/|\\|\+|\.', t[0])

                if len(token_seq) > 1 and all([len(elem) > 1 for elem in token_seq]):

                    for its, ts in enumerate(token_seq):
                        #print token_seq[:its]
                        charoffset_begin = int(t[1]["CharacterOffsetBegin"])
                        if token_seq[:its]:
                            charoffset_begin += sum([len(x) for x in token_seq[:its]])
                        charoffset_begin += its
                        charoffset_end = len(ts) + charoffset_begin
                        #logging.info(str(charoffset_begin) + ":" + str(charoffset_end))
                        ts_props = {"CharacterOffsetBegin": charoffset_begin,
                                    "CharacterOffsetEnd": charoffset_end,
                                    "PartOfSpeech": t[1]["PartOfSpeech"],
                                    "NamedEntityTag": t[1]["NamedEntityTag"],
                                    "Lemma": t[1]["Lemma"]}
                        self.create_newtoken(ts, ts_props)

                else:
                    self.create_newtoken(t[0], t[1])

    def create_newtoken(self, text, props):
        newtoken = Token2(text, order=len(self.tokens))
        try:
            newtoken.start = int(props["CharacterOffsetBegin"])
            newtoken.dstart = newtoken.start + self.offset
            newtoken.end = int(props["CharacterOffsetEnd"])
            newtoken.dend = newtoken.end + self.offset
            newtoken.pos = props["PartOfSpeech"]
            newtoken.tag = props["NamedEntityTag"]
            newtoken.lemma = props["Lemma"]
            # newtoken.stem = porter.stem_word(newtoken.text)
            newtoken.tid = self.sid + ".t" + str(len(self.tokens))
            self.tokens.append(newtoken)
            # print "|{}| <=> |{}|".format(text, self.text[newtoken.start:newtoken.end])
        except KeyError:
            logging.debug("error: text={} props={}".format(text, props))
            return None
        # logging.debug(newtoken.text)
        return newtoken

    def tag_entity(self, start, end, subtype="chemical", entity=None, totalchars=0, source="goldstandard", **kwargs):
        """Find the tokens that match this entity. start and end are relative to the sentence.
           Totalchars is the offset of the sentence on the document."""
        tlist = []
        #logging.debug("lets tag this entity")
        # print self.tokens
        for t in self.tokens:
            # discard tokens that intersect the entity for now
            # print t.start, t.end, t.text
            if t.start >= start and t.end <= end:
                tlist.append(t)
        if tlist:
            newtext = self.text[tlist[0].start:tlist[-1].end]
            if entity:
                entity.text = newtext
            if "text" in kwargs and newtext != kwargs["text"]:
                if newtext not in kwargs["text"] and kwargs["text"] not in newtext:
                    return None
                else:
                    logging.info("{}-{}|{}|=>|{}|{}-{}".format(tlist[0].start, tlist[-1].end, newtext, kwargs["text"],
                                 start, end))
                    logging.info("{} - {}".format(self.sid, self.text))
            #     print "tokens found:", [t.text for t in tlist]
                # sys.exit()
            # else:
            # print "found the tokens!", start, end, kwargs["text"], self.sid
            if self.entities.elist.get(source):
                eid = self.sid + ".e" + str(len(self.entities.elist[source]))
            else:
                eid = self.sid + ".e0"
            if entity:
                self.entities.add_entity(entity, source)
            elif subtype == "chemical":
                self.entities.add_entity(ChemdnerAnnotation(tlist, self.sid, text=newtext,
                                         did=self.did, eid=eid, subtype=subtype), source)
            elif subtype == "mirna":
                self.entities.add_entity(MirnaEntity(tlist, self.sid, text=newtext,
                                         did=self.did, eid=eid, subtype=subtype), source)
            elif subtype == "protein":
                self.entities.add_entity(ProteinEntity(tlist, self.sid, text=newtext,
                                         did=self.did, eid=eid, subtype=subtype), source)
            self.label_tokens(tlist, source, subtype)
            #logging.debug("added new entity to %s, now with %s entities" % (self.sid,
            #                                                                 len(self.entities.elist[source])))
            return eid
        else:
            print "no tokens found:"
            print start, end, kwargs.get("text")
            print [(t.start, t.end, t.text) for t in self.tokens]
            #

    def label_tokens(self, tlist, source, subtype="entity"):
        if len(tlist) == 1:
            tlist[0].tags[source] = "single"
            tlist[0].tags[source + "_subtype"] = subtype
        else:
            for t in range(len(tlist)):
                if t == 0:
                    tlist[t].tags[source] = "start"
                    tlist[t].tags[source + "_subtype"] = subtype
                elif t == len(tlist) - 1:
                    tlist[t].tags[source] = "end"
                    tlist[t].tags[source + "_subtype"] = subtype
                else:
                    tlist[t].tags[source] = "middle"
                    tlist[t].tags[source + "_subtype"] = subtype
        #logging.debug([t.tags for t in self.tokens])

    def write_bioc_results(self, parent, source):
        bioc_sentence = ET.SubElement(parent, "sentence")
        bioc_sentence_offset = ET.SubElement(bioc_sentence, "offset")
        bioc_sentence_offset.text = str(self.tokens[0].dstart)
        bioc_sentence_text = ET.SubElement(bioc_sentence, "text")
        bioc_sentence_text.text = self.text

        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                bioc_annotation = entity.write_bioc_annotation(bioc_sentence)
        return bioc_sentence

    def get_dic(self, source):
        dic = {}
        dic["id"] = self.sid
        dic["offset"] = str(self.tokens[0].dstart)
        dic["text"] = self.text
        dic["entities"] = []
        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                dic["entities"].append(entity.get_dic())
            dic["entities"] = sorted(dic["entities"], key=lambda k: k['offset'])
            for ei, e in enumerate(dic["entities"]):
                e["eid"] = self.sid + ".e{}".format(ei)
        dic["pairs"] = self.pairs.get_dic()
        return dic

    def find_tokens_between(self, start, end, relativeto="doc"):
        """Return list of tokens between offsets. Use relativeto to consider doc indexes or
           sentence indexes."""
        foundtokens = []
        for t in self.tokens:
            if relativeto.startswith("doc") and t.dstart >= start and t.dend <= end:
                foundtokens.append(t)
            elif relativeto.startswith("sent") and t.start >= start and t.end <= end:
                foundtokens.append(t)
        return foundtokens

    def test_relations(self, pairs, basemodel, classifiers=[relations.SLK_PRED, relations.SST_PRED],
                       tag="", backup=False, printstd=False):
        #data =  ddi_train_slk.model, ddi_train_sst.model
        tempfiles = []

        if relations.SLK_PRED in classifiers:
            logging.info("**Testing SLK classifier %s ..." % (tag,))
            #testpairdic = ddi_kernels.fromddiDic(testdocs)
            ddi_kernels.generatejSREdata(pairs, self, basemodel, tag + "ddi_test_jsre.txt")
            ddi_kernels.testjSRE(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                 model=tag + "all_ddi_train_slk.model")
            self.pairs.pairs = ddi_kernels.getjSREPredicitons(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                                      self.pairs.pairs)
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_jsre.txt")
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_result.txt")

        if relations.SST_PRED in classifiers:
            logging.info("****Testing SST classifier %s ..." % (tag,))
            self.pairs.pairs = ddi_kernels.testSVMTK(self, self.pairs.pairs, pairs,
                                             model=tag + "all_ddi_train_sst.model", tag=tag)
        for p in self.pairs.pairs:
            for r in self.pairs.pairs[p].recognized_by:
                if self.pairs.pairs[p].recognized_by[r] == 1:
                    p.relation = True
        return tempfiles
<commit_msg>Improve token separation<commit_after>import logging
import sys
from xml.etree import ElementTree as ET
import re
from text.protein_entity import ProteinEntity

from token2 import Token2
from entity import Entities
from classification.re.relations import Pairs
from classification.re import ddi_kernels
from classification.re import relations
from text.chemical_entity import ChemdnerAnnotation
from text.mirna_entity import MirnaEntity

class Sentence(object):
    """Sentence from a document, to be annotated"""
    def __init__(self, text, offset=0, **kwargs):
        self.text = text
        self.sid = kwargs.get("sid")
        self.did = kwargs.get("did")
        self.entities = Entities(sid=self.sid, did=self.did)
        self.offset = offset
        self.pairs = Pairs()
        self.parsetree = None
        self.tokens = []

    def process_corenlp_sentence(self, corenlpres):
        """
        Process the results obtained with CoreNLP for this sentence
        :param corenlpres:
        :return:
        """
        # self.sentences = []
        if len(corenlpres['sentences']) > 1:
            sys.exit("Number of sentences from CoreNLP is not 1.")
        if len(corenlpres['sentences']) == 0:
            self.tokens = []
            self.create_newtoken("", {})
            logging.debug("no sentences")
            logging.debug(self.text)
            return
        sentence = corenlpres['sentences'][0]
        # print sentence
        self.parsetree = sentence['parsetree']
        for t in sentence['words']:
            # print t[0]
            if t[0]:
                # separate "-" when between words with more than 1 chars
                token_seq = re.split(r'(\w+)(-|/|\\|\+|\.)(\w+)', t[0])

                if len(token_seq) > 1: # and all([len(elem) > 1 for elem in token_seq]):
                    #logging.info("{}: {}".format(t[0], "&".join(token_seq)))
                    for its, ts in enumerate(token_seq):
                        if ts.strip() != "":
                            charoffset_begin = int(t[1]["CharacterOffsetBegin"])
                            if token_seq[:its]: # not the first token
                                charoffset_begin += sum([len(x) for x in token_seq[:its]])
                            # charoffset_begin += its
                            charoffset_end = len(ts) + charoffset_begin
                            #logging.info(str(charoffset_begin) + ":" + str(charoffset_end))
                            ts_props = {"CharacterOffsetBegin": charoffset_begin,
                                        "CharacterOffsetEnd": charoffset_end,
                                        "PartOfSpeech": t[1]["PartOfSpeech"],
                                        "NamedEntityTag": t[1]["NamedEntityTag"],
                                        "Lemma": t[1]["Lemma"]}
                            self.create_newtoken(ts, ts_props)

                else:
                    self.create_newtoken(t[0], t[1])

    def create_newtoken(self, text, props):
        newtoken = Token2(text, order=len(self.tokens))
        try:
            newtoken.start = int(props["CharacterOffsetBegin"])
            newtoken.dstart = newtoken.start + self.offset
            newtoken.end = int(props["CharacterOffsetEnd"])
            newtoken.dend = newtoken.end + self.offset
            newtoken.pos = props["PartOfSpeech"]
            newtoken.tag = props["NamedEntityTag"]
            newtoken.lemma = props["Lemma"]
            # newtoken.stem = porter.stem_word(newtoken.text)
            newtoken.tid = self.sid + ".t" + str(len(self.tokens))
            self.tokens.append(newtoken)
            # print "|{}| <=> |{}|".format(text, self.text[newtoken.start:newtoken.end])
        except KeyError:
            logging.debug("error: text={} props={}".format(text, props))
            return None
        # logging.debug(newtoken.text)
        return newtoken

    def tag_entity(self, start, end, subtype="chemical", entity=None, totalchars=0, source="goldstandard", **kwargs):
        """Find the tokens that match this entity. start and end are relative to the sentence.
           Totalchars is the offset of the sentence on the document."""
        tlist = []
        #logging.debug("lets tag this entity")
        # print self.tokens
        for t in self.tokens:
            # discard tokens that intersect the entity for now
            # print t.start, t.end, t.text
            if t.start >= start and t.end <= end:
                tlist.append(t)
        if tlist:
            newtext = self.text[tlist[0].start:tlist[-1].end]
            if entity:
                entity.text = newtext
            if "text" in kwargs and newtext != kwargs["text"]:
                if newtext not in kwargs["text"] and kwargs["text"] not in newtext:
                    return None
                else:
                    logging.info("{}-{}|{}|=>|{}|{}-{}".format(tlist[0].start, tlist[-1].end, newtext, kwargs["text"],
                                 start, end))
                    logging.info("{} - {}".format(self.sid, self.text))
            #     print "tokens found:", [t.text for t in tlist]
                # sys.exit()
            # else:
            # print "found the tokens!", start, end, kwargs["text"], self.sid
            if self.entities.elist.get(source):
                eid = self.sid + ".e" + str(len(self.entities.elist[source]))
            else:
                eid = self.sid + ".e0"
            if entity:
                self.entities.add_entity(entity, source)
            elif subtype == "chemical":
                self.entities.add_entity(ChemdnerAnnotation(tlist, self.sid, text=newtext,
                                         did=self.did, eid=eid, subtype=subtype), source)
            elif subtype == "mirna":
                self.entities.add_entity(MirnaEntity(tlist, self.sid, text=newtext,
                                         did=self.did, eid=eid, subtype=subtype), source)
            elif subtype == "protein":
                self.entities.add_entity(ProteinEntity(tlist, self.sid, text=newtext,
                                         did=self.did, eid=eid, subtype=subtype), source)
            self.label_tokens(tlist, source, subtype)
            #logging.debug("added new entity to %s, now with %s entities" % (self.sid,
            #                                                                 len(self.entities.elist[source])))
            return eid
        else:
            print "no tokens found:"
            print start, end, kwargs.get("text")
            print [(t.start, t.end, t.text) for t in self.tokens]
            #

    def label_tokens(self, tlist, source, subtype="entity"):
        if len(tlist) == 1:
            tlist[0].tags[source] = "single"
            tlist[0].tags[source + "_subtype"] = subtype
        else:
            for t in range(len(tlist)):
                if t == 0:
                    tlist[t].tags[source] = "start"
                    tlist[t].tags[source + "_subtype"] = subtype
                elif t == len(tlist) - 1:
                    tlist[t].tags[source] = "end"
                    tlist[t].tags[source + "_subtype"] = subtype
                else:
                    tlist[t].tags[source] = "middle"
                    tlist[t].tags[source + "_subtype"] = subtype
        #logging.debug([t.tags for t in self.tokens])

    def write_bioc_results(self, parent, source):
        bioc_sentence = ET.SubElement(parent, "sentence")
        bioc_sentence_offset = ET.SubElement(bioc_sentence, "offset")
        bioc_sentence_offset.text = str(self.tokens[0].dstart)
        bioc_sentence_text = ET.SubElement(bioc_sentence, "text")
        bioc_sentence_text.text = self.text

        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                bioc_annotation = entity.write_bioc_annotation(bioc_sentence)
        return bioc_sentence

    def get_dic(self, source):
        dic = {}
        dic["id"] = self.sid
        dic["offset"] = str(self.tokens[0].dstart)
        dic["text"] = self.text
        dic["entities"] = []
        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                dic["entities"].append(entity.get_dic())
            dic["entities"] = sorted(dic["entities"], key=lambda k: k['offset'])
            for ei, e in enumerate(dic["entities"]):
                e["eid"] = self.sid + ".e{}".format(ei)
        dic["pairs"] = self.pairs.get_dic()
        return dic

    def find_tokens_between(self, start, end, relativeto="doc"):
        """Return list of tokens between offsets. Use relativeto to consider doc indexes or
           sentence indexes."""
        foundtokens = []
        for t in self.tokens:
            if relativeto.startswith("doc") and t.dstart >= start and t.dend <= end:
                foundtokens.append(t)
            elif relativeto.startswith("sent") and t.start >= start and t.end <= end:
                foundtokens.append(t)
        return foundtokens

    def test_relations(self, pairs, basemodel, classifiers=[relations.SLK_PRED, relations.SST_PRED],
                       tag="", backup=False, printstd=False):
        #data =  ddi_train_slk.model, ddi_train_sst.model
        tempfiles = []

        if relations.SLK_PRED in classifiers:
            logging.info("**Testing SLK classifier %s ..." % (tag,))
            #testpairdic = ddi_kernels.fromddiDic(testdocs)
            ddi_kernels.generatejSREdata(pairs, self, basemodel, tag + "ddi_test_jsre.txt")
            ddi_kernels.testjSRE(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                 model=tag + "all_ddi_train_slk.model")
            self.pairs.pairs = ddi_kernels.getjSREPredicitons(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                                      self.pairs.pairs)
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_jsre.txt")
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_result.txt")

        if relations.SST_PRED in classifiers:
            logging.info("****Testing SST classifier %s ..." % (tag,))
            self.pairs.pairs = ddi_kernels.testSVMTK(self, self.pairs.pairs, pairs,
                                             model=tag + "all_ddi_train_sst.model", tag=tag)
        for p in self.pairs.pairs:
            for r in self.pairs.pairs[p].recognized_by:
                if self.pairs.pairs[p].recognized_by[r] == 1:
                    p.relation = True
        return tempfiles
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Methods for handling playlists, clips..."""

from balistos.youtube import get_related_video
from datetime import datetime
from balistos.models.clip import PlaylistClip
from balistos.models.clip import PlaylistClipUser
from balistos.models.clip import Clip
from balistos.models.user import User
from pyramid_basemodel import Session

def get_playlist_videos(playlist, username=None):
    """
    Method that returns all the clips that are part of playlist

    :param    playlist: playlist of which we want to get videos
    :type     playlist: balistos.models.playlist.Playlist

    :returns: dict for each clip that is part of playlist
    :rtype:   list of dicts
    """
    result = []
    pclips = []
    user = None
    if username:
        user = User.get_by_username(username)
    if check_if_finished(PlaylistClip.get_active_playlist_clip(playlist)):
        play_next_clip(playlist)

    pclips.append(PlaylistClip.get_active_playlist_clip(playlist))
    next_pclip = PlaylistClip.get_queue_playlist_clip(playlist)
    if next_pclip:
        pclips.append(next_pclip)
    else:
        set_next_in_queue(playlist)
    wait_clips = PlaylistClip.get_by_playlist_waiting(playlist)
    if wait_clips:
        pclips = pclips + wait_clips
    for pclip in pclips:
        clip = pclip.clip
        if pclip.state == 2:
            start_time = (datetime.now() - pclip.started).total_seconds()
        else:
            start_time = 0
        liked = -2
        if user:
            pclipuser = PlaylistClipUser.get_by_playlist_clip_and_user(
                pclip,
                user
            )
            liked = pclipuser.liked
        result.append(
            {
                'id': clip.youtube_video_id,
                'title': clip.title,
                'likes': pclip.likes,
                'image': clip.image_url,
                'start_time': start_time,
                'liked': liked
            },
        )
    return result

def play_next_clip(playlist):
    """
    Play next clip (clip in queue) in playlist

    :param    playlist: current playlist
    :type     playlist: balistos.models.playlist.Playlist
    """

    next_pclip = PlaylistClip.get_queue_playlist_clip(playlist)
    if next_pclip:
        next_pclip.state = 2
        next_pclip.started = datetime.now()
        Session.flush()
        set_next_in_queue(playlist, next_pclip.clip.youtube_video_id)

def check_if_finished(pclip):
    """
    Check if currently active clip finished playing and remove it from playing,
    state if it is.

    :param    pclip: active playlist clip
    :type     pclip: balistos.models.playlist.PlaylistClip

    :returns: True if clip is finished and we removed it, False otherwise
    :rtype:   boolean
    """
    duration = pclip.clip.duration
    started = pclip.started
    if (datetime.now()-started).total_seconds() > duration:
        Session.delete(pclip)
        Session.flush()
        return True
    return False

def set_next_in_queue(playlist, youtube_video_id):
    """
    Set next video in queue based on likes, time added

    :param    playlist: playlist for which next clip we are setting
    :type     playlist: balistos.models.playlist.Playlist
    """
    pclips = PlaylistClip.get_by_playlist_waiting(playlist)
    if pclips:
        pclips[0].state = 1
    else:
        video = get_related_video(youtube_video_id)
        add_playlist_clip(
            playlist,
            video['title'],
            video['image_url'],
            video['youtube_video_id'],
            video['duration'],
            state=1,
        )
    Session.flush()

def add_playlist_clip(
    playlist,
    title,
    image_url,
    youtube_video_id,
    duration,
    state=0,
):
    """
    Add clip to playlist

    :param    playlist:         playlist of which we want to get videos
    :type     playlist:         balistos.models.playlist.Playlist
    :param    title:            title of video
    :type     title:            str
    :param    image_url:        url of image used for thumbnail
    :type     image_url:        str
    :param    youtube_video_id: id of video on youtube
    :type     youtube_video_id: str
    :param    duration:         duration of video
    :type     duration:         int
    :param    state:            state of video to be added
    :type     state:            int

    """

    clip = Clip.get(youtube_video_id)
    if not clip:
        clip = Clip(
            title=title,
            image_url=image_url,
            youtube_video_id=youtube_video_id,
            likes=0,
            duration=duration,
        )
        Session.add(clip)
        Session.flush()
    pclip = PlaylistClip.get_by_playlist_and_clip(playlist, clip)
    if not pclip:
        pclip = PlaylistClip(
            added=datetime.now(),
            likes=0,
            state=state,
            clip=clip,
            playlist=playlist,
        )
        Session.add(pclip)
        Session.flush()
    else:
        pclip.likes += 1

def remove_playlist_clip(playlist, youtube_video_id):
    """
    Remove clip from playlist

    :param    playlist: playlist we want to delete clip from
    :type     playlist: balistos.models.Playlist
    :param    clip:     youtube video id we want to delete from playlist
    :type     clip:     [clip type]

    :returns  True if removed, False otherwise
    :rtype    boolean
    """

    pclip = PlaylistClip.get_by_playlist_and_clip(
        playlist,
        Clip.get(youtube_video_id)
    )
    if not pclip:
        return False

    state = pclip.state
    Session.delete(pclip)

    if state == 2:
        play_next_clip(playlist)
    elif state == 1:
        set_next_in_queue(playlist, youtube_video_id)
    Session.flush()
    return True
<commit_msg>Add playlist settings method<commit_after># -*- coding: utf-8 -*-
"""Methods for handling playlists, clips..."""

from balistos.youtube import get_related_video
from datetime import datetime
from balistos.models.clip import PlaylistClip
from balistos.models.clip import PlaylistClipUser
from balistos.models.clip import Clip
from balistos.models.user import User
from pyramid_basemodel import Session

def get_playlist_videos(playlist, username=None):
    """
    Method that returns all the clips that are part of playlist

    :param    playlist: playlist of which we want to get videos
    :type     playlist: balistos.models.playlist.Playlist

    :returns: dict for each clip that is part of playlist
    :rtype:   list of dicts
    """
    result = []
    pclips = []
    user = None
    if username:
        user = User.get_by_username(username)
    if check_if_finished(PlaylistClip.get_active_playlist_clip(playlist)):
        play_next_clip(playlist)

    pclips.append(PlaylistClip.get_active_playlist_clip(playlist))
    next_pclip = PlaylistClip.get_queue_playlist_clip(playlist)
    if next_pclip:
        pclips.append(next_pclip)
    else:
        set_next_in_queue(playlist)
    wait_clips = PlaylistClip.get_by_playlist_waiting(playlist)
    if wait_clips:
        pclips = pclips + wait_clips
    for pclip in pclips:
        clip = pclip.clip
        if pclip.state == 2:
            start_time = (datetime.now() - pclip.started).total_seconds()
        else:
            start_time = 0
        liked = -2
        owner = PlaylistClipUser.get_playlist_clip_owner(pclip)
        owner = owner.username if owner else 'related'
        if user:
            pclipuser = PlaylistClipUser.get_by_playlist_clip_and_user(
                pclip,
                user
            )
            liked = pclipuser.liked
        result.append(
            {
                'id': clip.youtube_video_id,
                'title': clip.title,
                'likes': pclip.likes,
                'image': clip.image_url,
                'start_time': start_time,
                'liked': liked,
                'owner': owner
            },
        )
    return result

def get_playlist_settings(playlist, username=None):
    """
    Get settings for current playlist for this user

    :param    playlist: current playlist
    :type     playlist: balistos.models.playlist.Playlist
    :param    username: username of current user or None
    :type     username: str

    :returns: settings of playlist
    :rtype:   dict
    """
    return {
        'duration_limit': playlist.duration_limit
    }

def play_next_clip(playlist):
    """
    Play next clip (clip in queue) in playlist

    :param    playlist: current playlist
    :type     playlist: balistos.models.playlist.Playlist
    """

    next_pclip = PlaylistClip.get_queue_playlist_clip(playlist)
    if next_pclip:
        next_pclip.state = 2
        next_pclip.started = datetime.now()
        Session.flush()
        set_next_in_queue(playlist, next_pclip.clip.youtube_video_id)

def check_if_finished(pclip):
    """
    Check if currently active clip finished playing and remove it from playing,
    state if it is.

    :param    pclip: active playlist clip
    :type     pclip: balistos.models.playlist.PlaylistClip

    :returns: True if clip is finished and we removed it, False otherwise
    :rtype:   boolean
    """
    duration = pclip.clip.duration
    started = pclip.started
    if (datetime.now()-started).total_seconds() > duration:
        Session.delete(pclip)
        Session.flush()
        return True
    return False

def set_next_in_queue(playlist, youtube_video_id):
    """
    Set next video in queue based on likes, time added

    :param    playlist: playlist for which next clip we are setting
    :type     playlist: balistos.models.playlist.Playlist
    """
    pclips = PlaylistClip.get_by_playlist_waiting(playlist)
    if pclips:
        pclips[0].state = 1
    else:
        video = get_related_video(playlist, youtube_video_id)
        add_playlist_clip(
            playlist,
            video['title'],
            video['image_url'],
            video['youtube_video_id'],
            video['duration'],
            state=1,
        )
    Session.flush()

def add_playlist_clip(
    playlist,
    title,
    image_url,
    youtube_video_id,
    duration,
    username=None,
    state=0,
):
    """
    Add clip to playlist

    :param    playlist:         playlist of which we want to get videos
    :type     playlist:         balistos.models.playlist.Playlist
    :param    title:            title of video
    :type     title:            str
    :param    image_url:        url of image used for thumbnail
    :type     image_url:        str
    :param    youtube_video_id: id of video on youtube
    :type     youtube_video_id: str
    :param    duration:         duration of video
    :type     duration:         int
    :param    username:         username of user that added this clip
    :type     username:         str
    :param    state:            state of video to be added
    :type     state:            int

    """

    clip = Clip.get(youtube_video_id)
    if not clip:
        clip = Clip(
            title=title,
            image_url=image_url,
            youtube_video_id=youtube_video_id,
            likes=0,
            duration=duration,
        )
        Session.add(clip)
        Session.flush()
    pclip = PlaylistClip.get_by_playlist_and_clip(playlist, clip)
    if not pclip:
        pclip = PlaylistClip(
            added=datetime.now(),
            likes=0,
            state=state,
            clip=clip,
            playlist=playlist,
        )
        Session.add(pclip)
        Session.flush()
    else:
        pclip.likes += 1

def remove_playlist_clip(playlist, youtube_video_id):
    """
    Remove clip from playlist

    :param    playlist: playlist we want to delete clip from
    :type     playlist: balistos.models.Playlist
    :param    clip:     youtube video id we want to delete from playlist
    :type     clip:     [clip type]

    :returns  True if removed, False otherwise
    :rtype    boolean
    """

    pclip = PlaylistClip.get_by_playlist_and_clip(
        playlist,
        Clip.get(youtube_video_id)
    )
    if not pclip:
        return False

    state = pclip.state
    Session.delete(pclip)

    if state == 2:
        play_next_clip(playlist)
    elif state == 1:
        set_next_in_queue(playlist, youtube_video_id)
    Session.flush()
    return True
<|endoftext|>