<commit_before>"""
PatternGenerators based on bitmap images stored in files.

$Id$
"""

# PIL Image is imported as PIL because we have our own Image PatternGenerator
import Image as PIL
import ImageOps
import numpy
import copy

from numpy.oldnumeric import array, Float, sum, ravel, ones

from .. import param
from topo.param.parameterized import ParamOverrides,Oproperty

from topo.base.boundingregion import BoundingBox
from topo.base.patterngenerator import PatternGenerator
from topo.base.sheetcoords import SheetCoordinateSystem
from topo.transferfn.basic import DivisiveNormalizeLinf,TransferFn
from topo.misc.filepath import Filename

class ImageSampler(param.Parameterized):
    """
    A class of objects that, when called, sample an image.
    """
    __abstract=True

    def _get_image(self):
        # CB: In general, might need to consider caching to avoid
        # loading of image/creation of scs and application of wpofs
        # every time/whatever the sampler does to set up the image
        # before sampling
        return self._image

    def _set_image(self,image):
        self._image = image

    def _del_image(self):
        del self._image

    # As noted by JP in FastImageSampler, this isn't easy to figure out.
    def __call__(self,image,x,y,sheet_xdensity,sheet_ydensity,width=1.0,height=1.0):
        raise NotImplementedError

    image = Oproperty(_get_image,_set_image,_del_image)

# CEBALERT: ArraySampler?
class PatternSampler(ImageSampler):
    """
    When called, resamples - according to the size_normalization
    parameter - an image at the supplied (x,y) sheet coordinates.

    (x,y) coordinates outside the image are returned as the background
    value.
    """
    whole_pattern_output_fns = param.HookList(class_=TransferFn,default=[],doc="""
        Functions to apply to the whole image before any sampling is done.""")

    background_value_fn = param.Callable(default=None,doc="""
        Function to compute an appropriate background value. Must accept
        an array and return a scalar.""")

    size_normalization = param.Enumeration(default='original',
        available=['original','stretch_to_fit','fit_shortest','fit_longest'],
        doc="""
        Determines how the pattern is scaled initially, relative to the
        default retinal dimension of 1.0 in sheet coordinates:

        'stretch_to_fit': scale both dimensions of the pattern so they
        would fill a Sheet with bounds=BoundingBox(radius=0.5) (disregards
        the original's aspect ratio).

        'fit_shortest': scale the pattern so that its shortest dimension
        is made to fill the corresponding dimension on a Sheet with
        bounds=BoundingBox(radius=0.5) (maintains the original's aspect
        ratio, filling the entire bounding box).

        'fit_longest': scale the pattern so that its longest dimension is
        made to fill the corresponding dimension on a Sheet with
        bounds=BoundingBox(radius=0.5) (maintains the original's
        aspect ratio, fitting the image into the bounding box but not
        necessarily filling it).

        'original': no scaling is applied; each pixel of the pattern
        corresponds to one matrix unit of the Sheet on which the
        pattern being displayed.""")

    def _get_image(self):
        return self.scs.activity

    def _set_image(self,image):
        # Stores a SheetCoordinateSystem with an activity matrix
        # representing the image
        if not isinstance(image,numpy.ndarray):
            image = array(image,Float)

        rows,cols = image.shape
        self.scs = SheetCoordinateSystem(xdensity=1.0,ydensity=1.0,
                                         bounds=BoundingBox(points=((-cols/2.0,-rows/2.0),
                                                                    ( cols/2.0, rows/2.0))))
        self.scs.activity=image

    def _del_image(self):
        self.scs = None

    def __call__(self, image, x, y, sheet_xdensity, sheet_ydensity, width=1.0, height=1.0):
        """
        Return pixels from the supplied image at the given Sheet (x,y)
        coordinates.

        The image is assumed to be a NumPy array or other object that
        exports the NumPy buffer interface (i.e. can be converted to a
        NumPy array by passing it to numpy.array(), e.g. PIL.Image).
        The whole_pattern_output_fns are applied to the image before
        any sampling is done.

        To calculate the sample, the image is scaled according to the
        size_normalization parameter, and any supplied width and
        height. sheet_xdensity and sheet_ydensity are the xdensity and
        ydensity of the sheet on which the pattern is to be drawn.
        """
        self.image=image

        for wpof in self.whole_pattern_output_fns:
            wpof(self.image)
        if not self.background_value_fn:
            self.background_value = 0.0
        else:
            self.background_value = self.background_value_fn(self.image)

        # create new pattern sample, filled initially with the background value
        pattern_sample = ones(x.shape, Float)*self.background_value

        # if the height or width is zero, there's no pattern to display...
        if width==0 or height==0:
            return pattern_sample

        # scale the supplied coordinates to match the pattern being at density=1
        x=x*sheet_xdensity # deliberately don't operate in place (so as not to change supplied x & y)
        y=y*sheet_ydensity

        # scale according to initial pattern size_normalization selected (size_normalization)
        self.__apply_size_normalization(x,y,sheet_xdensity,sheet_ydensity,self.size_normalization)

        # scale according to user-specified width and height
        x/=width
        y/=height

        # convert the sheet (x,y) coordinates to matrixidx (r,c) ones
        r,c = self.scs.sheet2matrixidx(x,y)

        # now sample pattern at the (r,c) corresponding to the supplied (x,y)
        pattern_rows,pattern_cols = self.image.shape
        if pattern_rows==0 or pattern_cols==0:
            return pattern_sample
        else:
            # CEBALERT: is there a more NumPy way to do this that would be faster?
            rows,cols = pattern_sample.shape
            for i in xrange(rows):
                for j in xrange(cols):
                    # indexes outside the pattern are left with the background color
                    if self.scs.bounds.contains_exclusive(x[i,j],y[i,j]):
                        pattern_sample[i,j] = self.image[r[i,j],c[i,j]]

        return pattern_sample

    def __apply_size_normalization(self,x,y,sheet_xdensity,sheet_ydensity,size_normalization):
        pattern_rows,pattern_cols = self.image.shape

        # Instead of an if-test, could have a class of this type of
        # function (c.f. OutputFunctions, etc)...
        if size_normalization=='original':
            return

        elif size_normalization=='stretch_to_fit':
            x_sf,y_sf = pattern_cols/sheet_xdensity, pattern_rows/sheet_ydensity
            x*=x_sf; y*=y_sf

        elif size_normalization=='fit_shortest':
            if pattern_rows<pattern_cols:
                sf = pattern_rows/sheet_ydensity
            else:
                sf = pattern_cols/sheet_xdensity
            x*=sf;y*=sf

        elif size_normalization=='fit_longest':
            if pattern_rows<pattern_cols:
                sf = pattern_cols/sheet_xdensity
            else:
                sf = pattern_rows/sheet_ydensity
            x*=sf;y*=sf

from numpy.oldnumeric import sum,ravel
def edge_average(a):
    "Return the mean value around the edge of an array."

    if len(ravel(a)) < 2:
        return float(a[0])
    else:
        top_edge = a[0]
        bottom_edge = a[-1]
        left_edge = a[1:-1,0]
        right_edge = a[1:-1,-1]

        edge_sum = sum(top_edge) + sum(bottom_edge) + sum(left_edge) + sum(right_edge)
        num_values = len(top_edge)+len(bottom_edge)+len(left_edge)+len(right_edge)

        return float(edge_sum)/num_values

class FastImageSampler(ImageSampler):
    """
    A fast-n-dirty image sampler using Python Imaging Library
    routines.  Currently this sampler doesn't support user-specified
    size_normalization or cropping but rather simply scales and crops
    the image to fit the given matrix size without distorting the
    aspect ratio of the original picture.
    """

    sampling_method = param.Integer(default=PIL.NEAREST,doc="""
       Python Imaging Library sampling method for resampling an image.
       Defaults to Image.NEAREST.""")

    def _set_image(self,image):
        if not isinstance(image,PIL.Image):
            self._image = PIL.new('L',image.shape)
            self._image.putdata(image.ravel())
        else:
            self._image = image

    def __call__(self, image, x, y, sheet_xdensity, sheet_ydensity, width=1.0, height=1.0):
        self.image=image

        # JPALERT: Right now this ignores all options and just fits the image into given array.
        # It needs to be fleshed out to properly size and crop the
        # image given the options. (maybe this class needs to be
        # redesigned?  The interface to this function is pretty inscrutable.)
        im = ImageOps.fit(self.image,x.shape,self.sampling_method)
        return array(im,dtype=Float)

# Would be best called Image, but that causes confusion with PIL's Image
class GenericImage(PatternGenerator):
    """
    Generic 2D image generator.

    Generates a pattern from a Python Imaging Library image object.
    Subclasses should override the _get_image method to produce the
    image object.

    The background value is calculated as an edge average: see
    edge_average().  Black-bordered images therefore have a black
    background, and white-bordered images have a white
    background. Images with no border have a background that is less
    of a contrast than a white or black one.

    At present, rotation, size_normalization, etc. just resample; it
    would be nice to support some interpolation options as well.
    """

    __abstract = True

    aspect_ratio  = param.Number(default=1.0,bounds=(0.0,None),
        softbounds=(0.0,2.0),precedence=0.31,doc="""
        Ratio of width to height; size*aspect_ratio gives the width.""")

    size  = param.Number(default=1.0,bounds=(0.0,None),softbounds=(0.0,2.0),
        precedence=0.30,doc="""
        Height of the image.""")

    pattern_sampler = param.ClassSelector(class_=ImageSampler,
        default=PatternSampler(background_value_fn=edge_average,
                               size_normalization='fit_shortest',
                               whole_pattern_output_fns=[DivisiveNormalizeLinf()]),doc="""
        The PatternSampler to use to resample/resize the image.""")

    cache_image = param.Boolean(default=True,doc="""
        If False, discards the image and pattern_sampler after drawing the pattern each time,
        to make it possible to use very large databases of images without
        running out of memory.""")

    def _get_image(self,p):
        raise NotImplementedError

    # CEB: not currently possible, because _get_image needs access to p
    #image = property(_get_image,_set_image,_del_image,doc=" ")

    def function(self,p):
        height   = p.size
        width    = p.aspect_ratio*height

        result = p.pattern_sampler(self._get_image(p),p.pattern_x,p.pattern_y,float(p.xdensity),float(p.ydensity),
                                   float(width),float(height))

        if p.cache_image is False:
            self._image = None
            self.pattern_sampler.image = None

        return result

    ### support pickling of PIL.Image

    # CEBALERT: almost identical code to that in topo.plotting.bitmap.Bitmap.
    # Can we instead patch PIL? (Note that we can't use copy_reg as we do for
    # e.g. numpy ufuncs because PIL's Image is not a new-style class. So patching
    # PIL is probably the only option to handle this problem in one place.)

    # CEB: by converting to string and back, we probably incur some speed
    # penalty on copy()ing GenericImages (since __getstate__ and __setstate__ are
    # used for copying, unless __copy__ and __deepcopy__ are defined instead).
    def __getstate__(self):
        """
        Return the object's state (as in the superclass), but replace
        the '_image' attribute's Image with a string representation.
        """
        state = super(GenericImage,self).__getstate__()

        if '_image' in state and state['_image'] is not None:
            import StringIO
            f = StringIO.StringIO()
            image = state['_image']
            image.save(f,format=image.format or 'TIFF') # format could be None (we should probably just not save in that case)
            state['_image'] = f.getvalue()
            f.close()

        return state

    def __setstate__(self,state):
        """
        Load the object's state (as in the superclass), but replace
        the '_image' string with an actual Image object.
        """
        # CEBALERT: Need to figure out how state['_image'] could ever
        # actually be None; apparently it is sometimes (see SF
        # #2276819).
        if '_image' in state and state['_image'] is not None:
            import StringIO
            state['_image'] = PIL.open(StringIO.StringIO(state['_image']))
        super(GenericImage,self).__setstate__(state)

class FileImage(GenericImage):
    """
    2D Image generator that reads the image from a file.

    The image at the supplied filename is converted to grayscale if it
    is not already a grayscale image. See PIL's Image class for
    details of supported image file formats.
    """

    filename = Filename(default='images/ellen_arthur.pgm',precedence=0.9,doc="""
        File path (can be relative to Topographica's base path) to a bitmap image.
        The image can be in any format accepted by PIL, e.g. PNG, JPG, TIFF, or PGM.
        """)

    def __init__(self, **params):
        """
        Create the last_filename attribute, used to hold the last
        filename. This allows reloading an existing image to be
        avoided.
        """
        super(FileImage,self).__init__(**params)
        self.last_filename = None

    def _get_image(self,p):
        """
        If necessary as indicated by the parameters, get a new image,
        assign it to self._image and return True.  If no new image is
        needed, return False.
        """
        if p.filename!=self.last_filename or self._image is None:
            self.last_filename=p.filename
            self._image = ImageOps.grayscale(PIL.open(p.filename))
        return self._image

# PICKLEHACK (move to legacy)
# Temporary as of 12/2007, for backwards compatibility
Image=FileImage

<commit_msg>Fixed error deleting PatternSampler's cache of the image.<commit_after>"""
PatternGenerators based on bitmap images stored in files.

$Id$
"""

# PIL Image is imported as PIL because we have our own Image PatternGenerator
import Image as PIL
import ImageOps
import numpy
import copy

from numpy.oldnumeric import array, Float, sum, ravel, ones

from .. import param
from topo.param.parameterized import ParamOverrides,Oproperty

from topo.base.boundingregion import BoundingBox
from topo.base.patterngenerator import PatternGenerator
from topo.base.sheetcoords import SheetCoordinateSystem
from topo.transferfn.basic import DivisiveNormalizeLinf,TransferFn
from topo.misc.filepath import Filename

class ImageSampler(param.Parameterized):
    """
    A class of objects that, when called, sample an image.
    """
    __abstract=True

    def _get_image(self):
        # CB: In general, might need to consider caching to avoid
        # loading of image/creation of scs and application of wpofs
        # every time/whatever the sampler does to set up the image
        # before sampling
        return self._image

    def _set_image(self,image):
        self._image = image

    def _del_image(self):
        del self._image

    # As noted by JP in FastImageSampler, this isn't easy to figure out.
    def __call__(self,image,x,y,sheet_xdensity,sheet_ydensity,width=1.0,height=1.0):
        raise NotImplementedError

    image = Oproperty(_get_image,_set_image,_del_image)

# CEBALERT: ArraySampler?
class PatternSampler(ImageSampler):
    """
    When called, resamples - according to the size_normalization
    parameter - an image at the supplied (x,y) sheet coordinates.

    (x,y) coordinates outside the image are returned as the background
    value.
    """
    whole_pattern_output_fns = param.HookList(class_=TransferFn,default=[],doc="""
        Functions to apply to the whole image before any sampling is done.""")

    background_value_fn = param.Callable(default=None,doc="""
        Function to compute an appropriate background value. Must accept
        an array and return a scalar.""")

    size_normalization = param.Enumeration(default='original',
        available=['original','stretch_to_fit','fit_shortest','fit_longest'],
        doc="""
        Determines how the pattern is scaled initially, relative to the
        default retinal dimension of 1.0 in sheet coordinates:

        'stretch_to_fit': scale both dimensions of the pattern so they
        would fill a Sheet with bounds=BoundingBox(radius=0.5) (disregards
        the original's aspect ratio).

        'fit_shortest': scale the pattern so that its shortest dimension
        is made to fill the corresponding dimension on a Sheet with
        bounds=BoundingBox(radius=0.5) (maintains the original's aspect
        ratio, filling the entire bounding box).

        'fit_longest': scale the pattern so that its longest dimension is
        made to fill the corresponding dimension on a Sheet with
        bounds=BoundingBox(radius=0.5) (maintains the original's
        aspect ratio, fitting the image into the bounding box but not
        necessarily filling it).

        'original': no scaling is applied; each pixel of the pattern
        corresponds to one matrix unit of the Sheet on which the
        pattern being displayed.""")

    def _get_image(self):
        return self.scs.activity

    def _set_image(self,image):
        # Stores a SheetCoordinateSystem with an activity matrix
        # representing the image
        if not isinstance(image,numpy.ndarray):
            image = array(image,Float)

        rows,cols = image.shape
        self.scs = SheetCoordinateSystem(xdensity=1.0,ydensity=1.0,
                                         bounds=BoundingBox(points=((-cols/2.0,-rows/2.0),
                                                                    ( cols/2.0, rows/2.0))))
        self.scs.activity=image

    def _del_image(self):
        self.scs = None

    def __call__(self, image, x, y, sheet_xdensity, sheet_ydensity, width=1.0, height=1.0):
        """
        Return pixels from the supplied image at the given Sheet (x,y)
        coordinates.

        The image is assumed to be a NumPy array or other object that
        exports the NumPy buffer interface (i.e. can be converted to a
        NumPy array by passing it to numpy.array(), e.g. PIL.Image).
        The whole_pattern_output_fns are applied to the image before
        any sampling is done.

        To calculate the sample, the image is scaled according to the
        size_normalization parameter, and any supplied width and
        height. sheet_xdensity and sheet_ydensity are the xdensity and
        ydensity of the sheet on which the pattern is to be drawn.
        """
        self.image=image

        for wpof in self.whole_pattern_output_fns:
            wpof(self.image)
        if not self.background_value_fn:
            self.background_value = 0.0
        else:
            self.background_value = self.background_value_fn(self.image)

        # create new pattern sample, filled initially with the background value
        pattern_sample = ones(x.shape, Float)*self.background_value

        # if the height or width is zero, there's no pattern to display...
        if width==0 or height==0:
            return pattern_sample

        # scale the supplied coordinates to match the pattern being at density=1
        x=x*sheet_xdensity # deliberately don't operate in place (so as not to change supplied x & y)
        y=y*sheet_ydensity

        # scale according to initial pattern size_normalization selected (size_normalization)
        self.__apply_size_normalization(x,y,sheet_xdensity,sheet_ydensity,self.size_normalization)

        # scale according to user-specified width and height
        x/=width
        y/=height

        # convert the sheet (x,y) coordinates to matrixidx (r,c) ones
        r,c = self.scs.sheet2matrixidx(x,y)

        # now sample pattern at the (r,c) corresponding to the supplied (x,y)
        pattern_rows,pattern_cols = self.image.shape
        if pattern_rows==0 or pattern_cols==0:
            return pattern_sample
        else:
            # CEBALERT: is there a more NumPy way to do this that would be faster?
            rows,cols = pattern_sample.shape
            for i in xrange(rows):
                for j in xrange(cols):
                    # indexes outside the pattern are left with the background color
                    if self.scs.bounds.contains_exclusive(x[i,j],y[i,j]):
                        pattern_sample[i,j] = self.image[r[i,j],c[i,j]]

        return pattern_sample

    def __apply_size_normalization(self,x,y,sheet_xdensity,sheet_ydensity,size_normalization):
        pattern_rows,pattern_cols = self.image.shape

        # Instead of an if-test, could have a class of this type of
        # function (c.f. OutputFunctions, etc)...
        if size_normalization=='original':
            return

        elif size_normalization=='stretch_to_fit':
            x_sf,y_sf = pattern_cols/sheet_xdensity, pattern_rows/sheet_ydensity
            x*=x_sf; y*=y_sf

        elif size_normalization=='fit_shortest':
            if pattern_rows<pattern_cols:
                sf = pattern_rows/sheet_ydensity
            else:
                sf = pattern_cols/sheet_xdensity
            x*=sf;y*=sf

        elif size_normalization=='fit_longest':
            if pattern_rows<pattern_cols:
                sf = pattern_cols/sheet_xdensity
            else:
                sf = pattern_rows/sheet_ydensity
            x*=sf;y*=sf

from numpy.oldnumeric import sum,ravel
def edge_average(a):
    "Return the mean value around the edge of an array."

    if len(ravel(a)) < 2:
        return float(a[0])
    else:
        top_edge = a[0]
        bottom_edge = a[-1]
        left_edge = a[1:-1,0]
        right_edge = a[1:-1,-1]

        edge_sum = sum(top_edge) + sum(bottom_edge) + sum(left_edge) + sum(right_edge)
        num_values = len(top_edge)+len(bottom_edge)+len(left_edge)+len(right_edge)

        return float(edge_sum)/num_values

class FastImageSampler(ImageSampler):
    """
    A fast-n-dirty image sampler using Python Imaging Library
    routines.  Currently this sampler doesn't support user-specified
    size_normalization or cropping but rather simply scales and crops
    the image to fit the given matrix size without distorting the
    aspect ratio of the original picture.
    """

    sampling_method = param.Integer(default=PIL.NEAREST,doc="""
       Python Imaging Library sampling method for resampling an image.
       Defaults to Image.NEAREST.""")

    def _set_image(self,image):
        if not isinstance(image,PIL.Image):
            self._image = PIL.new('L',image.shape)
            self._image.putdata(image.ravel())
        else:
            self._image = image

    def __call__(self, image, x, y, sheet_xdensity, sheet_ydensity, width=1.0, height=1.0):
        self.image=image

        # JPALERT: Right now this ignores all options and just fits the image into given array.
        # It needs to be fleshed out to properly size and crop the
        # image given the options. (maybe this class needs to be
        # redesigned?  The interface to this function is pretty inscrutable.)
        im = ImageOps.fit(self.image,x.shape,self.sampling_method)
        return array(im,dtype=Float)

# Would be best called Image, but that causes confusion with PIL's Image
class GenericImage(PatternGenerator):
    """
    Generic 2D image generator.

    Generates a pattern from a Python Imaging Library image object.
    Subclasses should override the _get_image method to produce the
    image object.

    The background value is calculated as an edge average: see
    edge_average().  Black-bordered images therefore have a black
    background, and white-bordered images have a white
    background. Images with no border have a background that is less
    of a contrast than a white or black one.

    At present, rotation, size_normalization, etc. just resample; it
    would be nice to support some interpolation options as well.
    """

    __abstract = True

    aspect_ratio  = param.Number(default=1.0,bounds=(0.0,None),
        softbounds=(0.0,2.0),precedence=0.31,doc="""
        Ratio of width to height; size*aspect_ratio gives the width.""")

    size  = param.Number(default=1.0,bounds=(0.0,None),softbounds=(0.0,2.0),
        precedence=0.30,doc="""
        Height of the image.""")

    pattern_sampler = param.ClassSelector(class_=ImageSampler,
        default=PatternSampler(background_value_fn=edge_average,
                               size_normalization='fit_shortest',
                               whole_pattern_output_fns=[DivisiveNormalizeLinf()]),doc="""
        The PatternSampler to use to resample/resize the image.""")

    cache_image = param.Boolean(default=True,doc="""
        If False, discards the image and pattern_sampler after drawing the pattern each time,
        to make it possible to use very large databases of images without
        running out of memory.""")

    def _get_image(self,p):
        raise NotImplementedError

    # CEB: not currently possible, because _get_image needs access to p
    #image = property(_get_image,_set_image,_del_image,doc=" ")

    def function(self,p):
        height   = p.size
        width    = p.aspect_ratio*height

        result = p.pattern_sampler(self._get_image(p),p.pattern_x,p.pattern_y,float(p.xdensity),float(p.ydensity),
                                   float(width),float(height))

        if p.cache_image is False:
            self._image = None
            del self.pattern_sampler.image

        return result

    ### support pickling of PIL.Image

    # CEBALERT: almost identical code to that in topo.plotting.bitmap.Bitmap.
    # Can we instead patch PIL? (Note that we can't use copy_reg as we do for
    # e.g. numpy ufuncs because PIL's Image is not a new-style class. So patching
    # PIL is probably the only option to handle this problem in one place.)

    # CEB: by converting to string and back, we probably incur some speed
    # penalty on copy()ing GenericImages (since __getstate__ and __setstate__ are
    # used for copying, unless __copy__ and __deepcopy__ are defined instead).
    def __getstate__(self):
        """
        Return the object's state (as in the superclass), but replace
        the '_image' attribute's Image with a string representation.
        """
        state = super(GenericImage,self).__getstate__()

        if '_image' in state and state['_image'] is not None:
            import StringIO
            f = StringIO.StringIO()
            image = state['_image']
            image.save(f,format=image.format or 'TIFF') # format could be None (we should probably just not save in that case)
            state['_image'] = f.getvalue()
            f.close()

        return state

    def __setstate__(self,state):
        """
        Load the object's state (as in the superclass), but replace
        the '_image' string with an actual Image object.
        """
        # CEBALERT: Need to figure out how state['_image'] could ever
        # actually be None; apparently it is sometimes (see SF
        # #2276819).
        if '_image' in state and state['_image'] is not None:
            import StringIO
            state['_image'] = PIL.open(StringIO.StringIO(state['_image']))
        super(GenericImage,self).__setstate__(state)

class FileImage(GenericImage):
    """
    2D Image generator that reads the image from a file.

    The image at the supplied filename is converted to grayscale if it
    is not already a grayscale image. See PIL's Image class for
    details of supported image file formats.
    """

    filename = Filename(default='images/ellen_arthur.pgm',precedence=0.9,doc="""
        File path (can be relative to Topographica's base path) to a bitmap image.
        The image can be in any format accepted by PIL, e.g. PNG, JPG, TIFF, or PGM.
        """)

    def __init__(self, **params):
        """
        Create the last_filename attribute, used to hold the last
        filename. This allows reloading an existing image to be
        avoided.
        """
        super(FileImage,self).__init__(**params)
        self.last_filename = None

    def _get_image(self,p):
        """
        If necessary as indicated by the parameters, get a new image,
        assign it to self._image and return True.  If no new image is
        needed, return False.
        """
        if p.filename!=self.last_filename or self._image is None:
            self.last_filename=p.filename
            self._image = ImageOps.grayscale(PIL.open(p.filename))
        return self._image

# PICKLEHACK (move to legacy)
# Temporary as of 12/2007, for backwards compatibility
Image=FileImage

<|endoftext|>


<commit_before>#! /usr/env/python

"""Create 2D grid with randomly generated fractures.

fracture_grid: creates and returns a 2D grid with randomly generated fractures.
The grid contains the value 1 where fractures (one cell wide) exist, and
0 elsewhere. The idea is to use this for simulations based on weathering and
erosion of, and/or flow within, fracture networks.

The entry point is the function::

    make_frac_grid(frac_spacing, numrows=50, numcols=50, model_grid=None)

If called with a Landlab RasterModelGrid, the function returns the fracture
grid as a node array. Otherwise, it returns a numrows x numcols Numpy array.

Potential improvements:
    - Add doctests
    - Fractures could be defined by links rather than nodes (i.e., return a
        link array with a code indicating whether the link crosses a fracture
        or not)
    - Fractures could have a finite length rather than extending all the way
        across the grid
    - Use of starting position along either x or y axis makes fracture net
        somewhat asymmetric. One would need a different algorithm to make it
        fully (statistically) symmetric.

Created: September 2013 by Greg Tucker
Last significant modification: cleanup and unit tests Oct 2015 GT
"""

from numpy import tan, pi, size, zeros
import numpy as np
import random

def calculate_fracture_starting_position(shape, seed):
    """Choose a random starting position along the x or y axis (random choice).

    Parameters
    ----------
    shape : tuple of int
        Number of rows and columns in the grid
    seed : int
        Seeds the random number generator, so that a particular random
        sequence can be recreated.

    Returns
    -------
    (y, x) : tuple of int
        Fracture starting coordinates
    """
    random.seed(seed)

    if random.randint(0, 1) == 0:
        x = 0
        y = random.randint(0, shape[0] - 1)
    else:
        x = random.randint(0, shape[1] - 1)
        y = 0
    return (y, x)

def calculate_fracture_orientation(coords, seed):
    """Choose a random orientation for the fracture.

    Parameters
    ----------
    coords : tuple of int
        Starting coordinates (one of which should be zero) as *y*, *x*.
    seed : int
        Seed value for random number generator

    Returns
    -------
    ang : float
        Fracture angle relative to horizontal

    Notes
    -----
    If the fracture starts along the bottom of the grid (y=0), then the angle
    will be between 45 and 135 degrees from horizontal (counter-clockwise).
    Otherwise, it will be between -45 and 45 degrees.
    """
    y, x = coords

    np.random.seed(seed)
    ang = (pi/2)*np.random.rand()
    if y == 0:
        ang += pi/4
    else:
        ang -= pi/4

    return ang

def calculate_fracture_step_sizes(start_yx, ang):
    """
    Calculate the sizes of steps dx and dy to be used when "drawing" the
    fracture onto the grid.

    Parameters
    ----------
    start_yx : tuple of int
        Starting grid coordinates
    ang : float
        Fracture angle relative to horizontal (radians)

    Returns
    -------
    (dy, dx) : tuple of float
        Step sizes in y and x directions. One will always be unity, and the
        other will always be <1.
    """
    starty, startx = start_yx
    if startx==0:  # frac starts on left side
        dx = 1
        dy = tan(ang)
    else:  # frac starts on bottom side
        dy = 1
        dx = -tan(ang-pi/2)

    return (dy, dx)

def trace_fracture_through_grid(m, start_yx, spacing):
    """Create a 2D fracture in a grid.

    Creates a "fracture" in a 2D grid, m, by setting cell values to unity along
    the trace of the fracture (i.e., "drawing" a line throuh the grid).

    Parameters
    ----------
    m : 2D Numpy array
        Array that represents the grid
    start_yx : tuple of int
        Starting grid coordinates for fracture
    spacing : tuple of float
        Step sizes in y and x directions

    Returns
    -------
    None, but changes contents of m
    """
    y0, x0 = start_yx
    dy, dx = spacing

    x = x0
    y = y0

    while round(x) < size(m, 1) and round(y) < size(m, 0) \
            and round(x) >= 0 and round(y) >= 0:
        m[int(y + 0.5)][int(x + 0.5)] = 1
        x += dx
        y += dy

def make_frac_grid(frac_spacing, numrows=50, numcols=50, model_grid=None,
                   seed=0):
    """Create a grid that contains a network of random fractures.

    Creates and returns a grid containing a network of random fractures, which
    are represented as 1's embedded in a grid of 0's.

    Parameters
    ----------
    frac_spacing : int
        Average spacing of fractures (in grid cells)
    numrows : int, optional
        Number of rows in grid (if model_grid parameter is given,
        uses values from the model grid instead)
    numcols : int, optional
        Number of columns in grid (if model_grid parameter is given,
        uses values from the model grid instead)
    model_grid : Landlab RasterModelGrid object, optiona
        RasterModelGrid to use for grid size
    seed : int, optional
        Seed used for random number generator

    Returns
    -------
    m : Numpy array
        Array containing fracture grid, represented as 0's (matrix) and 1's
        (fractures). If model_grid parameter is given, returns a 1D array
        corresponding to a node-based array in the model grid. Otherwise,
        returns a 2D array with dimensions given by numrows, numcols.
    """
    # Make an initial grid of all zeros. If user specified a model grid,
    # use that. Otherwise, use the given dimensions.
    if model_grid is not None:
        numrows = model_grid.number_of_node_rows
        numcols = model_grid.number_of_node_columns
    m = zeros((numrows,numcols), dtype=int)

    # Add fractures to grid
    nfracs = (numrows+numcols)/frac_spacing
    for i in range(nfracs):

        (y, x) = calculate_fracture_starting_position((numrows, numcols), seed+i)
        ang = calculate_fracture_orientation((y, x), seed+i)
        (dy, dx) = calculate_fracture_step_sizes((y, x), ang)

        trace_fracture_through_grid(m, (y, x), (dy, dx))

    # If we have a model_grid, flatten the frac grid so it's equivalent to
    # a node array.
    if model_grid is not None:
        m.shape = (m.shape[0]*m.shape[1])

    return m
<commit_msg>Use integer division when calculating n fractures.<commit_after>#! /usr/env/python

"""Create 2D grid with randomly generated fractures.

fracture_grid: creates and returns a 2D grid with randomly generated fractures.
The grid contains the value 1 where fractures (one cell wide) exist, and
0 elsewhere. The idea is to use this for simulations based on weathering and
erosion of, and/or flow within, fracture networks.

The entry point is the function::

    make_frac_grid(frac_spacing, numrows=50, numcols=50, model_grid=None)

If called with a Landlab RasterModelGrid, the function returns the fracture
grid as a node array. Otherwise, it returns a numrows x numcols Numpy array.

Potential improvements:
    - Add doctests
    - Fractures could be defined by links rather than nodes (i.e., return a
        link array with a code indicating whether the link crosses a fracture
        or not)
    - Fractures could have a finite length rather than extending all the way
        across the grid
    - Use of starting position along either x or y axis makes fracture net
        somewhat asymmetric. One would need a different algorithm to make it
        fully (statistically) symmetric.

Created: September 2013 by Greg Tucker
Last significant modification: cleanup and unit tests Oct 2015 GT
"""

from numpy import tan, pi, size, zeros
import numpy as np
import random

def calculate_fracture_starting_position(shape, seed):
    """Choose a random starting position along the x or y axis (random choice).

    Parameters
    ----------
    shape : tuple of int
        Number of rows and columns in the grid
    seed : int
        Seeds the random number generator, so that a particular random
        sequence can be recreated.

    Returns
    -------
    (y, x) : tuple of int
        Fracture starting coordinates
    """
    random.seed(seed)

    if random.randint(0, 1) == 0:
        x = 0
        y = random.randint(0, shape[0] - 1)
    else:
        x = random.randint(0, shape[1] - 1)
        y = 0
    return (y, x)

def calculate_fracture_orientation(coords, seed):
    """Choose a random orientation for the fracture.

    Parameters
    ----------
    coords : tuple of int
        Starting coordinates (one of which should be zero) as *y*, *x*.
    seed : int
        Seed value for random number generator

    Returns
    -------
    ang : float
        Fracture angle relative to horizontal

    Notes
    -----
    If the fracture starts along the bottom of the grid (y=0), then the angle
    will be between 45 and 135 degrees from horizontal (counter-clockwise).
    Otherwise, it will be between -45 and 45 degrees.
    """
    y, x = coords

    np.random.seed(seed)
    ang = (pi/2)*np.random.rand()
    if y == 0:
        ang += pi/4
    else:
        ang -= pi/4

    return ang

def calculate_fracture_step_sizes(start_yx, ang):
    """
    Calculate the sizes of steps dx and dy to be used when "drawing" the
    fracture onto the grid.

    Parameters
    ----------
    start_yx : tuple of int
        Starting grid coordinates
    ang : float
        Fracture angle relative to horizontal (radians)

    Returns
    -------
    (dy, dx) : tuple of float
        Step sizes in y and x directions. One will always be unity, and the
        other will always be <1.
    """
    starty, startx = start_yx
    if startx==0:  # frac starts on left side
        dx = 1
        dy = tan(ang)
    else:  # frac starts on bottom side
        dy = 1
        dx = -tan(ang-pi/2)

    return (dy, dx)

def trace_fracture_through_grid(m, start_yx, spacing):
    """Create a 2D fracture in a grid.

    Creates a "fracture" in a 2D grid, m, by setting cell values to unity along
    the trace of the fracture (i.e., "drawing" a line throuh the grid).

    Parameters
    ----------
    m : 2D Numpy array
        Array that represents the grid
    start_yx : tuple of int
        Starting grid coordinates for fracture
    spacing : tuple of float
        Step sizes in y and x directions

    Returns
    -------
    None, but changes contents of m
    """
    y0, x0 = start_yx
    dy, dx = spacing

    x = x0
    y = y0

    while round(x) < size(m, 1) and round(y) < size(m, 0) \
            and round(x) >= 0 and round(y) >= 0:
        m[int(y + 0.5)][int(x + 0.5)] = 1
        x += dx
        y += dy

def make_frac_grid(frac_spacing, numrows=50, numcols=50, model_grid=None,
                   seed=0):
    """Create a grid that contains a network of random fractures.

    Creates and returns a grid containing a network of random fractures, which
    are represented as 1's embedded in a grid of 0's.

    Parameters
    ----------
    frac_spacing : int
        Average spacing of fractures (in grid cells)
    numrows : int, optional
        Number of rows in grid (if model_grid parameter is given,
        uses values from the model grid instead)
    numcols : int, optional
        Number of columns in grid (if model_grid parameter is given,
        uses values from the model grid instead)
    model_grid : Landlab RasterModelGrid object, optiona
        RasterModelGrid to use for grid size
    seed : int, optional
        Seed used for random number generator

    Returns
    -------
    m : Numpy array
        Array containing fracture grid, represented as 0's (matrix) and 1's
        (fractures). If model_grid parameter is given, returns a 1D array
        corresponding to a node-based array in the model grid. Otherwise,
        returns a 2D array with dimensions given by numrows, numcols.
    """
    # Make an initial grid of all zeros. If user specified a model grid,
    # use that. Otherwise, use the given dimensions.
    if model_grid is not None:
        numrows = model_grid.number_of_node_rows
        numcols = model_grid.number_of_node_columns
    m = zeros((numrows,numcols), dtype=int)

    # Add fractures to grid
    nfracs = (numrows + numcols) // frac_spacing
    for i in range(nfracs):

        (y, x) = calculate_fracture_starting_position((numrows, numcols), seed+i)
        ang = calculate_fracture_orientation((y, x), seed+i)
        (dy, dx) = calculate_fracture_step_sizes((y, x), ang)

        trace_fracture_through_grid(m, (y, x), (dy, dx))

    # If we have a model_grid, flatten the frac grid so it's equivalent to
    # a node array.
    if model_grid is not None:
        m.shape = (m.shape[0]*m.shape[1])

    return m
<|endoftext|>


<commit_before>import datetime

from django import forms
from django.conf import settings
from django.contrib.auth import forms as auth_forms
from django.forms.extras import widgets
from django.utils import safestring
from django.utils.translation import ugettext_lazy as _

from piplmesh.account import fields, formfields, models

class HorizontalRadioRenderer(forms.RadioSelect.renderer):
    """
    Renders horizontal radio buttons.
    Found `here
    <https://wikis.utexas.edu/display/~bm6432/Django-Modifying+RadioSelect+Widget+to+have+horizontal+buttons>`_.
    """

    def render(self):
        return safestring.mark_safe(u'\n'.join([u'%s\n' % widget for widget in self]))

class RegistrationForm(auth_forms.UserCreationForm):
    """
    Class with user registration form.
    """

    # Required data
    email = forms.EmailField(label=_("E-mail"))
    first_name = forms.CharField(label=_("First name"))
    last_name = forms.CharField(label=_("Last name"))

    # Additional information
    gender = forms.ChoiceField(label=_("Gender"), required=False, choices=fields.GENDER_CHOICES, widget=forms.RadioSelect(renderer=HorizontalRadioRenderer))
    birthdate = formfields.FormLimitedDateTimeField(upper_limit=datetime.datetime.today(), lower_limit=datetime.datetime.today() - datetime.timedelta(models.LOWER_DATE_LIMIT), label=_("Birth date"), required=False, widget=widgets.SelectDateWidget(years=[y for y in range(datetime.datetime.today().year, (datetime.datetime.today() - datetime.timedelta(models.LOWER_DATE_LIMIT)).year, -1)]))

    def clean_password2(self):
        # This method checks whether the passwords match
        if self.cleaned_data.has_key('password1') and self.cleaned_data['password1'] == self.cleaned_data['password2']:
            return self.cleaned_data['password2']
        raise forms.ValidationError(_("Passwords do not match."))

    def clean_username(self):
        # This method checks whether the username exists in case-insensitive manner
        username = self.cleaned_data['username']
        if models.User.objects(username__iexact=username).count():
            raise forms.ValidationError(_("A user with that username already exists."))
        return username

    def save(self):
        # We first have to save user to database
        new_user = models.User(
            username=self.cleaned_data['username'],
            first_name=self.cleaned_data['first_name'],
            last_name=self.cleaned_data['last_name'],
            email=self.cleaned_data['email'],
            gender=self.cleaned_data['gender'],
            birthdate=self.cleaned_data['birthdate'],
        )

        new_user.set_password(self.cleaned_data['password2'])
        new_user.save()

        return self.cleaned_data['username'], self.cleaned_data['password2']

    def validate_unique(self):
        # TODO: Check for errors
        # http://docs.nullpobug.com/django/trunk/django.forms.models-pysrc.html#BaseModelForm.validate_unique
        pass<commit_msg>GENDER_CHOICES moved to models.py. Removed bogus characters.<commit_after>import datetime

from django import forms
from django.conf import settings
from django.contrib.auth import forms as auth_forms
from django.forms.extras import widgets
from django.utils import safestring
from django.utils.translation import ugettext_lazy as _

from piplmesh.account import formfields, models

class HorizontalRadioRenderer(forms.RadioSelect.renderer):
    """
    Renders horizontal radio buttons.
    Found `here
    <https://wikis.utexas.edu/display/~bm6432/Django-Modifying+RadioSelect+Widget+to+have+horizontal+buttons>`_.
    """

    def render(self):
        return safestring.mark_safe(u'\n'.join([u'%s\n' % widget for widget in self]))

class RegistrationForm(auth_forms.UserCreationForm):
    """
    Class with user registration form.
    """

    # Required data
    email = forms.EmailField(label=_("E-mail"))
    first_name = forms.CharField(label=_("First name"))
    last_name = forms.CharField(label=_("Last name"))

    # Additional information
    gender = forms.ChoiceField(label=_("Gender"), required=False, choices=models.GENDER_CHOICES, widget=forms.RadioSelect(renderer=HorizontalRadioRenderer))
    birthdate = formfields.FormLimitedDateTimeField(upper_limit=datetime.datetime.today(), lower_limit=datetime.datetime.today() - datetime.timedelta(models.LOWER_DATE_LIMIT), label=_("Birth date"), required=False, widget=widgets.SelectDateWidget(years=[y for y in range(datetime.datetime.today().year, (datetime.datetime.today() - datetime.timedelta(models.LOWER_DATE_LIMIT)).year, -1)]))

    def clean_password2(self):
        # This method checks whether the passwords match
        if self.cleaned_data.has_key('password1') and self.cleaned_data['password1'] == self.cleaned_data['password2']:
            return self.cleaned_data['password2']
        raise forms.ValidationError(_("Passwords do not match."))

    def clean_username(self):
        # This method checks whether the username exists in case-insensitive manner
        username = self.cleaned_data['username']
        if models.User.objects(username__iexact=username).count():
            raise forms.ValidationError(_("A user with that username already exists."))
        return username

    def save(self):
        # We first have to save user to database
        new_user = models.User(
            username=self.cleaned_data['username'],
            first_name=self.cleaned_data['first_name'],
            last_name=self.cleaned_data['last_name'],
            email=self.cleaned_data['email'],
            gender=self.cleaned_data['gender'],
            birthdate=self.cleaned_data['birthdate'],
        )

        new_user.set_password(self.cleaned_data['password2'])
        new_user.save()

        return self.cleaned_data['username'], self.cleaned_data['password2']

    def validate_unique(self):
        # TODO: Check for errors
        # http://docs.nullpobug.com/django/trunk/django.forms.models-pysrc.html#BaseModelForm.validate_unique
        pass<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

"""Base class for the different queues used in Scootplayer."""

import Queue
import re
import numpy as np

class BaseQueue(object):

    """A set of common functions used on the queue classes of Scootplayer."""

    window_size = 5

    def __init__(self, *args, **kwargs):
        """Initialise a queue object with default values."""
        self.occupancy = []
        self.bandwidth = []
        self.url_bitrate = []
        self.stats = dict()
        self.report = dict(time_buffer=0, bandwidth=0, id=0, time_position=0,
                           moving_average_bandwidth=0, max_encoded_bitrate=0,
                           url_bitrate=0)
        self._previous_bandwidth = 0
        for key, val in kwargs.items():
            setattr(self, key, val)
        self.run = False
        self.queue = Queue.Queue()

    def pause(self):
        """Pause the operation of a queue."""
        self.run = False

    def resume(self):
        """Resume the operation of a queue."""
        self.run = True

    def _queue_analysis(self):
        """Analyse the occupancy of a queue."""

    def _url_parser(self, url):
        """Parse the URL to unreliably(!) determine the playback bitrate."""
        pattern = re.compile(ur'.*\_(.*kbit).*')
        match = re.match(pattern, url)
        self.report['url_bitrate'] = int(match.group(1).replace('kbit', ''))
        self.url_bitrate.append(self.report['url_bitrate'])
        self._object_analysis('url_bitrate', self.url_bitrate)

    def report_tick(self):
        """
        Called periodically by the player on a fixed interval.

        Append the latest values of bandwidth and occupancy to the object lists.
        Run the periodic statistical analysis of the bandwidth and occupancy
        lists.

        """
        self.bandwidth.append(self.report['bandwidth'])
        self.occupancy.append(self.report['time_buffer'])
        self._report_analysis('occupancy', self.occupancy)
        self._report_analysis('bandwidth', self.bandwidth)

    def calculate_stats(self):
        """Run fianl statistical analysis on the bandwidth and occupancy lists."""
        self._stats_analysis('occupancy', self.occupancy)
        self._stats_analysis('bandwidth', self.bandwidth)

    def _report_analysis(self, name, object_):
        """Calculate the moving average using NumPy."""
        self.report['moving_average_' + name] = np.average(object_[-self.window_size:])

    def _stats_analysis(self, name, object_):
        """Calculate various statistics using NumPy."""
        self.stats['min_' + name] = np.amin(object_)
        self.stats['max_' + name] = np.amax(object_)
        self.stats['changes_' + name] = self._changes(object_)
        self.stats['average_' + name] = np.average(object_)
        self.stats['std_' + name] = np.std(object_)
        self.stats['var_' + name] = np.var(object_)

    def _changes(self, list_):
        """Count the number of changes in a list object."""
        prev = list_[0]
        count = 0
        for item in list_[1:]:
            if not item == prev:
                count += 1
            prev = item
        return count
<commit_msg>Fixed function naming bug<commit_after>#!/usr/bin/env python2.7

"""Base class for the different queues used in Scootplayer."""

import Queue
import re
import numpy as np

class BaseQueue(object):

    """A set of common functions used on the queue classes of Scootplayer."""

    window_size = 5

    def __init__(self, *args, **kwargs):
        """Initialise a queue object with default values."""
        self.occupancy = []
        self.bandwidth = []
        self.url_bitrate = []
        self.stats = dict()
        self.report = dict(time_buffer=0, bandwidth=0, id=0, time_position=0,
                           moving_average_bandwidth=0, max_encoded_bitrate=0,
                           url_bitrate=0)
        self._previous_bandwidth = 0
        for key, val in kwargs.items():
            setattr(self, key, val)
        self.run = False
        self.queue = Queue.Queue()

    def pause(self):
        """Pause the operation of a queue."""
        self.run = False

    def resume(self):
        """Resume the operation of a queue."""
        self.run = True

    def _queue_analysis(self):
        """Analyse the occupancy of a queue."""

    def _url_parser(self, url):
        """Parse the URL to unreliably(!) determine the playback bitrate."""
        pattern = re.compile(ur'.*\_(.*kbit).*')
        match = re.match(pattern, url)
        self.report['url_bitrate'] = int(match.group(1).replace('kbit', ''))
        self.url_bitrate.append(self.report['url_bitrate'])
        self._stats_analysis('url_bitrate', self.url_bitrate)

    def report_tick(self):
        """
        Called periodically by the player on a fixed interval.

        Append the latest values of bandwidth and occupancy to the object lists.
        Run the periodic statistical analysis of the bandwidth and occupancy
        lists.

        """
        self.bandwidth.append(self.report['bandwidth'])
        self.occupancy.append(self.report['time_buffer'])
        self._report_analysis('occupancy', self.occupancy)
        self._report_analysis('bandwidth', self.bandwidth)

    def calculate_stats(self):
        """Run fianl statistical analysis on the bandwidth and occupancy lists."""
        self._stats_analysis('occupancy', self.occupancy)
        self._stats_analysis('bandwidth', self.bandwidth)

    def _report_analysis(self, name, object_):
        """Calculate the moving average using NumPy."""
        self.report['moving_average_' + name] = np.average(object_[-self.window_size:])

    def _stats_analysis(self, name, object_):
        """Calculate various statistics using NumPy."""
        self.stats['min_' + name] = np.amin(object_)
        self.stats['max_' + name] = np.amax(object_)
        self.stats['changes_' + name] = self._changes(object_)
        self.stats['average_' + name] = np.average(object_)
        self.stats['std_' + name] = np.std(object_)
        self.stats['var_' + name] = np.var(object_)

    def _changes(self, list_):
        """Count the number of changes in a list object."""
        prev = list_[0]
        count = 0
        for item in list_[1:]:
            if not item == prev:
                count += 1
            prev = item
        return count
<|endoftext|>


<commit_before>import logging
from typing import Tuple, List, Sequence

import sqlite3
from twisted.internet import defer
from twisted.enterprise import adbapi

from torba.hash import TXRefImmutable
from torba.basetransaction import BaseTransaction
from torba.baseaccount import BaseAccount

log = logging.getLogger(__name__)

def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
    sql, values = [], {}
    for key, constraint in constraints.items():
        col, op, key = key, '=', key.replace('.', '_')
        if key.startswith('$'):
            values[key] = constraint
            continue
        elif key.endswith('__not'):
            col, op = col[:-len('__not')], '!='
        elif key.endswith('__lt'):
            col, op = col[:-len('__lt')], '<'
        elif key.endswith('__lte'):
            col, op = col[:-len('__lte')], '<='
        elif key.endswith('__gt'):
            col, op = col[:-len('__gt')], '>'
        elif key.endswith('__like'):
            col, op = col[:-len('__like')], 'LIKE'
        elif key.endswith('__in') or key.endswith('__not_in'):
            if key.endswith('__in'):
                col, op = col[:-len('__in')], 'IN'
            else:
                col, op = col[:-len('__not_in')], 'NOT IN'
            if isinstance(constraint, (list, set)):
                items = ', '.join(
                    "'{}'".format(item) if isinstance(item, str) else str(item)
                    for item in constraint
                )
            elif isinstance(constraint, str):
                items = constraint
            else:
                raise ValueError("{} requires a list, set or string as constraint value.".format(col))
            sql.append('{} {} ({})'.format(col, op, items))
            continue
        elif key.endswith('__any'):
            where, subvalues = constraints_to_sql(constraint, ' OR ', key+'_')
            sql.append('({})'.format(where))
            values.update(subvalues)
            continue
        sql.append('{} {} :{}'.format(col, op, prepend_key+key))
        values[prepend_key+key] = constraint
    return joiner.join(sql) if sql else '', values

def query(select, **constraints):
    sql = [select]
    limit = constraints.pop('limit', None)
    offset = constraints.pop('offset', None)
    order_by = constraints.pop('order_by', None)

    constraints.pop('my_account', None)
    account = constraints.pop('account', None)
    if account is not None:
        if not isinstance(account, list):
            account = [account]
        constraints['account__in'] = [
            (a.public_key.address if isinstance(a, BaseAccount) else a) for a in account
        ]

    where, values = constraints_to_sql(constraints)
    if where:
        sql.append('WHERE')
        sql.append(where)

    if order_by is not None:
        sql.append('ORDER BY')
        if isinstance(order_by, str):
            sql.append(order_by)
        elif isinstance(order_by, list):
            sql.append(', '.join(order_by))
        else:
            raise ValueError("order_by must be string or list")

    if offset is not None:
        sql.append('OFFSET {}'.format(offset))

    if limit is not None:
        sql.append('LIMIT {}'.format(limit))

    return ' '.join(sql), values

def rows_to_dict(rows, fields):
    if rows:
        return [dict(zip(fields, r)) for r in rows]
    else:
        return []

def row_dict_or_default(rows, fields, default=None):
    dicts = rows_to_dict(rows, fields)
    return dicts[0] if dicts else default

class SQLiteMixin:

    CREATE_TABLES_QUERY: Sequence[str] = ()

    def __init__(self, path):
        self._db_path = path
        self.db: adbapi.ConnectionPool = None
        self.ledger = None

    def open(self):
        log.info("connecting to database: %s", self._db_path)
        self.db = adbapi.ConnectionPool(
            'sqlite3', self._db_path, cp_min=1, cp_max=1, check_same_thread=False
        )
        return self.db.runInteraction(
            lambda t: t.executescript(self.CREATE_TABLES_QUERY)
        )

    def close(self):
        self.db.close()
        return defer.succeed(True)

    @staticmethod
    def _insert_sql(table: str, data: dict) -> Tuple[str, List]:
        columns, values = [], []
        for column, value in data.items():
            columns.append(column)
            values.append(value)
        sql = "INSERT INTO {} ({}) VALUES ({})".format(
            table, ', '.join(columns), ', '.join(['?'] * len(values))
        )
        return sql, values

    @staticmethod
    def _update_sql(table: str, data: dict, where: str, constraints: list) -> Tuple[str, list]:
        columns, values = [], []
        for column, value in data.items():
            columns.append("{} = ?".format(column))
            values.append(value)
        values.extend(constraints)
        sql = "UPDATE {} SET {} WHERE {}".format(
            table, ', '.join(columns), where
        )
        return sql, values

    @defer.inlineCallbacks
    def query_one_value(self, query, params=None, default=None):
        result = yield self.run_query(query, params)
        if result:
            return result[0][0] or default
        else:
            return default

    @staticmethod
    def execute(t, sql, values):
        log.debug(sql)
        log.debug(values)
        return t.execute(sql, values)

    def run_operation(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runOperation(sql, values)

    def run_query(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runQuery(sql, values)

class BaseDatabase(SQLiteMixin):

    CREATE_PUBKEY_ADDRESS_TABLE = """
        create table if not exists pubkey_address (
            address text primary key,
            account text not null,
            chain integer not null,
            position integer not null,
            pubkey blob not null,
            history text,
            used_times integer not null default 0
        );
    """

    CREATE_TX_TABLE = """
        create table if not exists tx (
            txid text primary key,
            raw blob not null,
            height integer not null,
            position integer not null,
            is_verified boolean not null default 0
        );
    """

    CREATE_TXO_TABLE = """
        create table if not exists txo (
            txid text references tx,
            txoid text primary key,
            address text references pubkey_address,
            position integer not null,
            amount integer not null,
            script blob not null,
            is_reserved boolean not null default 0
        );
    """

    CREATE_TXI_TABLE = """
        create table if not exists txi (
            txid text references tx,
            txoid text references txo,
            address text references pubkey_address
        );
    """

    CREATE_TABLES_QUERY = (
        CREATE_TX_TABLE +
        CREATE_PUBKEY_ADDRESS_TABLE +
        CREATE_TXO_TABLE +
        CREATE_TXI_TABLE
    )

    @staticmethod
    def txo_to_row(tx, address, txo):
        return {
            'txid': tx.id,
            'txoid': txo.id,
            'address': address,
            'position': txo.position,
            'amount': txo.amount,
            'script': sqlite3.Binary(txo.script.source)
        }

    def save_transaction_io(self, save_tx, tx: BaseTransaction, address, txhash, history):

        def _steps(t):
            if save_tx == 'insert':
                self.execute(t, *self._insert_sql('tx', {
                    'txid': tx.id,
                    'raw': sqlite3.Binary(tx.raw),
                    'height': tx.height,
                    'position': tx.position,
                    'is_verified': tx.is_verified
                }))
            elif save_tx == 'update':
                self.execute(t, *self._update_sql("tx", {
                    'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
                }, 'txid = ?', (tx.id,)))

            existing_txos = [r[0] for r in self.execute(t, *query(
                "SELECT position FROM txo", txid=tx.id
            )).fetchall()]

            for txo in tx.outputs:
                if txo.position in existing_txos:
                    continue
                if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
                    self.execute(t, *self._insert_sql("txo", self.txo_to_row(tx, address, txo)))
                elif txo.script.is_pay_script_hash:
                    # TODO: implement script hash payments
                    print('Database.save_transaction_io: pay script hash is not implemented!')

            # lookup the address associated with each TXI (via its TXO)
            txoid_to_address = {r[0]: r[1] for r in self.execute(t, *query(
                "SELECT txoid, address FROM txo", txoid__in=[txi.txo_ref.id for txi in tx.inputs]
            )).fetchall()}

            # list of TXIs that have already been added
            existing_txis = [r[0] for r in self.execute(t, *query(
                "SELECT txoid FROM txi", txid=tx.id
            )).fetchall()]

            for txi in tx.inputs:
                txoid = txi.txo_ref.id
                new_txi = txoid not in existing_txis
                address_matches = txoid_to_address.get(txoid) == address
                if new_txi and address_matches:
                    self.execute(t, *self._insert_sql("txi", {
                        'txid': tx.id,
                        'txoid': txoid,
                        'address': address,
                    }))

            self._set_address_history(t, address, history)

        return self.db.runInteraction(_steps)

    def reserve_outputs(self, txos, is_reserved=True):
        txoids = [txo.id for txo in txos]
        return self.run_operation(
            "UPDATE txo SET is_reserved = ? WHERE txoid IN ({})".format(
                ', '.join(['?']*len(txoids))
            ), [is_reserved]+txoids
        )

    def release_outputs(self, txos):
        return self.reserve_outputs(txos, is_reserved=False)

    def rewind_blockchain(self, above_height):  # pylint: disable=no-self-use
        # TODO:
        # 1. delete transactions above_height
        # 2. update address histories removing deleted TXs
        return defer.succeed(True)

    def select_transactions(self, cols, account=None, **constraints):
        if 'txid' not in constraints and account is not None:
            constraints['$account'] = account.public_key.address
            constraints['txid__in'] = """
                SELECT txo.txid FROM txo
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
              UNION
                SELECT txi.txid FROM txi
                JOIN txo USING (txoid)
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
            """
        return self.run_query(*query("SELECT {} FROM tx".format(cols), **constraints))

    @defer.inlineCallbacks
    def get_transactions(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)

        tx_rows = yield self.select_transactions(
            'txid, raw, height, position, is_verified',
            order_by=["height DESC", "position DESC"],
            **constraints
        )

        txids, txs = [], []
        for row in tx_rows:
            txids.append(row[0])
            txs.append(self.ledger.transaction_class(
                raw=row[1], height=row[2], position=row[3], is_verified=row[4]
            ))

        annotated_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txid__in=txids
            ))
        }

        referenced_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txoid__in=query("SELECT txoid FROM txi", **{'txid__in': txids})[0]
            ))
        }

        for tx in txs:
            for txi in tx.inputs:
                txo = referenced_txos.get(txi.txo_ref.id)
                if txo:
                    txi.txo_ref = txo.ref
            for txo in tx.outputs:
                _txo = annotated_txos.get(txo.id)
                if _txo:
                    txo.update_annotations(_txo)
                else:
                    txo.update_annotations(None)

        return txs

    @defer.inlineCallbacks
    def get_transaction_count(self, **constraints):
        count = yield self.select_transactions('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_transaction(self, **constraints):
        txs = yield self.get_transactions(limit=1, **constraints)
        if txs:
            return txs[0]

    def select_txos(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM txo JOIN pubkey_address USING (address)".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_txos(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)
        if isinstance(my_account, BaseAccount):
            my_account = my_account.public_key.address
        rows = yield self.select_txos(
            "amount, script, txid, txo.position, chain, account", **constraints
        )
        output_class = self.ledger.transaction_class.output_class
        return [
            output_class(
                amount=row[0],
                script=output_class.script_class(row[1]),
                tx_ref=TXRefImmutable.from_id(row[2]),
                position=row[3],
                is_change=row[4] == 1,
                is_my_account=row[5] == my_account
            ) for row in rows
        ]

    @defer.inlineCallbacks
    def get_txo_count(self, **constraints):
        count = yield self.select_txos('count(*)', **constraints)
        return count[0][0]

    @staticmethod
    def constrain_utxo(constraints):
        constraints['is_reserved'] = False
        constraints['txoid__not_in'] = "SELECT txoid FROM txi"

    def get_utxos(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txos(**constraints)

    def get_utxos_count(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txo_count(**constraints)

    @defer.inlineCallbacks
    def get_balance(self, **constraints):
        self.constrain_utxo(constraints)
        balance = yield self.select_txos('SUM(amount)', **constraints)
        return balance[0][0] or 0

    def select_addresses(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM pubkey_address".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'), **constraints):
        addresses = yield self.select_addresses(', '.join(cols), **constraints)
        return rows_to_dict(addresses, cols)

    @defer.inlineCallbacks
    def get_address_count(self, **constraints):
        count = yield self.select_addresses('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_address(self, address):
        addresses = yield self.get_addresses(
            cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
            address=address, limit=1
        )
        if addresses:
            return addresses[0]

    def add_keys(self, account, chain, keys):
        sql = (
                  "insert into pubkey_address "
                  "(address, account, chain, position, pubkey) "
                  "values "
              ) + ', '.join(['(?, ?, ?, ?, ?)'] * len(keys))
        values = []
        for position, pubkey in keys:
            values.append(pubkey.address)
            values.append(account.public_key.address)
            values.append(chain)
            values.append(position)
            values.append(sqlite3.Binary(pubkey.pubkey_bytes))
        return self.run_operation(sql, values)

    @classmethod
    def _set_address_history(cls, t, address, history):
        cls.execute(
            t, "UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
            (history, history.count(':')//2, address)
        )

    def set_address_history(self, address, history):
        return self.db.runInteraction(lambda t: self._set_address_history(t, address, history))
<commit_msg>removed unused method and minor pylint fix<commit_after>import logging
from typing import Tuple, List, Sequence

import sqlite3
from twisted.internet import defer
from twisted.enterprise import adbapi

from torba.hash import TXRefImmutable
from torba.basetransaction import BaseTransaction
from torba.baseaccount import BaseAccount

log = logging.getLogger(__name__)

def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
    sql, values = [], {}
    for key, constraint in constraints.items():
        col, op, key = key, '=', key.replace('.', '_')
        if key.startswith('$'):
            values[key] = constraint
            continue
        elif key.endswith('__not'):
            col, op = col[:-len('__not')], '!='
        elif key.endswith('__lt'):
            col, op = col[:-len('__lt')], '<'
        elif key.endswith('__lte'):
            col, op = col[:-len('__lte')], '<='
        elif key.endswith('__gt'):
            col, op = col[:-len('__gt')], '>'
        elif key.endswith('__like'):
            col, op = col[:-len('__like')], 'LIKE'
        elif key.endswith('__in') or key.endswith('__not_in'):
            if key.endswith('__in'):
                col, op = col[:-len('__in')], 'IN'
            else:
                col, op = col[:-len('__not_in')], 'NOT IN'
            if isinstance(constraint, (list, set)):
                items = ', '.join(
                    "'{}'".format(item) if isinstance(item, str) else str(item)
                    for item in constraint
                )
            elif isinstance(constraint, str):
                items = constraint
            else:
                raise ValueError("{} requires a list, set or string as constraint value.".format(col))
            sql.append('{} {} ({})'.format(col, op, items))
            continue
        elif key.endswith('__any'):
            where, subvalues = constraints_to_sql(constraint, ' OR ', key+'_')
            sql.append('({})'.format(where))
            values.update(subvalues)
            continue
        sql.append('{} {} :{}'.format(col, op, prepend_key+key))
        values[prepend_key+key] = constraint
    return joiner.join(sql) if sql else '', values

def query(select, **constraints):
    sql = [select]
    limit = constraints.pop('limit', None)
    offset = constraints.pop('offset', None)
    order_by = constraints.pop('order_by', None)

    constraints.pop('my_account', None)
    account = constraints.pop('account', None)
    if account is not None:
        if not isinstance(account, list):
            account = [account]
        constraints['account__in'] = [
            (a.public_key.address if isinstance(a, BaseAccount) else a) for a in account
        ]

    where, values = constraints_to_sql(constraints)
    if where:
        sql.append('WHERE')
        sql.append(where)

    if order_by is not None:
        sql.append('ORDER BY')
        if isinstance(order_by, str):
            sql.append(order_by)
        elif isinstance(order_by, list):
            sql.append(', '.join(order_by))
        else:
            raise ValueError("order_by must be string or list")

    if offset is not None:
        sql.append('OFFSET {}'.format(offset))

    if limit is not None:
        sql.append('LIMIT {}'.format(limit))

    return ' '.join(sql), values

def rows_to_dict(rows, fields):
    if rows:
        return [dict(zip(fields, r)) for r in rows]
    else:
        return []

def row_dict_or_default(rows, fields, default=None):
    dicts = rows_to_dict(rows, fields)
    return dicts[0] if dicts else default

class SQLiteMixin:

    CREATE_TABLES_QUERY: Sequence[str] = ()

    def __init__(self, path):
        self._db_path = path
        self.db: adbapi.ConnectionPool = None
        self.ledger = None

    def open(self):
        log.info("connecting to database: %s", self._db_path)
        self.db = adbapi.ConnectionPool(
            'sqlite3', self._db_path, cp_min=1, cp_max=1, check_same_thread=False
        )
        return self.db.runInteraction(
            lambda t: t.executescript(self.CREATE_TABLES_QUERY)
        )

    def close(self):
        self.db.close()
        return defer.succeed(True)

    @staticmethod
    def _insert_sql(table: str, data: dict) -> Tuple[str, List]:
        columns, values = [], []
        for column, value in data.items():
            columns.append(column)
            values.append(value)
        sql = "INSERT INTO {} ({}) VALUES ({})".format(
            table, ', '.join(columns), ', '.join(['?'] * len(values))
        )
        return sql, values

    @staticmethod
    def _update_sql(table: str, data: dict, where: str, constraints: list) -> Tuple[str, list]:
        columns, values = [], []
        for column, value in data.items():
            columns.append("{} = ?".format(column))
            values.append(value)
        values.extend(constraints)
        sql = "UPDATE {} SET {} WHERE {}".format(
            table, ', '.join(columns), where
        )
        return sql, values

    @staticmethod
    def execute(t, sql, values):
        log.debug(sql)
        log.debug(values)
        return t.execute(sql, values)

    def run_operation(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runOperation(sql, values)

    def run_query(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runQuery(sql, values)

class BaseDatabase(SQLiteMixin):

    CREATE_PUBKEY_ADDRESS_TABLE = """
        create table if not exists pubkey_address (
            address text primary key,
            account text not null,
            chain integer not null,
            position integer not null,
            pubkey blob not null,
            history text,
            used_times integer not null default 0
        );
    """

    CREATE_TX_TABLE = """
        create table if not exists tx (
            txid text primary key,
            raw blob not null,
            height integer not null,
            position integer not null,
            is_verified boolean not null default 0
        );
    """

    CREATE_TXO_TABLE = """
        create table if not exists txo (
            txid text references tx,
            txoid text primary key,
            address text references pubkey_address,
            position integer not null,
            amount integer not null,
            script blob not null,
            is_reserved boolean not null default 0
        );
    """

    CREATE_TXI_TABLE = """
        create table if not exists txi (
            txid text references tx,
            txoid text references txo,
            address text references pubkey_address
        );
    """

    CREATE_TABLES_QUERY = (
        CREATE_TX_TABLE +
        CREATE_PUBKEY_ADDRESS_TABLE +
        CREATE_TXO_TABLE +
        CREATE_TXI_TABLE
    )

    @staticmethod
    def txo_to_row(tx, address, txo):
        return {
            'txid': tx.id,
            'txoid': txo.id,
            'address': address,
            'position': txo.position,
            'amount': txo.amount,
            'script': sqlite3.Binary(txo.script.source)
        }

    def save_transaction_io(self, save_tx, tx: BaseTransaction, address, txhash, history):

        def _steps(t):
            if save_tx == 'insert':
                self.execute(t, *self._insert_sql('tx', {
                    'txid': tx.id,
                    'raw': sqlite3.Binary(tx.raw),
                    'height': tx.height,
                    'position': tx.position,
                    'is_verified': tx.is_verified
                }))
            elif save_tx == 'update':
                self.execute(t, *self._update_sql("tx", {
                    'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
                }, 'txid = ?', (tx.id,)))

            existing_txos = [r[0] for r in self.execute(t, *query(
                "SELECT position FROM txo", txid=tx.id
            )).fetchall()]

            for txo in tx.outputs:
                if txo.position in existing_txos:
                    continue
                if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
                    self.execute(t, *self._insert_sql("txo", self.txo_to_row(tx, address, txo)))
                elif txo.script.is_pay_script_hash:
                    # TODO: implement script hash payments
                    print('Database.save_transaction_io: pay script hash is not implemented!')

            # lookup the address associated with each TXI (via its TXO)
            txoid_to_address = {r[0]: r[1] for r in self.execute(t, *query(
                "SELECT txoid, address FROM txo", txoid__in=[txi.txo_ref.id for txi in tx.inputs]
            )).fetchall()}

            # list of TXIs that have already been added
            existing_txis = [r[0] for r in self.execute(t, *query(
                "SELECT txoid FROM txi", txid=tx.id
            )).fetchall()]

            for txi in tx.inputs:
                txoid = txi.txo_ref.id
                new_txi = txoid not in existing_txis
                address_matches = txoid_to_address.get(txoid) == address
                if new_txi and address_matches:
                    self.execute(t, *self._insert_sql("txi", {
                        'txid': tx.id,
                        'txoid': txoid,
                        'address': address,
                    }))

            self._set_address_history(t, address, history)

        return self.db.runInteraction(_steps)

    def reserve_outputs(self, txos, is_reserved=True):
        txoids = [txo.id for txo in txos]
        return self.run_operation(
            "UPDATE txo SET is_reserved = ? WHERE txoid IN ({})".format(
                ', '.join(['?']*len(txoids))
            ), [is_reserved]+txoids
        )

    def release_outputs(self, txos):
        return self.reserve_outputs(txos, is_reserved=False)

    def rewind_blockchain(self, above_height):  # pylint: disable=no-self-use
        # TODO:
        # 1. delete transactions above_height
        # 2. update address histories removing deleted TXs
        return defer.succeed(True)

    def select_transactions(self, cols, account=None, **constraints):
        if 'txid' not in constraints and account is not None:
            constraints['$account'] = account.public_key.address
            constraints['txid__in'] = """
                SELECT txo.txid FROM txo
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
              UNION
                SELECT txi.txid FROM txi
                JOIN txo USING (txoid)
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
            """
        return self.run_query(*query("SELECT {} FROM tx".format(cols), **constraints))

    @defer.inlineCallbacks
    def get_transactions(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)

        tx_rows = yield self.select_transactions(
            'txid, raw, height, position, is_verified',
            order_by=["height DESC", "position DESC"],
            **constraints
        )

        txids, txs = [], []
        for row in tx_rows:
            txids.append(row[0])
            txs.append(self.ledger.transaction_class(
                raw=row[1], height=row[2], position=row[3], is_verified=row[4]
            ))

        annotated_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txid__in=txids
            ))
        }

        referenced_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txoid__in=query("SELECT txoid FROM txi", **{'txid__in': txids})[0]
            ))
        }

        for tx in txs:
            for txi in tx.inputs:
                txo = referenced_txos.get(txi.txo_ref.id)
                if txo:
                    txi.txo_ref = txo.ref
            for txo in tx.outputs:
                _txo = annotated_txos.get(txo.id)
                if _txo:
                    txo.update_annotations(_txo)
                else:
                    txo.update_annotations(None)

        return txs

    @defer.inlineCallbacks
    def get_transaction_count(self, **constraints):
        count = yield self.select_transactions('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_transaction(self, **constraints):
        txs = yield self.get_transactions(limit=1, **constraints)
        if txs:
            return txs[0]

    def select_txos(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM txo JOIN pubkey_address USING (address)".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_txos(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)
        if isinstance(my_account, BaseAccount):
            my_account = my_account.public_key.address
        rows = yield self.select_txos(
            "amount, script, txid, txo.position, chain, account", **constraints
        )
        output_class = self.ledger.transaction_class.output_class
        return [
            output_class(
                amount=row[0],
                script=output_class.script_class(row[1]),
                tx_ref=TXRefImmutable.from_id(row[2]),
                position=row[3],
                is_change=row[4] == 1,
                is_my_account=row[5] == my_account
            ) for row in rows
        ]

    @defer.inlineCallbacks
    def get_txo_count(self, **constraints):
        count = yield self.select_txos('count(*)', **constraints)
        return count[0][0]

    @staticmethod
    def constrain_utxo(constraints):
        constraints['is_reserved'] = False
        constraints['txoid__not_in'] = "SELECT txoid FROM txi"

    def get_utxos(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txos(**constraints)

    def get_utxos_count(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txo_count(**constraints)

    @defer.inlineCallbacks
    def get_balance(self, **constraints):
        self.constrain_utxo(constraints)
        balance = yield self.select_txos('SUM(amount)', **constraints)
        return balance[0][0] or 0

    def select_addresses(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM pubkey_address".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'), **constraints):
        addresses = yield self.select_addresses(', '.join(cols), **constraints)
        return rows_to_dict(addresses, cols)

    @defer.inlineCallbacks
    def get_address_count(self, **constraints):
        count = yield self.select_addresses('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_address(self, address):
        addresses = yield self.get_addresses(
            cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
            address=address, limit=1
        )
        if addresses:
            return addresses[0]

    def add_keys(self, account, chain, keys):
        sql = (
            "insert into pubkey_address "
            "(address, account, chain, position, pubkey) "
            "values "
        ) + ', '.join(['(?, ?, ?, ?, ?)'] * len(keys))
        values = []
        for position, pubkey in keys:
            values.append(pubkey.address)
            values.append(account.public_key.address)
            values.append(chain)
            values.append(position)
            values.append(sqlite3.Binary(pubkey.pubkey_bytes))
        return self.run_operation(sql, values)

    @classmethod
    def _set_address_history(cls, t, address, history):
        cls.execute(
            t, "UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
            (history, history.count(':')//2, address)
        )

    def set_address_history(self, address, history):
        return self.db.runInteraction(lambda t: self._set_address_history(t, address, history))
<|endoftext|>


<commit_before>"""
South Africa-specific Form helpers
"""

from django.core.validators import EMPTY_VALUES
from django.forms import ValidationError
from django.forms.fields import CharField, RegexField
from django.utils.checksums import luhn
from django.utils.translation import gettext as _
import re
from datetime import date

id_re = re.compile(r'^(?P<yy>\d\d)(?P<mm>\d\d)(?P<dd>\d\d)(?P<mid>\d{4})(?P<end>\d{3})')

class ZAIDField(CharField):
    """A form field for South African ID numbers -- the checksum is validated
    using the Luhn checksum, and uses a simlistic (read: not entirely accurate)
    check for the birthdate
    """
    default_error_messages = {
        'invalid': _(u'Enter a valid South African ID number'),
    }

    def clean(self, value):
        super(ZAIDField, self).clean(value)

        if value in EMPTY_VALUES:
            return u''

        # strip spaces and dashes
        value = value.strip().replace(' ', '').replace('-', '')

        match = re.match(id_re, value)

        if not match:
            raise ValidationError(self.error_messages['invalid'])

        g = match.groupdict()

        try:
            # The year 2000 is conveniently a leapyear.
            # This algorithm will break in xx00 years which aren't leap years
            # There is no way to guess the century of a ZA ID number
            d = date(int(g['yy']) + 2000, int(g['mm']), int(g['dd']))
        except ValueError:
            raise ValidationError(self.error_messages['invalid'])

        if not luhn(value):
            raise ValidationError(self.error_messages['invalid'])

        return value

class ZAPostCodeField(RegexField):
    default_error_messages = {
        'invalid': _(u'Enter a valid South African postal code'),
    }

    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
        super(ZAPostCodeField, self).__init__(r'^\d{4}$',
            max_length, min_length, *args, **kwargs)
<commit_msg>Fixed #18269 -- Applied unicode_literals for Python 3 compatibility.<commit_after>"""
South Africa-specific Form helpers
"""
from __future__ import unicode_literals

from django.core.validators import EMPTY_VALUES
from django.forms import ValidationError
from django.forms.fields import CharField, RegexField
from django.utils.checksums import luhn
from django.utils.translation import gettext as _
import re
from datetime import date

id_re = re.compile(r'^(?P<yy>\d\d)(?P<mm>\d\d)(?P<dd>\d\d)(?P<mid>\d{4})(?P<end>\d{3})')

class ZAIDField(CharField):
    """A form field for South African ID numbers -- the checksum is validated
    using the Luhn checksum, and uses a simlistic (read: not entirely accurate)
    check for the birthdate
    """
    default_error_messages = {
        'invalid': _('Enter a valid South African ID number'),
    }

    def clean(self, value):
        super(ZAIDField, self).clean(value)

        if value in EMPTY_VALUES:
            return ''

        # strip spaces and dashes
        value = value.strip().replace(' ', '').replace('-', '')

        match = re.match(id_re, value)

        if not match:
            raise ValidationError(self.error_messages['invalid'])

        g = match.groupdict()

        try:
            # The year 2000 is conveniently a leapyear.
            # This algorithm will break in xx00 years which aren't leap years
            # There is no way to guess the century of a ZA ID number
            d = date(int(g['yy']) + 2000, int(g['mm']), int(g['dd']))
        except ValueError:
            raise ValidationError(self.error_messages['invalid'])

        if not luhn(value):
            raise ValidationError(self.error_messages['invalid'])

        return value

class ZAPostCodeField(RegexField):
    default_error_messages = {
        'invalid': _('Enter a valid South African postal code'),
    }

    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
        super(ZAPostCodeField, self).__init__(r'^\d{4}$',
            max_length, min_length, *args, **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from tornado.httpclient import AsyncHTTPClient
from tornado.ioloop import IOLoop

class ESConnection:

    def __init__(self, host='', port= '9200', io_loop=None):
        self.io_loop = io_loop or IOLoop.instance()
        self.host = host
        self.port = port
        self.client = AsyncHTTPClient(self.io_loop)

    def get(self, callback, **kwargs):
        index = kwargs['index'] or '_all'
        type = '/' + kwargs.get('type') if kwargs.has_key('type') else ''
        field = kwargs.get('field','_all') + ':'
        value = kwargs.get('value', '')
        size = kwargs.get('size', 10)
        page = kwargs.get('page', 1)
        begin = (page-1)*size

        path = "/%(index)s%(type)s/_search?q=%(field)s%(value)s&from=%(begin)d&size=%(size)d" % {"index": index, "type": type, "field": field,
                                                                                "value": value, "begin": begin, "size":size}

        self.get_by_path(path, callback)

    def get_by_path(self, path, callback):
        url = 'http://%(host)s:%(porta)s%(path)s' % {"host": self.host, "porta": self.port, "path": path}
        self.client.fetch(url, callback)<commit_msg>melhoria na funcao get<commit_after># -*- coding: utf-8 -*-

from tornado.httpclient import AsyncHTTPClient
from tornado.ioloop import IOLoop

class ESConnection:

    def __init__(self, host='', port= '9200', io_loop=None):
        self.io_loop = io_loop or IOLoop.instance()
        self.host = host
        self.port = port
        self.client = AsyncHTTPClient(self.io_loop)

    def get(self, callback, **kwargs):
        index = kwargs.get('index', '_all')
        type = '/' + kwargs.get('type') if kwargs.has_key('type') else ''
        field = kwargs.get('field','_all') + ':'
        value = kwargs.get('value', '')
        size = kwargs.get('size', 10)
        page = kwargs.get('page', 1)
        begin = (page-1)*size

        path = "/%(index)s%(type)s/_search?q=%(field)s%(value)s&from=%(begin)d&size=%(size)d" % {"index": index, "type": type, "field": field,
                                                                                "value": value, "begin": begin, "size":size}

        self.get_by_path(path, callback)

    def get_by_path(self, path, callback):
        url = 'http://%(host)s:%(porta)s%(path)s' % {"host": self.host, "porta": self.port, "path": path}
        self.client.fetch(url, callback)<|endoftext|>


<commit_before>from flask_wtf import Form

from wtforms.fields import StringField, IntegerField, HiddenField, BooleanField
from wtforms import validators

from flask import request

class DonateForm(Form):
    first_name = StringField(u'First',
        [validators.required(message="Your first name is required.")])
    last_name = StringField(u'Last',
        [validators.required(message="Your last name is required.")])
    amount = IntegerField(u'Amount',
        [validators.required(message="Please choose a donation amount.")])
    reason = StringField(u'Encouraged to contribute by')
    installment_period = HiddenField(u'Installment Period')
    installments = HiddenField(u'Installments')
    openended_status = HiddenField(u'Openended Status')
    description = HiddenField(u'Description')
    pay_fees = BooleanField(u'Agree to pay fees')
    pay_fees_value = HiddenField(u'Pay Fees Value')

class TexasWeeklyForm(Form):
    first_name = StringField(u'First',
        [validators.required(message="Your first name is required.")])
    last_name = StringField(u'Last',
        [validators.required(message="Your last name is required.")])
    amount = IntegerField(u'Amount',
        [validators.required(message="Please choose an amount.")])
    description = HiddenField(u'Description')
<commit_msg>Change IntegerFields to DecimalFields<commit_after>from flask_wtf import Form

from wtforms.fields import StringField, HiddenField, BooleanField, DecimalField
from wtforms import validators

from flask import request

class DonateForm(Form):
    first_name = StringField(u'First',
        [validators.required(message="Your first name is required.")])
    last_name = StringField(u'Last',
        [validators.required(message="Your last name is required.")])
    amount = DecimalField(u'Amount',
        [validators.required(message="Please choose a donation amount.")])
    reason = StringField(u'Encouraged to contribute by')
    installment_period = HiddenField(u'Installment Period')
    installments = HiddenField(u'Installments')
    openended_status = HiddenField(u'Openended Status')
    description = HiddenField(u'Description')
    pay_fees = BooleanField(u'Agree to pay fees')
    pay_fees_value = HiddenField(u'Pay Fees Value')

class TexasWeeklyForm(Form):
    first_name = StringField(u'First',
        [validators.required(message="Your first name is required.")])
    last_name = StringField(u'Last',
        [validators.required(message="Your last name is required.")])
    amount = DecimalField(u'Amount',
        [validators.required(message="Please choose an amount.")])
    description = HiddenField(u'Description')
<|endoftext|>


<commit_before>#-*- coding=utf-8 -*-

__author___ = "HUANGBIAO"
__email__  = "19941222hb@gmail.com"
__version__ = "0.1"

from .tornado_session import *
from .tornado_hbredis import *

<commit_msg>version 0.2 released<commit_after>#-*- coding=utf-8 -*-

__author___ = "HUANGBIAO"
__email__  = "19941222hb@gmail.com"
__version__ = "0.2"

from .tornado_session import *
from .tornado_hbredis import *

<|endoftext|>