<commit_before>import hashlib
import numpy as np
import h5py

from ..database import db

class BrainData(object):
    def __init__(self, data, subject, **kwargs):
        if isinstance(data, str):
            import nibabel
            nib = nibabel.load(data)
            data = nib.get_data().T
        self._data = data
        try:
            basestring
        except NameError:
            subject = subject if isinstance(subject, str) else subject.decode('utf-8')
        self.subject = subject
        super(BrainData, self).__init__(**kwargs)

    @property
    def data(self):
        if isinstance(self._data, h5py.Dataset):
            return self._data.value
        return self._data

    @data.setter
    def data(self, data):
        self._data = data

    @property
    def name(self):
        '''Name of this BrainData, according to its hash'''
        return "__%s"%_hash(self.data)[:16]

    def exp(self):
        """Copy of this object with data exponentiated.
        """
        return self.copy(np.exp(self.data))

    def uniques(self, collapse=False):
        yield self

    def __hash__(self):
        return hash(_hash(self.data))

    def _write_hdf(self, h5, name=None):
        if name is None:
            name = self.name

        dgrp = h5.require_group("/data")
        if name in dgrp and "__%s"%_hash(dgrp[name].value)[:16] == name:
            #don't need to update anything, since it's the same data
            return h5.get("/data/%s"%name)

        node = _hdf_write(h5, self.data, name=name)
        node.attrs['subject'] = self.subject
        return node

    def to_json(self, simple=False):
        sdict = super(BrainData, self).to_json(simple=simple)
        if simple:
            sdict.update(dict(name=self.name,
                subject=self.subject,
                min=float(np.nan_to_num(self.data).min()),
                max=float(np.nan_to_num(self.data).max()),
                shape=self.shape))
        return sdict

    @classmethod
    def add_numpy_methods(cls):
        """Adds numpy operator methods (+, -, etc.) to this class to allow
        simple manipulation of the data, e.g. with VolumeData v:
        v + 1 # Returns new VolumeData with 1 added to data
        v ** 2 # Returns new VolumeData with data squared
        """
        # Binary operations
        npops = ["__add__", "__sub__", "__mul__", "__div__", "__pow__",
                 "__neg__", "__abs__"]

        def make_opfun(op): # function nesting creates closure containing op
            def opfun(self, *args):
                return self.copy(getattr(self.data, op)(*args))
            return opfun

        for op in npops:
            opfun = make_opfun(op)
            opfun.__name__ = op
            setattr(cls, opfun.__name__, opfun)

BrainData.add_numpy_methods()

class VolumeData(BrainData):
    def __init__(self, data, subject, xfmname, mask=None, **kwargs):
        """Three possible variables: volume, movie, vertex. Enumerated with size:
        volume movie: (t, z, y, x)
        volume image: (z, y, x)
        linear movie: (t, v)
        linear image: (v,)
        """
        if self.__class__ == VolumeData:
            raise TypeError('Cannot directly instantiate VolumeData objects')
        super(VolumeData, self).__init__(data, subject, **kwargs)
        try:
            basestring
        except NameError:
            xfmname = xfmname if isinstance(xfmname, str) else xfmname.decode('utf-8')
        self.xfmname = xfmname

        self._check_size(mask)
        self.masked = _masker(self)

    def to_json(self, simple=False):
        if simple:
            return super(VolumeData, self).to_json(simple=simple)

        xfm = db.get_xfm(self.subject, self.xfmname, 'coord').xfm
        sdict = dict(xfm=list(np.array(xfm).ravel()), data=[self.name])
        sdict.update(super(VolumeData, self).to_json())
        return sdict

    @classmethod
    def empty(cls, subject, xfmname, **kwargs):
        xfm = db.get_xfm(subject, xfmname)
        shape = xfm.shape
        return cls(np.zeros(shape), subject, xfmname, **kwargs)

    @classmethod
    def random(cls, subject, xfmname, **kwargs):
        xfm = db.get_xfm(subject, xfmname)
        shape = xfm.shape
        return cls(np.random.randn(*shape), subject, xfmname, **kwargs)

    def _check_size(self, mask):
        if self.data.ndim not in (1, 2, 3, 4):
            raise ValueError("Invalid data shape")

        self.linear = self.data.ndim in (1, 2)
        self.movie = self.data.ndim in (2, 4)

        if self.linear:
            #Guess the mask
            if mask is None:
                nvox = self.data.shape[-1]
                self._mask, self.mask = _find_mask(nvox, self.subject, self.xfmname)
            elif isinstance(mask, str):
                self.mask = db.get_mask(self.subject, self.xfmname, mask)
                self._mask = mask
            elif isinstance(mask, np.ndarray):
                self.mask = mask
                self._mask = mask

            self.shape = self.mask.shape
        else:
            self._mask = None
            shape = self.data.shape
            if self.movie:
                shape = shape[1:]
            xfm = db.get_xfm(self.subject, self.xfmname)
            if xfm.shape != shape:
                raise ValueError("Volumetric data (shape %s) is not the same shape as reference for transform (shape %s)" % (str(shape), str(xfm.shape)))
            self.shape = shape

    def map(self, projection="nearest"):
        """Convert this VolumeData into a VertexData using the given sampler
        """
        from .. import utils
        mapper = utils.get_mapper(self.subject, self.xfmname, projection)
        data = mapper(self)
        return data

    def __repr__(self):
        maskstr = "volumetric"
        if self.linear:
            name = self._mask
            if isinstance(self._mask, np.ndarray):
                name = "custom"
            maskstr = "%s masked"%name
        if self.movie:
            maskstr += " movie"
        maskstr = maskstr[0].upper()+maskstr[1:]
        return "<%s data for (%s, %s)>"%(maskstr, self.subject, self.xfmname)

    def copy(self, data):
        return super(VolumeData, self).copy(data, self.subject, self.xfmname, mask=self._mask)

    @property
    def volume(self):
        """Standardizes the VolumeData, ensuring that masked data are unmasked"""
        from .. import volume
        if self.linear:
            data = volume.unmask(self.mask, self.data[:])
        else:
            data = self.data[:]

        if not self.movie:
            data = data[np.newaxis]

        return data

    def save(self, filename, name=None):
        """Save the dataset into an hdf file with the provided name
        """
        if isinstance(filename, str):
            fname, ext = os.path.splitext(filename)
            if ext in (".hdf", ".h5"):
                h5 = h5py.File(filename, "a")
                self._write_hdf(h5, name=name)
                h5.close()
            else:
                raise TypeError('Unknown file type')
        elif isinstance(filename, h5py.Group):
            self._write_hdf(filename, name=name)

    def _write_hdf(self, h5, name=None):
        node = super(VolumeData, self)._write_hdf(h5, name=name)

        #write the mask into the file, as necessary
        if self._mask is not None:
            mask = self._mask
            if isinstance(self._mask, np.ndarray):
                mgrp = "/subjects/{subj}/transforms/{xfm}/masks/"
                mgrp = mgrp.format(subj=self.subject, xfm=self.xfmname)
                mname = "__%s" % _hash(self._mask)[:8]
                _hdf_write(h5, self._mask, name=mname, group=mgrp)
                mask = mname

            node.attrs['mask'] = mask

        return node

class VertexData(BrainData):
    def __init__(self, data, subject, **kwargs):
        """Represents `data` at each vertex on a `subject`s cortex.
        `data` shape possibilities:

        reg linear movie: (t, v)
        reg linear image: (v,)

        where t is the number of time points, c is colors (i.e. RGB), and v is the
        number of vertices (either in both hemispheres or one hemisphere).
        """
        if self.__class__ == VertexData:
            raise TypeError('Cannot directly instantiate VertexData objects')
        super(VertexData, self).__init__(data, subject, **kwargs)

        left, right = db.get_surf(self.subject, "fiducial")
        self.llen = len(left[0])
        self.rlen = len(right[0])
        self._set_data(data)

    @classmethod
    def empty(cls, subject, **kwargs):
        left, right = db.get_surf(subject, "fiducial")
        nverts = len(left[0]) + len(right[0])
        return cls(np.zeros((nverts,)), subject, **kwargs)

    @classmethod
    def random(cls, subject, **kwargs):
        left, right = db.get_surf(subject, "fiducial")
        nverts = len(left[0]) + len(right[0])
        return cls(np.random.randn(nverts), subject, **kwargs)

    def _set_data(self, data):
        """Stores data for this VertexData. Also sets flags if `data` appears to
        be in 'movie' or 'raw' format. See __init__ for `data` shape possibilities.
        """
        self._data = data
        self.movie = self.data.ndim > 1
        self.nverts = self.data.shape[-1]
        if self.llen == self.nverts:
            # Just data for left hemisphere
            self.hem = "left"
            rshape = list(self.data.shape)
            rshape[1 if self.movie else 0] = self.rlen
            self._data = np.hstack([self.data, np.zeros(rshape, dtype=self.data.dtype)])
        elif self.rlen == self.nverts:
            # Just data for right hemisphere
            self.hem = "right"
            lshape = list(self.data.shape)
            lshape[1 if self.movie else 0] = self.llen
            self._data = np.hstack([np.zeros(lshape, dtype=self.data.dtype), self.data])
        elif self.llen + self.rlen == self.nverts:
            # Data for both hemispheres
            self.hem = "both"
        else:
            raise ValueError('Invalid number of vertices for subject (given %d, should be %d for left hem, %d for right hem, or %d for both)' % (self.nverts, self.llen, self.rlen, self.llen+self.rlen))

    def copy(self, data):
        return super(VertexData, self).copy(data, self.subject)

    def volume(self, xfmname, projection='nearest', **kwargs):
        import warnings
        warnings.warn('Inverse mapping cannot be accurate')
        from .. import utils
        mapper = utils.get_mapper(self.subject, xfmname, projection)
        return mapper.backwards(self, **kwargs)

    def __repr__(self):
        maskstr = ""
        if self.movie:
            maskstr = "movie "
        return "<Vertex %sdata for %s>"%(maskstr, self.subject)

    def __getitem__(self, idx):
        if not self.movie:
            raise TypeError("Cannot index non-movie data")

        #return VertexData(self.data[idx], self.subject, **self.attrs)
        return self.copy(self.data[idx])

    @property
    def vertices(self):
        verts = self.data
        if not self.movie:
            verts = verts[np.newaxis]
        return verts

    @property
    def left(self):
        if self.movie:
            return self.data[:,:self.llen]
        else:
            return self.data[:self.llen]

    @property
    def right(self):
        if self.movie:
            return self.data[:,self.llen:]
        else:
            return self.data[self.llen:]

def _find_mask(nvox, subject, xfmname):
    import os
    import re
    import glob
    import nibabel
    files = db.get_paths(subject)['masks'].format(xfmname=xfmname, type="*")
    for fname in glob.glob(files):
        nib = nibabel.load(fname)
        mask = nib.get_data().T != 0
        if nvox == np.sum(mask):
            fname = os.path.split(fname)[1]
            name = re.compile(r'mask_([\w]+).nii.gz').search(fname)
            return name.group(1), mask

    raise ValueError('Cannot find a valid mask')

class _masker(object):
    def __init__(self, dv):
        self.dv = dv

        self.data = None
        if dv.linear:
            self.data = dv.data

    def __getitem__(self, masktype):
        s, x = self.dv.subject, self.dv.xfmname
        mask = db.get_mask(s, x, masktype)
        return self.dv.copy(self.dv.volume[:,mask].squeeze())

def _hash(array):
    '''A simple numpy hash function'''
    return hashlib.sha1(array.tostring()).hexdigest()

def _hdf_write(h5, data, name="data", group="/data"):
    try:
        node = h5.require_dataset("%s/%s"%(group, name), data.shape, data.dtype, exact=True)
    except TypeError:
        del h5[group][name]
        node = h5.create_dataset("%s/%s"%(group, name), data.shape, data.dtype, exact=True)

    node[:] = data
    return node
<commit_msg>use wm instead of fiducial to check size<commit_after>import hashlib
import numpy as np
import h5py

from ..database import db

class BrainData(object):
    def __init__(self, data, subject, **kwargs):
        if isinstance(data, str):
            import nibabel
            nib = nibabel.load(data)
            data = nib.get_data().T
        self._data = data
        try:
            basestring
        except NameError:
            subject = subject if isinstance(subject, str) else subject.decode('utf-8')
        self.subject = subject
        super(BrainData, self).__init__(**kwargs)

    @property
    def data(self):
        if isinstance(self._data, h5py.Dataset):
            return self._data.value
        return self._data

    @data.setter
    def data(self, data):
        self._data = data

    @property
    def name(self):
        '''Name of this BrainData, according to its hash'''
        return "__%s"%_hash(self.data)[:16]

    def exp(self):
        """Copy of this object with data exponentiated.
        """
        return self.copy(np.exp(self.data))

    def uniques(self, collapse=False):
        yield self

    def __hash__(self):
        return hash(_hash(self.data))

    def _write_hdf(self, h5, name=None):
        if name is None:
            name = self.name

        dgrp = h5.require_group("/data")
        if name in dgrp and "__%s"%_hash(dgrp[name].value)[:16] == name:
            #don't need to update anything, since it's the same data
            return h5.get("/data/%s"%name)

        node = _hdf_write(h5, self.data, name=name)
        node.attrs['subject'] = self.subject
        return node

    def to_json(self, simple=False):
        sdict = super(BrainData, self).to_json(simple=simple)
        if simple:
            sdict.update(dict(name=self.name,
                subject=self.subject,
                min=float(np.nan_to_num(self.data).min()),
                max=float(np.nan_to_num(self.data).max()),
                shape=self.shape))
        return sdict

    @classmethod
    def add_numpy_methods(cls):
        """Adds numpy operator methods (+, -, etc.) to this class to allow
        simple manipulation of the data, e.g. with VolumeData v:
        v + 1 # Returns new VolumeData with 1 added to data
        v ** 2 # Returns new VolumeData with data squared
        """
        # Binary operations
        npops = ["__add__", "__sub__", "__mul__", "__div__", "__pow__",
                 "__neg__", "__abs__"]

        def make_opfun(op): # function nesting creates closure containing op
            def opfun(self, *args):
                return self.copy(getattr(self.data, op)(*args))
            return opfun

        for op in npops:
            opfun = make_opfun(op)
            opfun.__name__ = op
            setattr(cls, opfun.__name__, opfun)

BrainData.add_numpy_methods()

class VolumeData(BrainData):
    def __init__(self, data, subject, xfmname, mask=None, **kwargs):
        """Three possible variables: volume, movie, vertex. Enumerated with size:
        volume movie: (t, z, y, x)
        volume image: (z, y, x)
        linear movie: (t, v)
        linear image: (v,)
        """
        if self.__class__ == VolumeData:
            raise TypeError('Cannot directly instantiate VolumeData objects')
        super(VolumeData, self).__init__(data, subject, **kwargs)
        try:
            basestring
        except NameError:
            xfmname = xfmname if isinstance(xfmname, str) else xfmname.decode('utf-8')
        self.xfmname = xfmname

        self._check_size(mask)
        self.masked = _masker(self)

    def to_json(self, simple=False):
        if simple:
            return super(VolumeData, self).to_json(simple=simple)

        xfm = db.get_xfm(self.subject, self.xfmname, 'coord').xfm
        sdict = dict(xfm=list(np.array(xfm).ravel()), data=[self.name])
        sdict.update(super(VolumeData, self).to_json())
        return sdict

    @classmethod
    def empty(cls, subject, xfmname, **kwargs):
        xfm = db.get_xfm(subject, xfmname)
        shape = xfm.shape
        return cls(np.zeros(shape), subject, xfmname, **kwargs)

    @classmethod
    def random(cls, subject, xfmname, **kwargs):
        xfm = db.get_xfm(subject, xfmname)
        shape = xfm.shape
        return cls(np.random.randn(*shape), subject, xfmname, **kwargs)

    def _check_size(self, mask):
        if self.data.ndim not in (1, 2, 3, 4):
            raise ValueError("Invalid data shape")

        self.linear = self.data.ndim in (1, 2)
        self.movie = self.data.ndim in (2, 4)

        if self.linear:
            #Guess the mask
            if mask is None:
                nvox = self.data.shape[-1]
                self._mask, self.mask = _find_mask(nvox, self.subject, self.xfmname)
            elif isinstance(mask, str):
                self.mask = db.get_mask(self.subject, self.xfmname, mask)
                self._mask = mask
            elif isinstance(mask, np.ndarray):
                self.mask = mask
                self._mask = mask

            self.shape = self.mask.shape
        else:
            self._mask = None
            shape = self.data.shape
            if self.movie:
                shape = shape[1:]
            xfm = db.get_xfm(self.subject, self.xfmname)
            if xfm.shape != shape:
                raise ValueError("Volumetric data (shape %s) is not the same shape as reference for transform (shape %s)" % (str(shape), str(xfm.shape)))
            self.shape = shape

    def map(self, projection="nearest"):
        """Convert this VolumeData into a VertexData using the given sampler
        """
        from .. import utils
        mapper = utils.get_mapper(self.subject, self.xfmname, projection)
        data = mapper(self)
        return data

    def __repr__(self):
        maskstr = "volumetric"
        if self.linear:
            name = self._mask
            if isinstance(self._mask, np.ndarray):
                name = "custom"
            maskstr = "%s masked"%name
        if self.movie:
            maskstr += " movie"
        maskstr = maskstr[0].upper()+maskstr[1:]
        return "<%s data for (%s, %s)>"%(maskstr, self.subject, self.xfmname)

    def copy(self, data):
        return super(VolumeData, self).copy(data, self.subject, self.xfmname, mask=self._mask)

    @property
    def volume(self):
        """Standardizes the VolumeData, ensuring that masked data are unmasked"""
        from .. import volume
        if self.linear:
            data = volume.unmask(self.mask, self.data[:])
        else:
            data = self.data[:]

        if not self.movie:
            data = data[np.newaxis]

        return data

    def save(self, filename, name=None):
        """Save the dataset into an hdf file with the provided name
        """
        if isinstance(filename, str):
            fname, ext = os.path.splitext(filename)
            if ext in (".hdf", ".h5"):
                h5 = h5py.File(filename, "a")
                self._write_hdf(h5, name=name)
                h5.close()
            else:
                raise TypeError('Unknown file type')
        elif isinstance(filename, h5py.Group):
            self._write_hdf(filename, name=name)

    def _write_hdf(self, h5, name=None):
        node = super(VolumeData, self)._write_hdf(h5, name=name)

        #write the mask into the file, as necessary
        if self._mask is not None:
            mask = self._mask
            if isinstance(self._mask, np.ndarray):
                mgrp = "/subjects/{subj}/transforms/{xfm}/masks/"
                mgrp = mgrp.format(subj=self.subject, xfm=self.xfmname)
                mname = "__%s" % _hash(self._mask)[:8]
                _hdf_write(h5, self._mask, name=mname, group=mgrp)
                mask = mname

            node.attrs['mask'] = mask

        return node

class VertexData(BrainData):
    def __init__(self, data, subject, **kwargs):
        """Represents `data` at each vertex on a `subject`s cortex.
        `data` shape possibilities:

        reg linear movie: (t, v)
        reg linear image: (v,)

        where t is the number of time points, c is colors (i.e. RGB), and v is the
        number of vertices (either in both hemispheres or one hemisphere).
        """
        if self.__class__ == VertexData:
            raise TypeError('Cannot directly instantiate VertexData objects')
        super(VertexData, self).__init__(data, subject, **kwargs)
        try:
            left, right = db.get_surf(self.subject, "wm")
        except IOError:
            left, right = db.get_surf(self.subject, "fiducial")
        self.llen = len(left[0])
        self.rlen = len(right[0])
        self._set_data(data)

    @classmethod
    def empty(cls, subject, **kwargs):
        try:
            left, right = db.get_surf(self.subject, "wm")
        except IOError:
            left, right = db.get_surf(self.subject, "fiducial")
        nverts = len(left[0]) + len(right[0])
        return cls(np.zeros((nverts,)), subject, **kwargs)

    @classmethod
    def random(cls, subject, **kwargs):
        try:
            left, right = db.get_surf(self.subject, "wm")
        except IOError:
            left, right = db.get_surf(self.subject, "fiducial")
        nverts = len(left[0]) + len(right[0])
        return cls(np.random.randn(nverts), subject, **kwargs)

    def _set_data(self, data):
        """Stores data for this VertexData. Also sets flags if `data` appears to
        be in 'movie' or 'raw' format. See __init__ for `data` shape possibilities.
        """
        self._data = data
        self.movie = self.data.ndim > 1
        self.nverts = self.data.shape[-1]
        if self.llen == self.nverts:
            # Just data for left hemisphere
            self.hem = "left"
            rshape = list(self.data.shape)
            rshape[1 if self.movie else 0] = self.rlen
            self._data = np.hstack([self.data, np.zeros(rshape, dtype=self.data.dtype)])
        elif self.rlen == self.nverts:
            # Just data for right hemisphere
            self.hem = "right"
            lshape = list(self.data.shape)
            lshape[1 if self.movie else 0] = self.llen
            self._data = np.hstack([np.zeros(lshape, dtype=self.data.dtype), self.data])
        elif self.llen + self.rlen == self.nverts:
            # Data for both hemispheres
            self.hem = "both"
        else:
            raise ValueError('Invalid number of vertices for subject (given %d, should be %d for left hem, %d for right hem, or %d for both)' % (self.nverts, self.llen, self.rlen, self.llen+self.rlen))

    def copy(self, data):
        return super(VertexData, self).copy(data, self.subject)

    def volume(self, xfmname, projection='nearest', **kwargs):
        import warnings
        warnings.warn('Inverse mapping cannot be accurate')
        from .. import utils
        mapper = utils.get_mapper(self.subject, xfmname, projection)
        return mapper.backwards(self, **kwargs)

    def __repr__(self):
        maskstr = ""
        if self.movie:
            maskstr = "movie "
        return "<Vertex %sdata for %s>"%(maskstr, self.subject)

    def __getitem__(self, idx):
        if not self.movie:
            raise TypeError("Cannot index non-movie data")

        #return VertexData(self.data[idx], self.subject, **self.attrs)
        return self.copy(self.data[idx])

    @property
    def vertices(self):
        verts = self.data
        if not self.movie:
            verts = verts[np.newaxis]
        return verts

    @property
    def left(self):
        if self.movie:
            return self.data[:,:self.llen]
        else:
            return self.data[:self.llen]

    @property
    def right(self):
        if self.movie:
            return self.data[:,self.llen:]
        else:
            return self.data[self.llen:]

def _find_mask(nvox, subject, xfmname):
    import os
    import re
    import glob
    import nibabel
    files = db.get_paths(subject)['masks'].format(xfmname=xfmname, type="*")
    for fname in glob.glob(files):
        nib = nibabel.load(fname)
        mask = nib.get_data().T != 0
        if nvox == np.sum(mask):
            fname = os.path.split(fname)[1]
            name = re.compile(r'mask_([\w]+).nii.gz').search(fname)
            return name.group(1), mask

    raise ValueError('Cannot find a valid mask')

class _masker(object):
    def __init__(self, dv):
        self.dv = dv

        self.data = None
        if dv.linear:
            self.data = dv.data

    def __getitem__(self, masktype):
        s, x = self.dv.subject, self.dv.xfmname
        mask = db.get_mask(s, x, masktype)
        return self.dv.copy(self.dv.volume[:,mask].squeeze())

def _hash(array):
    '''A simple numpy hash function'''
    return hashlib.sha1(array.tostring()).hexdigest()

def _hdf_write(h5, data, name="data", group="/data"):
    try:
        node = h5.require_dataset("%s/%s"%(group, name), data.shape, data.dtype, exact=True)
    except TypeError:
        del h5[group][name]
        node = h5.create_dataset("%s/%s"%(group, name), data.shape, data.dtype, exact=True)

    node[:] = data
    return node
<|endoftext|>


<commit_before>from models import Player, Session, Card, CardDetail, Stack
from models import CARD_KIND_WEAPON, CARD_KIND_POTION, CARD_KIND_MONSTER, CARD_KIND_SCRAP, CARD_KIND_TREASURE

import random

def roll(min, max):
    return random.randint(min, max)

def get_random_card_id_in_value_range(min, max, offset):
    card_id = roll(
        min + offset,
        max + offset)

    return card_id

def get_random_weapon_id_in_value_range(min, max):
    if min < 2 or max > 10:
        return None

    return get_random_card_id_in_value_range(min, max, 1)

def get_random_potion_id_in_value_range(min, max):
    if min < 2 or max > 10:
        return None

    return get_random_card_id_in_value_range(min, max, 10)

def get_random_monster_id_in_value_range(min, max):
    if min < 2 or max > 14:
        return None

    return get_random_card_id_in_value_range(min, max, 19)

def start(player):
    """
    Attempts creating a new game session for the given player.
    """
    if player is None:
        return None

    # Initialize all the stacks.
    room_stack = Stack()
    room_stack.save()

    you_stack = Stack()
    you_stack.save()

    equipment_stack = Stack()
    equipment_stack.save()

    forge_stack = Stack()
    forge_stack.save()

    treasure_stack = Stack()
    treasure_stack.save()

    discard_stack = Stack()
    discard_stack.save()

    # Begin a new session.
    session = Session(
        # Important to note that a session has to be tied to a player. Same goes for
        # cards and stacks; they must, ultimately, be tied to a session. Otherwise
        # it would be possible to move cards between sessions.
        belongs_to_player = player,

        room_stack = room_stack,
        you_stack = you_stack,
        equipment_stack = equipment_stack,
        forge_stack = forge_stack,
        treasure_stack = treasure_stack,
        discard_stack = discard_stack
    )

    session.save()

    # Draw the first 5 cards.
    initial_room_cards = draw(session, 5)

    # Put the initial cards in place.
    room_stack.push_many(initial_room_cards)

    # If everything went as expected, activate the session by hooking it up to the player.
    player.active_session = session
    player.save()

    return session

def draw_single(session, properties=None):
    """
    Attempts drawing a single card.
    Can optionally be given specific properties, determined randomly otherwise.
    """
    if session is None:
        return None

    if properties is None:
        card_should_be_beneficial = roll(0, 100) >= 60 # 40 chance of not being a monster card
        card_should_be_special = roll(0, 100) >= 95    # 5% chance of being special

        details_id = None

        if card_should_be_beneficial:
            luck = roll(0, 100)

            weapon_range = range(0, 45)
            weapon_part_range = range(45, 75)
            potion_range = range(75, 90)
            treasure_range = range(90, 100)

            if luck in weapon_part_range:
                # Weapon Part
                details_id = 1
            elif luck in treasure_range:
                # Treasure
                details_id = 2
            elif luck in weapon_range:
                # Weapon (2-10)
                details_id = get_random_weapon_id_in_value_range(2, 10)
            elif luck in potion_range:
                # Potion (2-10)
                details_id = get_random_potion_id_in_value_range(2, 10)
        else:
            # Monster (2-14)
            details_id = get_random_monster_id_in_value_range(2, 14)

        if details_id is None:
            return None

        try:
            properties = CardDetail.objects.get(pk=details_id)
        except CardDetail.DoesNotExist:
            return None

    try:
        card = Card(
            belongs_to_session=session,
            details=properties,
            is_special=card_should_be_special
        )

        card.save()
    except:
        return None

    return card

def draw(session, amount):
    """
    Attempts drawing a specific amount of cards.
    """
    if session is None:
        return None

    if amount <= 0:
        return None

    cards = []

    for i in range(0, amount):
        card = draw_single(session)

        if card:
            cards.append(card)

    return cards

def can_activate_stack(session, stack):
    """
    Determines whether a stack can be activated in its current state.
    """
    if not session or not stack:
        return False

    # Assuming an empty stack can never be activated.
    if stack.is_empty():
        return False

    if stack == session.room_stack:
        # The current room can never be activated.
        return False

    if stack == session.discard_stack:
        # The discarded stack can never be activated.
        return False

    if stack == session.equipment_stack or stack == session.you_stack:
        # No special rules for weapons/monsters.
        pass

    if stack == session.forge_stack:
        all_forgeable_cards = stack.get_all_cards()

        if all_forgeable_cards:
            amount_of_forgeable_cards = len(all_forgeable_cards)

            if amount_of_forgeable_cards < 2:
                # The forge stack can only be activated when at least 2 scrap cards are placed here.
                return False

            # todo: should forged cards always be special? (this means you can always override the current stack! potential game changer)
            if not session.equipment_stack.is_empty():
                return False

    return True

def activate_stack(session, stack):
    """
    Attempts activating/clearing a stack.
    """
    if not session or not stack:
        return False

    if not can_activate_stack(session, stack):
        return False

    if stack == session.equipment_stack or stack == session.you_stack:
        # todo: discard all and score points
        pass

    if stack == session.forge_stack:
        # Draw a new weapon card that is valued depending on how many cards were spent.
        # value = session.forge_stack.count()

        # Attempt discarding all cards that were spent creating a weapon.
        value = discard_many(session, session.forge_stack.get_all_cards())

        if value <= 0:
            return False

        details_id = get_random_weapon_id_in_value_range(value, value)

        if details_id is None:
            return False

        try:
            properties = CardDetail.objects.get(pk=details_id)
        except CardDetail.DoesNotExist:
            return False

        # Draw the actual card, given the specific properties determined previously.
        weapon_card = draw_single(session, properties)

        # Attempt placing the new weapon on the equipment stack. Keep in mind that it is assumed
        # that the equipment stack is empty when reaching this point.
        if not session.equipment_stack.push(weapon_card):
            return False

    return True

def can_activate_card(session, card):
    """
    Determines whether a card has properties that allow it to be activated.
    """
    if not session or not card:
        return False

    if card.details.kind is CARD_KIND_POTION:
        if card.stack != session.you_stack:
            # Can only be activated when placed on the You stack.
            return False

    if card.details.kind is CARD_KIND_MONSTER:
        if card.stack != session.you_stack:
            # Can only be activated when placed on the You stack.
            return False

    return True

def activate_card(session, card):
    """
    Attempts activating a card.
    This usually occurs when a card has been successfully moved from the current room.
    """
    if not session or not card:
        return False

    if not can_activate_card(session, card):
        return False

    if card.details.kind is CARD_KIND_POTION:
        restored_health = card.value
        current_health = session.health

        current_health += restored_health

        if current_health > 20:
            current_health = 20

        try:
            session.health = current_health
            session.save()
        except:
            return False

    if card.details.kind is CARD_KIND_MONSTER:
        most_recently_played_weapon_card = session.equipment_stack.get_top()

        if most_recently_played_weapon_card and most_recently_played_weapon_card.is_special:
            try:
                # Disable special status as soon as a monster has been placed.
                most_recently_played_weapon_card.is_special = False
                most_recently_played_weapon_card.save()
            except:
                return False

        damage = card.value

        if damage:
            if most_recently_played_weapon_card:
                damage -= most_recently_played_weapon_card.value

            if damage > 0:
                try:
                    # todo: need to figure out what happens in terms of rollover..
                    # since `health` is unsigned small int does it roll over to maxval when going below 0, like in c?
                    session.health -= damage
                    session.save()
                except:
                    return False

        if not most_recently_played_weapon_card:
            # Monsters only stack if player has a weapon equipped
            discard(session, card)

    return True

def can_move(session, card, to_stack):
    """
    Determines whether a card can be moved to a given stack.
    """
    if (not session or
        not card or
        not to_stack):
        return False

    if to_stack == session.room_stack:
        # you can't move cards to the room...
        return False

    if to_stack == session.treasure_stack:
        if card.details.kind is not CARD_KIND_TREASURE:
            # Not a treasure card, bail out...
            return False

        if len(session.treasure_stack.get_all_cards()) >= 10:
            # Treasure stack already holds maximum amount of treasure
            return False

    if to_stack == session.forge_stack:
        if card.details.kind is not CARD_KIND_SCRAP:
            # Not a scrap card, bail out...
            return False

        if len(session.forge_stack.get_all_cards()) >= 10:
            # Forge stack already holds maximum amount of scraps
            return False

    if to_stack == session.equipment_stack:
        if card.details.kind is not CARD_KIND_WEAPON:
            # Not a weapon card, bail out...
            return False

        most_recently_played_weapon_card = session.equipment_stack.get_top()

        if most_recently_played_weapon_card is not None:
            if not card.is_special:
                # Only special cards can be placed on top of the previous weapon as a score multiplier.
                return False

    if to_stack == session.you_stack:
        if card.details.kind is not CARD_KIND_MONSTER or card.details.kind is not CARD_KIND_POTION:
            # Only monsters or potions can be placed here
            return False

        if card.details.kind is CARD_KIND_MONSTER:
            # Card is a monster
            most_recently_played_monster_card = session.you_stack.get_top()

            if most_recently_played_monster_card is not None:
                if card.value >= most_recently_played_monster_card:
                    most_recently_played_weapon_card = session.equipment_stack.get_top()

                    if most_recently_played_weapon_card and not most_recently_played_weapon_card.is_special:
                        # Basically, you can only place monsters of higher value on other monsters if
                        # the current weapon is special.
                        return False

    return True

def move(session, card, to_stack):
    """
    Attempts moving a card into a given stack.
    """
    if not session:
        return False

    if not card or not to_stack:
        return False

    if not card.can_be_moved(to_stack):
        return False

    if not can_move(session, card, to_stack):
        return False

    if not to_stack.push(card):
        return False

    if to_stack == session.room_stack:
        try:
            session.amount_of_cards_moved_since_last_skip += 1
            session.save()
        except:
            return False

    if not activate_card(session, card):
        return False

    return True

def discard(session, card):
    """
    Discards a card from a game session. The 10 most recently discarded cards are stored.
    """
    if session is None:
        return False

    if move(session, card, session.discard_stack):
        all_discarded_cards = session.discard_stack.get_all_cards()

        if all_discarded_cards.count() >= 10:
            oldest_discarded_card = all_discarded_cards.reverse()[:1]
            oldest_discarded_card.stack = None
            oldest_discarded_card.save()

        return True

    return False

def discard_many(session, cards):
    """
    Attempts discarding several cards at once.
    """
    if session is None or cards is None:
        return False

    amount_discarded = 0

    for card in cards:
        if discard(session, card):
            amount_discarded += 1

    return amount_discarded

def can_skip_on_next_move(session):
    """
    Determines whether a game can have its current room skipped on the following turn.
    """
    if session is None:
        return False

    if (session.amount_of_cards_moved_since_last_skip == 4):
        return True

    return False

def can_skip(session):
    """
    Determines whether a game can have its current room skipped or not.
    """
    if session is None:
        return False

    if (session.amount_of_cards_moved_since_last_skip == -1 or
        session.amount_of_cards_moved_since_last_skip >= 5):
        return True

    return False

def skip(session):
    """
    Attempts skipping the current room.
    """
    if session is None:
        return False

    if can_skip(session):
        room_cards = session.room_stack.get_all_cards()
        amount_discarded = discard_many(session, room_cards)

        if amount_discarded > 0:
            replacement_cards = draw(session, amount_discarded)
            session.room_stack.push_many(replacement_cards)

        try:
            session.amount_of_cards_moved_since_last_skip = 0
            session.save()
        except:
            # This would be bad :S Cards probably were drawn and attached.. so needs some cleanup code here
            return False

        return True

    return False<commit_msg>Fixed a bug where discarded cards were accidentally getting activated.<commit_after>import logging

logger = logging.getLogger(__name__)

from models import Player, Session, Card, CardDetail, Stack
from models import CARD_KIND_WEAPON, CARD_KIND_POTION, CARD_KIND_MONSTER, CARD_KIND_SCRAP, CARD_KIND_TREASURE

import random

def roll(min, max):
    return random.randint(min, max)

def get_random_card_id_in_value_range(min, max, offset):
    card_id = roll(
        min + offset,
        max + offset)

    return card_id

def get_random_weapon_id_in_value_range(min, max):
    if min < 2 or max > 10:
        return None

    return get_random_card_id_in_value_range(min, max, 1)

def get_random_potion_id_in_value_range(min, max):
    if min < 2 or max > 10:
        return None

    return get_random_card_id_in_value_range(min, max, 10)

def get_random_monster_id_in_value_range(min, max):
    if min < 2 or max > 14:
        return None

    return get_random_card_id_in_value_range(min, max, 19)

def start(player):
    """
    Attempts creating a new game session for the given player.
    """
    if player is None:
        return None

    # Initialize all the stacks.
    room_stack = Stack()
    room_stack.save()

    you_stack = Stack()
    you_stack.save()

    equipment_stack = Stack()
    equipment_stack.save()

    forge_stack = Stack()
    forge_stack.save()

    treasure_stack = Stack()
    treasure_stack.save()

    discard_stack = Stack()
    discard_stack.save()

    # Begin a new session.
    session = Session(
        # Important to note that a session has to be tied to a player. Same goes for
        # cards and stacks; they must, ultimately, be tied to a session. Otherwise
        # it would be possible to move cards between sessions.
        belongs_to_player = player,

        room_stack = room_stack,
        you_stack = you_stack,
        equipment_stack = equipment_stack,
        forge_stack = forge_stack,
        treasure_stack = treasure_stack,
        discard_stack = discard_stack
    )

    session.save()

    # Draw the first 5 cards.
    initial_room_cards = draw(session, 5)

    # Put the initial cards in place.
    room_stack.push_many(initial_room_cards)

    # If everything went as expected, activate the session by hooking it up to the player.
    player.active_session = session
    player.save()

    return session

def draw_single(session, properties=None):
    """
    Attempts drawing a single card.
    Can optionally be given specific properties, determined randomly otherwise.
    """
    if session is None:
        return None

    if properties is None:
        card_should_be_beneficial = roll(0, 100) >= 60 # 40 chance of not being a monster card
        card_should_be_special = roll(0, 100) >= 95    # 5% chance of being special

        details_id = None

        if card_should_be_beneficial:
            luck = roll(0, 100)

            weapon_range = range(0, 45)
            weapon_part_range = range(45, 75)
            potion_range = range(75, 90)
            treasure_range = range(90, 100)

            if luck in weapon_part_range:
                # Weapon Part
                details_id = 1
            elif luck in treasure_range:
                # Treasure
                details_id = 2
            elif luck in weapon_range:
                # Weapon (2-10)
                details_id = get_random_weapon_id_in_value_range(2, 10)
            elif luck in potion_range:
                # Potion (2-10)
                details_id = get_random_potion_id_in_value_range(2, 10)
        else:
            # Monster (2-14)
            details_id = get_random_monster_id_in_value_range(2, 14)

        if details_id is None:
            return None

        try:
            properties = CardDetail.objects.get(pk=details_id)
        except CardDetail.DoesNotExist:
            return None

    try:
        card = Card(
            belongs_to_session=session,
            details=properties,
            is_special=card_should_be_special
        )

        card.save()
    except:
        return None

    return card

def draw(session, amount):
    """
    Attempts drawing a specific amount of cards.
    """
    if session is None:
        return None

    if amount <= 0:
        return None

    cards = []

    for i in range(0, amount):
        card = draw_single(session)

        if card is not None:
            cards.append(card)

    return cards

def can_activate_stack(session, stack):
    """
    Determines whether a stack can be activated in its current state.
    """
    if not session or not stack:
        return False

    # Assuming an empty stack can never be activated.
    if stack.is_empty():
        return False

    if stack == session.room_stack:
        # The current room can never be activated.
        return False

    if stack == session.discard_stack:
        # The discarded stack can never be activated.
        return False

    if stack == session.equipment_stack or stack == session.you_stack:
        # No special rules for weapons/monsters.
        pass

    if stack == session.forge_stack:
        all_forgeable_cards = stack.get_all_cards()

        if all_forgeable_cards:
            amount_of_forgeable_cards = len(all_forgeable_cards)

            if amount_of_forgeable_cards < 2:
                # The forge stack can only be activated when at least 2 scrap cards are placed here.
                return False

            # todo: should forged cards always be special? (this means you can always override the current stack! potential game changer)
            if not session.equipment_stack.is_empty():
                return False

    return True

def activate_stack(session, stack):
    """
    Attempts activating/clearing a stack.
    """
    if not session or not stack:
        return False

    if not can_activate_stack(session, stack):
        return False

    if stack == session.equipment_stack or stack == session.you_stack:
        # todo: discard all and score points
        pass

    if stack == session.forge_stack:
        # Draw a new weapon card that is valued depending on how many cards were spent.
        # value = session.forge_stack.count()

        # Attempt discarding all cards that were spent creating a weapon.
        value = discard_many(session, session.forge_stack.get_all_cards())

        if value <= 0:
            return False

        details_id = get_random_weapon_id_in_value_range(value, value)

        if details_id is None:
            return False

        try:
            properties = CardDetail.objects.get(pk=details_id)
        except CardDetail.DoesNotExist:
            return False

        # Draw the actual card, given the specific properties determined previously.
        weapon_card = draw_single(session, properties)

        # Attempt placing the new weapon on the equipment stack. Keep in mind that it is assumed
        # that the equipment stack is empty when reaching this point.
        if not session.equipment_stack.push(weapon_card):
            return False

    return True

def can_activate_card(session, card):
    """
    Determines whether a card has properties that allow it to be activated.
    """
    if not session or not card:
        return False

    if card.details.kind is CARD_KIND_POTION:
        if card.stack != session.you_stack:
            # Can only be activated when placed on the You stack.
            return False

    if card.details.kind is CARD_KIND_MONSTER:
        if card.stack != session.you_stack:
            # Can only be activated when placed on the You stack.
            return False

    return True

def activate_card(session, card):
    """
    Attempts activating a card.
    This usually occurs when a card has been successfully moved from the current room.
    """
    if not session or not card:
        return False

    if not can_activate_card(session, card):
        return False

    if card.details.kind is CARD_KIND_POTION:
        restored_health = card.value
        current_health = session.health

        current_health += restored_health

        if current_health > 20:
            current_health = 20

        try:
            session.health = current_health
            session.save()
        except:
            return False

    if card.details.kind is CARD_KIND_MONSTER:
        most_recently_played_weapon_card = session.equipment_stack.get_top()

        if most_recently_played_weapon_card and most_recently_played_weapon_card.is_special:
            try:
                # Disable special status as soon as a monster has been placed.
                most_recently_played_weapon_card.is_special = False
                most_recently_played_weapon_card.save()
            except:
                return False

        damage = card.value

        if damage:
            if most_recently_played_weapon_card:
                damage -= most_recently_played_weapon_card.value

            if damage > 0:
                try:
                    # todo: need to figure out what happens in terms of rollover..
                    # since `health` is unsigned small int does it roll over to maxval when going below 0, like in c?
                    session.health -= damage
                    session.save()
                except:
                    return False

        if not most_recently_played_weapon_card:
            # Monsters only stack if player has a weapon equipped
            discard(session, card)

    return True

def can_move(session, card, to_stack):
    """
    Determines whether a card can be moved to a given stack.
    """
    if (not session or
        not card or
        not to_stack):
        return False

    if to_stack == session.room_stack:
        # you can't move cards to the room...
        return False

    if to_stack == session.treasure_stack:
        if card.details.kind is not CARD_KIND_TREASURE:
            # Not a treasure card, bail out...
            return False

        if len(session.treasure_stack.get_all_cards()) >= 10:
            # Treasure stack already holds maximum amount of treasure
            return False

    if to_stack == session.forge_stack:
        if card.details.kind is not CARD_KIND_SCRAP:
            # Not a scrap card, bail out...
            return False

        if len(session.forge_stack.get_all_cards()) >= 10:
            # Forge stack already holds maximum amount of scraps
            return False

    if to_stack == session.equipment_stack:
        if card.details.kind is not CARD_KIND_WEAPON:
            # Not a weapon card, bail out...
            return False

        most_recently_played_weapon_card = session.equipment_stack.get_top()

        if most_recently_played_weapon_card is not None:
            if not card.is_special:
                # Only special cards can be placed on top of the previous weapon as a score multiplier.
                return False

    if to_stack == session.you_stack:
        if card.details.kind is not CARD_KIND_MONSTER or card.details.kind is not CARD_KIND_POTION:
            # Only monsters or potions can be placed here
            return False

        if card.details.kind is CARD_KIND_MONSTER:
            # Card is a monster
            most_recently_played_monster_card = session.you_stack.get_top()

            if most_recently_played_monster_card is not None:
                if card.value >= most_recently_played_monster_card:
                    most_recently_played_weapon_card = session.equipment_stack.get_top()

                    if most_recently_played_weapon_card and not most_recently_played_weapon_card.is_special:
                        # Basically, you can only place monsters of higher value on other monsters if
                        # the current weapon is special.
                        return False

    return True

def move(session, card, to_stack):
    """
    Attempts moving a card into a given stack.
    """
    if not session:
        return False

    if not card or not to_stack:
        return False

    if not card.can_be_moved(to_stack):
        logger.error(' * card can not be moved!')
        return False

    if not can_move(session, card, to_stack):
        logger.error(' * could not allow moving card!')
        return False

    if not to_stack.push(card):
        logger.error(' * could not push card on stack!')
        return False

    if to_stack == session.room_stack:
        try:
            session.amount_of_cards_moved_since_last_skip += 1
            session.save()
        except:
            logger.error(' * could not increment "cards_moved_since_last_skip"!')
            return False

    if to_stack != session.discard_stack:
        if not activate_card(session, card):
            logger.error(' * could not activate card!')
            return False

    return True

def discard(session, card):
    """
    Discards a card from a game session. The 10 most recently discarded cards are stored.
    """
    if session is None:
        return False

    logger.info('  trying to move %s to "discard_stack"' % (card))

    if move(session, card, session.discard_stack):
        logger.info('    success!')
        all_discarded_cards = session.discard_stack.get_all_cards()

        if all_discarded_cards.count() >= 10:
            oldest_discarded_card = all_discarded_cards.reverse()[:1]

            oldest_discarded_card.stack = None
            oldest_discarded_card.save()

        return True

    return False

def discard_many(session, cards):
    """
    Attempts discarding several cards at once.
    """
    if session is None or cards is None:
        return False

    amount_discarded = 0

    for card in cards:
        logger.info('discarding %s' % (card))
        if discard(session, card):
            logger.info('  success!')
            amount_discarded += 1
        else:
            logger.info('  fail!')

    return amount_discarded

def can_skip_on_next_move(session):
    """
    Determines whether a game can have its current room skipped on the following turn.
    """
    if session is None:
        return False

    if (session.amount_of_cards_moved_since_last_skip == 4):
        return True

    return False

def can_skip(session):
    """
    Determines whether a game can have its current room skipped or not.
    """
    if session is None:
        return False

    if (session.amount_of_cards_moved_since_last_skip == -1 or
        session.amount_of_cards_moved_since_last_skip >= 5):
        return True

    return False

def skip(session):
    """
    Attempts skipping the current room.
    """
    if session is None:
        return False

    if can_skip(session):
        room_cards = session.room_stack.get_all_cards()
        logger.info('skipping %d cards...' % (len(room_cards)))
        amount_discarded = discard_many(session, room_cards)
        logger.info('discarded %d cards!' % (amount_discarded))

        if amount_discarded > 0:
            replacement_cards = draw(session, amount_discarded)
            logger.info('%d new cards were drawn...' % (len(replacement_cards)))
            session.room_stack.push_many(replacement_cards)

        try:
            session.amount_of_cards_moved_since_last_skip = 0
            session.save()
        except:
            return False

        return True

    return False<|endoftext|>


<commit_before># test asynchat

import asyncore, asynchat, socket, threading, time
import unittest
import sys
from test import test_support

# Skip tests if thread module does not exist.
test_support.import_module('thread')

HOST = test_support.HOST
SERVER_QUIT = 'QUIT\n'

class echo_server(threading.Thread):
    # parameter to determine the number of bytes passed back to the
    # client each send
    chunk_size = 1

    def __init__(self, event):
        threading.Thread.__init__(self)
        self.event = event
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.port = test_support.bind_port(self.sock)

    def run(self):
        self.sock.listen(1)
        self.event.set()
        conn, client = self.sock.accept()
        self.buffer = ""
        # collect data until quit message is seen
        while SERVER_QUIT not in self.buffer:
            data = conn.recv(1)
            if not data:
                break
            self.buffer = self.buffer + data

        # remove the SERVER_QUIT message
        self.buffer = self.buffer.replace(SERVER_QUIT, '')

        # re-send entire set of collected data
        try:
            # this may fail on some tests, such as test_close_when_done, since
            # the client closes the channel when it's done sending
            while self.buffer:
                n = conn.send(self.buffer[:self.chunk_size])
                time.sleep(0.001)
                self.buffer = self.buffer[n:]
        except:
            pass

        conn.close()
        self.sock.close()

class echo_client(asynchat.async_chat):

    def __init__(self, terminator, server_port):
        asynchat.async_chat.__init__(self)
        self.contents = []
        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect((HOST, server_port))
        self.set_terminator(terminator)
        self.buffer = ''

    def handle_connect(self):
        pass

    if sys.platform == 'darwin':
        # select.poll returns a select.POLLHUP at the end of the tests
        # on darwin, so just ignore it
        def handle_expt(self):
            pass

    def collect_incoming_data(self, data):
        self.buffer += data

    def found_terminator(self):
        self.contents.append(self.buffer)
        self.buffer = ""

def start_echo_server():
    event = threading.Event()
    s = echo_server(event)
    s.start()
    event.wait()
    event.clear()
    time.sleep(0.01) # Give server time to start accepting.
    return s, event

class TestAsynchat(unittest.TestCase):
    usepoll = False

    def setUp (self):
        pass

    def tearDown (self):
        pass

    def line_terminator_check(self, term, server_chunk):
        event = threading.Event()
        s = echo_server(event)
        s.chunk_size = server_chunk
        s.start()
        event.wait()
        event.clear()
        time.sleep(0.01) # Give server time to start accepting.
        c = echo_client(term, s.port)
        c.push("hello ")
        c.push("world%s" % term)
        c.push("I'm not dead yet!%s" % term)
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "I'm not dead yet!"])

    # the line terminator tests below check receiving variously-sized
    # chunks back from the server in order to exercise all branches of
    # async_chat.handle_read

    def test_line_terminator1(self):
        # test one-character terminator
        for l in (1,2,3):
            self.line_terminator_check('\n', l)

    def test_line_terminator2(self):
        # test two-character terminator
        for l in (1,2,3):
            self.line_terminator_check('\r\n', l)

    def test_line_terminator3(self):
        # test three-character terminator
        for l in (1,2,3):
            self.line_terminator_check('qqq', l)

    def numeric_terminator_check(self, termlen):
        # Try reading a fixed number of bytes
        s, event = start_echo_server()
        c = echo_client(termlen, s.port)
        data = "hello world, I'm not dead yet!\n"
        c.push(data)
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, [data[:termlen]])

    def test_numeric_terminator1(self):
        # check that ints & longs both work (since type is
        # explicitly checked in async_chat.handle_read)
        self.numeric_terminator_check(1)
        self.numeric_terminator_check(1L)

    def test_numeric_terminator2(self):
        self.numeric_terminator_check(6L)

    def test_none_terminator(self):
        # Try reading a fixed number of bytes
        s, event = start_echo_server()
        c = echo_client(None, s.port)
        data = "hello world, I'm not dead yet!\n"
        c.push(data)
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, [])
        self.assertEqual(c.buffer, data)

    def test_simple_producer(self):
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        data = "hello world\nI'm not dead yet!\n"
        p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8)
        c.push_with_producer(p)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "I'm not dead yet!"])

    def test_string_producer(self):
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        data = "hello world\nI'm not dead yet!\n"
        c.push_with_producer(data+SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "I'm not dead yet!"])

    def test_empty_line(self):
        # checks that empty lines are handled correctly
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        c.push("hello world\n\nI'm not dead yet!\n")
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "", "I'm not dead yet!"])

    def test_close_when_done(self):
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        c.push("hello world\nI'm not dead yet!\n")
        c.push(SERVER_QUIT)
        c.close_when_done()
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, [])
        # the server might have been able to send a byte or two back, but this
        # at least checks that it received something and didn't just fail
        # (which could still result in the client not having received anything)
        self.assertTrue(len(s.buffer) > 0)

class TestAsynchat_WithPoll(TestAsynchat):
    usepoll = True

class TestHelperFunctions(unittest.TestCase):
    def test_find_prefix_at_end(self):
        self.assertEqual(asynchat.find_prefix_at_end("qwerty\r", "\r\n"), 1)
        self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0)

class TestFifo(unittest.TestCase):
    def test_basic(self):
        f = asynchat.fifo()
        f.push(7)
        f.push('a')
        self.assertEqual(len(f), 2)
        self.assertEqual(f.first(), 7)
        self.assertEqual(f.pop(), (1, 7))
        self.assertEqual(len(f), 1)
        self.assertEqual(f.first(), 'a')
        self.assertEqual(f.is_empty(), False)
        self.assertEqual(f.pop(), (1, 'a'))
        self.assertEqual(len(f), 0)
        self.assertEqual(f.is_empty(), True)
        self.assertEqual(f.pop(), (0, None))

    def test_given_list(self):
        f = asynchat.fifo(['x', 17, 3])
        self.assertEqual(len(f), 3)
        self.assertEqual(f.pop(), (1, 'x'))
        self.assertEqual(f.pop(), (1, 17))
        self.assertEqual(f.pop(), (1, 3))
        self.assertEqual(f.pop(), (0, None))

def test_main(verbose=None):
    test_support.run_unittest(TestAsynchat, TestAsynchat_WithPoll,
                              TestHelperFunctions, TestFifo)

if __name__ == "__main__":
    test_main(verbose=True)
<commit_msg>Try to fix transient refleaks in test_asynchat.<commit_after># test asynchat

import asyncore, asynchat, socket, threading, time
import unittest
import sys
from test import test_support

# Skip tests if thread module does not exist.
test_support.import_module('thread')

HOST = test_support.HOST
SERVER_QUIT = 'QUIT\n'

class echo_server(threading.Thread):
    # parameter to determine the number of bytes passed back to the
    # client each send
    chunk_size = 1

    def __init__(self, event):
        threading.Thread.__init__(self)
        self.event = event
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.port = test_support.bind_port(self.sock)

    def run(self):
        self.sock.listen(1)
        self.event.set()
        conn, client = self.sock.accept()
        self.buffer = ""
        # collect data until quit message is seen
        while SERVER_QUIT not in self.buffer:
            data = conn.recv(1)
            if not data:
                break
            self.buffer = self.buffer + data

        # remove the SERVER_QUIT message
        self.buffer = self.buffer.replace(SERVER_QUIT, '')

        # re-send entire set of collected data
        try:
            # this may fail on some tests, such as test_close_when_done, since
            # the client closes the channel when it's done sending
            while self.buffer:
                n = conn.send(self.buffer[:self.chunk_size])
                time.sleep(0.001)
                self.buffer = self.buffer[n:]
        except:
            pass

        conn.close()
        self.sock.close()

class echo_client(asynchat.async_chat):

    def __init__(self, terminator, server_port):
        asynchat.async_chat.__init__(self)
        self.contents = []
        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect((HOST, server_port))
        self.set_terminator(terminator)
        self.buffer = ''

    def handle_connect(self):
        pass

    if sys.platform == 'darwin':
        # select.poll returns a select.POLLHUP at the end of the tests
        # on darwin, so just ignore it
        def handle_expt(self):
            pass

    def collect_incoming_data(self, data):
        self.buffer += data

    def found_terminator(self):
        self.contents.append(self.buffer)
        self.buffer = ""

def start_echo_server():
    event = threading.Event()
    s = echo_server(event)
    s.start()
    event.wait()
    event.clear()
    time.sleep(0.01) # Give server time to start accepting.
    return s, event

class TestAsynchat(unittest.TestCase):
    usepoll = False

    def setUp (self):
        self._threads = test_support.threading_setup()

    def tearDown (self):
        test_support.threading_cleanup(*self._threads)

    def line_terminator_check(self, term, server_chunk):
        event = threading.Event()
        s = echo_server(event)
        s.chunk_size = server_chunk
        s.start()
        event.wait()
        event.clear()
        time.sleep(0.01) # Give server time to start accepting.
        c = echo_client(term, s.port)
        c.push("hello ")
        c.push("world%s" % term)
        c.push("I'm not dead yet!%s" % term)
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "I'm not dead yet!"])

    # the line terminator tests below check receiving variously-sized
    # chunks back from the server in order to exercise all branches of
    # async_chat.handle_read

    def test_line_terminator1(self):
        # test one-character terminator
        for l in (1,2,3):
            self.line_terminator_check('\n', l)

    def test_line_terminator2(self):
        # test two-character terminator
        for l in (1,2,3):
            self.line_terminator_check('\r\n', l)

    def test_line_terminator3(self):
        # test three-character terminator
        for l in (1,2,3):
            self.line_terminator_check('qqq', l)

    def numeric_terminator_check(self, termlen):
        # Try reading a fixed number of bytes
        s, event = start_echo_server()
        c = echo_client(termlen, s.port)
        data = "hello world, I'm not dead yet!\n"
        c.push(data)
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, [data[:termlen]])

    def test_numeric_terminator1(self):
        # check that ints & longs both work (since type is
        # explicitly checked in async_chat.handle_read)
        self.numeric_terminator_check(1)
        self.numeric_terminator_check(1L)

    def test_numeric_terminator2(self):
        self.numeric_terminator_check(6L)

    def test_none_terminator(self):
        # Try reading a fixed number of bytes
        s, event = start_echo_server()
        c = echo_client(None, s.port)
        data = "hello world, I'm not dead yet!\n"
        c.push(data)
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, [])
        self.assertEqual(c.buffer, data)

    def test_simple_producer(self):
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        data = "hello world\nI'm not dead yet!\n"
        p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8)
        c.push_with_producer(p)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "I'm not dead yet!"])

    def test_string_producer(self):
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        data = "hello world\nI'm not dead yet!\n"
        c.push_with_producer(data+SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "I'm not dead yet!"])

    def test_empty_line(self):
        # checks that empty lines are handled correctly
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        c.push("hello world\n\nI'm not dead yet!\n")
        c.push(SERVER_QUIT)
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, ["hello world", "", "I'm not dead yet!"])

    def test_close_when_done(self):
        s, event = start_echo_server()
        c = echo_client('\n', s.port)
        c.push("hello world\nI'm not dead yet!\n")
        c.push(SERVER_QUIT)
        c.close_when_done()
        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
        s.join()

        self.assertEqual(c.contents, [])
        # the server might have been able to send a byte or two back, but this
        # at least checks that it received something and didn't just fail
        # (which could still result in the client not having received anything)
        self.assertTrue(len(s.buffer) > 0)

class TestAsynchat_WithPoll(TestAsynchat):
    usepoll = True

class TestHelperFunctions(unittest.TestCase):
    def test_find_prefix_at_end(self):
        self.assertEqual(asynchat.find_prefix_at_end("qwerty\r", "\r\n"), 1)
        self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0)

class TestFifo(unittest.TestCase):
    def test_basic(self):
        f = asynchat.fifo()
        f.push(7)
        f.push('a')
        self.assertEqual(len(f), 2)
        self.assertEqual(f.first(), 7)
        self.assertEqual(f.pop(), (1, 7))
        self.assertEqual(len(f), 1)
        self.assertEqual(f.first(), 'a')
        self.assertEqual(f.is_empty(), False)
        self.assertEqual(f.pop(), (1, 'a'))
        self.assertEqual(len(f), 0)
        self.assertEqual(f.is_empty(), True)
        self.assertEqual(f.pop(), (0, None))

    def test_given_list(self):
        f = asynchat.fifo(['x', 17, 3])
        self.assertEqual(len(f), 3)
        self.assertEqual(f.pop(), (1, 'x'))
        self.assertEqual(f.pop(), (1, 17))
        self.assertEqual(f.pop(), (1, 3))
        self.assertEqual(f.pop(), (0, None))

def test_main(verbose=None):
    test_support.run_unittest(TestAsynchat, TestAsynchat_WithPoll,
                              TestHelperFunctions, TestFifo)

if __name__ == "__main__":
    test_main(verbose=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rethinkdb as r
import optparse
import sys

if __name__ == "__main__":
    parser = optparse.OptionParser()
    parser.add_option("-P", "--port", dest="port",
                      help="rethinkdb port", default=30815)
    (options, args) = parser.parse_args()
    conn = r.connect('localhost', int(options.port), db='materialscommons')
    all_processes = r.table('processes').has_fields('inputs').run(conn)
    for process in all_processes:
        inputs = process['inputs']
        outputs = process['outputs']
        if inputs == []:
            print 'no inputs'
        else:
            for i in inputs:
                if i['attribute'] == 'file':
                    df_dnorm = {}
                    df_dnorm['df_id'] = i['properties']['id']['value']
                    df_dnorm['df_name'] = i['properties']['name']['value']
                    df_dnorm['process_id'] = process['id']
                    df_dnorm['process_name'] = process['name']
                    df_dnorm['project_id'] = process['project']
                    df_dnorm['file_type'] = 'input'
                    r.table('datafiles_denorm').insert(df_dnorm).run(conn)
                elif i['attribute'] == 'sample':
                    sample_dnorm = {}
                    sample_dnorm['sample_id'] = i['properties']['id']['value']
                    sample_dnorm['sample_name'] = i['properties']['name']['value']
                    sample_dnorm['process_id'] = process['id']
                    sample_dnorm['project_id'] = process['project']
                    sample_dnorm['process_name'] = process['name']
                    sample_dnorm['file_type'] = 'input'
                    r.table('samples_denorm').insert(sample_dnorm).run(conn)
        if outputs == []:
            print 'no outputs'
        else:
            for o in outputs:
                if o['attribute'] == 'file':
                    df_dnorm = {}
                    df_dnorm['df_id'] = o['properties']['id']['value']
                    df_dnorm['df_name'] = o['properties']['name']['value']
                    df_dnorm['process_id'] = process['id']
                    df_dnorm['process_name'] = process['name']
                    df_dnorm['project_id'] = process['project']
                    df_dnorm['file_type'] = 'output'
                    r.table('datafiles_denorm').insert(df_dnorm).run(conn)
                elif o['attribute'] == 'sample':
                    sample_dnorm = {}
                    sample_dnorm['sample_id'] = o['properties']['id']['value']
                    sample_dnorm['sample_name'] = o['properties']['name']['value']
                    sample_dnorm['process_id'] = process['id']
                    sample_dnorm['project_id'] = process['project']
                    sample_dnorm['process_name'] = process['name']
                    sample_dnorm['file_type'] = 'output'
                    r.table('samples_denorm').insert(sample_dnorm).run(conn)    <commit_msg>changes to the df denorm<commit_after>#!/usr/bin/env python

import rethinkdb as r
import optparse
import sys

if __name__ == "__main__":
    parser = optparse.OptionParser()
    parser.add_option("-P", "--port", dest="port",
                      help="rethinkdb port", default=30815)
    (options, args) = parser.parse_args()
    conn = r.connect('localhost', int(options.port), db='materialscommons')
    all_processes = r.table('processes').has_fields('inputs').run(conn)
    for process in all_processes:
        inputs = process['inputs']
        outputs = process['outputs']
        if inputs == []:
            print 'no inputs'
        else:
            for i in inputs:
                if i['attribute'] == 'file':
                    df_dnorm = {}
                    df_dnorm['df_id'] = i['properties']['id']['value']
                    df_dnorm['df_name'] = i['properties']['name']['value']
                    df_dnorm['process_id'] = process['id']
                    df_dnorm['process_name'] = process['name']
                    df_dnorm['project_id'] = process['project']
                    df_dnorm['file_type'] = 'input'
                    r.table('datafiles_denorm').insert(df_dnorm).run(conn)
        if outputs == []:
            print 'no outputs'
        else:
            for o in outputs:
                if o['attribute'] == 'file':
                    df_dnorm = {}
                    df_dnorm['df_id'] = o['properties']['id']['value']
                    df_dnorm['df_name'] = o['properties']['name']['value']
                    df_dnorm['process_id'] = process['id']
                    df_dnorm['process_name'] = process['name']
                    df_dnorm['project_id'] = process['project']
                    df_dnorm['file_type'] = 'output'
                    r.table('datafiles_denorm').insert(df_dnorm).run(conn)<|endoftext|>


<commit_before>import unittest
import BayesianModel as bm
import networkx as nx

class TestModel(unittest.TestCase):

    def setUp(self):
        self.G = bm.BayesianModel()

    def test_class_init(self):
        self.assertIsInstance(self.G, nx.DiGraph)

    def test_add_nodes_string(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.assertSameElements(self.G.nodes(), ['a', 'b', 'c', 'd'])

    def test_add_nodes_non_string(self):
        self.assertRaises(TypeError, self.G.add_nodes, [1, 2, 3, 4])

    def test_add_edges_both_tuples(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges(('a', 'b'), ('c', 'd'))
        self.assertSameElements(self.G.edges(), [('a', 'c'), ('a', 'd'),
            ('b', 'c'), ('b', 'd')])

    def test_add_edges_tail_string(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges('a', ('b', 'c'))
        self.assertSameElements(self.G.edges(), [('a', 'b'), ('a', 'c')])

    def test_add_edges_head_string(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges(('a', 'b'), 'c')
        self.assertSameElements(self.G.edges(), [('a', 'c'), ('b', 'c')])

    def test_add_edges_both_string(self):
        self.G.add_nodes('a', 'b')
        self.G.add_edges('a', 'b')
        self.assertSameElements(self.G.edges(), [('a', 'b')])

    def test_add_edges_multiple_times(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges('a', ('c', 'd'))
        self.G.add_edges('b', ('c', 'd'))
        self.assertSameElements(self.G.edges(), [('a', 'c'), ('a', 'd'),
            ('b', 'c'), ('b', 'd')])

    def tearDown(self):
        del self.G

class TestNodeProperties(unittest.TestCase):

    def setUp(self):
        self.G = bm.BayesianModel()
        self.G.add_nodes('a', 'b', 'c', 'd', 'e')
        self.G.add_edges(('a', 'b'), ('c', 'd'))

    def test_parents(self):
        self.assertSameElements(self.G.node['c']['_parents'], ['a', 'b'])
        self.assertSameElements(self.G.node['d']['_parents'], ['a', 'b'])
#        self.assertRaises(KeyError, self.G.node['a']['_parents'])
#        self.assertRaises(KeyError, self.G.node['b']['_parents'])

    def tearDown(self):
        del self.G

if __name__ == '__main__':
        unittest.main()
<commit_msg>Added tests<commit_after>import unittest
import BayesianModel as bm
import networkx as nx

class TestModel(unittest.TestCase):

    def setUp(self):
        self.G = bm.BayesianModel()

    def test_class_init(self):
        self.assertIsInstance(self.G, nx.DiGraph)

    def test_add_nodes_string(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.assertSameElements(self.G.nodes(), ['a', 'b', 'c', 'd'])

    def test_add_nodes_non_string(self):
        self.assertRaises(TypeError, self.G.add_nodes, [1, 2, 3, 4])

    def test_add_edges_both_tuples(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges(('a', 'b'), ('c', 'd'))
        self.assertSameElements(self.G.edges(), [('a', 'c'), ('a', 'd'),
            ('b', 'c'), ('b', 'd')])

    def test_add_edges_tail_string(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges('a', ('b', 'c'))
        self.assertSameElements(self.G.edges(), [('a', 'b'), ('a', 'c')])

    def test_add_edges_head_string(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges(('a', 'b'), 'c')
        self.assertSameElements(self.G.edges(), [('a', 'c'), ('b', 'c')])

    def test_add_edges_both_string(self):
        self.G.add_nodes('a', 'b')
        self.G.add_edges('a', 'b')
        self.assertSameElements(self.G.edges(), [('a', 'b')])

    def test_add_edges_multiple_times(self):
        self.G.add_nodes('a', 'b', 'c', 'd')
        self.G.add_edges('a', ('c', 'd'))
        self.G.add_edges('b', ('c', 'd'))
        self.assertSameElements(self.G.edges(), [('a', 'c'), ('a', 'd'),
            ('b', 'c'), ('b', 'd')])

    def tearDown(self):
        del self.G

class TestNodeProperties(unittest.TestCase):

    def setUp(self):
        self.G = bm.BayesianModel()
        self.G.add_nodes('a', 'b', 'c', 'd', 'e')
        self.G.add_edges(('a', 'b'), ('c', 'd'))

    def test_parents(self):
        self.assertSameElements(self.G.node['c']['_parents'], ['a', 'b'])
        self.assertSameElements(self.G.node['d']['_parents'], ['a', 'b'])
# TODO       self.assertRaises(KeyError, self.G.node['a']['_parents'])
# TODO       self.assertRaises(KeyError, self.G.node['b']['_parents'])

# TODO add test_default_rule

    def test_add_states(self):
        self.G.add_states('a', ('test_state_3', 'test_state_1',
            'test_state_2'))
        self.G.add_states('b', ('test_state_2', 'test_state_1'))
        self.G.add_states('c', ('test_state_1',))
        self.G.add_states('d', ('test_state_1', 'test_state_2'))
        self.assertEqual(self.G.node['a']['_states'], [['test_state_1', 0],
            ['test_state_2', 0], ['test_state_3', 0]])
        self.assertEqual(self.G.node['b']['_states'], [['test_state_1', 0],
            ['test_state_2', 0]])
        self.assertEqual(self.G.node['c']['_states'], [['test_state_1', 0]])
        self.assertEqual(self.G.node['d']['_states'], [['test_state_1', 0],
            ['test_state_2', 0]])

# TODO add test_default_rule_for_states

# TODO    def test_rule_for_states

    def test_states_function(self):
        self.G.add_states('a', ('test_state_3', 'test_state_1',
            'test_state_2'))
        self.G.add_states('b', ('test_state_2', 'test_state_1'))
        self.G.add_states('c', ('test_state_1',))
        self.G.add_states('d', ('test_state_1', 'test_state_2'))
        states = {'a':[], 'b':[], 'c':[], 'd':[]}
        nodes = ['a', 'b', 'c', 'd']
        for node in nodes:
            for state in self.G.states(node):
                states[node].append(state)
        self.assertEqual(states['a'], ['test_state_1', 'test_state_2',
            'test_state_3'])
        self.assertEqual(states['b'], ['test_state_1', 'test_state_2'])
        self.assertEqual(states['c'], ['test_state_1'])
        self.assertEqual(states['d'], ['test_state_1', 'test_state_2'])

    def tearDown(self):
        del self.G

if __name__ == '__main__':
        unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import json
from datetime import datetime

from optparse import make_option

from django.core.management.base import BaseCommand, CommandError

from portal.models import Xref
from portal.models import Database
from portal.models import Accession
from portal.models import Reference_map
from portal.models import Release

class MgiImporter(object):
    def __init__(self):
        self.database = Database.objects.get(descr='MGI')
        self.release = Release.objects.get(db_id=self.database.id)
        self.reference = self.database.references()[0]

    def delete_current_data(self):
        Xref.objects.filter(db_id=self.database.id).delete()
        Reference_map.objects.filter(accession__database=self.database.descr).delete()
        Accession.objects.filter(database=self.database.descr).delete()

    def xref(self, entry):
        return Xref(
            db_id=self.database.id,
            accession_id=entry['accession'],
            created_id=self.release.id,
            last_id=self.release.id,
            upi_id=entry['rnacentral_id'],
            version_i=1,
            deleted='N',
            timestamp=datetime.now(),
            userstamp='RNACEN',
            version=1,
            taxid=entry['ncbi_taxon_id'],
        )

    def accession(self, entry):
        return Accession(
            accession=entry['accession'],
            description=entry['name'],
            division=entry['division'],
            species=entry['species'],
            is_composite='N',
            database=entry['database'],
            classification=entry['lineage'],
            external_id=entry['accession'],
            feature_name=entry['feature_name'],
            ncrna_class=entry['ncrna_class'],
            gene=entry['gene'],
            note=json.dumps(entry['note']),
            xref=json.dumps(entry['xref_data']),
        )

    def references(self, entry):
        return Reference_map(
            accession_id=entry['acccession'],
            data_id=self.reference.id
        )

    def __call__(self, filename):
        with open(filename, 'rb') as raw:
            data = json.load(raw)

        self.delete_current_data()
        for entry in data:
            self.xref(entry).create_or_update()
            self.accession(entry).create_or_update()
            self.references(entry).create_or_update()

class Command(BaseCommand):
    """
    Handle command line options.
    """

    option_list = BaseCommand.option_list + (
        make_option(
            '-i',
            '--input',
            dest='json_file',
            default=False,
            help='[Required] Path to MGI JSON file with RNAcentral identifiers'),
    )
    # shown with -h, --help
    help = ('Import HGNC cross-references into RNAcentral database')

    def handle(self, *args, **options):
        """
        Django entry point
        """
        if not options['json_file']:
            raise CommandError('Please specify MGI input file')
        MgiImporter()(options['json_file'])
<commit_msg>Simplify MGI importing script<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import json
from datetime import datetime

from optparse import make_option

from django.core.management.base import BaseCommand, CommandError

from portal.models import Xref
from portal.models import Database
from portal.models import Release

class MgiImporter(object):
    def __init__(self):
        self.database = Database.objects.get(descr='MGI')
        self.release = Release.objects.get(db_id=self.database.id)

    def delete_current_data(self):
        Xref.objects.filter(db_id=self.database.id).delete()

    def save_xref(self, entry):
        return Xref.create_or_update(
            db_id=self.database.id,
            accession_id=entry['accession'],
            created_id=self.release.id,
            last_id=self.release.id,
            upi_id=entry['rnacentral_id'],
            version_i=1,
            deleted='N',
            timestamp=datetime.now(),
            userstamp='RNACEN',
            version=1,
            taxid=entry['ncbi_taxon_id'],
        )

    def __call__(self, filename):
        with open(filename, 'rb') as raw:
            data = json.load(raw)

        self.delete_current_data()
        for entry in data:
            self.save_xref(entry)

class Command(BaseCommand):
    """
    Handle command line options.
    """

    option_list = BaseCommand.option_list + (
        make_option(
            '-i',
            '--input',
            dest='json_file',
            default=False,
            help='[Required] Path to MGI JSON file with RNAcentral identifiers'),
    )
    # shown with -h, --help
    help = ('Import HGNC cross-references into RNAcentral database')

    def handle(self, *args, **options):
        """
        Django entry point
        """
        if not options['json_file']:
            raise CommandError('Please specify MGI input file')
        MgiImporter()(options['json_file'])
<|endoftext|>


<commit_before>#!/usr/bin/env python

from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)

import argparse
import csv
import sys
import itertools
from collections import defaultdict, Counter
from math import log2
from sklearn.feature_extraction import DictVectorizer
from sklearn.metrics.pairwise import cosine_similarity as sim
from operator import itemgetter
import concurrent.futures

WEIGHT = {
    'tf': lambda w, words: float(Counter(words)[w]),
    'idf': lambda w, words: idf.get(w, 1.),
    'tfidf': lambda w, words: float(Counter(words)[w]) * idf.get(w, 1.)
}

parser = argparse.ArgumentParser()
parser.add_argument('--synsets', required=True, type=argparse.FileType('r'))
parser.add_argument('--isas', required=True, type=argparse.FileType('r'))
parser.add_argument('--weight', choices=WEIGHT.keys(), default='tfidf')
parser.add_argument('-k', nargs='?', type=int, default=6)
args = parser.parse_args()

synsets, index, lexicon = {}, defaultdict(lambda: list()), set()

with args.synsets as f:
    reader = csv.reader(f, delimiter='\t', quoting=csv.QUOTE_NONE)

    for row in reader:
        synsets[int(row[0])] = [word for word in row[2].split(', ') if word]

        for word in synsets[int(row[0])]:
            index[word].append(int(row[0]))

        lexicon.update(synsets[int(row[0])])

index = {word: {id: i + 1 for i, id in enumerate(ids)} for word, ids in index.items()}

isas = defaultdict(lambda: set())

with args.isas as f:
    reader = csv.reader(f, delimiter='\t', quoting=csv.QUOTE_NONE)

    for row in reader:
        if len(row) > 1 and row[0] in lexicon and row[1] in lexicon:
            isas[row[0]].add(row[1])

idf, D = defaultdict(lambda: 0), .0

for words in synsets.values():
    hypernyms = [isas[word] for word in words if word in isas]

    if not hypernyms:
        continue

    for hypernym in set.union(*hypernyms):
        idf[hypernym] += 1

    D += 1

idf = {hypernym: log2(D / df) for hypernym, df in idf.items()}

weight = WEIGHT[args.weight]

hctx = {}

for id, words in synsets.items():
    hypernyms = list(itertools.chain(*(isas[word] for word in words if word in isas)))

    if not hypernyms:
        continue

    hctx[id] = {word: weight(word, hypernyms) for word in hypernyms}

v = DictVectorizer().fit(hctx.values())

def emit(id):
    if not id in hctx:
        return (id, {})

    hvector, candidates = v.transform(hctx[id]), Counter()

    for hypernym in hctx[id]:
        hsenses = Counter({hid: sim(v.transform({word: weight(word, synsets[hid]) for word in synsets[hid]}), hvector).item(0) for hid in index[hypernym]})

        for hid, cosine in hsenses.most_common(1):
            candidates[(hypernym, hid)] = cosine

    matches = [(hypernym, hid) for (hypernym, hid), _ in candidates.most_common(args.k) if hypernym not in synsets[id]]

    return (id, matches)

with concurrent.futures.ProcessPoolExecutor() as executor:
    futures = (executor.submit(emit, id) for id in synsets)

    for i, future in enumerate(concurrent.futures.as_completed(futures)):
        id, matches = future.result()

        senses = [(word, index[word][id]) for word in synsets[id]]
        senses_str = ', '.join(('%s#%d' % sense for sense in senses))

        isas = [(word, index[word][hid]) for word, hid in matches]
        isas_str = ', '.join(('%s#%d' % sense for sense in isas))

        print('\t'.join((str(id), str(len(synsets[id])), senses_str, str(len(isas)), isas_str)))

        if (i + 1) % 1000 == 0:
            print('%d entries out of %d done.' % (i + 1, len(synsets)), file=sys.stderr, flush=True)

if len(synsets) % 1000 != 0:
    print('%d entries out of %d done.' % (len(synsets), len(synsets)), file=sys.stderr, flush=True)
<commit_msg>Omit irrelevant hypernyms in link.py<commit_after>#!/usr/bin/env python

from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)

import argparse
import csv
import sys
import itertools
from collections import defaultdict, Counter
from math import log2
from sklearn.feature_extraction import DictVectorizer
from sklearn.metrics.pairwise import cosine_similarity as sim
from operator import itemgetter
import concurrent.futures

WEIGHT = {
    'tf': lambda w, words: float(Counter(words)[w]),
    'idf': lambda w, words: idf.get(w, 1.),
    'tfidf': lambda w, words: float(Counter(words)[w]) * idf.get(w, 1.)
}

parser = argparse.ArgumentParser()
parser.add_argument('--synsets', required=True, type=argparse.FileType('r'))
parser.add_argument('--isas', required=True, type=argparse.FileType('r'))
parser.add_argument('--weight', choices=WEIGHT.keys(), default='tfidf')
parser.add_argument('-k', nargs='?', type=int, default=6)
args = parser.parse_args()

synsets, index, lexicon = {}, defaultdict(lambda: list()), set()

with args.synsets as f:
    reader = csv.reader(f, delimiter='\t', quoting=csv.QUOTE_NONE)

    for row in reader:
        synsets[int(row[0])] = [word for word in row[2].split(', ') if word]

        for word in synsets[int(row[0])]:
            index[word].append(int(row[0]))

        lexicon.update(synsets[int(row[0])])

index = {word: {id: i + 1 for i, id in enumerate(ids)} for word, ids in index.items()}

isas = defaultdict(lambda: set())

with args.isas as f:
    reader = csv.reader(f, delimiter='\t', quoting=csv.QUOTE_NONE)

    for row in reader:
        if len(row) > 1 and row[0] in lexicon and row[1] in lexicon:
            isas[row[0]].add(row[1])

idf, D = defaultdict(lambda: 0), .0

for words in synsets.values():
    hypernyms = [isas[word] for word in words if word in isas]

    if not hypernyms:
        continue

    for hypernym in set.union(*hypernyms):
        idf[hypernym] += 1

    D += 1

idf = {hypernym: log2(D / df) for hypernym, df in idf.items()}

weight = WEIGHT[args.weight]

hctx = {}

for id, words in synsets.items():
    hypernyms = list(itertools.chain(*(isas[word] for word in words if word in isas)))

    if not hypernyms:
        continue

    hctx[id] = {word: weight(word, hypernyms) for word in hypernyms}

v = DictVectorizer().fit(hctx.values())

def emit(id):
    if not id in hctx:
        return (id, {})

    hvector, candidates = v.transform(hctx[id]), Counter()

    for hypernym in hctx[id]:
        hsenses = Counter({hid: sim(v.transform({word: weight(word, synsets[hid]) for word in synsets[hid]}), hvector).item(0) for hid in index[hypernym]})

        for hid, cosine in hsenses.most_common(1):
            if cosine > 0:
                candidates[(hypernym, hid)] = cosine

    matches = [(hypernym, hid) for (hypernym, hid), _ in candidates.most_common(args.k) if hypernym not in synsets[id]]

    return (id, matches)

with concurrent.futures.ProcessPoolExecutor() as executor:
    futures = (executor.submit(emit, id) for id in synsets)

    for i, future in enumerate(concurrent.futures.as_completed(futures)):
        id, matches = future.result()

        senses = [(word, index[word][id]) for word in synsets[id]]
        senses_str = ', '.join(('%s#%d' % sense for sense in senses))

        isas = [(word, index[word][hid]) for word, hid in matches]
        isas_str = ', '.join(('%s#%d' % sense for sense in isas))

        print('\t'.join((str(id), str(len(synsets[id])), senses_str, str(len(isas)), isas_str)))

        if (i + 1) % 1000 == 0:
            print('%d entries out of %d done.' % (i + 1, len(synsets)), file=sys.stderr, flush=True)

if len(synsets) % 1000 != 0:
    print('%d entries out of %d done.' % (len(synsets), len(synsets)), file=sys.stderr, flush=True)
<|endoftext|>