<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2008-2009 Benoit Chesneau <benoitc@e-engura.com>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
from calendar import timegm
import datetime
import decimal
import time

import couchdbkit
from couchdbkit.properties import Property
from couchdbkit.properties_map import value_to_json, value_to_python
from couchdbkit.schema import DocumentSchema, ALLOWED_PROPERTY_TYPES
from couchdbkit.exceptions import *

__all__ = ['SchemaProperty', 'ListProperty', 'DictProperty']

class SchemaProperty(Property):
    """ Schema property. It allow you add a DocumentSchema instance
    a member of a Document object. It return a
    :class:`simplecouchdb.schemaDocumentSchema` object.

    Exemple :

        >>> from simplecouchdb.schema import *
        >>> class Blog(DocumentSchema):
        ...     title = StringProperty()
        ...     author = StringProperty(default="me")
        ...
        >>> class Entry(Document):
        ...     title = StringProperty()
        ...     body = StringProperty()
        ...     blog = SchemaProperty(Blog())
        ...
        >>> test = Entry()
        >>> test._doc
        {'body': None, 'doc_type': 'Entry', 'title': None, 'blog': {'doc_type': 'Blog', 'author': u'me', 'title': None}}
        >>> test.blog.title = "Mon Blog"
        >>> test._doc
        {'body': None, 'doc_type': 'Entry', 'title': None, 'blog': {'doc_type': 'Blog', 'author': u'me', 'title': u'Mon Blog'}}
        >>> test.blog.title
        u'Mon Blog'
        >>> from simplecouchdb import Server
        >>> s = Server()
        >>> db = s.create_db('simplecouchdb_test')
        >>> Entry._db = db
        >>> test.save()
        >>> doc = Entry.objects.get(test.id)
        >>> doc.blog.title
        u'Mon Blog'
        >>> del s['simplecouchdb_test']

    """

    def __init__(self, schema, verbose_name=None, name=None,
            required=False, validators=None, default=None):

        Property.__init__(self, verbose_name=None,
            name=None, required=False, validators=None)

        use_instance = True
        if type(schema) == couchdbkit.schema.SchemaProperties:
            use_instance = False

        elif not isinstance(schema, DocumentSchema):
            raise TypeError('schema should be a DocumentSchema instance')

        elif schema.__class__.__name__ == 'DocumentSchema':
            use_instance = False
            properties = schema._dynamic_properties.copy()
            schema = DocumentSchema.build(**properties)

        self._use_instance = use_instance
        self._schema = schema

    def default_value(self):
        if not self._use_instance:
            return self._schema()
        return self._schema.clone()

    def empty(self, value):
        if not hasattr(value, '_doc'):
            return True
        if not value._doc or value._doc is None:
            return True
        return False

    def validate(self, value, required=True):
        value.validate(required=required)
        value = super(SchemaProperty, self).validate(value)

        if value is None:
            return Value

        if not isinstance(value, DocumentSchema):
            raise BadValueError(
                'Property %s must be DocumentSchema instance, not a %s' % (self.name,
                type(value).__name__))

        return value

    def to_python(self, value):
        if not self._use_instance:
            schema = self._schema()
        else:
            schema = self._schema.clone()
        return schema.wrap(value)

    def to_json(self, value):
        if not isinstance(value, DocumentSchema):
            if not self._use_instance:
                schema = self._schema()
            else:
                schema = self._schema.clone()

            if not isinstance(value, dict):
                raise BadValueError("%s is not a dict" % str(value))
            value = schema(**value)
        return value._doc

def validate_list_contents(value):
    for item in value:
        item = validate_content(item)
    return value

def validate_dict_contents(value):
    for k, v in value.iteritems():
        value[k] = validate_content(v)
    return value

def validate_content(value):
    print value
    if type(value) not in ALLOWED_PROPERTY_TYPES:
        raise BadValueError(
            'Items  must all be in %s' %
                (ALLOWED_PROPERTY_TYPES))
    if isinstance(value, list):
        value = validate_list_content(value)
    elif isinstance(value, dict):
        value = validate_dict_content(value)
    return value

class DictProperty(Property):
    """ A property that stores a dict of things"""

    def __init__(self, verbose_name=None, default=None, **kwds):
        """
        :args verbose_name: Optional verbose name.
        :args default: Optional default value; if omitted, an empty list is used.
        :args**kwds: Optional additional keyword arguments, passed to base class.

        Note that the only permissible value for 'required' is True.
        """
        if 'required' in kwds and kwds['required'] is not True:
             raise ValueError('dict values must be required')

        if default is None:
            default = {}

        Property.__init__(self, verbose_name, default=default,
            required=True, **kwds)

    data_type = dict

    def validate(self, value, required=True):
        value = super(DictProperty, self).validate(value)
        if value is not None:
            if not isinstance(value, dict):
                raise BadValueError('Property %s must be a dict' % self.name)
            value = self.validate_dict_contents(value)
        return value

    def validate_dict_contents(self, value):
        try:
            value = validate_dict_contents(value)
        except BadValueError:
            raise BadValueError(
                'Items of %s dict must all be in %s' %
                    (self.name, ALLOWED_PROPERTY_TYPES))
        return value

    def empty(self, value):
        """Is list property empty.

        {} is not an empty value.

        Returns:
          True if value is None, else false.
        """
        return value is None

    def default_value(self):
        """Default value for list.

        Because the property supplied to 'default' is a static value,
        that value must be shallow copied to prevent all fields with
        default values from sharing the same instance.

        Returns:
          Copy of the default value.
        """
        value = super(DictProperty, self).default_value()
        if value is None:
            value = {}
        return dict(value)

    def to_python(self, value):
        return self.DictProxy(value)

    def to_json(self, value):
        return value_to_json(value)

    class DictProxy(dict):

        def __init__(self, d):
            self._dict = d

        def __eq__(self, other):
            return self._dict is other

        def __getitem__(self, key):
            return value_to_python(self._dict[key])

        def __setitem__(self, key, value):
            self._dict[key] = value_to_json(value)

        def __delitem__(self, key):
            del self._dict[key]

        def get(self, key, default=None, type=None):
            if key in self._dict:
                if type is not None:
                    try:
                        return type(self._dict[key])
                    except ValueError:
                        pass
                return self._dict[key]
            return default

        def iteritems(self):
            for key, value in self._dict.iteritems():
                yield key, value_to_python(value)

        def itervalues(self):
            for value in self._dict.itervalues():
                yield value_to_python(value)

        def values(self):
            return list(self.itervalues())

        def items():
            return list(self.iteritems())

        def keys(self):
            return self._dict.keys()

        def iterkeys(self):
            return iter(self.keys())

        def copy(self):
            return self.__class__(self._dict)

        def pop(self, key, default=None):
            return self._dict.pop(key, default=default)

        def __len__(self):
            return len(self.keys())

        def __contains__(self, key):
            if key in self._dict:
                return True
            return False

        __iter__ = iteritems

        def setdefault(self, key, default):
            if key in self._dict:
                return value_to_python(self._dict[key])

            self._dict.setdefault(key, value_to_json(default))
            return default

        def update(self, value):
            for k, v in value.items():
                value[k] = value_to_json(v)
            self._dict.update(value)

        def popitem(self, value):
            value[0] = value_to_json(value[0])
            value = self._dic.popitem(value)
            value[0] = value_to_python(value[0])
            return value

        def clear(self):
            self._dict.clear()

class ListProperty(Property):
    """A property that stores a list of things.

      """
    def __init__(self, verbose_name=None, default=None, **kwds):
        """Construct ListProperty.

         :args verbose_name: Optional verbose name.
         :args default: Optional default value; if omitted, an empty list is used.
         :args**kwds: Optional additional keyword arguments, passed to base class.

        Note that the only permissible value for 'required' is True.

        """
        if 'required' in kwds and kwds['required'] is not True:
             raise ValueError('List values must be required')
        if default is None:
            default = []

        Property.__init__(self, verbose_name, default=default,
            required=True, **kwds)

    data_type = list

    def validate(self, value, required=True):
        value = super(ListProperty, self).validate(value)
        if value is not None:
            if not isinstance(value, list):
                raise BadValueError('Property %s must be a list' % self.name)
            value = self.validate_list_contents(value)
        return value

    def validate_list_contents(self, value):
        try:
            value = validate_list_contents(value)
        except BadValueError:
            raise BadValueError(
                'Items of %s list must all be in %s' %
                    (self.name, ALLOWED_PROPERTY_TYPES))
        return value

    def empty(self, value):
        """Is list property empty.

        [] is not an empty value.

        Returns:
          True if value is None, else false.
        """
        return value is None

    def default_value(self):
        """Default value for list.

        Because the property supplied to 'default' is a static value,
        that value must be shallow copied to prevent all fields with
        default values from sharing the same instance.

        Returns:
          Copy of the default value.
        """
        value = super(ListProperty, self).default_value()
        if value is None:
            value = []
        return list(value)

    def to_python(self, value):
        return self.ListProxy(value)

    def to_json(self, value):
        return [value_to_json(item) for item in value]

    class ListProxy(list):
        """ ProxyList. Idee taken from couchdb-python."""
        def __init__(self, l):
            self._list = l

        def __lt__(self, other):
            return self._list < other

        def __le__(self, other):
            return self._list <= other

        def __eq__(self, other):
            return self._list == other

        def __ne__(self, other):
            return self._list != other

        def __gt__(self, other):
            return self._list > other

        def __ge__(self, other):
            return self._list >= other

        def __repr__(self):
            return repr(list(self._list))

        def __str__(self):
            return str(self._list)

        def __unicode__(self):
            return unicode(self._list)

        def __getslice__(self, i, j):
            return self.__getitem__(slice(i, j))

        def __setslice__(self, i, j, seq):
            return self.__setitem__(slice(i, j), seq)

        def __delslice__(self, i, j):
            return self.__delitem__(slice(i, j))

        def __delitem__(self, index):
            del self._list[index]

        def __getitem__(self, index):
            return value_to_python(self._list[index])

        def __setitem__(self, index, value):
            self._list[index] = value_to_json(value)

        def __iter__(self):
            for index in range(len(self)):
                yield self[index]

        def __len__(self):
            return len(self._list)

        def __nonzero__(self):
            return bool(self._list)

        def append(self, *args, **kwargs):
            if args:
                assert len(args) == 1
                value = args[0]
            else:
                value = kwargs
            value = value_to_json(value)
            self._list.append(value)

        def extend(self, list):
            for item in list:
                self.append(item)<commit_msg>useless debug info<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2008-2009 Benoit Chesneau <benoitc@e-engura.com>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
from calendar import timegm
import datetime
import decimal
import time

import couchdbkit
from couchdbkit.properties import Property
from couchdbkit.properties_map import value_to_json, value_to_python
from couchdbkit.schema import DocumentSchema, ALLOWED_PROPERTY_TYPES
from couchdbkit.exceptions import *

__all__ = ['SchemaProperty', 'ListProperty', 'DictProperty']

class SchemaProperty(Property):
    """ Schema property. It allow you add a DocumentSchema instance
    a member of a Document object. It return a
    :class:`simplecouchdb.schemaDocumentSchema` object.

    Exemple :

        >>> from simplecouchdb.schema import *
        >>> class Blog(DocumentSchema):
        ...     title = StringProperty()
        ...     author = StringProperty(default="me")
        ...
        >>> class Entry(Document):
        ...     title = StringProperty()
        ...     body = StringProperty()
        ...     blog = SchemaProperty(Blog())
        ...
        >>> test = Entry()
        >>> test._doc
        {'body': None, 'doc_type': 'Entry', 'title': None, 'blog': {'doc_type': 'Blog', 'author': u'me', 'title': None}}
        >>> test.blog.title = "Mon Blog"
        >>> test._doc
        {'body': None, 'doc_type': 'Entry', 'title': None, 'blog': {'doc_type': 'Blog', 'author': u'me', 'title': u'Mon Blog'}}
        >>> test.blog.title
        u'Mon Blog'
        >>> from simplecouchdb import Server
        >>> s = Server()
        >>> db = s.create_db('simplecouchdb_test')
        >>> Entry._db = db
        >>> test.save()
        >>> doc = Entry.objects.get(test.id)
        >>> doc.blog.title
        u'Mon Blog'
        >>> del s['simplecouchdb_test']

    """

    def __init__(self, schema, verbose_name=None, name=None,
            required=False, validators=None, default=None):

        Property.__init__(self, verbose_name=None,
            name=None, required=False, validators=None)

        use_instance = True
        if type(schema) == couchdbkit.schema.SchemaProperties:
            use_instance = False

        elif not isinstance(schema, DocumentSchema):
            raise TypeError('schema should be a DocumentSchema instance')

        elif schema.__class__.__name__ == 'DocumentSchema':
            use_instance = False
            properties = schema._dynamic_properties.copy()
            schema = DocumentSchema.build(**properties)

        self._use_instance = use_instance
        self._schema = schema

    def default_value(self):
        if not self._use_instance:
            return self._schema()
        return self._schema.clone()

    def empty(self, value):
        if not hasattr(value, '_doc'):
            return True
        if not value._doc or value._doc is None:
            return True
        return False

    def validate(self, value, required=True):
        value.validate(required=required)
        value = super(SchemaProperty, self).validate(value)

        if value is None:
            return Value

        if not isinstance(value, DocumentSchema):
            raise BadValueError(
                'Property %s must be DocumentSchema instance, not a %s' % (self.name,
                type(value).__name__))

        return value

    def to_python(self, value):
        if not self._use_instance:
            schema = self._schema()
        else:
            schema = self._schema.clone()
        return schema.wrap(value)

    def to_json(self, value):
        if not isinstance(value, DocumentSchema):
            if not self._use_instance:
                schema = self._schema()
            else:
                schema = self._schema.clone()

            if not isinstance(value, dict):
                raise BadValueError("%s is not a dict" % str(value))
            value = schema(**value)
        return value._doc

def validate_list_contents(value):
    for item in value:
        item = validate_content(item)
    return value

def validate_dict_contents(value):
    for k, v in value.iteritems():
        value[k] = validate_content(v)
    return value

def validate_content(value):
    if type(value) not in ALLOWED_PROPERTY_TYPES:
        raise BadValueError(
            'Items  must all be in %s' %
                (ALLOWED_PROPERTY_TYPES))
    if isinstance(value, list):
        value = validate_list_content(value)
    elif isinstance(value, dict):
        value = validate_dict_content(value)
    return value

class DictProperty(Property):
    """ A property that stores a dict of things"""

    def __init__(self, verbose_name=None, default=None, **kwds):
        """
        :args verbose_name: Optional verbose name.
        :args default: Optional default value; if omitted, an empty list is used.
        :args**kwds: Optional additional keyword arguments, passed to base class.

        Note that the only permissible value for 'required' is True.
        """
        if 'required' in kwds and kwds['required'] is not True:
             raise ValueError('dict values must be required')

        if default is None:
            default = {}

        Property.__init__(self, verbose_name, default=default,
            required=True, **kwds)

    data_type = dict

    def validate(self, value, required=True):
        value = super(DictProperty, self).validate(value)
        if value is not None:
            if not isinstance(value, dict):
                raise BadValueError('Property %s must be a dict' % self.name)
            value = self.validate_dict_contents(value)
        return value

    def validate_dict_contents(self, value):
        try:
            value = validate_dict_contents(value)
        except BadValueError:
            raise BadValueError(
                'Items of %s dict must all be in %s' %
                    (self.name, ALLOWED_PROPERTY_TYPES))
        return value

    def empty(self, value):
        """Is list property empty.

        {} is not an empty value.

        Returns:
          True if value is None, else false.
        """
        return value is None

    def default_value(self):
        """Default value for list.

        Because the property supplied to 'default' is a static value,
        that value must be shallow copied to prevent all fields with
        default values from sharing the same instance.

        Returns:
          Copy of the default value.
        """
        value = super(DictProperty, self).default_value()
        if value is None:
            value = {}
        return dict(value)

    def to_python(self, value):
        return self.DictProxy(value)

    def to_json(self, value):
        return value_to_json(value)

    class DictProxy(dict):

        def __init__(self, d):
            self._dict = d

        def __eq__(self, other):
            return self._dict is other

        def __getitem__(self, key):
            return value_to_python(self._dict[key])

        def __setitem__(self, key, value):
            self._dict[key] = value_to_json(value)

        def __delitem__(self, key):
            del self._dict[key]

        def get(self, key, default=None, type=None):
            if key in self._dict:
                if type is not None:
                    try:
                        return type(self._dict[key])
                    except ValueError:
                        pass
                return self._dict[key]
            return default

        def iteritems(self):
            for key, value in self._dict.iteritems():
                yield key, value_to_python(value)

        def itervalues(self):
            for value in self._dict.itervalues():
                yield value_to_python(value)

        def values(self):
            return list(self.itervalues())

        def items():
            return list(self.iteritems())

        def keys(self):
            return self._dict.keys()

        def iterkeys(self):
            return iter(self.keys())

        def copy(self):
            return self.__class__(self._dict)

        def pop(self, key, default=None):
            return self._dict.pop(key, default=default)

        def __len__(self):
            return len(self.keys())

        def __contains__(self, key):
            if key in self._dict:
                return True
            return False

        __iter__ = iteritems

        def setdefault(self, key, default):
            if key in self._dict:
                return value_to_python(self._dict[key])

            self._dict.setdefault(key, value_to_json(default))
            return default

        def update(self, value):
            for k, v in value.items():
                value[k] = value_to_json(v)
            self._dict.update(value)

        def popitem(self, value):
            value[0] = value_to_json(value[0])
            value = self._dic.popitem(value)
            value[0] = value_to_python(value[0])
            return value

        def clear(self):
            self._dict.clear()

class ListProperty(Property):
    """A property that stores a list of things.

      """
    def __init__(self, verbose_name=None, default=None, **kwds):
        """Construct ListProperty.

         :args verbose_name: Optional verbose name.
         :args default: Optional default value; if omitted, an empty list is used.
         :args**kwds: Optional additional keyword arguments, passed to base class.

        Note that the only permissible value for 'required' is True.

        """
        if 'required' in kwds and kwds['required'] is not True:
             raise ValueError('List values must be required')
        if default is None:
            default = []

        Property.__init__(self, verbose_name, default=default,
            required=True, **kwds)

    data_type = list

    def validate(self, value, required=True):
        value = super(ListProperty, self).validate(value)
        if value is not None:
            if not isinstance(value, list):
                raise BadValueError('Property %s must be a list' % self.name)
            value = self.validate_list_contents(value)
        return value

    def validate_list_contents(self, value):
        try:
            value = validate_list_contents(value)
        except BadValueError:
            raise BadValueError(
                'Items of %s list must all be in %s' %
                    (self.name, ALLOWED_PROPERTY_TYPES))
        return value

    def empty(self, value):
        """Is list property empty.

        [] is not an empty value.

        Returns:
          True if value is None, else false.
        """
        return value is None

    def default_value(self):
        """Default value for list.

        Because the property supplied to 'default' is a static value,
        that value must be shallow copied to prevent all fields with
        default values from sharing the same instance.

        Returns:
          Copy of the default value.
        """
        value = super(ListProperty, self).default_value()
        if value is None:
            value = []
        return list(value)

    def to_python(self, value):
        return self.ListProxy(value)

    def to_json(self, value):
        return [value_to_json(item) for item in value]

    class ListProxy(list):
        """ ProxyList. Idee taken from couchdb-python."""
        def __init__(self, l):
            self._list = l

        def __lt__(self, other):
            return self._list < other

        def __le__(self, other):
            return self._list <= other

        def __eq__(self, other):
            return self._list == other

        def __ne__(self, other):
            return self._list != other

        def __gt__(self, other):
            return self._list > other

        def __ge__(self, other):
            return self._list >= other

        def __repr__(self):
            return repr(list(self._list))

        def __str__(self):
            return str(self._list)

        def __unicode__(self):
            return unicode(self._list)

        def __getslice__(self, i, j):
            return self.__getitem__(slice(i, j))

        def __setslice__(self, i, j, seq):
            return self.__setitem__(slice(i, j), seq)

        def __delslice__(self, i, j):
            return self.__delitem__(slice(i, j))

        def __delitem__(self, index):
            del self._list[index]

        def __getitem__(self, index):
            return value_to_python(self._list[index])

        def __setitem__(self, index, value):
            self._list[index] = value_to_json(value)

        def __iter__(self):
            for index in range(len(self)):
                yield self[index]

        def __len__(self):
            return len(self._list)

        def __nonzero__(self):
            return bool(self._list)

        def append(self, *args, **kwargs):
            if args:
                assert len(args) == 1
                value = args[0]
            else:
                value = kwargs
            value = value_to_json(value)
            self._list.append(value)

        def extend(self, list):
            for item in list:
                self.append(item)<|endoftext|>


<commit_before>
# coding: utf-8

# This notebook provides examples of analysis of resting fMRI.
#
# To grab the data, you should first do the following:
#
# bash ~/fmri-analysis-vm/get_resting.sh
#
# This will grab 4 sessions of resting data from ses-105, along with the field maps for this session.  These data have been motion-corrected to a common target so that they are aligned to one another. The motion parameter files are derived separately from each session, rather than using the common target.
#
# Credit:
# - nilearn examples: https://nilearn.github.io/auto_examples/04_manipulating_images/plot_nifti_simple.html#sphx-glr-auto-examples-04-manipulating-images-plot-nifti-simple-py
#

# In[6]:

import os,glob
import nibabel
import numpy
import sklearn
import nilearn.input_data
from nilearn.plotting import plot_roi
from nilearn.image.image import mean_img
from nilearn.plotting import plot_stat_map, show
from nilearn.image import index_img,clean_img
from nilearn.decomposition import CanICA

import matplotlib.pyplot as plt
from nipype.interfaces import fsl, nipy
from nipype.caching import Memory
mem = Memory(base_dir='.')

import sys
sys.path.append('/home/vagrant/fmri-analysis-vm/analysis/utils')
from compute_fd_dvars import compute_fd,compute_dvars

get_ipython().magic('matplotlib inline')

# Load the data, clean it, and compute a mask using the nilearn NiftiMasker function

# In[7]:

rsfmri_basedir='/home/vagrant/data/ds031/sub-01/ses-105/mcflirt'
rsfmri_files=glob.glob(os.path.join(rsfmri_basedir,'sub*.nii.gz'))

rsfmri_files.sort()

# load the first image and create the masker object
rsfmri_img=nibabel.load(rsfmri_files[0])
masker= nilearn.input_data.NiftiMasker(mask_strategy='epi')
masker.fit(rsfmri_img)
mask_img = masker.mask_img_

rsfmri={}  # nifti handle to cleaned image
fmri_masked=None
# load and clean each image
for f in rsfmri_files:
    rsfmri_img=nibabel.load(f)
    runnum=int(f.split('_')[3].split('-')[1])
    rsfmri[runnum]=nilearn.image.clean_img(rsfmri_img)
    print('loaded run',runnum)
    motparfile=f.replace('nii.gz','par')
    mp=numpy.loadtxt(motparfile)
    if fmri_masked is None:
        fmri_masked=masker.transform(rsfmri[runnum])
        motpars=mp
    else:
        fmri_masked=numpy.vstack((fmri_masked,masker.transform(rsfmri[runnum])))
        motpars=numpy.vstack((motpars,mp))

# Visualize the mask

# In[8]:

# calculate mean image for the background
mean_func_img = '/home/vagrant/data/ds031/sub-01/ses-105/mcflirt/mcflirt_target.nii.gz'

plot_roi(mask_img, mean_func_img, display_mode='y',
         cut_coords=4, title="Mask")

# Compute framewise displacement and plot it.

# In[9]:

fd=compute_fd(motpars)
numpy.where(fd>1)
plt.figure(figsize=(12,4))
# remove first timepoint from each session
fd[240]=0
fd[480]=0
fd[720]=0
plt.plot(fd)
for c in [240,480,720]:
    plt.plot([c,c],
             [0,numpy.max(fd)*1.1],'k--')

# Run ICA on the masked data, using the CanICA tool from nilearn.  Just arbitrarily set 10 components for now.

# In[10]:

n_components=10

canica = CanICA(n_components=n_components, smoothing_fwhm=6.,
                memory="nilearn_cache", memory_level=2,
                threshold=3., verbose=10, random_state=0)
canica.fit(rsfmri_files)

# Retrieve the independent components in brain space
components_img = canica.masker_.inverse_transform(canica.components_)

# In[11]:

# Visualize results
for i in range(n_components):
    img=index_img(components_img, i)
    img_masked=masker.transform(img)
    ts=fmri_masked.dot(img_masked.T)
    plot_stat_map(img, mean_func_img,
              display_mode='z', cut_coords=4, threshold=0.01,
              title="Component %d"%i)
    plt.figure(figsize=(8,3))
    plt.plot(ts)
    for c in [240,480,720]:
        plt.plot([c,c],
                 [numpy.min(ts)*1.1,
                  numpy.max(ts)*1.1],
                'k--')

# ### Extracting a parcellated connectome
#
# Next we will extract data from a set of atlas regions in order to calculate a connectome across regions.  First we need to align our data to the 2mm MNI space since the atlases from nilearn are in that space.  We already have our T1 image aligned to MNI, so we just need to align our EPI image to the T1 image.

# In[14]:

# clean up memory
try:
    del canica
except:
    pass
try:
    del img_masked
except:
    pass
try:
    del components_img
except:
    pass
try:
    del rsfmri
except:
    pass
try:
    del fmri_masked
except:
    pass

datadir='/home/vagrant/data'
anat_subject='ses018'
anatomydir=os.path.join(datadir,'ds031/sub00001',anat_subject,
        'anatomy')
t1file=os.path.join(anatomydir,'sub00001_ses018_T1w_001.nii.gz')

# strip mean EPI
boldbet = mem.cache(fsl.BET)
bet_results = boldbet(functional=True,
              in_file=mean_func_img,
              mask=True)

# strip anatomy
t1bet = mem.cache(fsl.BET)
t1bet_results = t1bet(in_file=t1file,
              mask=True)

# EPIreg
epi_reg = mem.cache(fsl.EpiReg)
epi_reg_results = epi_reg(epi=bet_results.outputs.out_file,
                          t1_head=t1file,
                          t1_brain=t1bet_results.outputs.out_file,
                          out_base="epi2struct")

# In[20]:

for f in rsfmri_files:
    epi_to_t1 = mem.cache(fsl.FLIRT)
    epi_to_t1_results = epi_to_t1(in_file=f,
                                  reference=t1file,
                                  apply_xfm=True,
                                  in_matrix_file=epi_reg_results.outputs.epi2str_mat)

    t1_to_mni = mem.cache(ants.ApplyTransforms)
    t1_to_mni_results = t1_to_mni(input_image=epi_to_t1_results.outputs.out_file,
                                  reference_image=os.path.join(os.getenv('FSLDIR'),'data/standard/MNI152_T1_2mm_brain.nii.gz'),
                                  transforms='../../results/t1_to_mni_warp.h5',
                                  interpolation="BSpline")
    t1_to_mni_results.outputs

# In[17]:

epi_reg_results.outputs

# In[ ]:

<commit_msg>cleanup<commit_after><|endoftext|>


<commit_before>import cPickle as pickle
import numpy
import glob, os, stat, time, datetime
import re

class TSPLException(Exception):
  def __init__(self,arg):
    self.value=arg
    print self.value

class TSPLBase:
  def __init__(self,file,k1,k2):
    self.f=open(file)
    self.j=pickle.load(self.f)
    self.f.close()

    try:
      self.wayness=int(re.findall('\d+',self.j.acct['granted_pe'])[0])
    except KeyError:
      try:
        self.wayness=self.j.acct['cores']/self.j.acct['nodes']
      except ZeroDivisionError:
        print "Read zero nodes, assuming 16 way job"
        self.wayness=16

    try:
      self.owner=self.j.acct['owner']
    except KeyError:
      self.owner=self.j.acct['uid']

    self.numhosts=len(self.j.hosts.keys())

    if self.numhosts == 0:
      raise TSPLException('No hosts')
    elif 'amd64_core' in self.j.hosts.values()[0].stats:
      self.pmc_type='amd64'
    elif 'intel_pmc3' in self.j.hosts.values()[0].stats:
      self.pmc_type='intel'
    else:
      raise TSPLException('No PMC data for: ' + self.j.id)

    if self.pmc_type in k1:
      self.k1=k1[self.pmc_type]
      self.k2=k2[self.pmc_type]
    else:
      self.k1=k1
      self.k2=k2

    self.t=(self.j.times-self.j.times[0])

    if len(k1) != len(k2):
      raise TSPLException('Lengths don\'t match')

    self.index=[ self.j.get_schema(self.k1[i])[self.k2[i]].index
                 for i in range(len(self.k1))]

    g=self.j.hosts[self.j.hosts.keys()[0]]
    self.size=len(g.stats[self.k1[0]].values()[0])

    d=datetime.datetime.fromtimestamp(self.j.acct['end_time'])
    self.end_date=d.strftime('%Y-%m-%d %H:%M:%S')

    self.title='ID: %(ID)s, u: %(u)s, N: %(name)s, D: %(date)s, NH: %(nh)d' % \
           { 'ID' : self.j.id,'u': self.owner,
             'name': self.j.acct['name'], 'nh' : self.numhosts,
             'date': self.end_date }

    # Create an array of dictionaries of lists initialized and constructed using
    # derived class methods for the keys of interest.
    # self.index embedds the location of self.k2 in the sechma
    self.data=[]
    for i in range(len(self.k1)):
      self.data.append({})
      for k in self.j.hosts.keys():
        h=self.j.hosts[k]
        self.data[i][k]=self.data_init()
        for s in h.stats[self.k1[i]].values():
          self.data_assign(self.data[i][k],s[:,self.index[i]])

  # Initialize to an empty array and accumulate with appending
  def data_init(self):
    return []
  def data_assign(self,d,v):
    d.append(v)

  # Generate a label for title strings
  def label(self,k1,k2,mod=1.):
    u=''
    if mod==1e9 or mod == 1024.**3:
      u='G'
    elif mod==1e6 or mod == 1024.**2:
      u='M'

    l=k1 + ' ' + k2
    s=self.j.get_schema(k1)[k2]
    if not s.unit is None:
      l+=' ' + u + s.unit

    if len(l) > 10:
      l=k1 + '\n' + k2
      s=self.j.get_schema(k1)[k2]
      if not s.unit is None:
        l+=' ' + u + s.unit

    return l

  # These iterator fuctions iterate linearly over the array of dictionaries. We
  # should probably create a sorted version, but this works for now.
  def __iter__(self):
    self.ind=-1
    self.a=len(self.data)
    self.b=len(self.data[0].keys())
    self.c=len(self.data[0][self.data[0].keys()[0]])

    return(self)

  def next(self):
    if self.ind == self.a*self.b*self.c-1:
      raise StopIteration
    self.ind += 1
    inds=numpy.unravel_index(self.ind,(self.a,self.b,self.c))
    k=self.data[inds[0]].keys()[inds[1]]
    return self.data[inds[0]][k][inds[2]]

# Load a job file and sum a socket-based or core-based counter into
# time-dependent arrays for each key pair. Takes a tacc stats pickle file and
# two lists of keys.

class TSPLSum(TSPLBase):
  def __init__(self,file,k1,k2):
    TSPLBase.__init__(self,file,k1,k2)

  # Initialize with an zero array and accumuluate to the first list element with
  # a sum
  def data_init(self):
    return [numpy.zeros(self.size)]
  def data_assign(self,d,v):
    d[0]+=v

<commit_msg>clean up for multiple PMC counter types. This needs to be generalized in a less dumb way<commit_after>import cPickle as pickle
import numpy
import glob, os, stat, time, datetime
import re

class TSPLException(Exception):
  def __init__(self,arg):
    self.value=arg
    print self.value

class TSPLBase:
  def __init__(self,file,k1,k2):
    self.f=open(file)
    self.j=pickle.load(self.f)
    self.f.close()

    try:
      self.wayness=int(re.findall('\d+',self.j.acct['granted_pe'])[0])
    except KeyError:
      try:
        self.wayness=self.j.acct['cores']/self.j.acct['nodes']
      except ZeroDivisionError:
        print "Read zero nodes, assuming 16 way job"
        self.wayness=16
      except TypeError:
        raise TSPLException('Something is funny with job ' +str(self.j.id) +
                            ' ' + self.j.acct['cores'] + ' ' +
                            self.j.acct['nodes'])

    try:
      self.owner=self.j.acct['owner']
    except KeyError:
      self.owner=self.j.acct['uid']

    self.numhosts=len(self.j.hosts.keys())

    if self.numhosts == 0:
      raise TSPLException('No hosts')
    elif 'amd64_core' in self.j.hosts.values()[0].stats:
      self.pmc_type='amd64'
    elif 'intel_pmc3' in self.j.hosts.values()[0].stats:
      self.pmc_type='intel'
    elif 'intel_snb' in self.j.hosts.values()[0].stats:
      self.pmc_type='intel_snb'

    if isinstance(k1,dict) and self.pmc_type in k1:
      self.k1=k1[self.pmc_type]
      self.k2=k2[self.pmc_type]
    else:
      self.k1=k1
      self.k2=k2

    if self.j.get_schema(self.k1[0]) is None:
      raise TSPLException(self.k1[0]+' not supported for job '+str(self.j.id))

    self.t=(self.j.times-self.j.times[0])

    if len(k1) != len(k2):
      raise TSPLException('Lengths don\'t match')

    self.index=[ self.j.get_schema(self.k1[i])[self.k2[i]].index
                 for i in range(len(self.k1))]

    g=self.j.hosts[self.j.hosts.keys()[0]]
    self.size=len(g.stats[self.k1[0]].values()[0])

    d=datetime.datetime.fromtimestamp(self.j.acct['end_time'])
    self.end_date=d.strftime('%Y-%m-%d %H:%M:%S')

    self.title='ID: %(ID)s, u: %(u)s, N: %(name)s, D: %(date)s, NH: %(nh)d' % \
           { 'ID' : self.j.id,'u': self.owner,
             'name': self.j.acct['name'], 'nh' : self.numhosts,
             'date': self.end_date }

    # Create an array of dictionaries of lists initialized and constructed using
    # derived class methods for the keys of interest.
    # self.index embedds the location of self.k2 in the sechma
    self.data=[]
    for i in range(len(self.k1)):
      self.data.append({})
      for k in self.j.hosts.keys():
        h=self.j.hosts[k]
        self.data[i][k]=self.data_init()
        for s in h.stats[self.k1[i]].values():
          self.data_assign(self.data[i][k],s[:,self.index[i]])

  # Initialize to an empty array and accumulate with appending
  def data_init(self):
    return []
  def data_assign(self,d,v):
    d.append(v)

  # Generate a label for title strings
  def label(self,k1,k2,mod=1.):
    u=''
    if mod==1e9 or mod == 1024.**3:
      u='G'
    elif mod==1e6 or mod == 1024.**2:
      u='M'

    l=k1 + ' ' + k2
    s=self.j.get_schema(k1)[k2]
    if not s.unit is None:
      l+=' ' + u + s.unit

    if len(l) > 10:
      l=k1 + '\n' + k2
      s=self.j.get_schema(k1)[k2]
      if not s.unit is None:
        l+=' ' + u + s.unit

    return l

  # These iterator fuctions iterate linearly over the array of dictionaries. We
  # should probably create a sorted version, but this works for now.
  def __iter__(self):
    self.ind=-1
    self.a=len(self.data)
    self.b=len(self.data[0].keys())
    self.c=len(self.data[0][self.data[0].keys()[0]])

    return(self)

  def next(self):
    if self.ind == self.a*self.b*self.c-1:
      raise StopIteration
    self.ind += 1
    inds=numpy.unravel_index(self.ind,(self.a,self.b,self.c))
    k=self.data[inds[0]].keys()[inds[1]]
    return self.data[inds[0]][k][inds[2]]

# Load a job file and sum a socket-based or core-based counter into
# time-dependent arrays for each key pair. Takes a tacc stats pickle file and
# two lists of keys.

class TSPLSum(TSPLBase):
  def __init__(self,file,k1,k2):
    TSPLBase.__init__(self,file,k1,k2)

  # Initialize with an zero array and accumuluate to the first list element with
  # a sum
  def data_init(self):
    return [numpy.zeros(self.size)]
  def data_assign(self,d,v):
    d[0]+=v

<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django import VERSION as DJANGO_VERSION
from django.utils import six
from django.utils.translation import ugettext_lazy as _
from django.db.models import Lookup, Transform
from django.db.models.fields import *

from rdkit.Chem import AllChem as Chem
from rdkit import DataStructs
from rdkit.DataStructs import ExplicitBitVect

__all__ = ["MolField", "RxnField", "BfpField", "SfpField",]

##########################################
# Molecule Field
# Updated 12/22/2018 to allow for non-SMILES strings to be parsed & fixed compatibility of mol objects with operators

class MolField(Field):

    description = _("Molecule")

    def db_type(self, connection):
        return 'mol'

    def get_placeholder(self, value, compiler, connection):
        if hasattr(value, 'as_sql'):
            if DJANGO_VERSION > (2,):
                return '%s'
            else:
                # No value used for expressions, substitute in
                # the column name instead.
                sql, _ = compiler.compile(value)
                return sql
        else:
            return 'mol_from_pkl(%s)'

    def select_format(self, compiler, sql, params):
        return 'mol_to_pkl(%s)' % sql, params

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return Chem.Mol(bytes(value))

    def to_python(self, value):
        if value is None or isinstance(value, Chem.Mol):
            return value
        elif isinstance(value, six.string_types):
            # The string case. Assume a SMILES is passed.
            mol = Chem.MolFromSmiles(str(value))
            if mol is None:
                mol = Chem.MolFromMolBlock(str(value))
            if mol is None:
                mol = Chem.inchi.MolFromInchi(str(value))
            if mol is None:
                raise ValidationError("Invalid input for a Mol instance")
            return mol
        elif isinstance(value, six.buffer_types):
            return Chem.Mol(bytes(value))
        else:
            raise ValidationError("Invalid input for a Mol instance")

    def get_prep_value(self, value):
        # convert the Molecule instance to the value used by the
        # db driver
        if isinstance(value, six.string_types):
            # The string case. Don't assume SMILES
            value = self.to_python(str(value))
        if isinstance(value, Chem.Mol):
            value = six.memoryview(value.ToBinary())
        return value

    def get_prep_lookup(self, lookup_type, value):
        "Perform preliminary non-db specific lookup checks and conversions"
        supported_lookup_types = (
            ['hassubstruct', 'issubstruct', 'exact', 'isnull',] +
            [T.lookup_name for T in MOL_DESCRIPTOR_TRANFORMS]
        )
        if lookup_type in supported_lookup_types:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

##########################################
# Reaction Field

class RxnField(Field):

    description = _("Reaction")

    def db_type(self, connection):
        return 'reaction'

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return Chem.ReactionFromSmarts(value, useSmiles=True)

    def to_python(self, value):
        if value is None or isinstance(value, Chem.ChemicalReaction):
            return value
        elif isinstance(value, six.string_types):
            # The string case. A reaction SMILES is expected.
            return Chem.ReactionFromSmarts(str(value), useSmiles=True)
        else:
            raise ValidationError("Invalid input for a ChemicalReaction instance")

    def get_prep_value(self, value):
        if isinstance(value, Chem.ChemicalReaction):
            value = Chem.ReactionToSmiles(value)
        return value

    def get_prep_lookup(self, lookup_type, value):
        "Perform preliminary non-db specific lookup checks and conversions"
        supported_lookup_types = (
            ['hassubstruct', 'issubstruct', 'isnull',] + #'exact',] +
            [T.lookup_name for T in RXN_DESCRIPTOR_TRANFORMS]
        )
        if lookup_type in supported_lookup_types:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

########################################################
# Binary Fingerprint Field

class BfpField(Field):

    description = _("Binary Fingerprint")

    def db_type(self, connection):
        return 'bfp'

    def get_placeholder(self, value, compiler, connection):
        if hasattr(value, 'as_sql'):
            if DJANGO_VERSION > (2,):
                return '%s'
            else:
                # No value used for expressions, substitute in
                # the column name instead.
                sql, _ = compiler.compile(value)
                return sql
        else:
            return 'bfp_from_binary_text(%s)'

    def select_format(self, compiler, sql, params):
        return 'bfp_to_binary_text(%s)' % sql, params

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return DataStructs.CreateFromBinaryText(bytes(value))

    def to_python(self, value):
        if value is None or isinstance(value, ExplicitBitVect):
            return value
        elif isinstance(value, six.buffer_types):
            return DataStructs.CreateFromBinaryText(bytes(value))
        else:
            raise ValidationError("Invalid input for a Bfp instance")

    def get_prep_value(self, value):
        # convert the ExplicitBitVect instance to the value used by the
        # db driver
        if isinstance(value, ExplicitBitVect):
            value = six.memoryview(DataStructs.BitVectToBinaryText(value))
        return value

    def get_prep_lookup(self, lookup_type, value):
        if lookup_type in [
                'lt', 'lte', 'exact', 'isnull', 'gte', 'gt', 'ne',
                'tanimoto', 'dice']:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

########################################################
# Sparse Integer Vector Fingerprint Field

class SfpField(Field):

    description = _("Sparse Integer Vector Fingerprint")

    def db_type(self, connection):
        return 'sfp'

    def get_prep_lookup(self, lookup_type, value):
        if lookup_type in [
                'lt', 'lte', 'exact', 'isnull', 'gte', 'gt', 'ne',
                'tanimoto', 'dice']:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

###################################################################
# MolField/RxnField lookup operations, substruct and exact searches

class HasSubstruct(Lookup):

    lookup_name = 'hassubstruct'
    prepare_rhs = True

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s @> mol_from_pkl(%s)' % (lhs, rhs), params

MolField.register_lookup(HasSubstruct)
RxnField.register_lookup(HasSubstruct)

class HasSubstructFP(Lookup):

    lookup_name = 'hassubstructfp'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s ?> mol_from_pkl(%s)' % (lhs, rhs), params

RxnField.register_lookup(HasSubstructFP)

class IsSubstruct(Lookup):

    lookup_name = 'issubstruct'
    prepare_rhs = True

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s <@ mol_from_pkl(%s)' % (lhs, rhs), params

MolField.register_lookup(IsSubstruct)
RxnField.register_lookup(IsSubstruct)

class IsSubstructFP(Lookup):

    lookup_name = 'issubstructfp'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s ?< mol_from_pkl(%s)' % (lhs, rhs), params

RxnField.register_lookup(IsSubstructFP)

class SameStructure(Lookup):

    lookup_name = 'exact'
    prepare_rhs = True

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        #return '%s @= %s' % (lhs, rhs), params
        return '%s <@ mol_from_pkl(%s) AND %s @> mol_from_pkl(%s)' % (lhs, rhs, lhs, rhs), params + params

MolField.register_lookup(SameStructure)

################
# descriptors utils

def make_descriptor_mixin(name, prefix, field):
    return type(
        str('{0}_Mixin'.format(name.upper())),
        (object,),
        {
            'descriptor_name': name,
            'function': '{0}_{1}'.format(prefix, name),
            'default_output_field': field,
        },
    )

class DescriptorTransform(Transform):

    def as_sql(self, qn, connection):
        lhs, params = qn.compile(self.lhs)
        return "%s(mol_from_pkl(%s))" % (self.function, lhs), params

##########################################
# MolField transforms and descriptors

MOL_DESCRIPTORS = [
    ('hba', IntegerField),
    ('hbd', IntegerField),
    ('numatoms', IntegerField),
    ('numheavyatoms', IntegerField),
    ('numrotatablebonds', IntegerField),
    ('numheteroatoms', IntegerField),
    ('numrings', IntegerField),
    ('numaromaticrings', IntegerField),
    ('numaliphaticrings', IntegerField),
    ('numsaturatedrings', IntegerField),
    ('numaromaticheterocycles', IntegerField),
    ('numaliphaticheterocycles', IntegerField),
    ('numsaturatedheterocycles', IntegerField),
    ('numaromaticcarbocycles', IntegerField),
    ('numaliphaticcarbocycles', IntegerField),
    ('numsaturatedcarbocycles', IntegerField),
    ('amw', FloatField),
    ('logp', FloatField),
    ('tpsa', FloatField),
    ('fractioncsp3', FloatField),
    ('chi0v', FloatField),
    ('chi1v', FloatField),
    ('chi2v', FloatField),
    ('chi3v', FloatField),
    ('chi4v', FloatField),
    ('chi0n', FloatField),
    ('chi1n', FloatField),
    ('chi2n', FloatField),
    ('chi3n', FloatField),
    ('chi4n', FloatField),
    ('kappa1', FloatField),
    ('kappa2', FloatField),
    ('kappa3', FloatField),
    ('murckoscaffold', MolField),
]

MOL_DESCRIPTOR_MIXINS = [
    make_descriptor_mixin(d, 'mol', fieldkls())
    for d, fieldkls in MOL_DESCRIPTORS
]

MOL_DESCRIPTOR_TRANFORMS = [
    type(str('{0}_Transform'.format(mixin.descriptor_name.upper())),
         (mixin, DescriptorTransform,),
         { 'lookup_name': mixin.descriptor_name,
           'output_field': mixin.default_output_field, }
     )
    for mixin in MOL_DESCRIPTOR_MIXINS
]

for Transform in MOL_DESCRIPTOR_TRANFORMS:
    MolField.register_lookup(Transform)

##########################################
# RxnField transforms and descriptors

RXN_DESCRIPTORS = [
    ('numreactants', IntegerField),
    ('numproducts', IntegerField),
    ('numagents', IntegerField),
]

RXN_DESCRIPTOR_MIXINS = [
    make_descriptor_mixin(d, 'reaction', fieldkls())
    for d, fieldkls in RXN_DESCRIPTORS
]

RXN_DESCRIPTOR_TRANFORMS = [
    type(str('{0}_Transform'.format(mixin.descriptor_name.upper())),
         (mixin, DescriptorTransform,),
         { 'lookup_name': mixin.descriptor_name,
           'output_field': mixin.default_output_field, }
     )
    for mixin in RXN_DESCRIPTOR_MIXINS
]

for Transform in RXN_DESCRIPTOR_TRANFORMS:
    RxnField.register_lookup(Transform)

####################################################################
# Fingerprint Fields lookup operations, similarity searches

class TanimotoSimilar(Lookup):

    lookup_name = 'tanimoto'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s %%%% mol_from_pkl(%s)' % (lhs, rhs), params

BfpField.register_lookup(TanimotoSimilar)
SfpField.register_lookup(TanimotoSimilar)

class DiceSimilar(Lookup):

    lookup_name = 'dice'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s # mol_from_pkl(%s)' % (lhs, rhs), params

BfpField.register_lookup(DiceSimilar)
SfpField.register_lookup(DiceSimilar)

class NotEqual(Lookup):

    lookup_name = 'ne'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s <> mol_from_pkl(%s)' % (lhs, rhs), params

BfpField.register_lookup(NotEqual)
SfpField.register_lookup(NotEqual)
<commit_msg>Added TextField to accommodate molBlock input<commit_after>from __future__ import unicode_literals

from django import VERSION as DJANGO_VERSION
from django.utils import six
from django.utils.translation import ugettext_lazy as _
from django.db.models import Lookup, Transform
from django.db.models.fields import *

from rdkit.Chem import AllChem as Chem
from rdkit import DataStructs
from rdkit.DataStructs import ExplicitBitVect

__all__ = ["MolField", "RxnField", "BfpField", "SfpField",]

##########################################
# Molecule Field
# Updated 10/22/2018 to allow for non-SMILES strings to be parsed & fixed compatibility of mol objects with operators
# UPDATED 10/23/2018 to specify an appropriate default form field input

class MolField(Field):

    description = _("Molecule")

    def db_type(self, connection):
        return 'mol'

    def get_placeholder(self, value, compiler, connection):
        if hasattr(value, 'as_sql'):
            if DJANGO_VERSION > (2,):
                return '%s'
            else:
                # No value used for expressions, substitute in
                # the column name instead.
                sql, _ = compiler.compile(value)
                return sql
        else:
            return 'mol_from_pkl(%s)'

    def select_format(self, compiler, sql, params):
        return 'mol_to_pkl(%s)' % sql, params

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return Chem.Mol(bytes(value))

    def to_python(self, value):
        if value is None or isinstance(value, Chem.Mol):
            return value
        elif isinstance(value, six.string_types):
            # The string case. Assume a SMILES is passed.
            mol = Chem.MolFromSmiles(str(value))
            if mol is None:
                mol = Chem.MolFromMolBlock(str(value))
            if mol is None:
                mol = Chem.inchi.MolFromInchi(str(value))
            if mol is None:
                raise ValidationError("Invalid input for a Mol instance")
            return mol
        elif isinstance(value, six.buffer_types):
            return Chem.Mol(bytes(value))
        else:
            raise ValidationError("Invalid input for a Mol instance")

    def get_prep_value(self, value):
        # convert the Molecule instance to the value used by the
        # db driver
        if isinstance(value, six.string_types):
            # The string case. Don't assume SMILES
            value = self.to_python(str(value))
        if isinstance(value, Chem.Mol):
            value = six.memoryview(value.ToBinary())
        return value

    def get_prep_lookup(self, lookup_type, value):
        "Perform preliminary non-db specific lookup checks and conversions"
        supported_lookup_types = (
            ['hassubstruct', 'issubstruct', 'exact', 'isnull',] +
            [T.lookup_name for T in MOL_DESCRIPTOR_TRANFORMS]
        )
        if lookup_type in supported_lookup_types:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

    def formfield(self, **kwargs):
        # Use TextField as default input form to accommodate line breaks needed for molBlocks
        defaults = {'form_class': forms.CharField, 'strip': False, 'widget':forms.Textarea}
        defaults.update(kwargs)
        return super().formfield(**defaults)

##########################################
# Reaction Field

class RxnField(Field):

    description = _("Reaction")

    def db_type(self, connection):
        return 'reaction'

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return Chem.ReactionFromSmarts(value, useSmiles=True)

    def to_python(self, value):
        if value is None or isinstance(value, Chem.ChemicalReaction):
            return value
        elif isinstance(value, six.string_types):
            # The string case. A reaction SMILES is expected.
            return Chem.ReactionFromSmarts(str(value), useSmiles=True)
        else:
            raise ValidationError("Invalid input for a ChemicalReaction instance")

    def get_prep_value(self, value):
        if isinstance(value, Chem.ChemicalReaction):
            value = Chem.ReactionToSmiles(value)
        return value

    def get_prep_lookup(self, lookup_type, value):
        "Perform preliminary non-db specific lookup checks and conversions"
        supported_lookup_types = (
            ['hassubstruct', 'issubstruct', 'isnull',] + #'exact',] +
            [T.lookup_name for T in RXN_DESCRIPTOR_TRANFORMS]
        )
        if lookup_type in supported_lookup_types:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

########################################################
# Binary Fingerprint Field

class BfpField(Field):

    description = _("Binary Fingerprint")

    def db_type(self, connection):
        return 'bfp'

    def get_placeholder(self, value, compiler, connection):
        if hasattr(value, 'as_sql'):
            if DJANGO_VERSION > (2,):
                return '%s'
            else:
                # No value used for expressions, substitute in
                # the column name instead.
                sql, _ = compiler.compile(value)
                return sql
        else:
            return 'bfp_from_binary_text(%s)'

    def select_format(self, compiler, sql, params):
        return 'bfp_to_binary_text(%s)' % sql, params

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return DataStructs.CreateFromBinaryText(bytes(value))

    def to_python(self, value):
        if value is None or isinstance(value, ExplicitBitVect):
            return value
        elif isinstance(value, six.buffer_types):
            return DataStructs.CreateFromBinaryText(bytes(value))
        else:
            raise ValidationError("Invalid input for a Bfp instance")

    def get_prep_value(self, value):
        # convert the ExplicitBitVect instance to the value used by the
        # db driver
        if isinstance(value, ExplicitBitVect):
            value = six.memoryview(DataStructs.BitVectToBinaryText(value))
        return value

    def get_prep_lookup(self, lookup_type, value):
        if lookup_type in [
                'lt', 'lte', 'exact', 'isnull', 'gte', 'gt', 'ne',
                'tanimoto', 'dice']:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

########################################################
# Sparse Integer Vector Fingerprint Field

class SfpField(Field):

    description = _("Sparse Integer Vector Fingerprint")

    def db_type(self, connection):
        return 'sfp'

    def get_prep_lookup(self, lookup_type, value):
        if lookup_type in [
                'lt', 'lte', 'exact', 'isnull', 'gte', 'gt', 'ne',
                'tanimoto', 'dice']:
            return value
        raise TypeError("Field has invalid lookup: %s" % lookup_type)

###################################################################
# MolField/RxnField lookup operations, substruct and exact searches

class HasSubstruct(Lookup):

    lookup_name = 'hassubstruct'
    prepare_rhs = True

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s @> mol_from_pkl(%s)' % (lhs, rhs), params

MolField.register_lookup(HasSubstruct)
RxnField.register_lookup(HasSubstruct)

class HasSubstructFP(Lookup):

    lookup_name = 'hassubstructfp'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s ?> mol_from_pkl(%s)' % (lhs, rhs), params

RxnField.register_lookup(HasSubstructFP)

class IsSubstruct(Lookup):

    lookup_name = 'issubstruct'
    prepare_rhs = True

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s <@ mol_from_pkl(%s)' % (lhs, rhs), params

MolField.register_lookup(IsSubstruct)
RxnField.register_lookup(IsSubstruct)

class IsSubstructFP(Lookup):

    lookup_name = 'issubstructfp'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s ?< mol_from_pkl(%s)' % (lhs, rhs), params

RxnField.register_lookup(IsSubstructFP)

class SameStructure(Lookup):

    lookup_name = 'exact'
    prepare_rhs = True

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        #return '%s @= %s' % (lhs, rhs), params
        return '%s <@ mol_from_pkl(%s) AND %s @> mol_from_pkl(%s)' % (lhs, rhs, lhs, rhs), params + params

MolField.register_lookup(SameStructure)

################
# descriptors utils

def make_descriptor_mixin(name, prefix, field):
    return type(
        str('{0}_Mixin'.format(name.upper())),
        (object,),
        {
            'descriptor_name': name,
            'function': '{0}_{1}'.format(prefix, name),
            'default_output_field': field,
        },
    )

class DescriptorTransform(Transform):

    def as_sql(self, qn, connection):
        lhs, params = qn.compile(self.lhs)
        return "%s(mol_from_pkl(%s))" % (self.function, lhs), params

##########################################
# MolField transforms and descriptors

MOL_DESCRIPTORS = [
    ('hba', IntegerField),
    ('hbd', IntegerField),
    ('numatoms', IntegerField),
    ('numheavyatoms', IntegerField),
    ('numrotatablebonds', IntegerField),
    ('numheteroatoms', IntegerField),
    ('numrings', IntegerField),
    ('numaromaticrings', IntegerField),
    ('numaliphaticrings', IntegerField),
    ('numsaturatedrings', IntegerField),
    ('numaromaticheterocycles', IntegerField),
    ('numaliphaticheterocycles', IntegerField),
    ('numsaturatedheterocycles', IntegerField),
    ('numaromaticcarbocycles', IntegerField),
    ('numaliphaticcarbocycles', IntegerField),
    ('numsaturatedcarbocycles', IntegerField),
    ('amw', FloatField),
    ('logp', FloatField),
    ('tpsa', FloatField),
    ('fractioncsp3', FloatField),
    ('chi0v', FloatField),
    ('chi1v', FloatField),
    ('chi2v', FloatField),
    ('chi3v', FloatField),
    ('chi4v', FloatField),
    ('chi0n', FloatField),
    ('chi1n', FloatField),
    ('chi2n', FloatField),
    ('chi3n', FloatField),
    ('chi4n', FloatField),
    ('kappa1', FloatField),
    ('kappa2', FloatField),
    ('kappa3', FloatField),
    ('murckoscaffold', MolField),
]

MOL_DESCRIPTOR_MIXINS = [
    make_descriptor_mixin(d, 'mol', fieldkls())
    for d, fieldkls in MOL_DESCRIPTORS
]

MOL_DESCRIPTOR_TRANFORMS = [
    type(str('{0}_Transform'.format(mixin.descriptor_name.upper())),
         (mixin, DescriptorTransform,),
         { 'lookup_name': mixin.descriptor_name,
           'output_field': mixin.default_output_field, }
     )
    for mixin in MOL_DESCRIPTOR_MIXINS
]

for Transform in MOL_DESCRIPTOR_TRANFORMS:
    MolField.register_lookup(Transform)

##########################################
# RxnField transforms and descriptors

RXN_DESCRIPTORS = [
    ('numreactants', IntegerField),
    ('numproducts', IntegerField),
    ('numagents', IntegerField),
]

RXN_DESCRIPTOR_MIXINS = [
    make_descriptor_mixin(d, 'reaction', fieldkls())
    for d, fieldkls in RXN_DESCRIPTORS
]

RXN_DESCRIPTOR_TRANFORMS = [
    type(str('{0}_Transform'.format(mixin.descriptor_name.upper())),
         (mixin, DescriptorTransform,),
         { 'lookup_name': mixin.descriptor_name,
           'output_field': mixin.default_output_field, }
     )
    for mixin in RXN_DESCRIPTOR_MIXINS
]

for Transform in RXN_DESCRIPTOR_TRANFORMS:
    RxnField.register_lookup(Transform)

####################################################################
# Fingerprint Fields lookup operations, similarity searches

class TanimotoSimilar(Lookup):

    lookup_name = 'tanimoto'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s %%%% mol_from_pkl(%s)' % (lhs, rhs), params

BfpField.register_lookup(TanimotoSimilar)
SfpField.register_lookup(TanimotoSimilar)

class DiceSimilar(Lookup):

    lookup_name = 'dice'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s # mol_from_pkl(%s)' % (lhs, rhs), params

BfpField.register_lookup(DiceSimilar)
SfpField.register_lookup(DiceSimilar)

class NotEqual(Lookup):

    lookup_name = 'ne'

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s <> mol_from_pkl(%s)' % (lhs, rhs), params

BfpField.register_lookup(NotEqual)
SfpField.register_lookup(NotEqual)
<|endoftext|>


<commit_before>import openpnm.models as mods
from openpnm.geometry import GenericGeometry

class StickAndBall2D(GenericGeometry):
    r"""
    2D Stick and Ball subclass of GenericGeometry.  This subclass is meant as a
    basic default geometry to get started quickly.

    Pore diameters are randomly assigned between 0 and the largest sphere that
    does not overlap with it's nearest neighbor.

    Throat diameters are half the diameter of the smaller of it's two
    neighboring pores.

    Parameters
    ----------
    network : OpenPNM Network object
        The network with which this Geometry should be associated

    project : OpenPNM Project object, optional
        Can be supplied instead of a ``network``

    pores : array_like
        The pores in the domain where this Geometry applies

    throats : array_like
        The throats in the domain where this Geometry applies

    name : string
        The name of the object, which is also used as the label where this
        geometry is defined.

    Examples
    --------
    Geometry objects (along with Physics objects) can be applied to a subset
    of pores and/or throats.  This allows for different geometrical property
    models to be applied in different regions.  This is illustrated in the
    following code:

    >>> import numpy as np
    >>> import scipy as sp
    >>> import openpnm as op
    >>> import matplotlib.pyplot as plt
    >>> pn = op.network.CubicDual(shape=[5, 5, 5])
    >>> Ps = pn.pores('primary')
    >>> Ts = pn.throats('primary')
    >>> geo1 = op.geometry.StickAndBall2D(network=pn, pores=Ps, throats=Ts)
    >>> Ps = pn.pores('secondary')
    >>> Ts = pn.throats(['secondary', 'interconnect'])
    >>> geo2 = op.geometry.StickAndBall2D(network=pn, pores=Ps, throats=Ts)

    Now override the 'pore.diameter' values on the ``geo2`` object:

    >>> geo2.remove_model('pore.diameter')  # Remove model and data
    >>> geo2['pore.diameter'] = np.random.rand(geo2.Np)*0.05

    Look at the 'pore.diameter' distributions on each object:

    >>> fig = plt.hist(geo1['pore.diameter'], bins=20, alpha=0.5)
    >>> fig = plt.hist(geo2['pore.diameter'], bins=20, alpha=0.5)

    The resulting figure shows that these two Geometry object each have a
    different pore size distribution, with ``geo2`` being much smaller:

    .. image:: /../docs/_static/images/stick_and_ball_histogram.png
        :align: center

    Notes
    -----
    The table below gives a summary of the all the pore-scale models that are
    included on this class.

    All of these parameters can be adjusted manually by editing the entries in
    the **ModelsDict** stored in the ``models`` attribute of the object.

    For a full listing of models and their parameters use ``print(obj.models)``
    where ``obj`` is the handle to the object.

    In addition to these models, this class also has a number of constant
    values assigned to it which can be found by running
    ``props(mode='constants')``.

    +----+----------------------+------------------+--------------------------+
    | #  | Property Name        | Parameter        | Value                    |
    +====+======================+==================+==========================+
    | 1  | pore.seed            | model:           | random                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | element          | pore                     |
    +----+----------------------+------------------+--------------------------+
    |    |                      | num_range        | [0, 0.1]                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | seed             | None                     |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 2  | pore.max_size        | model:           | largest_sphere           |
    +----+----------------------+------------------+--------------------------+
    |    |                      | iters            | 10                       |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | fixed_diameter   | pore.fixed_diameter      |
    +----+----------------------+------------------+--------------------------+
    | 3  | pore.diameter        | model:           | product                  |
    +----+----------------------+------------------+--------------------------+
    |    |                      | prop1            | pore.max_size            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | prop2            | pore.seed                |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 4  | pore.area            | model:           | sphere                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_diameter    | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 5  | pore.volume          | model:           | sphere                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_diameter    | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 6  | throat.max_size      | model:           | from_neighbor_pores      |
    +----+----------------------+------------------+--------------------------+
    |    |                      | mode             | min                      |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_prop        | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 7  | throat.diameter      | model:           | scaled                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | factor           | 0.5                      |
    +----+----------------------+------------------+--------------------------+
    |    |                      | prop             | throat.max_size          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 8  | throat.length        | model:           | piecewise                |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_diameter    | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 9  | throat.surface_area  | model:           | cylinder                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_diameter  | throat.diameter          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_length    | throat.length            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 10 | throat.volume        | model:           | cylinder                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_diameter  | throat.diameter          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_length    | throat.length            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 11 | throat.area          | model:           | cylinder                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_diameter  | throat.diameter          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+

    """

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.add_model(propname='pore.seed',
                       model=mods.misc.random,
                       element='pore',
                       num_range=[0.2, 0.7],
                       seed=None)

        self.add_model(propname='pore.max_size',
                       model=mods.geometry.pore_size.largest_sphere,
                       iters=10)

        self.add_model(propname='pore.diameter',
                       model=mods.misc.product,
                       props=['pore.max_size', 'pore.seed'])

        self.add_model(propname='pore.area',
                       model=mods.geometry.pore_cross_sectional_area.circle,
                       pore_diameter='pore.diameter')

        self.add_model(propname='pore.volume',
                       model=mods.geometry.pore_volume.circle,
                       pore_diameter='pore.diameter')

        self.add_model(propname='throat.max_size',
                       model=mods.misc.from_neighbor_pores,
                       mode='min',
                       prop='pore.diameter')

        self.add_model(propname='throat.diameter',
                       model=mods.misc.scaled,
                       factor=0.5,
                       prop='throat.max_size')

        self.add_model(propname='throat.endpoints',
                       model=mods.geometry.throat_endpoints.circular_pores,
                       pore_diameter='pore.diameter',
                       throat_diameter='throat.diameter')

        self.add_model(propname='throat.length',
                       model=mods.geometry.throat_length.piecewise,
                       throat_endpoints='throat.endpoints')

        self.add_model(propname='throat.surface_area',
                       model=mods.geometry.throat_surface_area.rectangle,
                       throat_length='throat.length')

        self.add_model(propname='throat.volume',
                       model=mods.geometry.throat_volume.rectangle,
                       throat_diameter='throat.diameter',
                       throat_length='throat.length')

        self.add_model(propname='throat.area',
                       model=mods.geometry.throat_cross_sectional_area.rectangle,
                       throat_diameter='throat.diameter')

        self.add_model(propname='throat.conduit_lengths',
                       model=mods.geometry.throat_length.conduit_lengths,
                       throat_endpoints='throat.endpoints',
                       throat_length='throat.length')
        self.add_model(propname='throat.hydraulic_size_factors',
                       model=mods.geometry.hydraulic_size_factors.cones_and_cylinders)
        self.add_model(propname='throat.diffusive_size_factors',
                       model=mods.geometry.diffusive_size_factors.cones_and_cylinders)
<commit_msg>Calling circle and rectangle size factors in StickAndBall2D now<commit_after>import openpnm.models as mods
from openpnm.geometry import GenericGeometry

class StickAndBall2D(GenericGeometry):
    r"""
    2D Stick and Ball subclass of GenericGeometry.  This subclass is meant as a
    basic default geometry to get started quickly.

    Pore diameters are randomly assigned between 0 and the largest sphere that
    does not overlap with it's nearest neighbor.

    Throat diameters are half the diameter of the smaller of it's two
    neighboring pores.

    Parameters
    ----------
    network : OpenPNM Network object
        The network with which this Geometry should be associated

    project : OpenPNM Project object, optional
        Can be supplied instead of a ``network``

    pores : array_like
        The pores in the domain where this Geometry applies

    throats : array_like
        The throats in the domain where this Geometry applies

    name : string
        The name of the object, which is also used as the label where this
        geometry is defined.

    Examples
    --------
    Geometry objects (along with Physics objects) can be applied to a subset
    of pores and/or throats.  This allows for different geometrical property
    models to be applied in different regions.  This is illustrated in the
    following code:

    >>> import numpy as np
    >>> import scipy as sp
    >>> import openpnm as op
    >>> import matplotlib.pyplot as plt
    >>> pn = op.network.CubicDual(shape=[5, 5, 5])
    >>> Ps = pn.pores('primary')
    >>> Ts = pn.throats('primary')
    >>> geo1 = op.geometry.StickAndBall2D(network=pn, pores=Ps, throats=Ts)
    >>> Ps = pn.pores('secondary')
    >>> Ts = pn.throats(['secondary', 'interconnect'])
    >>> geo2 = op.geometry.StickAndBall2D(network=pn, pores=Ps, throats=Ts)

    Now override the 'pore.diameter' values on the ``geo2`` object:

    >>> geo2.remove_model('pore.diameter')  # Remove model and data
    >>> geo2['pore.diameter'] = np.random.rand(geo2.Np)*0.05

    Look at the 'pore.diameter' distributions on each object:

    >>> fig = plt.hist(geo1['pore.diameter'], bins=20, alpha=0.5)
    >>> fig = plt.hist(geo2['pore.diameter'], bins=20, alpha=0.5)

    The resulting figure shows that these two Geometry object each have a
    different pore size distribution, with ``geo2`` being much smaller:

    .. image:: /../docs/_static/images/stick_and_ball_histogram.png
        :align: center

    Notes
    -----
    The table below gives a summary of the all the pore-scale models that are
    included on this class.

    All of these parameters can be adjusted manually by editing the entries in
    the **ModelsDict** stored in the ``models`` attribute of the object.

    For a full listing of models and their parameters use ``print(obj.models)``
    where ``obj`` is the handle to the object.

    In addition to these models, this class also has a number of constant
    values assigned to it which can be found by running
    ``props(mode='constants')``.

    +----+----------------------+------------------+--------------------------+
    | #  | Property Name        | Parameter        | Value                    |
    +====+======================+==================+==========================+
    | 1  | pore.seed            | model:           | random                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | element          | pore                     |
    +----+----------------------+------------------+--------------------------+
    |    |                      | num_range        | [0, 0.1]                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | seed             | None                     |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 2  | pore.max_size        | model:           | largest_sphere           |
    +----+----------------------+------------------+--------------------------+
    |    |                      | iters            | 10                       |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | fixed_diameter   | pore.fixed_diameter      |
    +----+----------------------+------------------+--------------------------+
    | 3  | pore.diameter        | model:           | product                  |
    +----+----------------------+------------------+--------------------------+
    |    |                      | prop1            | pore.max_size            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | prop2            | pore.seed                |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 4  | pore.area            | model:           | sphere                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_diameter    | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 5  | pore.volume          | model:           | sphere                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_diameter    | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 6  | throat.max_size      | model:           | from_neighbor_pores      |
    +----+----------------------+------------------+--------------------------+
    |    |                      | mode             | min                      |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_prop        | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 7  | throat.diameter      | model:           | scaled                   |
    +----+----------------------+------------------+--------------------------+
    |    |                      | factor           | 0.5                      |
    +----+----------------------+------------------+--------------------------+
    |    |                      | prop             | throat.max_size          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 8  | throat.length        | model:           | piecewise                |
    +----+----------------------+------------------+--------------------------+
    |    |                      | pore_diameter    | pore.diameter            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 9  | throat.surface_area  | model:           | cylinder                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_diameter  | throat.diameter          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_length    | throat.length            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 10 | throat.volume        | model:           | cylinder                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_diameter  | throat.diameter          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_length    | throat.length            |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+
    | 11 | throat.area          | model:           | cylinder                 |
    +----+----------------------+------------------+--------------------------+
    |    |                      | throat_diameter  | throat.diameter          |
    +----+----------------------+------------------+--------------------------+
    |    |                      | regen_mode       | normal                   |
    +----+----------------------+------------------+--------------------------+

    """

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.add_model(propname='pore.seed',
                       model=mods.misc.random,
                       element='pore',
                       num_range=[0.2, 0.7],
                       seed=None)

        self.add_model(propname='pore.max_size',
                       model=mods.geometry.pore_size.largest_sphere,
                       iters=10)

        self.add_model(propname='pore.diameter',
                       model=mods.misc.product,
                       props=['pore.max_size', 'pore.seed'])

        self.add_model(propname='pore.area',
                       model=mods.geometry.pore_cross_sectional_area.circle,
                       pore_diameter='pore.diameter')

        self.add_model(propname='pore.volume',
                       model=mods.geometry.pore_volume.circle,
                       pore_diameter='pore.diameter')

        self.add_model(propname='throat.max_size',
                       model=mods.misc.from_neighbor_pores,
                       mode='min',
                       prop='pore.diameter')

        self.add_model(propname='throat.diameter',
                       model=mods.misc.scaled,
                       factor=0.5,
                       prop='throat.max_size')

        self.add_model(propname='throat.endpoints',
                       model=mods.geometry.throat_endpoints.circular_pores,
                       pore_diameter='pore.diameter',
                       throat_diameter='throat.diameter')

        self.add_model(propname='throat.length',
                       model=mods.geometry.throat_length.piecewise,
                       throat_endpoints='throat.endpoints')

        self.add_model(propname='throat.surface_area',
                       model=mods.geometry.throat_surface_area.rectangle,
                       throat_length='throat.length')

        self.add_model(propname='throat.volume',
                       model=mods.geometry.throat_volume.rectangle,
                       throat_diameter='throat.diameter',
                       throat_length='throat.length')

        self.add_model(propname='throat.area',
                       model=mods.geometry.throat_cross_sectional_area.rectangle,
                       throat_diameter='throat.diameter')

        self.add_model(propname='throat.conduit_lengths',
                       model=mods.geometry.throat_length.conduit_lengths,
                       throat_endpoints='throat.endpoints',
                       throat_length='throat.length')
        self.add_model(propname='throat.hydraulic_size_factors',
                       model=mods.geometry.hydraulic_size_factors.circles_and_rectangles)
        self.add_model(propname='throat.diffusive_size_factors',
                       model=mods.geometry.diffusive_size_factors.circles_and_rectangles)
<|endoftext|>


<commit_before>from datetime import datetime

import geojson
from shapely.geometry import asShape

_max_length = None

def _get_max_lengths(field_map):
    from ..models import Submission
    res = {}

    for field in field_map:
        res[field] = Submission._meta.get_field(field).max_length

    return res

def normalize(prop):
    '''Convert old format to the new one'''

    global _max_length

    field_map = dict(city='kota',
                     company='perusahaan',
                     number='nomor',
                     origin='berangkat',
                     destination='jurusan')
    for new, old in field_map.items():
        if old in prop:
            prop[new] = prop[old]

    # Cut values that are too long
    if _max_length is None:
        _max_length = _get_max_lengths(field_map)

    for field in field_map:
        max_length = _max_length[field]
        if field in prop and prop[field] is not None:
            prop[field] = prop[field][:max_length]

    return prop

def parseData(submission):
    # Parse
    doc = submission.raw_geojson
    data = geojson.loads(doc, object_hook=geojson.GeoJSON.to_instance)

    # Fill properties
    prop = normalize(data.properties)
    fields = ['city', 'company', 'number', 'origin', 'destination']
    for field in fields:
        if field in prop:
            setattr(submission, field, prop[field])
        else:
            setattr(submission, field, None)

    # Fill route
    if len(data.geometry.coordinates):
        geometry = asShape(data.geometry)
        submission.route = geometry.wkt
    else:
        submission.route = None

def validateData(submission):
    # required fields
    fields = ['city', 'number']
    for field in fields:
        if getattr(submission, field) is None:
            raise ValueError('Incomplete data')

def process(submission):
    try:
        parseData(submission)
        validateData(submission)
        submission.parsed_ok = True
        submission.parsed_error = None
    except StandardError, e:
        submission.parsed_error = str(e)
        submission.parsed_ok = False

    submission.parsed_date = datetime.now()

def processAndSave(submission):
    process(submission)
    submission.save()

<commit_msg>Store province to the database<commit_after>from datetime import datetime

import geojson
from shapely.geometry import asShape

_max_length = None

def _get_max_lengths(field_map):
    from ..models import Submission
    res = {}

    for field in field_map:
        res[field] = Submission._meta.get_field(field).max_length

    return res

def normalize(prop):
    '''Convert old format to the new one'''

    global _max_length

    field_map = dict(city='kota',
                     company='perusahaan',
                     number='nomor',
                     origin='berangkat',
                     destination='jurusan')
    for new, old in field_map.items():
        if old in prop:
            prop[new] = prop[old]

    # Cut values that are too long
    if _max_length is None:
        _max_length = _get_max_lengths(field_map)

    for field in field_map:
        max_length = _max_length[field]
        if field in prop and prop[field] is not None:
            prop[field] = prop[field][:max_length]

    return prop

def parseData(submission):
    # Parse
    doc = submission.raw_geojson
    data = geojson.loads(doc, object_hook=geojson.GeoJSON.to_instance)

    # Fill properties
    prop = normalize(data.properties)
    fields = ['province', 'city', 'company', 'number', 'origin', 'destination']
    for field in fields:
        if field in prop:
            setattr(submission, field, prop[field])
        else:
            setattr(submission, field, None)

    # Fill route
    if len(data.geometry.coordinates):
        geometry = asShape(data.geometry)
        submission.route = geometry.wkt
    else:
        submission.route = None

def validateData(submission):
    # required fields
    fields = ['province', 'city', 'number']
    for field in fields:
        if getattr(submission, field) is None:
            raise ValueError('Incomplete data')

def process(submission):
    try:
        parseData(submission)
        validateData(submission)
        submission.parsed_ok = True
        submission.parsed_error = None
    except StandardError, e:
        submission.parsed_error = str(e)
        submission.parsed_ok = False

    submission.parsed_date = datetime.now()

def processAndSave(submission):
    process(submission)
    submission.save()

<|endoftext|>