<commit_before>from django.db import models
from django.core.files.base import ContentFile
from django.core.files.images import get_image_dimensions
from django.conf import settings
import datetime
import StringIO
import Image
import uuid
from django.db.models import Q

from mysite.customs import ohloh

def get_image_data_scaled(image_data, width):
    # scale it
    image_fd = StringIO.StringIO(image_data)
    im = Image.open(image_fd)
    image_fd.seek(0)

    w, h = get_image_dimensions(image_fd)

    new_w = width
    new_h = (h * 1.0 / w) * width

    smaller = im.resize((new_w, new_h),
                        Image.ANTIALIAS)

    # "Save" it to memory
    new_image_fd = StringIO.StringIO()
    smaller.save(new_image_fd, format='PNG')
    new_image_fd.seek(0)

    # pull data out
    image_data = new_image_fd.getvalue()
    return image_data

# Create your models here.
class Project(models.Model):

    @staticmethod
    def generate_random_icon_path(instance, filename):
        # MEDIA_ROOT is prefixed automatically.
        return 'images/icons/projects/%s.png' % uuid.uuid4().hex

    name = models.CharField(max_length=200, unique = True)
    language = models.CharField(max_length=200)

    # FIXME: Replace this with 'icon'
    icon_url = models.URLField(max_length=200)

    # In case we need it
    # dont_use_ohloh_icon = models.BooleanField(default=False)
    icon = models.ImageField(
            upload_to=lambda a,b: Project.generate_random_icon_path(a, b),
            null=True,
            default=None)

    date_icon_was_fetched_from_ohloh = models.DateTimeField(null=True, default=None)

    icon_smaller_for_badge = models.ImageField(
        upload_to=lambda a,b: Project.generate_random_icon_path(a,b),
        null=True,
        default=None)

    icon_for_search_result = models.ImageField(
        upload_to=lambda a,b: Project.generate_random_icon_path(a,b),
        null=True,
        default=None)

    def populate_icon_from_ohloh(self):

        oh = ohloh.get_ohloh()
        try:
            icon_data = oh.get_icon_for_project(self.name)
            self.date_icon_was_fetched_from_ohloh = datetime.datetime.utcnow()
        except ValueError:
            self.date_icon_was_fetched_from_ohloh = datetime.datetime.utcnow()
            return None

        # if you want to scale, use get_image_data_scaled(icon_data)
        self.icon.save('', ContentFile(icon_data))

        # Since we are saving an icon, also update our scaled-down version of
        # that icon for the badge.
        self.update_scaled_icons_from_self_icon()

    def get_url_of_icon_or_generic(self):
        if self.icon:
            return self.icon.url
        else:
            return settings.MEDIA_URL + 'no-project-icon.png'

    def get_url_of_badge_size_icon_or_generic(self):
        if self.icon_smaller_for_badge:
            return self.icon_smaller_for_badge.url
        else:
            return settings.MEDIA_URL + 'no-project-icon-w=40.png'

    def get_url_of_search_result_icon_or_generic(self):
        if self.icon_for_search_result:
            return self.icon_for_search_result.url
        else:
            return settings.MEDIA_URL + 'no-project-icon-w=20.png'

    def update_scaled_icons_from_self_icon(self):
        '''This method should be called when you update the Project.icon attribute.
        Side-effect: Saves a scaled-down version of that icon in the
        Project.icon_smaller_for_badge field.'''
        # First of all, do nothing if self.icon is a false value.
        if not self.icon:
            return
        # Okay, now we must have some normal-sized icon.

        normal_sized_icon_data = self.icon.file.read()

        # Scale it down to badge size, which
        # happens to be width=40
        badge_icon_data = get_image_data_scaled(normal_sized_icon_data, 40)
        self.icon_smaller_for_badge.save('', ContentFile(badge_icon_data))

        # Scale normal-sized icon down to a size that fits in the search results--20px by 20px
        search_result_icon_data = get_image_data_scaled(normal_sized_icon_data, 20)
        self.icon_for_search_result.save('', ContentFile(search_result_icon_data))

    def get_contributors(self):
        """Return a list of Person objects who are contributors to
        this Project."""
        from mysite.profile.models import PortfolioEntry
        # What portfolio entries point to this project?
        pf_entries = PortfolioEntry.objects.filter(
                Q(project=self), Q(is_deleted=False),
                Q(is_published=True) )
        # List the owners of those portfolio entries.
        return [pf_entry.person for pf_entry in pf_entries]

    def get_n_other_contributors_than(self, n, person):
        from mysite.profile.models import PortfolioEntry
        pf_entries = list(PortfolioEntry.objects.filter(Q(project=self),
                ~Q(person=person),
                Q(is_deleted=False),
                Q(is_published=True),
                ))
        import random
        random.shuffle(pf_entries)
        pf_entries = pf_entries[:n] # Slicing the pf entries has the same effect as
                                    # slicing the list of people.
        other_contributors = [p.person for p in pf_entries]
        return other_contributors

    def __unicode__(self):
        return "<Project name='%s' language='%s'>" % (self.name, self.language)

def populate_icon_on_project_creation(instance, created, *args, **kwargs):
    if created and not instance.icon:
        instance.populate_icon_from_ohloh()

models.signals.post_save.connect(populate_icon_on_project_creation, Project)

# An easy way to find

class OpenBugsManager(models.Manager):
    def get_query_set(self):
        return super(OpenBugsManager, self).get_query_set().filter(
                looks_closed=False)

class Bug(models.Model):
    project = models.ForeignKey(Project)
    title = models.CharField(max_length=200)
    description = models.TextField()
    status = models.CharField(max_length=200)
    importance = models.CharField(max_length=200)
    people_involved = models.IntegerField()
    date_reported = models.DateTimeField()
    last_touched = models.DateTimeField()
    last_polled = models.DateTimeField()
    submitter_username = models.CharField(max_length=200)
    submitter_realname = models.CharField(max_length=200)
    canonical_bug_link = models.URLField(max_length=200)
    good_for_newcomers = models.BooleanField(default=False)
    looks_closed = models.BooleanField(default=False)

    all_bugs = models.Manager()
    open_ones = OpenBugsManager()

    def __unicode__(self):
        return "<Bug title='%s' project='%s' project__language='%s' description='%s...'>" % (self.title, self.project.name, self.project.language, self.description[:50])

    # on save hook for seems_closed
    # does it

# vim: set ai ts=4 nu:
<commit_msg>Remove comments that refer to nothing<commit_after>from django.db import models
from django.core.files.base import ContentFile
from django.core.files.images import get_image_dimensions
from django.conf import settings
import datetime
import StringIO
import Image
import uuid
from django.db.models import Q

from mysite.customs import ohloh

def get_image_data_scaled(image_data, width):
    # scale it
    image_fd = StringIO.StringIO(image_data)
    im = Image.open(image_fd)
    image_fd.seek(0)

    w, h = get_image_dimensions(image_fd)

    new_w = width
    new_h = (h * 1.0 / w) * width

    smaller = im.resize((new_w, new_h),
                        Image.ANTIALIAS)

    # "Save" it to memory
    new_image_fd = StringIO.StringIO()
    smaller.save(new_image_fd, format='PNG')
    new_image_fd.seek(0)

    # pull data out
    image_data = new_image_fd.getvalue()
    return image_data

# Create your models here.
class Project(models.Model):

    @staticmethod
    def generate_random_icon_path(instance, filename):
        # MEDIA_ROOT is prefixed automatically.
        return 'images/icons/projects/%s.png' % uuid.uuid4().hex

    name = models.CharField(max_length=200, unique = True)
    language = models.CharField(max_length=200)

    # FIXME: Replace this with 'icon'
    icon_url = models.URLField(max_length=200)

    # In case we need it
    # dont_use_ohloh_icon = models.BooleanField(default=False)
    icon = models.ImageField(
            upload_to=lambda a,b: Project.generate_random_icon_path(a, b),
            null=True,
            default=None)

    date_icon_was_fetched_from_ohloh = models.DateTimeField(null=True, default=None)

    icon_smaller_for_badge = models.ImageField(
        upload_to=lambda a,b: Project.generate_random_icon_path(a,b),
        null=True,
        default=None)

    icon_for_search_result = models.ImageField(
        upload_to=lambda a,b: Project.generate_random_icon_path(a,b),
        null=True,
        default=None)

    def populate_icon_from_ohloh(self):

        oh = ohloh.get_ohloh()
        try:
            icon_data = oh.get_icon_for_project(self.name)
            self.date_icon_was_fetched_from_ohloh = datetime.datetime.utcnow()
        except ValueError:
            self.date_icon_was_fetched_from_ohloh = datetime.datetime.utcnow()
            return None

        # if you want to scale, use get_image_data_scaled(icon_data)
        self.icon.save('', ContentFile(icon_data))

        # Since we are saving an icon, also update our scaled-down version of
        # that icon for the badge.
        self.update_scaled_icons_from_self_icon()

    def get_url_of_icon_or_generic(self):
        if self.icon:
            return self.icon.url
        else:
            return settings.MEDIA_URL + 'no-project-icon.png'

    def get_url_of_badge_size_icon_or_generic(self):
        if self.icon_smaller_for_badge:
            return self.icon_smaller_for_badge.url
        else:
            return settings.MEDIA_URL + 'no-project-icon-w=40.png'

    def get_url_of_search_result_icon_or_generic(self):
        if self.icon_for_search_result:
            return self.icon_for_search_result.url
        else:
            return settings.MEDIA_URL + 'no-project-icon-w=20.png'

    def update_scaled_icons_from_self_icon(self):
        '''This method should be called when you update the Project.icon attribute.
        Side-effect: Saves a scaled-down version of that icon in the
        Project.icon_smaller_for_badge field.'''
        # First of all, do nothing if self.icon is a false value.
        if not self.icon:
            return
        # Okay, now we must have some normal-sized icon.

        normal_sized_icon_data = self.icon.file.read()

        # Scale it down to badge size, which
        # happens to be width=40
        badge_icon_data = get_image_data_scaled(normal_sized_icon_data, 40)
        self.icon_smaller_for_badge.save('', ContentFile(badge_icon_data))

        # Scale normal-sized icon down to a size that fits in the search results--20px by 20px
        search_result_icon_data = get_image_data_scaled(normal_sized_icon_data, 20)
        self.icon_for_search_result.save('', ContentFile(search_result_icon_data))

    def get_contributors(self):
        """Return a list of Person objects who are contributors to
        this Project."""
        from mysite.profile.models import PortfolioEntry
        # What portfolio entries point to this project?
        pf_entries = PortfolioEntry.objects.filter(
                Q(project=self), Q(is_deleted=False),
                Q(is_published=True) )
        # List the owners of those portfolio entries.
        return [pf_entry.person for pf_entry in pf_entries]

    def get_n_other_contributors_than(self, n, person):
        from mysite.profile.models import PortfolioEntry
        pf_entries = list(PortfolioEntry.objects.filter(Q(project=self),
                ~Q(person=person),
                Q(is_deleted=False),
                Q(is_published=True),
                ))
        import random
        random.shuffle(pf_entries)
        pf_entries = pf_entries[:n] # Slicing the pf entries has the same effect as
                                    # slicing the list of people.
        other_contributors = [p.person for p in pf_entries]
        return other_contributors

    def __unicode__(self):
        return "<Project name='%s' language='%s'>" % (self.name, self.language)

def populate_icon_on_project_creation(instance, created, *args, **kwargs):
    if created and not instance.icon:
        instance.populate_icon_from_ohloh()

models.signals.post_save.connect(populate_icon_on_project_creation, Project)

# An easy way to find

class OpenBugsManager(models.Manager):
    def get_query_set(self):
        return super(OpenBugsManager, self).get_query_set().filter(
                looks_closed=False)

class Bug(models.Model):
    project = models.ForeignKey(Project)
    title = models.CharField(max_length=200)
    description = models.TextField()
    status = models.CharField(max_length=200)
    importance = models.CharField(max_length=200)
    people_involved = models.IntegerField()
    date_reported = models.DateTimeField()
    last_touched = models.DateTimeField()
    last_polled = models.DateTimeField()
    submitter_username = models.CharField(max_length=200)
    submitter_realname = models.CharField(max_length=200)
    canonical_bug_link = models.URLField(max_length=200)
    good_for_newcomers = models.BooleanField(default=False)
    looks_closed = models.BooleanField(default=False)

    all_bugs = models.Manager()
    open_ones = OpenBugsManager()

    def __unicode__(self):
        return "<Bug title='%s' project='%s' project__language='%s' description='%s...'>" % (self.title, self.project.name, self.project.language, self.description[:50])

# vim: set ai ts=4 nu:
<|endoftext|>


<commit_before>"""soft reset demo."""

from nanpy.arduinotree import ArduinoTree
from nanpy.serialmanager import SerialManager

def print_millis(a):
    print ('uptime: %s sec' % (a.api.millis() / 1000.0))

def reset_demo():
    connection = SerialManager()
    a = ArduinoTree(connection=connection)
    print_millis(a)
    print ('soft reset')
    a.soft_reset()
    print_millis(a)

if __name__ == '__main__':
    reset_demo()
<commit_msg>soft reset demo: comment<commit_after>"""soft reset demo.

https://github.com/nanpy/nanpy/issues/72

watchdog doesn't work with some bootloaders.
Soft_reset() is based on watchdog.

More information:
http://arduino.stackexchange.com/questions/2922/watchdog-timer-stuck-in-restart-loop-green-led-flashing
"The OP is using an Adriuni Pro Mini clone.
The included bootloader on the Arduino Pro Mini
does not support system restarts by the WDT.
Essentially, on a device with a bootloader
that does not support WDT restarts,
the board will restart but the timer / reset will not causing
the board to continuously reset on reboot.
There are alternative bootloaders for arduino boards
that may resolve this issue."

"""

from nanpy.arduinotree import ArduinoTree
from nanpy.serialmanager import SerialManager

def print_millis(a):
    print ('uptime: %s sec' % (a.api.millis() / 1000.0))

def reset_demo():
    connection = SerialManager()
    a = ArduinoTree(connection=connection)
    print_millis(a)
    print ('soft reset')
    a.soft_reset()
    print_millis(a)

if __name__ == '__main__':
    reset_demo()
<|endoftext|>


<commit_before>from __future__ import division

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$Aug 09, 2017 17:12$"

import itertools

import numpy

import dask
import dask.array
import dask.array.linalg

from builtins import range as irange

from nanshe.imp.filters.wavelet import transform as _wavelet_transform
from nanshe.imp.segment import extract_f0 as _extract_f0

def extract_f0(new_data,
               half_window_size,
               which_quantile,
               temporal_smoothing_gaussian_filter_stdev,
               temporal_smoothing_gaussian_filter_window_size,
               spatial_smoothing_gaussian_filter_stdev,
               spatial_smoothing_gaussian_filter_window_size,
               bias=None,
               return_f0=False,
               **parameters):
    """
        Compute ``extract_f0`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    if "out" in parameters:
        raise TypeError("Got an unexpected keyword argument 'out'.")

    new_data = new_data.astype(numpy.float32)

    if bias is None:
        bias = 1 - new_data.min()

    depth = numpy.zeros((new_data.ndim,), dtype=int)

    depth[0] = int(numpy.ceil(
        temporal_smoothing_gaussian_filter_window_size *
        temporal_smoothing_gaussian_filter_stdev
    ))
    depth[1:] = int(numpy.ceil(
        spatial_smoothing_gaussian_filter_window_size *
        spatial_smoothing_gaussian_filter_stdev
    ))
    depth[0] = max(depth[0], half_window_size)

    depth = tuple(depth.tolist())

    boundary = len(depth) * ["none"]
    # Workaround for a bug in Dask with 0 depth.
    #
    # ref: https://github.com/dask/dask/issues/2258
    #
    for i in irange(len(depth)):
        if boundary[i] == "none" and depth[i] == 0:
            boundary[i] = "reflect"

    boundary = tuple(boundary)

    result = new_data.map_overlap(
        func=_extract_f0,
        depth=depth,
        boundary=boundary,
        dtype=numpy.float32,
        half_window_size=half_window_size,
        which_quantile=which_quantile,
        temporal_smoothing_gaussian_filter_stdev=temporal_smoothing_gaussian_filter_stdev,
        temporal_smoothing_gaussian_filter_window_size=temporal_smoothing_gaussian_filter_window_size,
        spatial_smoothing_gaussian_filter_stdev=spatial_smoothing_gaussian_filter_stdev,
        spatial_smoothing_gaussian_filter_window_size=spatial_smoothing_gaussian_filter_window_size,
        bias=bias,
        return_f0=return_f0,
        **parameters
    )

    return(result)

def wavelet_transform(im0,
                      scale=5):
    """
        Compute ``wavelet_transform`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    im0 = im0.astype(numpy.float32)

    try:
        scale_iter = enumerate(scale)
    except TypeError:
        scale_iter = enumerate(itertools.repeat(scale, im0.ndim))

    depth = list(itertools.repeat(0, im0.ndim))
    for i, each_scale in scale_iter:
        depth_i = 0
        for j in irange(1, 1 + each_scale):
            depth_i += 2 ** j

        depth[i] = depth_i

    depth = tuple(depth)

    boundary = len(depth) * ["none"]
    # Workaround for a bug in Dask with 0 depth.
    #
    # ref: https://github.com/dask/dask/issues/2258
    #
    for i in irange(len(depth)):
        if boundary[i] == "none" and depth[i] == 0:
            boundary[i] = "reflect"

    boundary = tuple(boundary)

    result = im0.map_overlap(
        func=_wavelet_transform,
        depth=depth,
        boundary=boundary,
        dtype=numpy.float32,
        scale=scale
    )

    return(result)

def zeroed_mean_images(new_data):
    """
        Compute ``zeroed_mean_images`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    new_data_means = new_data.mean(
        axis=tuple(irange(1, new_data.ndim)),
        keepdims=True
    )

    new_data_zeroed = new_data - new_data_means

    return new_data_zeroed

def normalize_data(new_data, **parameters):
    """
        Compute ``normalize_data`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    if "out" in parameters:
        raise TypeError("Got an unexpected keyword argument 'out'.")

    ord = parameters.get("renormalized_images", {}).get("ord", 2)

    if ord is None:
        ord = 2

    new_data_means = new_data.mean(
        axis=tuple(irange(1, new_data.ndim)),
        keepdims=True
    )

    new_data_zeroed = new_data - new_data_means

    new_data_norms = dask.array.linalg.norm(
        new_data_zeroed.reshape((len(new_data_zeroed), -1)),
        ord=ord,
        axis=1
    )
    new_data_norms = new_data_norms[
        (slice(None),) + (new_data_zeroed.ndim - new_data_norms.ndim) * (None,)
    ]

    new_data_renormed = dask.array.where(
        new_data_norms != 0, new_data_zeroed / new_data_norms, new_data_zeroed
    )

    return(new_data_renormed)
<commit_msg>Use zeroed_mean_images in normalize_data<commit_after>from __future__ import division

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$Aug 09, 2017 17:12$"

import itertools

import numpy

import dask
import dask.array
import dask.array.linalg

from builtins import range as irange

from nanshe.imp.filters.wavelet import transform as _wavelet_transform
from nanshe.imp.segment import extract_f0 as _extract_f0

def extract_f0(new_data,
               half_window_size,
               which_quantile,
               temporal_smoothing_gaussian_filter_stdev,
               temporal_smoothing_gaussian_filter_window_size,
               spatial_smoothing_gaussian_filter_stdev,
               spatial_smoothing_gaussian_filter_window_size,
               bias=None,
               return_f0=False,
               **parameters):
    """
        Compute ``extract_f0`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    if "out" in parameters:
        raise TypeError("Got an unexpected keyword argument 'out'.")

    new_data = new_data.astype(numpy.float32)

    if bias is None:
        bias = 1 - new_data.min()

    depth = numpy.zeros((new_data.ndim,), dtype=int)

    depth[0] = int(numpy.ceil(
        temporal_smoothing_gaussian_filter_window_size *
        temporal_smoothing_gaussian_filter_stdev
    ))
    depth[1:] = int(numpy.ceil(
        spatial_smoothing_gaussian_filter_window_size *
        spatial_smoothing_gaussian_filter_stdev
    ))
    depth[0] = max(depth[0], half_window_size)

    depth = tuple(depth.tolist())

    boundary = len(depth) * ["none"]
    # Workaround for a bug in Dask with 0 depth.
    #
    # ref: https://github.com/dask/dask/issues/2258
    #
    for i in irange(len(depth)):
        if boundary[i] == "none" and depth[i] == 0:
            boundary[i] = "reflect"

    boundary = tuple(boundary)

    result = new_data.map_overlap(
        func=_extract_f0,
        depth=depth,
        boundary=boundary,
        dtype=numpy.float32,
        half_window_size=half_window_size,
        which_quantile=which_quantile,
        temporal_smoothing_gaussian_filter_stdev=temporal_smoothing_gaussian_filter_stdev,
        temporal_smoothing_gaussian_filter_window_size=temporal_smoothing_gaussian_filter_window_size,
        spatial_smoothing_gaussian_filter_stdev=spatial_smoothing_gaussian_filter_stdev,
        spatial_smoothing_gaussian_filter_window_size=spatial_smoothing_gaussian_filter_window_size,
        bias=bias,
        return_f0=return_f0,
        **parameters
    )

    return(result)

def wavelet_transform(im0,
                      scale=5):
    """
        Compute ``wavelet_transform`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    im0 = im0.astype(numpy.float32)

    try:
        scale_iter = enumerate(scale)
    except TypeError:
        scale_iter = enumerate(itertools.repeat(scale, im0.ndim))

    depth = list(itertools.repeat(0, im0.ndim))
    for i, each_scale in scale_iter:
        depth_i = 0
        for j in irange(1, 1 + each_scale):
            depth_i += 2 ** j

        depth[i] = depth_i

    depth = tuple(depth)

    boundary = len(depth) * ["none"]
    # Workaround for a bug in Dask with 0 depth.
    #
    # ref: https://github.com/dask/dask/issues/2258
    #
    for i in irange(len(depth)):
        if boundary[i] == "none" and depth[i] == 0:
            boundary[i] = "reflect"

    boundary = tuple(boundary)

    result = im0.map_overlap(
        func=_wavelet_transform,
        depth=depth,
        boundary=boundary,
        dtype=numpy.float32,
        scale=scale
    )

    return(result)

def zeroed_mean_images(new_data):
    """
        Compute ``zeroed_mean_images`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    new_data_means = new_data.mean(
        axis=tuple(irange(1, new_data.ndim)),
        keepdims=True
    )

    new_data_zeroed = new_data - new_data_means

    return new_data_zeroed

def normalize_data(new_data, **parameters):
    """
        Compute ``normalize_data`` on Dask Arrays.

        See the nanshe function for more details

        Returns:
            Dask Array:    A lazily computed result.
    """

    if "out" in parameters:
        raise TypeError("Got an unexpected keyword argument 'out'.")

    ord = parameters.get("renormalized_images", {}).get("ord", 2)

    if ord is None:
        ord = 2

    new_data_zeroed = zeroed_mean_images(new_data)

    new_data_norms = dask.array.linalg.norm(
        new_data_zeroed.reshape((len(new_data_zeroed), -1)),
        ord=ord,
        axis=1
    )
    new_data_norms = new_data_norms[
        (slice(None),) + (new_data_zeroed.ndim - new_data_norms.ndim) * (None,)
    ]

    new_data_renormed = dask.array.where(
        new_data_norms != 0, new_data_zeroed / new_data_norms, new_data_zeroed
    )

    return(new_data_renormed)
<|endoftext|>


<commit_before><commit_msg>Refactor `gzip_compress` utility function<commit_after>import gzip
import io

def gzip_compress(data, compresslevel=6):
    compressed = io.BytesIO()
    with gzip.GzipFile(fileobj=compressed,
                       mode="wb",
                       compresslevel=compresslevel) as compressor:
        compressor.write(data)
    return compressed.getvalue()
<|endoftext|>


<commit_before>
try:
    import chardet
except ImportError:
    chardet = None

class Writer(object):
    '''
    A generator friendly, unicode aware CSV encoder class built for speed.
    '''

    # What to put between fields
    SEP = u','
    # What to wrap fields in, if they contain SEP
    QUOTE = u'"'
    # What to replace a QUOTE in a field with
    ESCQUOTE = QUOTE + QUOTE
    # What to put between records
    LINEBREAK = u'\n'
    ENCODING = 'utf-8'

    fields = []

    def __init__(self, **opts):
        '''
        opts MUST contain 'fields', a list of field names.
        opts may also include 'headers', a list of field headings.
        opts MAY override any of the above configurables.
        '''
        self.__dict__.update(opts)

    def write(self, values):
        '''Write a row of values'''
        def escape_field(val, SEP=self.SEP, QUOTE=self.QUOTE, ESCQ=self.ESCQUOTE):
            '''
            Escape separator and quote values, and wrap with quotes if needed

            Inlined for speed
            '''
            # escape quotes in the value
            val = val.replace(QUOTE, ESCQ)
            # if needed, wrap quotes around value
            if SEP in val or QUOTE in val:
                return QUOTE + val + QUOTE
            return val

        line = self.SEP.join(map(escape_field, values)) + self.LINEBREAK
        if self.ENCODING:
            line = line.encode(self.ENCODING)
        return line

    def write_dict(self, values):
        '''Write a row, getting values from a dict.'''
        return self.write(map(values.get, self.fields))

    def write_headers(self):
        '''Write a row of headers.'''
        return self.write(self.headers or self.fields)

class Reader(object):

    # What to put between fields
    SEP = u','
    # What to wrap fields in, if they contain SEP
    QUOTE = u'"'
    # What to replace a QUOTE in a field with
    ESCQUOTE = QUOTE + QUOTE
    # What to put between records
    LINEBREAK = u'\n'
    ENCODING = 'utf-8'

    def __init__(self, source, **opts):
        self.source = iter(source)
        self.__dict__.update(opts)
        if self.ENCODING == 'auto' and chardet is None:
            raise ImportError('Encoding auto-detect requires chardet installed.')

    def __iter__(self):
        return self

    def next(self):
        line = self.source.next().rstrip('\n')
        if self.ENCODING == 'auto':
            line = line.decode(chardet.detect(line)['encoding'] or 'ascii')
        else:
            line = line.decode(self.ENCODING)

        # Split on all separators
        fragments = line.split(self.SEP)
        # Now check each fragment for quoting
        fields = []
        frags = iter(fragments)
        qs = len(self.QUOTE)
        for frag in frags:
            while True:
                if frag.startswith(self.QUOTE):
                    if not frag.endswith(self.QUOTE):
                        frag = self.SEP.join([frag, frags.next()])
                        continue
                    frag = frag[qs:-qs]
                    frag.replace(self.ESCQUOTE, self.QUOTE)
                fields.append(frag)
                break
        return fields
<commit_msg>Remove bogus CSV Reader<commit_after>
class Writer(object):
    '''
    A generator friendly, unicode aware CSV encoder class built for speed.
    '''

    # What to put between fields
    SEP = u','
    # What to wrap fields in, if they contain SEP
    QUOTE = u'"'
    # What to replace a QUOTE in a field with
    ESCQUOTE = QUOTE + QUOTE
    # What to put between records
    LINEBREAK = u'\n'
    ENCODING = 'utf-8'

    fields = []

    def __init__(self, **opts):
        '''
        opts MUST contain 'fields', a list of field names.
        opts may also include 'headers', a list of field headings.
        opts MAY override any of the above configurables.
        '''
        self.__dict__.update(opts)

    def write(self, values):
        '''Write a row of values'''
        def escape_field(val, SEP=self.SEP, QUOTE=self.QUOTE, ESCQ=self.ESCQUOTE):
            '''
            Escape separator and quote values, and wrap with quotes if needed

            Inlined for speed
            '''
            # escape quotes in the value
            val = val.replace(QUOTE, ESCQ)
            # if needed, wrap quotes around value
            if SEP in val or QUOTE in val:
                return QUOTE + val + QUOTE
            return val

        line = self.SEP.join(map(escape_field, values)) + self.LINEBREAK
        if self.ENCODING:
            line = line.encode(self.ENCODING)
        return line

    def write_dict(self, values):
        '''Write a row, getting values from a dict.'''
        return self.write(map(values.get, self.fields))

    def write_headers(self):
        '''Write a row of headers.'''
        return self.write(self.headers or self.fields)
<|endoftext|>


<commit_before>__version__ = '0.2'<commit_msg>register needs to be imported for adding to builtins<commit_after>from registry import register
__version__ = '0.2'<|endoftext|>


<commit_before>import json
import logging
import os
import sys

import numpy
import tensorflow as tf
import tensorflow.contrib.slim as slim #tensorflow.contrib.framework ???

import compat
import training_progress

def init_or_restore_variables(config, sess, ensemble_scope=None, train=False):
    # Construct a mapping between saved variable names and names in the current
    # scope. There are two reasons why names might be different:
    #
    #   1. This model is part of an ensemble, in which case a model-specific
    #       name scope will be active.
    #
    #   2. The saved model is from an old version of Nematus (before deep model
    #        support was added) and uses a different variable naming scheme
    #        for the GRUs.
    variables = slim.get_variables_to_restore()
    var_map = {}
    for v in variables:
        name = v.name.split(':')[0]
        if ensemble_scope == None:
            saved_name = name
        elif v.name.startswith(ensemble_scope.name + "/"):
            saved_name = name[len(ensemble_scope.name)+1:]
            # The ensemble scope is repeated for Adam variables. See
            # https://github.com/tensorflow/tensorflow/issues/8120
            if saved_name.startswith(ensemble_scope.name + "/"):
                saved_name = saved_name[len(ensemble_scope.name)+1:]
        else: # v belongs to a different model in the ensemble.
            continue
        if config.model_version == 0.1:
            # Backwards compatibility with the old variable naming scheme.
            saved_name = compat.revert_variable_name(saved_name, 0.1)
        var_map[saved_name] = v
    saver = tf.train.Saver(var_map, max_to_keep=None)

    # compute reload model filename
    reload_filename = None
    if config.reload == 'latest_checkpoint':
        checkpoint_dir = os.path.dirname(config.saveto)
        reload_filename = tf.train.latest_checkpoint(checkpoint_dir)
        if reload_filename != None:
            if (os.path.basename(reload_filename).rsplit('-', 1)[0] !=
                os.path.basename(config.saveto)):
                logging.error("Mismatching model filename found in the same directory while reloading from the latest checkpoint")
                sys.exit(1)
            logging.info('Latest checkpoint found in directory ' + os.path.abspath(checkpoint_dir))
    elif config.reload != None:
        reload_filename = config.reload
    if (reload_filename == None) and (config.prior_model != None):
        logging.info('Initializing model parameters from prior')
        reload_filename = config.prior_model

    # initialize or reload training progress
    if train:
        progress = training_progress.TrainingProgress()
        progress.bad_counter = 0
        progress.uidx = 0
        progress.eidx = 0
        progress.estop = False
        progress.history_errs = []
        progress.valid_script_scores = []
        if reload_filename and config.reload_training_progress:
            path = reload_filename + '.progress.json'
            if os.path.exists(path):
                logging.info('Reloading training progress')
                progress.load_from_json(path)
                if (progress.estop == True or
                    progress.eidx > config.max_epochs or
                    progress.uidx >= config.finish_after):
                    logging.warning('Training is already complete. Disable reloading of training progress (--no_reload_training_progress) or remove or modify progress file (%s) to train anyway.' % reload_path)
                    sys.exit(0)

    # load prior model
    if train and config.prior_model != None:
        load_prior(config, sess, saver)

    # initialize or restore model
    if reload_filename == None:
        logging.info('Initializing model parameters from scratch...')
        init_op = tf.global_variables_initializer()
        sess.run(init_op)
    else:
        logging.info('Loading model parameters from file ' + os.path.abspath(reload_filename))
        saver.restore(sess, os.path.abspath(reload_filename))

    logging.info('Done')

    if train:
        return saver, progress
    else:
        return saver

def load_prior(config, sess, saver):
     logging.info('Loading prior model parameters from file ' + os.path.abspath(config.prior_model))
     saver.restore(sess, os.path.abspath(config.prior_model))

     # fill prior variables with the loaded values
     prior_variables = tf.get_collection_ref('prior_variables')
     prior_variables_dict = dict([(v.name, v) for v in prior_variables])
     assign_tensors = []
     with tf.variable_scope('prior'):
         for v in tf.trainable_variables():
             prior_name = 'loss/prior/'+v.name
             prior_variable = prior_variables_dict[prior_name]
             assign_tensors.append(prior_variable.assign(v))
     tf.variables_initializer(prior_variables)
     sess.run(assign_tensors)
<commit_msg>Fix warning message<commit_after>import json
import logging
import os
import sys

import numpy
import tensorflow as tf
import tensorflow.contrib.slim as slim #tensorflow.contrib.framework ???

import compat
import training_progress

def init_or_restore_variables(config, sess, ensemble_scope=None, train=False):
    # Construct a mapping between saved variable names and names in the current
    # scope. There are two reasons why names might be different:
    #
    #   1. This model is part of an ensemble, in which case a model-specific
    #       name scope will be active.
    #
    #   2. The saved model is from an old version of Nematus (before deep model
    #        support was added) and uses a different variable naming scheme
    #        for the GRUs.
    variables = slim.get_variables_to_restore()
    var_map = {}
    for v in variables:
        name = v.name.split(':')[0]
        if ensemble_scope == None:
            saved_name = name
        elif v.name.startswith(ensemble_scope.name + "/"):
            saved_name = name[len(ensemble_scope.name)+1:]
            # The ensemble scope is repeated for Adam variables. See
            # https://github.com/tensorflow/tensorflow/issues/8120
            if saved_name.startswith(ensemble_scope.name + "/"):
                saved_name = saved_name[len(ensemble_scope.name)+1:]
        else: # v belongs to a different model in the ensemble.
            continue
        if config.model_version == 0.1:
            # Backwards compatibility with the old variable naming scheme.
            saved_name = compat.revert_variable_name(saved_name, 0.1)
        var_map[saved_name] = v
    saver = tf.train.Saver(var_map, max_to_keep=None)

    # compute reload model filename
    reload_filename = None
    if config.reload == 'latest_checkpoint':
        checkpoint_dir = os.path.dirname(config.saveto)
        reload_filename = tf.train.latest_checkpoint(checkpoint_dir)
        if reload_filename != None:
            if (os.path.basename(reload_filename).rsplit('-', 1)[0] !=
                os.path.basename(config.saveto)):
                logging.error("Mismatching model filename found in the same directory while reloading from the latest checkpoint")
                sys.exit(1)
            logging.info('Latest checkpoint found in directory ' + os.path.abspath(checkpoint_dir))
    elif config.reload != None:
        reload_filename = config.reload
    if (reload_filename == None) and (config.prior_model != None):
        logging.info('Initializing model parameters from prior')
        reload_filename = config.prior_model

    # initialize or reload training progress
    if train:
        progress = training_progress.TrainingProgress()
        progress.bad_counter = 0
        progress.uidx = 0
        progress.eidx = 0
        progress.estop = False
        progress.history_errs = []
        progress.valid_script_scores = []
        if reload_filename and config.reload_training_progress:
            path = reload_filename + '.progress.json'
            if os.path.exists(path):
                logging.info('Reloading training progress')
                progress.load_from_json(path)
                if (progress.estop == True or
                    progress.eidx > config.max_epochs or
                    progress.uidx >= config.finish_after):
                    logging.warning('Training is already complete. Disable reloading of training progress (--no_reload_training_progress) or remove or modify progress file (%s) to train anyway.' % path)
                    sys.exit(0)

    # load prior model
    if train and config.prior_model != None:
        load_prior(config, sess, saver)

    # initialize or restore model
    if reload_filename == None:
        logging.info('Initializing model parameters from scratch...')
        init_op = tf.global_variables_initializer()
        sess.run(init_op)
    else:
        logging.info('Loading model parameters from file ' + os.path.abspath(reload_filename))
        saver.restore(sess, os.path.abspath(reload_filename))

    logging.info('Done')

    if train:
        return saver, progress
    else:
        return saver

def load_prior(config, sess, saver):
     logging.info('Loading prior model parameters from file ' + os.path.abspath(config.prior_model))
     saver.restore(sess, os.path.abspath(config.prior_model))

     # fill prior variables with the loaded values
     prior_variables = tf.get_collection_ref('prior_variables')
     prior_variables_dict = dict([(v.name, v) for v in prior_variables])
     assign_tensors = []
     with tf.variable_scope('prior'):
         for v in tf.trainable_variables():
             prior_name = 'loss/prior/'+v.name
             prior_variable = prior_variables_dict[prior_name]
             assign_tensors.append(prior_variable.assign(v))
     tf.variables_initializer(prior_variables)
     sess.run(assign_tensors)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""

Support for intan tech rhd file.

See http://intantech.com/files/Intan_RHD2000_data_file_formats.pdf

Author: Samuel Garcia

"""
from __future__ import print_function, division, absolute_import
# from __future__ import unicode_literals is not compatible with numpy.dtype both py2 py3

from .baserawio import (BaseRawIO, _signal_channel_dtype, _unit_channel_dtype,
                        _event_channel_dtype)

import numpy as np
from collections import OrderedDict

class IntanRawIO(BaseRawIO):
    """

    """
    extensions = ['rdh']
    rawmode = 'one-dir'
    <commit_msg>WIP intan<commit_after># -*- coding: utf-8 -*-
"""

Support for intan tech rhd  and rhs files.

This 2 formats are more or less the same with some variance in headers.

See:
  * http://intantech.com/files/Intan_RHD2000_data_file_formats.pdf
  * http://intantech.com/files/Intan_RHS2000_data_file_formats.pdf

Author: Samuel Garcia

"""
from __future__ import print_function, division, absolute_import
# from __future__ import unicode_literals is not compatible with numpy.dtype both py2 py3

from .baserawio import (BaseRawIO, _signal_channel_dtype, _unit_channel_dtype,
                        _event_channel_dtype)

import numpy as np
from collections import OrderedDict

class IntanRawIO(BaseRawIO):
    """

    """
    extensions = ['rhd', 'rhs']
    rawmode = 'one-file'

    def __init__(self, filename=''):
        BaseRawIO.__init__(self)
        self.filename = filename

    def _source_name(self):
        return self.filename

    def _parse_header(self):

        if self.filename.endswith('.rhs'):
            info = read_rhs(self.filename)
        elif self.filename.endswith('.rhd'):
            info = read_rh(self.filename)

        exit()

        # signals
        sig_channels = []
        for c in range(nb_channel):
            name = 'ch{}grp{}'.format(c, channel_group[c])
            chan_id = c
            units = 'mV'
            offset = 0.
            group_id = 0
            sig_channels.append((name, chan_id, self._sampling_rate,
                                 sig_dtype, units, gain, offset, group_id))
        sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype)

        # No events
        event_channels = []
        event_channels = np.array(event_channels, dtype=_event_channel_dtype)

        # No spikes
        unit_channels = []
        unit_channels = np.array(unit_channels, dtype=_unit_channel_dtype)

        # fille into header dict
        self.header = {}
        self.header['nb_block'] = 1
        self.header['nb_segment'] = [1]
        self.header['signal_channels'] = sig_channels
        self.header['unit_channels'] = unit_channels
        self.header['event_channels'] = event_channels

        self._generate_minimal_annotations()

    def _segment_t_start(self, block_index, seg_index):
        return 0.

    def _segment_t_stop(self, block_index, seg_index):
        t_stop = self._raw_signals.shape[0] / self._sampling_rate
        return t_stop

    def _get_signal_size(self, block_index, seg_index, channel_indexes):
        return self._raw_signals.shape[0]

    def _get_signal_t_start(self, block_index, seg_index, channel_indexes):
        return 0.

    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, channel_indexes):
        if channel_indexes is None:
            channel_indexes = slice(None)
        raw_signals = self._raw_signals[slice(i_start, i_stop), channel_indexes]
        return raw_signals

def read_qstring(f):
    a = ''
    length = np.fromfile(f, dtype='uint32', count=1)[0]

    print('length', length, hex(length))
    #~ exit()

    if length == 0xFFFFFFFF or length == '':
        return ''

    txt =''
    txt = f.read(length // 2)  #.decode('utf-16')
    #~ print(txt)
    #~ txt = txt.decode()

    #~ for ii in range(length):
        #~ print('ii', ii)
        #~ newchar = np.fromfile(f, 'u2', 1)[0]
        #~ print(newchar)
        #~ a += newchar.tostring().decode('utf-16')
    return txt

import struct
import os
import sys
def read_qstring(fid):
    """Read Qt style QString.

    The first 32-bit unsigned number indicates the length of the string (in bytes).
    If this number equals 0xFFFFFFFF, the string is null.

    Strings are stored as unicode.
    """

    length, = struct.unpack('<I', fid.read(4))
    print(length)
    if length == int('ffffffff', 16): return ""

    if length > (os.fstat(fid.fileno()).st_size - fid.tell() + 1) :
        print(length)
        raise Exception('Length too long.')

    # convert length from bytes to 16-bit Unicode words
    length = int(length / 2)

    data = []
    for i in range(0, length):
        c, = struct.unpack('<H', fid.read(2))
        data.append(c)

    if sys.version_info >= (3,0):
        a = ''.join([chr(c) for c in data])
    else:
        a = ''.join([unichr(c) for c in data])

    return a

rhs_global_header =[
    ('magic_number', 'uint32'),  # 0xD69127AC for rhs   0xC6912702 for rdh

    ('major_version', 'int16'),
    ('minor_version', 'int16'),

    ('sampling_rate', 'float32'),

    ('dsp_enabled', 'int16'),

    ('actual_dsp_cutoff_frequency', 'float32'),
    ('actual_lower_bandwidth', 'float32'),
    ('actual_lower_settle_bandwidth', 'float32'),   #######
    ('actual_upper_bandwidth', 'float32'),
    ('desired_dsp_cutoff_frequency', 'float32'),
    ('desired_lower_bandwidth', 'float32'),
    ('desired_lower_settle_bandwidth', 'float32'),   #####
    ('desired_upper_bandwidth', 'float32'),

    ('notch_filter_mode', 'int16'), # 0 :no filter 1: 50Hz 2 : 60Hz

    ('desired_impedance_test_frequency', 'float32'),
    ('actual_impedance_test_frequency', 'float32'),

    ('amp_settle_mode', 'int16'),  ####
    ('charge_recovery_mode', 'int16'), ####

    ('stim_step_size', 'float32'),  ####
    ('recovery_current_limit', 'float32'),  ####
    ('recovery_target_voltage', 'float32'), ####

    ('note1', 'QString'),
    ('note2', 'QString'),
    ('note3', 'QString'),

    ('dc_amplifier_data_saved', 'int16'),  ###### nb_temp_sensor

    ('board_mode', 'int16'),

    ('ref_channel_name', 'QString'),

    ('nb_signal_group', 'int16'),

]

signal_group_header = [
    ('signal_group_name', 'QString'),
    ('signal_group_prefix', 'QString'),
    ('signal_group_enabled', 'int16'),
    ('channel_num', 'int16'),
    ('amplified_channel_num', 'int16'),
]

signal_channel_header = [
    ('native_channel_name', 'QString'),
    ('custom_channel_name', 'QString'),
    ('native_order', 'int16'),
    ('custom_order', 'int16'),
    ('signal_type', 'int16'),
    ('channel_enabled', 'int16'),
    ('chip_channel_num', 'int16'),
    ('board_stream_num', 'int16'),
    ('spike_scope_trigger_mode', 'int16'),
    ('spike_scope_voltage_thresh', 'int16'),
    ('spike_scope_digital_trigger_channel', 'int16'),
    ('spike_scope_digital_edge_polarity', 'int16'),
    ('electrode_impedance_magnitude', 'float32'),
    ('electrode_impedance_phase', 'float32'),

]

def read_variable_header(f, header):
    info = {}
    for field_name, field_type in header:

        if field_type == 'QString':
            field_value = read_qstring(f)
            print(field_name, field_type,  len(field_value), field_value)
        else:
            field_value = np.fromfile(f, dtype=field_type, count=1)[0]
            print(field_name, field_type,  field_value)

        info[field_name] = field_value

    return info

def read_rhd(filename):
    return

def read_rhs(filename):
    with open(filename, mode='rb') as f:
        info = read_variable_header(f, rhs_global_header)

    <|endoftext|>


<commit_before>from neon.datasets.dataset import Dataset
import logging
import numpy as np
import scipy.io

logger = logging.getLogger(__name__)

class Matlab(Dataset):
    """
    Sets up a Matlab dataset.
    """

    def __init__(self, data_file, inputs_name = 'features', targets_name = 'targets', shuffle = True,
            train_size = None, test_size = None, normalize_mean = True, normalize_std = True, **kwargs):
        self.__dict__.update(kwargs)
        self.data_file = data_file
        self.inputs_name = inputs_name
        self.targets_name = targets_name
        self.shuffle = shuffle
        self.train_size = train_size
        self.test_size = test_size
        self.normalize_mean = normalize_mean
        self.normalize_std = normalize_std

    def load(self, backend=None, experiment=None):
        # don't try to load data again if already loaded
        if self.inputs['train'] is not None:
            return

        # load Matlab file and extract matrices
        logger.info('loading: %s', self.data_file)
        data = scipy.io.loadmat(self.data_file)
        inputs = data[self.inputs_name].astype(float)
        targets = data[self.targets_name]
        logger.info('inputs: %s, targets: %s', str(inputs.shape), str(targets.shape))
        assert inputs.shape[0] == targets.shape[0]
        nr_samples = inputs.shape[0]

        if self.shuffle:
            indices = np.random.permutation(nr_samples)
        else:
            indices = range(nr_samples)

        # split data into training and test set
        if self.train_size is not None and isinstance(self.train_size, float):
            assert 0 <= self.train_size <= 1
            self.train_size = int(self.train_size * nr_samples)
        if self.test_size is not None and isinstance(self.test_size, float):
            assert 0 <= self.test_size <= 1
            self.test_size = int(self.test_size * nr_samples)

        if self.train_size is not None and self.test_size is not None:
            self.train_size = nr_samples - self.test_size
        elif self.test_size is None and self.train_size is not None:
            self.test_size = nr_samples - self.train_size

        assert 0 <= self.train_size <= nr_samples
        assert 0 <= self.test_size <= nr_samples
        assert (self.train_size + self.test_size) <= nr_samples

        self.inputs['train'] = inputs[indices[:self.train_size], ...]
        self.inputs['test'] = inputs[indices[self.train_size:(self.train_size + self.test_size)], ...]
        self.targets['train'] = targets[indices[:self.train_size], ...]
        self.targets['test'] = targets[indices[self.train_size:(self.train_size + self.test_size)], ...]
        logger.info('training set size: %d, test set size: %d', self.inputs['train'].shape[0], self.inputs['test'].shape[0])

        # normalize to zero mean and unit variance
        mean = np.mean(self.inputs['train'], axis=0)
        std = np.std(self.inputs['train'], axis=0)
        assert len(mean.shape) == 1 and mean.shape[0] == inputs.shape[1]
        assert len(std.shape) == 1 and std.shape[0] == inputs.shape[1]
        std[std == 0.0] = 1.0

        self.inputs['train'] -= mean
        self.inputs['train'] /= std
        self.inputs['test'] -= mean
        self.inputs['test'] /= std

        self.format()
<commit_msg>Fixed bug - training set was always nr_samples - test_size.<commit_after>from neon.datasets.dataset import Dataset
import logging
import numpy as np
import scipy.io

logger = logging.getLogger(__name__)

class Matlab(Dataset):
    """
    Sets up a Matlab dataset.
    """

    def __init__(self, data_file, inputs_name = 'features', targets_name = 'targets', shuffle = True,
            train_size = None, test_size = None, normalize_mean = True, normalize_std = True, **kwargs):
        self.__dict__.update(kwargs)
        self.data_file = data_file
        self.inputs_name = inputs_name
        self.targets_name = targets_name
        self.shuffle = shuffle
        self.train_size = train_size
        self.test_size = test_size
        self.normalize_mean = normalize_mean
        self.normalize_std = normalize_std

    def load(self, backend=None, experiment=None):
        # don't try to load data again if already loaded
        if self.inputs['train'] is not None:
            return

        # load Matlab file and extract matrices
        logger.info('loading: %s', self.data_file)
        data = scipy.io.loadmat(self.data_file)
        inputs = data[self.inputs_name].astype(float)
        targets = data[self.targets_name]
        logger.info('inputs: %s, targets: %s', str(inputs.shape), str(targets.shape))
        assert inputs.shape[0] == targets.shape[0]
        nr_samples = inputs.shape[0]

        if self.shuffle:
            indices = np.random.permutation(nr_samples)
        else:
            indices = range(nr_samples)

        # split data into training and test set
        if self.train_size is not None and isinstance(self.train_size, float):
            assert 0 <= self.train_size <= 1
            self.train_size = int(self.train_size * nr_samples)
        if self.test_size is not None and isinstance(self.test_size, float):
            assert 0 <= self.test_size <= 1
            self.test_size = int(self.test_size * nr_samples)

        if self.train_size is None and self.test_size is not None:
            self.train_size = nr_samples - self.test_size
        elif self.test_size is None and self.train_size is not None:
            self.test_size = nr_samples - self.train_size

        assert 0 <= self.train_size <= nr_samples
        assert 0 <= self.test_size <= nr_samples
        assert (self.train_size + self.test_size) <= nr_samples

        self.inputs['train'] = inputs[indices[:self.train_size], ...]
        self.inputs['test'] = inputs[indices[self.train_size:(self.train_size + self.test_size)], ...]
        self.targets['train'] = targets[indices[:self.train_size], ...]
        self.targets['test'] = targets[indices[self.train_size:(self.train_size + self.test_size)], ...]
        logger.info('training set size: %d, test set size: %d', self.inputs['train'].shape[0], self.inputs['test'].shape[0])

        # normalize to zero mean and unit variance
        mean = np.mean(self.inputs['train'], axis=0)
        std = np.std(self.inputs['train'], axis=0)
        assert len(mean.shape) == 1 and mean.shape[0] == inputs.shape[1]
        assert len(std.shape) == 1 and std.shape[0] == inputs.shape[1]
        std[std == 0.0] = 1.0

        self.inputs['train'] -= mean
        self.inputs['train'] /= std
        self.inputs['test'] -= mean
        self.inputs['test'] /= std

        self.format()
<|endoftext|>


<commit_before>from __future__ import division
import re,struct
from iotbx.detectors.detectorbase import DetectorImageBase
import os,re,time
import logging, logging.handlers

class MARImage(DetectorImageBase):
  def __init__(self,filename):
    DetectorImageBase.__init__(self,filename)
    self.vendortype = "MARCCD"

    byte_order = str(open(self.filename,"rb").read(2))
    if byte_order == 'II':
      self.endian = 0
    else:
      self.endian = 1

    assert not self.isCompressed()

  def isCompressed(self):
    if self.getEndian(): format = '>'
    else: format = '<'

    f = open(self.filename,"rb")
    try:
      f.seek(4)
      rawdata = f.read(4)
      ifd = struct.unpack(format+'i',rawdata)[0]

      # if ifd is 0 then there are no more Image File Directories
      while ifd:
        f.seek(ifd)

        rawdata = f.read(2)
        # get the number of directory entries in the IFD
        numentries = struct.unpack(format+'h',rawdata)[0]

        # search for compression tag
        for x in range(numentries):
          f.seek(ifd+x*12+2)
          rawdata = f.read(2)
          tag = struct.unpack(format+'h',rawdata)[0]
          if tag == 259: # value of compression tag
            f.seek(ifd+x*12+2+8) # seek to value offset
            # value is left justified in 4 byte field
            # read two bytes so can unpack as short
            rawdata = f.read(2)
            value = struct.unpack(format+'h',rawdata)[0]
            #print value
            if value == 1: # no compression
              return 0
            else:
              return 1

        f.seek(ifd+numentries*12+2)
        rawdata = f.read(4)
        ifd = struct.unpack(format+'i',rawdata)[0]

    finally:
      f.close()

    # control should never reach this point
    assert 1==0

  # returns 0 for little endian 'II'
  # returns 1 for big endian 'MM'
  def getEndian(self):
    return self.endian

  def readHeader(self,offset=1024):
    if not self.parameters:
      if self.getEndian(): format = '>'
      else: format = '<'

      f = open(self.filename,"rb")

      try:
        f.seek(2464) # seek to file_comments
        file_comments = f.read(512) # read file_comments

        parameters={}
        for item in [('Detector Serial Number','DETECTOR_SN')]: #expected integers
          pattern = re.compile(item[0]+' = '+r'(.*)')
          matches = pattern.findall(file_comments)
          if len(matches) > 0:
            parameters[item[1]] = int(matches[-1])
          else:
            parameters[item[1]] = 0

        f.seek(offset+28)
        rawdata = f.read(8)
        header_byte_order,data_byte_order = struct.unpack(format+'ii',rawdata)

        f.seek(offset+80)
        rawdata = f.read(8)
        parameters['SIZE1'],parameters['SIZE2'] = struct.unpack(format+'ii',rawdata)
        assert parameters['SIZE1'] == parameters['SIZE2']
        f.seek(offset+88)
        rawdata = f.read(4)
        self.depth = struct.unpack(format+'i',rawdata)[0]

        f.seek(offset+104)
        rawdata = f.read(4)
        parameters['CCD_IMAGE_SATURATION'] = struct.unpack(format+'i',rawdata)[0]

        f.seek(offset+116)
        rawdata = f.read(12)
        origin,orientation,view_direction = struct.unpack(format+'iii',rawdata)

        f.seek(offset+696)
        rawdata = f.read(4)
        start_xtal_to_detector = struct.unpack(format+'i',rawdata)[0]/1000.
        f.seek(offset+728)
        rawdata = f.read(4)
        end_xtal_to_detector = struct.unpack(format+'i',rawdata)[0]/1000.
        #assert start_xtal_to_detector == end_xtal_to_detector
        #that assertion would've been nice but ESRF BM14 frames fail; instead:
        #assert the distance is greater than zero in _read_header_asserts
        parameters['DISTANCE'] = start_xtal_to_detector

        f.seek(offset+772)
        rawdata = f.read(8)
        pixelsize_x,pixelsize_y = struct.unpack(format+'ii',rawdata)
        assert pixelsize_x == pixelsize_y
        parameters['PIXEL_SIZE'] = pixelsize_x*1.0e-6 # convert from nano to milli

        f.seek(offset+644)
        rawdata = f.read(8)
        beam_center_x,beam_center_y = struct.unpack(format+'ii',rawdata)
        parameters['BEAM_CENTER_X'] = beam_center_x/1000.*parameters['PIXEL_SIZE']
        parameters['BEAM_CENTER_Y'] = beam_center_y/1000.*parameters['PIXEL_SIZE']

        # ----- phi analysis
        f.seek(offset+684)
        rawdata = f.read(4)
        parameters['OSC_START'] = struct.unpack(format+'i',rawdata)[0]/1000.

        f.seek(offset+716)
        rawdata = f.read(4)
        end_phi = struct.unpack(format+'i',rawdata)[0]/1000.

        #parameters['OSC_RANGE'] = end_phi - parameters['OSC_START']
        #would have thought this would work; but turns out unreliable because
        # software doesn't always fill in the end_phi

        # ----- rotation analysis
        f.seek(offset+736)
        rawdata = f.read(4)
        rotation_range = struct.unpack(format+'i',rawdata)[0]/1000.
        parameters['OSC_RANGE'] = rotation_range

        f.seek(offset+732)
        rawdata = f.read(4)
        rotation_axis = struct.unpack(format+'i',rawdata)[0]
        print rotation_axis
	#assert rotation_axis == 4 # if it isn't phi; go back and recode to cover all cases

        # ----- omega analysis
        f.seek(offset+672)
        rawdata = f.read(4)
        parameters['OMEGA_START'] = struct.unpack(format+'i',rawdata)[0]/1000.

        f.seek(offset+704)
        rawdata = f.read(4)
        parameters['OMEGA_END'] = struct.unpack(format+'i',rawdata)[0]/1000.

        if rotation_axis == 4: # rotation axis is phi
          pass
        #elif rotation_axis == 1: # rotation about omega
	elif rotation_axis in [1,2]:
          parameters['OSC_START'] = parameters['OMEGA_START']

        f.seek(offset+668)
        rawdata = f.read(4)
        start_twotheta = struct.unpack(format+'i',rawdata)[0]/1000.
        f.seek(offset+700)
        rawdata = f.read(4)
        end_twotheta = struct.unpack(format+'i',rawdata)[0]/1000.
        self._assert_matching_twothetas = start_twotheta == end_twotheta
        parameters['TWOTHETA'] = start_twotheta

        f.seek(offset+908)
        rawdata = f.read(4)
        parameters['WAVELENGTH'] = struct.unpack(format+'i',rawdata)[0]*1.0e-5 # convert from femto to angstrom

        f.seek(offset+656)
        rawdata = f.read(4)
        parameters['TIME'] = struct.unpack(format+'i',rawdata)[0]/1000.

        #Put date stamp together
        f.seek(2368)
        rawdata = f.read(32)
        mo = rawdata[:2]
        d = rawdata[2:4]
        h = rawdata[4:6]
        mi = rawdata[6:8]
        y = rawdata[8:12]
        s = rawdata[13:15]
        time = ':'.join((h,mi,s))
        date =  '-'.join((y,mo,d))
        parameters['DATE'] = 'T'.join((date,time))

      finally:
        f.close()

      self.parameters=parameters

      self._read_header_asserts()

  def _read_header_asserts(self):
    ''' move a couple asserts here that aren't always desireable for un-initialized data '''
    assert self.parameters['DISTANCE'] > 0
    assert self._assert_matching_twothetas

  def dataoffset(self):
    return 4096

  def integerdepth(self):
    return self.depth

def MarReadHeader(image,
                  mode=None,
                  run_id=None,
                  place_in_run=None,
                  logger=False):

  """
  Given a full file name for a Piltus image (as a string), read the header and
  return a dict with all the header info
  """
  # print "determine_flux %s" % image
  if logger:
    logger.debug('MarReadHeader ' % image)

  m = MARImage(image)
  m.read()
  header = m.parameters

  header_items = { 'detector_sn'  : str(m.serial_number),
                   'date'         : str(header['DATE']),
                   'pixel_size'   : float(m.pixel_size),
                   'time'         : float(header['TIME']),
                   #'period'       : ("^# Exposure_period\s*([\d\.]+) s", lambda x: float(x)),
                   'count_cutoff' : int(m.saturation),
                   'wavelength'   : float(m.wavelength),
                   'distance'     : float(m.distance),
                   #'transmission' : ("^# Filter_transmission\s*([\d\.]+)", lambda x: float(x)),
                   'osc_start'    : float(m.osc_start),
                   'osc_range'    : float(header['OSC_RANGE']),
                   'twotheta'     : float(header['TWOTHETA']),
                   'size1'        : int(header['SIZE1']),
                   'size2'        : int(header['SIZE2']),
                   'omega_end'    : float(header['OMEGA_END']),
                   'omega_start'  : float(header['OMEGA_START']),
                   #Flipped!!
                   'beam_center_x': float(header['BEAM_CENTER_Y']),
                   'beam_center_y': float(header['BEAM_CENTER_X']),
                   #'vendortype'   : m.vendortype,
                   }

  #Figure out which MAR detector was used
  #if m.vendortype == 'MARCCD':
  if header_items['size1'] == 3840:
    det = 'ray300'
  else:
    det = 'mar300'

  #try:
  #tease out the info from the file name
  base = os.path.basename(image)
  ubc = base.count('_')
  if ubc == 0:
    ip =  base.split(".")[0]
    rn = None
  elif ubc == 1:
    ip = base.split("_")[0]
    rn = base[base.rfind("_")+1:base.rfind('.')]
  else:
    ip = "_".join(base.split("_")[0:-1])
    rn = base[base.rfind("_")+1:base.rfind('.')]

  parameters = {'fullname'     : image,
                'detector'     : det,
                'directory'    : os.path.dirname(image),
                #'image_prefix' : "_".join(base.split("_")[0:-2]),
                'image_prefix' : str(ip),
		#'run_number'   : str(base.split("_")[-1]),
                'run_number'   : str(rn),
		'image_number' : int(base.split(".")[-1]),
                'axis'         : 'omega',
                'collect_mode' : mode,
                'run_id'       : run_id,
                'place_in_run' : place_in_run,
                }
  """
  for label,pat in header_items.iteritems():
    # print label
    pattern = re.compile(pat[0], re.MULTILINE)
    matches = pattern.findall(header)
    if len(matches) > 0:
      parameters[label] = pat[1](matches[-1])
    else:
      parameters[label] = None
  """
  parameters.update(header_items)
  return(parameters)
  """
  except:
    if logger:
      logger.exception('Error reading the header for image %s' % image)
  """
if __name__ == "__main__":
    """
    #testing
    #Test PilatusMonitor
    def notify(input):
        print input
    P = PilatusMonitor(beamline='C',notify=notify,reconnect=None,logger=None)
    """
    #Test the header reading
    test_image = '/panfs/panfs0.localdomain/raw/BM_16_03_03_staff_staff/Tryp/SERX12_Pn1_r1_1.0001'
    header = MarReadHeader(test_image)
    import pprint
    P = pprint.PrettyPrinter()
    P.pprint(header)
<commit_msg>cleaned up<commit_after>from __future__ import division
import re,struct
from iotbx.detectors.detectorbase import DetectorImageBase
import os,re,time
import logging, logging.handlers

class MARImage(DetectorImageBase):
  def __init__(self,filename):
    DetectorImageBase.__init__(self,filename)
    self.vendortype = "MARCCD"

    byte_order = str(open(self.filename,"rb").read(2))
    if byte_order == 'II':
      self.endian = 0
    else:
      self.endian = 1

    assert not self.isCompressed()

  def isCompressed(self):
    if self.getEndian(): format = '>'
    else: format = '<'

    f = open(self.filename,"rb")
    try:
      f.seek(4)
      rawdata = f.read(4)
      ifd = struct.unpack(format+'i',rawdata)[0]

      # if ifd is 0 then there are no more Image File Directories
      while ifd:
        f.seek(ifd)

        rawdata = f.read(2)
        # get the number of directory entries in the IFD
        numentries = struct.unpack(format+'h',rawdata)[0]

        # search for compression tag
        for x in range(numentries):
          f.seek(ifd+x*12+2)
          rawdata = f.read(2)
          tag = struct.unpack(format+'h',rawdata)[0]
          if tag == 259: # value of compression tag
            f.seek(ifd+x*12+2+8) # seek to value offset
            # value is left justified in 4 byte field
            # read two bytes so can unpack as short
            rawdata = f.read(2)
            value = struct.unpack(format+'h',rawdata)[0]
            #print value
            if value == 1: # no compression
              return 0
            else:
              return 1

        f.seek(ifd+numentries*12+2)
        rawdata = f.read(4)
        ifd = struct.unpack(format+'i',rawdata)[0]

    finally:
      f.close()

    # control should never reach this point
    assert 1==0

  # returns 0 for little endian 'II'
  # returns 1 for big endian 'MM'
  def getEndian(self):
    return self.endian

  def readHeader(self,offset=1024):
    if not self.parameters:
      if self.getEndian(): format = '>'
      else: format = '<'

      f = open(self.filename,"rb")

      try:
        f.seek(2464) # seek to file_comments
        file_comments = f.read(512) # read file_comments

        parameters={}
        for item in [('Detector Serial Number','DETECTOR_SN')]: #expected integers
          pattern = re.compile(item[0]+' = '+r'(.*)')
          matches = pattern.findall(file_comments)
          if len(matches) > 0:
            parameters[item[1]] = int(matches[-1])
          else:
            parameters[item[1]] = 0

        f.seek(offset+28)
        rawdata = f.read(8)
        header_byte_order,data_byte_order = struct.unpack(format+'ii',rawdata)

        f.seek(offset+80)
        rawdata = f.read(8)
        parameters['SIZE1'],parameters['SIZE2'] = struct.unpack(format+'ii',rawdata)
        assert parameters['SIZE1'] == parameters['SIZE2']
        f.seek(offset+88)
        rawdata = f.read(4)
        self.depth = struct.unpack(format+'i',rawdata)[0]

        f.seek(offset+104)
        rawdata = f.read(4)
        parameters['CCD_IMAGE_SATURATION'] = struct.unpack(format+'i',rawdata)[0]

        f.seek(offset+116)
        rawdata = f.read(12)
        origin,orientation,view_direction = struct.unpack(format+'iii',rawdata)

        f.seek(offset+696)
        rawdata = f.read(4)
        start_xtal_to_detector = struct.unpack(format+'i',rawdata)[0]/1000.
        f.seek(offset+728)
        rawdata = f.read(4)
        end_xtal_to_detector = struct.unpack(format+'i',rawdata)[0]/1000.
        #assert start_xtal_to_detector == end_xtal_to_detector
        #that assertion would've been nice but ESRF BM14 frames fail; instead:
        #assert the distance is greater than zero in _read_header_asserts
        parameters['DISTANCE'] = start_xtal_to_detector

        f.seek(offset+772)
        rawdata = f.read(8)
        pixelsize_x,pixelsize_y = struct.unpack(format+'ii',rawdata)
        assert pixelsize_x == pixelsize_y
        parameters['PIXEL_SIZE'] = pixelsize_x*1.0e-6 # convert from nano to milli

        f.seek(offset+644)
        rawdata = f.read(8)
        beam_center_x,beam_center_y = struct.unpack(format+'ii',rawdata)
        parameters['BEAM_CENTER_X'] = beam_center_x/1000.*parameters['PIXEL_SIZE']
        parameters['BEAM_CENTER_Y'] = beam_center_y/1000.*parameters['PIXEL_SIZE']

        # ----- phi analysis
        f.seek(offset+684)
        rawdata = f.read(4)
        parameters['OSC_START'] = struct.unpack(format+'i',rawdata)[0]/1000.

        f.seek(offset+716)
        rawdata = f.read(4)
        end_phi = struct.unpack(format+'i',rawdata)[0]/1000.

        #parameters['OSC_RANGE'] = end_phi - parameters['OSC_START']
        #would have thought this would work; but turns out unreliable because
        # software doesn't always fill in the end_phi

        # ----- rotation analysis
        f.seek(offset+736)
        rawdata = f.read(4)
        rotation_range = struct.unpack(format+'i',rawdata)[0]/1000.
        parameters['OSC_RANGE'] = rotation_range

        f.seek(offset+732)
        rawdata = f.read(4)
        rotation_axis = struct.unpack(format+'i',rawdata)[0]
        print rotation_axis
	#assert rotation_axis == 4 # if it isn't phi; go back and recode to cover all cases

        # ----- omega analysis
        f.seek(offset+672)
        rawdata = f.read(4)
        parameters['OMEGA_START'] = struct.unpack(format+'i',rawdata)[0]/1000.

        f.seek(offset+704)
        rawdata = f.read(4)
        parameters['OMEGA_END'] = struct.unpack(format+'i',rawdata)[0]/1000.

        if rotation_axis == 4: # rotation axis is phi
          pass
        #elif rotation_axis == 1: # rotation about omega
	elif rotation_axis in [1,2]:
          parameters['OSC_START'] = parameters['OMEGA_START']

        f.seek(offset+668)
        rawdata = f.read(4)
        start_twotheta = struct.unpack(format+'i',rawdata)[0]/1000.
        f.seek(offset+700)
        rawdata = f.read(4)
        end_twotheta = struct.unpack(format+'i',rawdata)[0]/1000.
        self._assert_matching_twothetas = start_twotheta == end_twotheta
        parameters['TWOTHETA'] = start_twotheta

        f.seek(offset+908)
        rawdata = f.read(4)
        parameters['WAVELENGTH'] = struct.unpack(format+'i',rawdata)[0]*1.0e-5 # convert from femto to angstrom

        f.seek(offset+656)
        rawdata = f.read(4)
        parameters['TIME'] = struct.unpack(format+'i',rawdata)[0]/1000.

        #Put date stamp together
        f.seek(2368)
        rawdata = f.read(32)
        mo = rawdata[:2]
        d = rawdata[2:4]
        h = rawdata[4:6]
        mi = rawdata[6:8]
        y = rawdata[8:12]
        s = rawdata[13:15]
        time = ':'.join((h,mi,s))
        date =  '-'.join((y,mo,d))
        parameters['DATE'] = 'T'.join((date,time))

      finally:
        f.close()

      self.parameters=parameters

      self._read_header_asserts()

  def _read_header_asserts(self):
    ''' move a couple asserts here that aren't always desireable for un-initialized data '''
    assert self.parameters['DISTANCE'] > 0
    assert self._assert_matching_twothetas

  def dataoffset(self):
    return 4096

  def integerdepth(self):
    return self.depth

def MarReadHeader(image,
                  mode=None,
                  run_id=None,
                  place_in_run=None,
                  logger=False):

  """
  Given a full file name for a Piltus image (as a string), read the header and
  return a dict with all the header info
  """
  # print "determine_flux %s" % image
  if logger:
    logger.debug('MarReadHeader ' % image)

  m = MARImage(image)
  m.read()
  header = m.parameters

  header_items = { 'detector_sn'  : str(m.serial_number),
                   'date'         : str(header['DATE']),
                   'pixel_size'   : float(m.pixel_size),
                   'time'         : float(header['TIME']),
                   #'period'       : ("^# Exposure_period\s*([\d\.]+) s", lambda x: float(x)),
                   'count_cutoff' : int(m.saturation),
                   'wavelength'   : float(m.wavelength),
                   'distance'     : float(m.distance),
                   #'transmission' : ("^# Filter_transmission\s*([\d\.]+)", lambda x: float(x)),
                   'osc_start'    : float(m.osc_start),
                   'osc_range'    : float(header['OSC_RANGE']),
                   'twotheta'     : float(header['TWOTHETA']),
                   'size1'        : int(header['SIZE1']),
                   'size2'        : int(header['SIZE2']),
                   'omega_end'    : float(header['OMEGA_END']),
                   'omega_start'  : float(header['OMEGA_START']),
                   #Flipped!!
                   'beam_center_x': float(header['BEAM_CENTER_Y']),
                   'beam_center_y': float(header['BEAM_CENTER_X']),
                   #'vendortype'   : m.vendortype,
                   }

  #Figure out which MAR detector was used
  if header_items['size1'] == 3840:
    det = 'ray300'
  else:
    det = 'mar300'

  #try:
  #tease out the info from the file name
  base = os.path.basename(image)
  ubc = base.count('_')
  if ubc == 0:
    ip =  base.split(".")[0]
    rn = None
  elif ubc == 1:
    ip = base.split("_")[0]
    rn = base[base.rfind("_")+1:base.rfind('.')]
  else:
    ip = "_".join(base.split("_")[0:-1])
    rn = base[base.rfind("_")+1:base.rfind('.')]

  parameters = {'fullname'     : image,
                'detector'     : det,
                'directory'    : os.path.dirname(image),
                'image_prefix' : str(ip),
                'run_number'   : str(rn),
		'image_number' : int(base.split(".")[-1]),
                'axis'         : 'omega',
                'collect_mode' : mode,
                'run_id'       : run_id,
                'place_in_run' : place_in_run,
                }
  """
  for label,pat in header_items.iteritems():
    # print label
    pattern = re.compile(pat[0], re.MULTILINE)
    matches = pattern.findall(header)
    if len(matches) > 0:
      parameters[label] = pat[1](matches[-1])
    else:
      parameters[label] = None
  """
  parameters.update(header_items)
  return(parameters)
  """
  except:
    if logger:
      logger.exception('Error reading the header for image %s' % image)
  """
if __name__ == "__main__":
    """
    #testing
    #Test PilatusMonitor
    def notify(input):
        print input
    P = PilatusMonitor(beamline='C',notify=notify,reconnect=None,logger=None)
    """
    #Test the header reading
    test_image = '/panfs/panfs0.localdomain/raw/BM_16_03_03_staff_staff/Tryp/SERX12_Pn1_r1_1.0001'
    header = MarReadHeader(test_image)
    import pprint
    P = pprint.PrettyPrinter()
    P.pprint(header)
<|endoftext|>