<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#    This file is part of FLOSS Talks.
#
#    FLOSS Talks is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    FLOSS Talks is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with FLOSS Talks.  If not, see <http://www.gnu.org/licenses/>.
import datetime
from django.db import models

class Project(models.Model):
    name = models.CharField(max_length=100)
    nice_url = models.CharField(max_length=100, null=True, blank=True)
    description = models.TextField()
    description_source = models.URLField("Source of the description", null=True, blank=True)
    url = models.URLField("Website", null=True, blank=True)
    #TODO: Create a mechanism to retrieve a copy of the logo on the server
    # to prevent depending on external websites
    logo_url = models.URLField(null=True, blank=True)
    ohloh_id = models.IntegerField("Ohloh ID", null=True, blank=True)
    skip_ohloh = models.NullBooleanField("Skip looking on Ohloh")
    STATUSES = (
        ('NW', 'New'),
        ('PD', 'Pending'),
        ('VF', 'Verified'),
        ('HD', 'Hidden'),
    )
    status = models.CharField(max_length=2, choices=STATUSES, default="VF")

    def __unicode__(self):
        return self.name

    def number_of_resources(self):
        return self.resource_set.count()

class Series(models.Model):
    name = models.CharField(max_length=100)
    nice_url = models.CharField(max_length=100, null=True, blank=True)
    description = models.TextField()
    url = models.URLField("Website", null=True, blank=True)
    MODES = (
        ('AU', 'Auto-add'),
        ('QU', 'Queue'),
    )
    mode = models.CharField(max_length=2, choices=MODES, default="QU")

    class Meta:
        verbose_name_plural = "series"

    def __unicode__(self):
        return self.name

    def number_of_resources(self):
        return self.resource_set.count()

class Resource(models.Model):
    name = models.CharField(max_length=100)
    description = models.TextField()
    url = models.URLField("Web page", null=True, blank=True)
    projects = models.ManyToManyField(Project)
    series = models.ForeignKey(Series, null=True, blank=True)
    STATUSES = (
        ('NW', 'New'),
        ('SG', 'Suggested'),
        ('VF', 'Verified'),
        ('PD', 'Pending'),
        ('IG', 'Ignored'),
    )
    status = models.CharField(max_length=2, choices=STATUSES, default="VF")
    external_id = models.CharField(max_length=200)
    #TODO: use length
    length = models.CharField(max_length=10, null=True, blank=True)
    #TODO: use pub_date
    pub_date = models.DateTimeField("Date published"
                                    , default=datetime.datetime.now())

    def __unicode__(self):
        return self.name

    def list_of_projects(self):
        projects = []
        for p in self.projects.get_query_set():
            projects.append(p.name)
        return ", ".join(projects)

class GenericURL(models.Model):
    MEDIA_TYPES = (
        ('Audio', (
                ('AUS', 'Audio'),
                ('AUL', 'Audio (low quality)'),
                ('AUH', 'Audio (high quality)'),
            )
        ),
        ('Video', (
                ('VIS', 'Video'),
                ('VIL', 'Video (low quality)'),
                ('VIH', 'Video (high quality)'),
            )
        ),
    )
    media_type = models.CharField(max_length=3, choices=MEDIA_TYPES)
    FORMATS = (
        ('Audio', (
                ('UNA', 'Unspecified audio format'),
                ('OGA', 'Ogg'),
                ('MP3', 'MP3'),
            )
        ),
        ('Video', (
                ('UNV', 'Unspecified video format'),
                ('OGV', 'Ogg'),
                ('MP4', 'MP4'),
            )
        ),
    )
    format = models.CharField(max_length=3, choices=FORMATS)
    url = models.URLField("Resource download URL")

class SeriesFeedURL(GenericURL):
    series = models.ForeignKey(Series)

    class Meta:
        verbose_name = "series' feed URL"
        verbose_name_plural = "series' feed URLs"

    def __unicode__(self):
        return "%s's %s feed" % (self.series.name, self.get_media_type_display())

class ResourceDownloadURL(GenericURL):
    resource = models.ForeignKey(Resource)

    class Meta:
        verbose_name = "resource download URL"
        verbose_name_plural = "resource download URLs"

    def __unicode__(self):
        return "%s for %s" % (self.get_media_type_display(), self.resource.name)

<commit_msg>Add Speex audio format<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#    This file is part of FLOSS Talks.
#
#    FLOSS Talks is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    FLOSS Talks is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with FLOSS Talks.  If not, see <http://www.gnu.org/licenses/>.
import datetime
from django.db import models

class Project(models.Model):
    name = models.CharField(max_length=100)
    nice_url = models.CharField(max_length=100, null=True, blank=True)
    description = models.TextField()
    description_source = models.URLField("Source of the description", null=True, blank=True)
    url = models.URLField("Website", null=True, blank=True)
    #TODO: Create a mechanism to retrieve a copy of the logo on the server
    # to prevent depending on external websites
    logo_url = models.URLField(null=True, blank=True)
    ohloh_id = models.IntegerField("Ohloh ID", null=True, blank=True)
    skip_ohloh = models.NullBooleanField("Skip looking on Ohloh")
    STATUSES = (
        ('NW', 'New'),
        ('PD', 'Pending'),
        ('VF', 'Verified'),
        ('HD', 'Hidden'),
    )
    status = models.CharField(max_length=2, choices=STATUSES, default="VF")

    def __unicode__(self):
        return self.name

    def number_of_resources(self):
        return self.resource_set.count()

class Series(models.Model):
    name = models.CharField(max_length=100)
    nice_url = models.CharField(max_length=100, null=True, blank=True)
    description = models.TextField()
    url = models.URLField("Website", null=True, blank=True)
    MODES = (
        ('AU', 'Auto-add'),
        ('QU', 'Queue'),
    )
    mode = models.CharField(max_length=2, choices=MODES, default="QU")

    class Meta:
        verbose_name_plural = "series"

    def __unicode__(self):
        return self.name

    def number_of_resources(self):
        return self.resource_set.count()

class Resource(models.Model):
    name = models.CharField(max_length=100)
    description = models.TextField()
    url = models.URLField("Web page", null=True, blank=True)
    projects = models.ManyToManyField(Project)
    series = models.ForeignKey(Series, null=True, blank=True)
    STATUSES = (
        ('NW', 'New'),
        ('SG', 'Suggested'),
        ('VF', 'Verified'),
        ('PD', 'Pending'),
        ('IG', 'Ignored'),
    )
    status = models.CharField(max_length=2, choices=STATUSES, default="VF")
    external_id = models.CharField(max_length=200)
    #TODO: use length
    length = models.CharField(max_length=10, null=True, blank=True)
    #TODO: use pub_date
    pub_date = models.DateTimeField("Date published"
                                    , default=datetime.datetime.now())

    def __unicode__(self):
        return self.name

    def list_of_projects(self):
        projects = []
        for p in self.projects.get_query_set():
            projects.append(p.name)
        return ", ".join(projects)

class GenericURL(models.Model):
    MEDIA_TYPES = (
        ('Audio', (
                ('AUS', 'Audio'),
                ('AUL', 'Audio (low quality)'),
                ('AUH', 'Audio (high quality)'),
            )
        ),
        ('Video', (
                ('VIS', 'Video'),
                ('VIL', 'Video (low quality)'),
                ('VIH', 'Video (high quality)'),
            )
        ),
    )
    media_type = models.CharField(max_length=3, choices=MEDIA_TYPES)
    FORMATS = (
        ('Audio', (
                ('UNA', 'Unspecified audio format'),
                ('OGA', 'Ogg'),
                ('SPX', 'Speex'),
                ('MP3', 'MP3'),
            )
        ),
        ('Video', (
                ('UNV', 'Unspecified video format'),
                ('OGV', 'Ogg'),
                ('MP4', 'MP4'),
            )
        ),
    )
    format = models.CharField(max_length=3, choices=FORMATS)
    url = models.URLField("Resource download URL")

class SeriesFeedURL(GenericURL):
    series = models.ForeignKey(Series)

    class Meta:
        verbose_name = "series' feed URL"
        verbose_name_plural = "series' feed URLs"

    def __unicode__(self):
        return "%s's %s feed" % (self.series.name, self.get_media_type_display())

class ResourceDownloadURL(GenericURL):
    resource = models.ForeignKey(Resource)

    class Meta:
        verbose_name = "resource download URL"
        verbose_name_plural = "resource download URLs"

    def __unicode__(self):
        return "%s for %s" % (self.get_media_type_display(), self.resource.name)

<|endoftext|>


<commit_before>"""

.. currentmodule:: cmac

X-SAPR CMAC function for determing gate ids, detect second trip returns and
more.

Functions
=========

.. autosummary::
    :toctree: generated/

    config_xsapr_i4
    config_xsapr_i5
    config_xsapr_i6
    cmac
    quicklooks
    snr_and_sounding
    get_texture
    cum_score_fuzzy_logic
    do_my_fuzz
    return_csu_kdp
    retrieve_qvp
    xsapr_clutter

"""

from .config import config_xsapr_i4, config_xsapr_i5, config_xsapr_i6
from .cmac_xsapr import cmac
from .cmac_quicklooks import quicklooks
from .processing_code import snr_and_sounding, do_my_fuzz
from .processing_code import get_texture, cum_score_fuzzy_logic
from .processing_code import return_csu_kdp, retrieve_qvp
from .data_catalouging import get_sounding_times, get_sounding_file_name
from .xsapr_clutter import xsapr_clutter

__all__ = [s for s in dir() if not s.startswith('_')]
<commit_msg>MAINT: Updated all imports to reflect new module names.<commit_after>"""
====
CMAC
====

CMAC functions for determing gate ids, detect second trip returns and
more.

    config_xsapr_i4
    config_xsapr_i5
    config_xsapr_i6
    cmac
    quicklooks
    snr_and_sounding
    get_texture
    cum_score_fuzzy_logic
    do_my_fuzz
    return_csu_kdp
    retrieve_qvp
    tall_clutter

"""

from .config import config_xsapr_i4, config_xsapr_i5, config_xsapr_i6
from .cmac_radar import cmac
from .cmac_quicklooks import quicklooks
from .cmac_processing import snr_and_sounding, do_my_fuzz
from .cmac_processing import get_texture, cum_score_fuzzy_logic
from .cmac_processing import return_csu_kdp, retrieve_qvp
from .data_catalouging import get_sounding_times, get_sounding_file_name
from .radar_clutter import tall_clutter

__all__ = [s for s in dir() if not s.startswith('_')]
<|endoftext|>


<commit_before>
#
# Copyright 2014-2015 University of Southern California
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
#

"""Volume data support.

Volume data is loaded as an ND-array with axes ZYXC and shape
(D,H,W,C) using available volumetric image file readers.

The ImageCropper class encapsulates the state necessary to:

  1. load a volume image file

  2. prepare 3D texture data of an appropriate texture storage format

  3. prepare 3D bounding box geometry

The ImageCropper also understands image file metadata returned by the
image reader to configure voxel aspect ratio for a spatial
interpretation of the volume data.

"""

import os
import numpy as np
import math

from vispy import gloo

from .util import load_image, bin_reduce
from .geometry import make_cube_clipped

class wrapper (np.ndarray):
    """Subtype to allow extra attributes"""
    pass

class ImageManager (object):

    def __init__(self, filename, reform_data=None):
        I, self.meta = load_image(filename)

        try:
            voxel_size = I.micron_spacing
        except AttributeError:
            voxel_size = (1., 1., 1.)

        try:
            view_grid_microns = tuple(map(float, os.getenv('ZYX_VIEW_GRID').split(",")))
            assert len(view_grid_microns) == 3
        except:
            view_grid_microns = (0.25, 0.25, 0.25)
        print "Using %s micron view grid. Override with ZYX_VIEW_GRID='float,float,float'" % (view_grid_microns,)

        view_reduction = tuple(map(lambda vs, ps: max(int(ps/vs), 1), voxel_size, view_grid_microns))

        # temporary pre-processing hacks to investigate XY-correlated sensor artifacts...
        try:
            ntile = int(os.getenv('ZNOISE_PERCENTILE'))
            I = I.force()
            zerofield = np.percentile(I, ntile, axis=1)
            print 'Image %d percentile value over Z-axis ranges [%f,%f]' % (ntile, zerofield.min(), zerofield.max())
            I -= zerofield
            print 'Image offset by %d percentile XY value to new range [%f,%f]' % (ntile, I.min(), I.max())
            zero = float(os.getenv('ZNOISE_ZERO_LEVEL'), 0)
            I = I * (I >= 0.)
            print 'Image clamped to range [%f,%f]' % (I.min(), I.max())
        except:
            pass

        I = I.transpose(1,2,3,0)
        if isinstance(I, np.ndarray):
            # temporarily maintain micron_spacing after percentile hack above...
            I2 = wrapper(shape=I.shape, dtype=I.dtype)
            I2[:,:,:,:] = I[:,:,:,:]
            I = I2
            setattr(I, 'micron_spacing', voxel_size)

        if reform_data is not None:
            I = reform_data(I, self.meta, view_reduction)

        voxel_size = map(lambda a, b: a*b, voxel_size, view_reduction)
        self.Zaspect = voxel_size[0] / voxel_size[2]

        # allow user to select a bounding box region of interest
        bbox = os.getenv('ZYX_SLICE')
        if bbox:
            bbox = bbox.split(",")
            assert len(bbox) == 3, "ZYX_SLICE must have comma-separated slices for 3 axes Z,Y,X"
            bbox = [s.split(":") for s in bbox]
            for p in bbox:
                assert len(p) == 2, "ZYX_SLICE must have START:STOP pair for each axial slice"
            for start, stop in bbox:
                assert int(start) >= 0, "ZYX_SLICE START must be 0 or greater"
                assert int(stop) >= 1, "ZYX_SLICE STOP must be 1 or greater"
            bbox = tuple(map(
                lambda slc, vr: slice(int(slc[0])/vr, int(slc[1])/vr),
                bbox,
                view_reduction
            )) + (slice(None),)
            datamin = I.min()
            datamax = I.max()
            I = I[bbox]
            I[0,0,0,0] = datamin
            I[-1,-1,-1,0] = datamax

        if I.shape[2] % 16:
            # trim for 16-pixel row alignment
            slc = tuple([
                slice(None),
                slice(None),
                slice(0,I.shape[2]/16*16),
                slice(None)
            ])
            if hasattr(I, 'lazyget'):
                I = I.lazyget(slc)
            else:
                I = I[slc]

        self.data = I

        self.last_channels = None
        self.channels = None
        self.set_view()

    def min_pixel_step_size(self, outtexture=None):
        if outtexture is not None:
            D, H, W, C = outtexture.shape
        else:
            D, H, W, C = self.data.shape

        span = max(W, H, D*self.Zaspect)

        return 1./span

    def set_view(self, anti_view=None, channels=None):
        if anti_view is not None:
            self.anti_view = anti_view
        if channels is not None:
            # use caller-specified sequence of channels
            assert type(channels) is tuple
            assert len(channels) <= 4
            self.channels = channels
        else:
            # default to first N channels u to 4 for RGBA direct mapping
            self.channels = tuple(range(0, min(self.data.shape[3], 4)))
        for c in self.channels:
            assert c >= 0
            assert c < self.data.shape[3]

    def _get_texture3d_format(self):
        I0 = self.data
        nc = len(self.channels)

        if I0.dtype == np.uint8:
            bps = 1
        elif I0.dtype == np.uint16 or I0.dtype == np.int16:
            bps = 2
        else:
            assert I0.dtype == np.float16 or I0.dtype == np.float32
            bps = 2
            #bps = 4

        print (nc, bps)
        return {
            (1,1): ('luminance', 'red'),
            (1,2): ('luminance', 'r16f'),
            (1,4): ('luminance', 'r16f'),
            (2,1): ('rg', 'rg'),
            (2,2): ('rg', 'rg32f'),
            (2,4): ('rg', 'rg32f'),
            (3,1): ('rgb', 'rgb'),
            (3,2): ('rgb', 'rgb16f'),
            (3,4): ('rgb', 'rgb16f'),
            (4,1): ('rgba', 'rgba'),
            (4,2): ('rgba', 'rgba16f'),
            (4,4): ('rgba', 'rgba16f')
        }[(nc, bps)]

    def get_texture3d(self, outtexture=None):
        """Pack N-channel image data into R, RG, RGB, RGBA Texture3D using self.channels projection.

           outtexture:
             None:     allocate new Texture3D
             not None: use existing Texture3D

           sets data in outtexture and returns the texture.
        """
        I0 = self.data

        # choose size for texture data
        D, H, W = self.data.shape[0:3]
        C = len(self.channels)

        if outtexture is None:
            format, internalformat = self._get_texture3d_format()
            print 'allocating texture3D', (D, H, W, C), internalformat
            outtexture = gloo.Texture3D(shape=(D, H, W, C), format=format, internalformat=internalformat)
        elif self.last_channels == self.channels:
            print 'reusing texture'
            return outtexture
        else:
            print 'regenerating texture'

        print (D, H, W, C), '<-', I0.shape, list(self.channels), I0.dtype

        # normalize for OpenGL [0,1.0] or [0,2**N-1] and zero black-level
        maxval = I0.max()
        minval = I0.min()
        scale = 1.0/(float(maxval) - float(minval))
        if I0.dtype == np.uint8 or I0.dtype == np.int8:
            tmpout = np.zeros((D, H, W, C), dtype=np.uint8)
            scale *= float(2**8-1)
        else:
            assert I0.dtype == np.float16 or I0.dtype == np.float32 or I0.dtype == np.uint16 or I0.dtype == np.int16
            tmpout = np.zeros((D, H, W, C), dtype=np.uint16 )
            scale *= (2.0**16-1)

        # pack selected channels into texture
        for i in range(C):
            tmpout[:,:,:,i] = (I0[:,:,:,self.channels[i]].astype(np.float32) - minval) * scale

        self.last_channels = self.channels
        outtexture.set_data(tmpout)
        return outtexture

    def make_cube_clipped(self, dataplane=None):
        """Generate cube clipped against plane equation 4-tuple.

            Excludes semi-space beneath plane, i.e. with negative plane
            distance.  Omitting plane produces regular unclipped cube.
        """
        shape = self.data.shape[0:3]
        return make_cube_clipped(shape, self.Zaspect, 2, dataplane)

<commit_msg>process ZYX_SLICE env. var before running app's reform_data function<commit_after>
#
# Copyright 2014-2015 University of Southern California
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
#

"""Volume data support.

Volume data is loaded as an ND-array with axes ZYXC and shape
(D,H,W,C) using available volumetric image file readers.

The ImageCropper class encapsulates the state necessary to:

  1. load a volume image file

  2. prepare 3D texture data of an appropriate texture storage format

  3. prepare 3D bounding box geometry

The ImageCropper also understands image file metadata returned by the
image reader to configure voxel aspect ratio for a spatial
interpretation of the volume data.

"""

import os
import numpy as np
import math

from vispy import gloo

from .util import load_image, bin_reduce
from .geometry import make_cube_clipped

class wrapper (np.ndarray):
    """Subtype to allow extra attributes"""
    pass

class ImageManager (object):

    def __init__(self, filename, reform_data=None):
        I, self.meta = load_image(filename)

        try:
            voxel_size = I.micron_spacing
        except AttributeError:
            voxel_size = (1., 1., 1.)

        try:
            view_grid_microns = tuple(map(float, os.getenv('ZYX_VIEW_GRID').split(",")))
            assert len(view_grid_microns) == 3
        except:
            view_grid_microns = (0.25, 0.25, 0.25)
        print "Using %s micron view grid. Override with ZYX_VIEW_GRID='float,float,float'" % (view_grid_microns,)

        view_reduction = tuple(map(lambda vs, ps: max(int(ps/vs), 1), voxel_size, view_grid_microns))

        # temporary pre-processing hacks to investigate XY-correlated sensor artifacts...
        try:
            ntile = int(os.getenv('ZNOISE_PERCENTILE'))
            I = I.force()
            zerofield = np.percentile(I, ntile, axis=1)
            print 'Image %d percentile value over Z-axis ranges [%f,%f]' % (ntile, zerofield.min(), zerofield.max())
            I -= zerofield
            print 'Image offset by %d percentile XY value to new range [%f,%f]' % (ntile, I.min(), I.max())
            zero = float(os.getenv('ZNOISE_ZERO_LEVEL'), 0)
            I = I * (I >= 0.)
            print 'Image clamped to range [%f,%f]' % (I.min(), I.max())
        except:
            pass

        I = I.transpose(1,2,3,0)

        # allow user to select a bounding box region of interest
        bbox = os.getenv('ZYX_SLICE')
        if bbox:
            bbox = bbox.split(",")
            assert len(bbox) == 3, "ZYX_SLICE must have comma-separated slices for 3 axes Z,Y,X"
            bbox = [s.split(":") for s in bbox]
            for p in bbox:
                assert len(p) == 2, "ZYX_SLICE must have START:STOP pair for each axial slice"
            for start, stop in bbox:
                assert int(start) >= 0, "ZYX_SLICE START must be 0 or greater"
                assert int(stop) >= 1, "ZYX_SLICE STOP must be 1 or greater"
            bbox = tuple(map(
                lambda slc, vr: slice(int(slc[0])/vr, int(slc[1])/vr),
                bbox,
                view_reduction
            )) + (slice(None),)
            datamin = I.min()
            datamax = I.max()
            I = I[bbox]
            I[0,0,0,0] = datamin
            I[-1,-1,-1,0] = datamax

        if I.shape[2] % 16:
            # trim for 16-pixel row alignment
            slc = tuple([
                slice(None),
                slice(None),
                slice(0,I.shape[2]/16*16),
                slice(None)
            ])
            if hasattr(I, 'lazyget'):
                I = I.lazyget(slc)
            else:
                I = I[slc]

        if isinstance(I, np.ndarray):
            # temporarily maintain micron_spacing after munging above...
            I2 = wrapper(shape=I.shape, dtype=I.dtype)
            I2[:,:,:,:] = I[:,:,:,:]
            I = I2
            setattr(I, 'micron_spacing', voxel_size)

        if reform_data is not None:
            I = reform_data(I, self.meta, view_reduction)

        voxel_size = map(lambda a, b: a*b, voxel_size, view_reduction)
        self.Zaspect = voxel_size[0] / voxel_size[2]

        self.data = I

        self.last_channels = None
        self.channels = None
        self.set_view()

    def min_pixel_step_size(self, outtexture=None):
        if outtexture is not None:
            D, H, W, C = outtexture.shape
        else:
            D, H, W, C = self.data.shape

        span = max(W, H, D*self.Zaspect)

        return 1./span

    def set_view(self, anti_view=None, channels=None):
        if anti_view is not None:
            self.anti_view = anti_view
        if channels is not None:
            # use caller-specified sequence of channels
            assert type(channels) is tuple
            assert len(channels) <= 4
            self.channels = channels
        else:
            # default to first N channels u to 4 for RGBA direct mapping
            self.channels = tuple(range(0, min(self.data.shape[3], 4)))
        for c in self.channels:
            assert c >= 0
            assert c < self.data.shape[3]

    def _get_texture3d_format(self):
        I0 = self.data
        nc = len(self.channels)

        if I0.dtype == np.uint8:
            bps = 1
        elif I0.dtype == np.uint16 or I0.dtype == np.int16:
            bps = 2
        else:
            assert I0.dtype == np.float16 or I0.dtype == np.float32
            bps = 2
            #bps = 4

        print (nc, bps)
        return {
            (1,1): ('luminance', 'red'),
            (1,2): ('luminance', 'r16f'),
            (1,4): ('luminance', 'r16f'),
            (2,1): ('rg', 'rg'),
            (2,2): ('rg', 'rg32f'),
            (2,4): ('rg', 'rg32f'),
            (3,1): ('rgb', 'rgb'),
            (3,2): ('rgb', 'rgb16f'),
            (3,4): ('rgb', 'rgb16f'),
            (4,1): ('rgba', 'rgba'),
            (4,2): ('rgba', 'rgba16f'),
            (4,4): ('rgba', 'rgba16f')
        }[(nc, bps)]

    def get_texture3d(self, outtexture=None):
        """Pack N-channel image data into R, RG, RGB, RGBA Texture3D using self.channels projection.

           outtexture:
             None:     allocate new Texture3D
             not None: use existing Texture3D

           sets data in outtexture and returns the texture.
        """
        I0 = self.data

        # choose size for texture data
        D, H, W = self.data.shape[0:3]
        C = len(self.channels)

        if outtexture is None:
            format, internalformat = self._get_texture3d_format()
            print 'allocating texture3D', (D, H, W, C), internalformat
            outtexture = gloo.Texture3D(shape=(D, H, W, C), format=format, internalformat=internalformat)
        elif self.last_channels == self.channels:
            print 'reusing texture'
            return outtexture
        else:
            print 'regenerating texture'

        print (D, H, W, C), '<-', I0.shape, list(self.channels), I0.dtype

        # normalize for OpenGL [0,1.0] or [0,2**N-1] and zero black-level
        maxval = I0.max()
        minval = I0.min()
        scale = 1.0/(float(maxval) - float(minval))
        if I0.dtype == np.uint8 or I0.dtype == np.int8:
            tmpout = np.zeros((D, H, W, C), dtype=np.uint8)
            scale *= float(2**8-1)
        else:
            assert I0.dtype == np.float16 or I0.dtype == np.float32 or I0.dtype == np.uint16 or I0.dtype == np.int16
            tmpout = np.zeros((D, H, W, C), dtype=np.uint16 )
            scale *= (2.0**16-1)

        # pack selected channels into texture
        for i in range(C):
            tmpout[:,:,:,i] = (I0[:,:,:,self.channels[i]].astype(np.float32) - minval) * scale

        self.last_channels = self.channels
        outtexture.set_data(tmpout)
        return outtexture

    def make_cube_clipped(self, dataplane=None):
        """Generate cube clipped against plane equation 4-tuple.

            Excludes semi-space beneath plane, i.e. with negative plane
            distance.  Omitting plane produces regular unclipped cube.
        """
        shape = self.data.shape[0:3]
        return make_cube_clipped(shape, self.Zaspect, 2, dataplane)

<|endoftext|>


<commit_before>"""Walk functions for the Python walker.
"""

import sys
from .walker import get_fallback_walker_method_list
from .low import Limitation

__all__ = ['TooDeep', 'walk_object', 'walk_modules']

class TooDeep(Limitation):
    def __init__(self, what, depth=None):
        self.depth = depth
        self.what = what

    def __str__(self):
        return ("{0} (depth {1})".format(self.what, self.depth) if self.depth
                else self.what)

def walk_object(root, visitor=lambda o, d, p, n: d,
                fabricator=lambda o: None, combiner=lambda d1, d2: None,
                walkers=None, seen=None, maxdepth=100,
                root_parent=None, root_name=None):
    # Walk an object and its children.
    #
    # For an object which has not been seen already, each function in
    # walkers is called, and should return either an iterator over
    # (name, value) tuples, or None.  Each element of all the
    # iterators is walked recursively, with the results of the walk
    # being merged pairwise by the combiner function.  If the object
    # has already been seen, or there were no children (all of the
    # walkers return None), then some data is fabricated by calling
    # fabricator on the object.  Finally the visitor is called with
    # the object, the data from the children or the fabricator, the
    # parent object and the name in the parent, and its retuirn value
    # is the value of walk.
    #
    # For the root object the root_parent and root_name variables are
    # used as the parent and name-in-parent arguments.
    #
    # The seen argument is a set containing the id()s of object which
    # should be considered to have already been seen, or None.  The
    # ids of objects are used because very many objects are not
    # hashable, although their ids are (hello, Earth to Python?).
    #
    # maxdepth says how deep to go before raising a TooDeep exception.
    #
    # There are defaults for the visitor, combiner and fabricator
    # which are uninteresting but mean you can call walk(thing) and it
    # will work to exercise the walkers.
    #
    # Note that this is a post-order walk: the object is visited after
    # its children.
    #
    # Note that objects which have already been visited are visited
    # again if they appear as children of other nodes, but their
    # children are then *not* visited.  This allows the graph of
    # dependencies to be stitched together without following loops
    # forever.
    #
    if walkers is None:
        walkers = get_fallback_walker_method_list()

    if seen is None:
        seen = set()

    def walk_into(it, parent, name, depth):
        if depth >= maxdepth:
            raise TooDeep("too deep", depth)

        hashable = id(it)
        data = None
        seenp = False
        walked_children_p = False

        if hashable in seen:
            seenp = True
        else:
            seen.add(hashable)

        if not seenp:
            for walker in walkers:
                iterator = walker(it)
                if iterator:
                    for (n, v) in iterator:
                        walked_children_p = True
                        result = walk_into(v, it, n, depth + 1)
                        data = combiner(data, result) if data else result

        return visitor(it, data if walked_children_p else fabricator(it),
                       parent, name)

    return walk_into(root, root_parent, root_name, 0)

def walk_modules(modules=sys.modules,
                 visitor=lambda o, d, p, n: d,
                 fabricator=lambda o: None, combiner=lambda d1, d2: None,
                 walkers=None, seen=None, maxdepth=100,
                 root_parent=None, root_name=None, for_side_effect=False):

    seen = set() if not seen else seen

    def wlk(name, mod):
        return walk_object(mod, visitor=visitor,
                           fabricator=fabricator, combiner=combiner,
                           walkers=walkers, seen=seen, maxdepth=maxdepth,
                           root_parent=None, root_name=name)

    if for_side_effect:
        for (name, mod) in modules.iteritems():
            wlk(name, mod)
        return None
    else:
        return [wlk(name, mod)
                for (name, mod) in modules.iteritems()
                if mod]
<commit_msg>User-provided identity function.<commit_after>"""Walk functions for the Python walker.
"""

import sys
from .walker import get_fallback_walker_method_list
from .low import Limitation

__all__ = ['TooDeep', 'walk_object', 'walk_modules']

class TooDeep(Limitation):
    def __init__(self, what, depth=None):
        self.depth = depth
        self.what = what

    def __str__(self):
        return ("{0} (depth {1})".format(self.what, self.depth) if self.depth
                else self.what)

def walk_object(root, visitor=lambda o, d, p, n: d,
                fabricator=lambda o: None, combiner=lambda d1, d2: None,
                identity=id, walkers=None, seen=None, maxdepth=100,
                root_parent=None, root_name=None):
    # Walk an object and its children.
    #
    # For an object which has not been seen already, each function in
    # walkers is called, and should return either an iterator over
    # (name, value) tuples, or None.  Each element of all the
    # iterators is walked recursively, with the results of the walk
    # being merged pairwise by the combiner function.  If the object
    # has already been seen, or there were no children (all of the
    # walkers return None), then some data is fabricated by calling
    # fabricator on the object.  Finally the visitor is called with
    # the object, the data from the children or the fabricator, the
    # parent object and the name in the parent, and its retuirn value
    # is the value of walk.
    #
    # For the root object the root_parent and root_name variables are
    # used as the parent and name-in-parent arguments.
    #
    # An occurs check is needed to prevent loops (if a->b->a, then the
    # function must not loop, and the visit order is for a walk
    # starting at a is: a with b as parent, b with a as parent and a
    # with no parent (starting from b it would be b with a as parent,
    # a with b as parent, b with no parent).  To do this check the
    # function must keep track of what it has seen already, which it
    # does using a set.  Not all objects are hashable in Python, and
    # in any case whether two objects represent the same thing is
    # application-dependent.  This is supported with two arguments:
    # seen and identity.
    #
    # The identity argument is a function which should return a
    # hashable value from each object, in such a way that two objects
    # which are the same for the purposes of the program have the same
    # value.  The default value of this is the id function, which
    # returns a unique, hashable value for any Python object.
    #
    # The seen argument is a set containing the values returned by the
    # identity function of objects which should be considered to have
    # already been seen.  If no value is given (or it is given as
    # None) then a set will be constructed.  The function mutates this set.
    #
    # You only need to provide the seen argument if you want to make
    # several walks (see walk_modules for an example): normally it's
    # fine to let the function fabricate one.  The notion of identity
    # is complex: if you use the default (id) then, for instance, not
    # all numbers with the same value and type will be considered
    # identical, and strings with the same content will also often not
    # be identical.  Python unfortunately has not thought to provide
    # many useful tools here (no real equivalent of EQL or EQL
    # hashtables, for instance).
    #
    # maxdepth says how deep to go before raising a TooDeep exception.
    #
    # There are defaults for the visitor, combiner and fabricator
    # which are uninteresting but mean you can call walk(thing) and it
    # will work to exercise the walkers.
    #
    # Note that this is a post-order walk: the object is visited after
    # its children, so that it can combine the values from its
    # children.
    #
    # Note that objects which have already been visited are visited
    # again if they appear as children of other nodes, but their
    # children are then *not* visited.  This allows the graph of
    # dependencies to be stitched together without following loops
    # forever.
    #
    if walkers is None:
        walkers = get_fallback_walker_method_list()

    if seen is None:
        seen = set()

    def walk_into(it, parent, name, depth):
        if depth >= maxdepth:
            raise TooDeep("too deep", depth)

        hashable = identity(it)
        data = None
        seenp = False
        walked_children_p = False

        if hashable in seen:
            seenp = True
        else:
            seen.add(hashable)

        if not seenp:
            for walker in walkers:
                iterator = walker(it)
                if iterator:
                    for (n, v) in iterator:
                        walked_children_p = True
                        result = walk_into(v, it, n, depth + 1)
                        data = combiner(data, result) if data else result

        return visitor(it, data if walked_children_p else fabricator(it),
                       parent, name)

    return walk_into(root, root_parent, root_name, 0)

def walk_modules(modules=sys.modules,
                 visitor=lambda o, d, p, n: d,
                 fabricator=lambda o: None, combiner=lambda d1, d2: None,
                 identity=id, walkers=None, seen=None, maxdepth=100,
                 root_parent=None, root_name=None, for_side_effect=False):

    seen = set() if not seen else seen

    def wlk(name, mod):
        return walk_object(mod, visitor=visitor,
                           fabricator=fabricator, combiner=combiner,
                           identity=identity, walkers=walkers, seen=seen,
                           maxdepth=maxdepth, root_parent=None, root_name=name)

    if for_side_effect:
        for (name, mod) in modules.iteritems():
            wlk(name, mod)
        return None
    else:
        return [wlk(name, mod)
                for (name, mod) in modules.iteritems()
                if mod]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2017 Weitian LI <liweitianux@live.com>
# MIT license

"""
Calculate the coordinate of the emission centroid within the image.

The image are smoothed first, and then an iterative procedure with
two phases is applied to determine the emission centroid.
"""

import os
import sys
import argparse
import subprocess
import tempfile

from _context import acispy
from acispy.manifest import get_manifest
from acispy.pfiles import setup_pfiles
from acispy.ds9 import ds9_view
from acispy.region import Regions

def smooth_image(infile, outfile=None,
                 kernelspec="lib:gaus(2,5,1,10,10)", method="fft",
                 clobber=False):
    """
    Smooth the image by a Gaussian kernel using the ``aconvolve`` tool.

    Parameters
    ----------
    infile : str
        Path to the input image file
    outfile : str, optional
        Filename/path of the output smoothed image
        (default: build in format ``<infile_basename>_aconv.fits``)
    kernelspec : str, optional
        Kernel specification for ``aconvolve``
    method : str, optional
        Smooth method for ``aconvolve``

    Returns
    -------
    outfile : str
        Filename/path of the smoothed image
    """
    clobber = "yes" if clobber else "no"
    if outfile is None:
        outfile = os.path.splitext(infile)[0] + "_aconv.fits"
    subprocess.check_call(["punlearn", "aconvolve"])
    subprocess.check_call([
        "aconvolve", "infile=%s" % infile, "outfile=%s" % outfile,
        "kernelspec=%s" % kernelspec, "method=%s" % method,
        "clobber=%s" % clobber
    ])
    return outfile

def get_peak(image, center=None, radius=300):
    """
    Get the peak coordinate on the image within the circle if specified.

    Parameters
    ----------
    image : str
        Path to the image file.
    center : 2-float tuple
        Central (physical) coordinate of the circle.
    radius : float
        Radius (pixel) of the circle.

    Returns
    -------
    peak : 2-float tuple
        (Physical) coordinate of the peak.
    """
    subprocess.check_call(["punlearn", "dmstat"])
    subprocess.check_call([
        "dmstat", "infile=%s" % image,
        "centroid=no", "media=no", "sigma=no", "clip=no"
    ])
    peak = subprocess.check_output([
        "pget", "dmstat", "out_max_loc"
    ]).decode("utf-8").strip()
    peak = peak.split(",")
    return (float(peak[0]), float(peak[1]))

def get_centroid(image, center, radius=100):
    """
    Calculate the centroid on image within the specified circle.

    Parameters
    ----------
    image : str
        Path to the image file.
    center : 2-float tuple
        Central (physical) coordinate of the circle.
    radius : float
        Radius (pixel) of the circle.

    Returns
    -------
    centroid : 2-float tuple
        (Physical) coordinate of the centroid.
    """
    x, y = center
    with tempfile.NamedTemporaryFile(mode="w+") as fp:
        fp.file.write("circle(%f,%f,%f)\n" % (x, y, radius))
        fp.file.flush()
        subprocess.check_call(["punlearn", "dmstat"])
        subprocess.check_call([
            "dmstat", "infile=%s[sky=region(%s)]" % (image, fp.name),
            "centroid=yes", "media=no", "sigma=no", "clip=no"
        ])
    centroid = subprocess.check_output([
        "pget", "dmstat", "out_cntrd_phys"
    ]).decode("utf-8").strip()
    centroid = centroid.split(",")
    return (float(centroid[0]), float(centroid[1]))

def main():
    parser = argparse.ArgumentParser(
        description="Calculate the emission centroid within the image")
    parser.add_argument("-i", "--infile", dest="infile", required=True,
                        help="input image file (e.g., 0.7-2.0 keV)")
    parser.add_argument("-o", "--outfile", dest="outfile",
                        default="centroid.reg",
                        help="output centroid region file " +
                        "(default: centroid.reg")
    parser.add_argument("-R", "--radius1", dest="radius1",
                        type=float, default=300,
                        help="circle radius [pixel] for first phase " +
                        "centroid calculation (default: 300 pixel)")
    parser.add_argument("-r", "--radius2", dest="radius2",
                        type=float, default=100,
                        help="circle radius [pixel] for second phase " +
                        "calculation to tune centroid (default: 100 pixel)")
    parser.add_argument("-n", "--niter", dest="niter",
                        type=int, default=5,
                        help="iterations for each phase (default: 5)")
    parser.add_argument("-s", "--start", dest="start",
                        help="a region file containing a circle/point " +
                        "that specifies the starting point " +
                        "(default: using the peak of the image)")
    parser.add_argument("-V", "--view", dest="view", action="store_true",
                        help="open DS9 to view output centroid")
    parser.add_argument("-C", "--clobber", dest="clobber", action="store_true",
                        help="overwrite existing files")
    args = parser.parse_args()

    setup_pfiles(["aconvolve", "dmstat"])

    print("Smooth input image using 'aconvolve' ...", file=sys.stderr)
    img_smoothed = smooth_image(args.infile, clobber=args.clobber)

    if args.start:
        print("Get starting point from region file: %s" % args.start,
              file=sys.stderr)
        region = Regions(args.start).regions[0]
        center = (region.xc, region.yc)
    else:
        print("Use peak as the starting point ...", file=sys.stderr)
        center = get_peak(img_smoothed)
    print("Starting point: (%f, %f)" % center, file=sys.stderr)

    centroid = center
    for phase, radius in enumerate([args.radius1, args.radius2]):
        print("Calculate centroid phase %d (circle radius: %.1f)" %
              (phase+1, radius), file=sys.stderr)
        for i in range(args.niter):
            print("%d..." % (i+1), end="", flush=True, file=sys.stderr)
            centroid = get_centroid(img_smoothed, center=centroid,
                                    radius=radius)
        print("Done!", file=sys.stderr)

    with open(args.outfile, "w") as f:
        f.write("point(%f,%f)\n" % centroid)
    print("Saved centroid to file:", args.outfile, file=sys.stderr)
    if args.view:
        ds9_view(img_smoothed, regfile=args.outfile)

    # Add calculated centroid region to manifest
    manifest = get_manifest()
    key = "reg_centroid"
    manifest.setpath(key, args.outfile)

if __name__ == "__main__":
    main()
<commit_msg>calc_centroid.py: Update default arguments for better results<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2017 Weitian LI <liweitianux@live.com>
# MIT license

"""
Calculate the coordinate of the emission centroid within the image.

The image are smoothed first, and then an iterative procedure with
two phases is applied to determine the emission centroid.
"""

import os
import sys
import argparse
import subprocess
import tempfile

from _context import acispy
from acispy.manifest import get_manifest
from acispy.pfiles import setup_pfiles
from acispy.ds9 import ds9_view
from acispy.region import Regions

def smooth_image(infile, outfile=None,
                 kernelspec="lib:gaus(2,5,1,10,10)", method="fft",
                 clobber=False):
    """
    Smooth the image by a Gaussian kernel using the ``aconvolve`` tool.

    Parameters
    ----------
    infile : str
        Path to the input image file
    outfile : str, optional
        Filename/path of the output smoothed image
        (default: build in format ``<infile_basename>_aconv.fits``)
    kernelspec : str, optional
        Kernel specification for ``aconvolve``
    method : str, optional
        Smooth method for ``aconvolve``

    Returns
    -------
    outfile : str
        Filename/path of the smoothed image
    """
    clobber = "yes" if clobber else "no"
    if outfile is None:
        outfile = os.path.splitext(infile)[0] + "_aconv.fits"
    subprocess.check_call(["punlearn", "aconvolve"])
    subprocess.check_call([
        "aconvolve", "infile=%s" % infile, "outfile=%s" % outfile,
        "kernelspec=%s" % kernelspec, "method=%s" % method,
        "clobber=%s" % clobber
    ])
    return outfile

def get_peak(image):
    """
    Get the peak coordinate on the image.

    Returns
    -------
    peak : 2-float tuple
        (Physical) coordinate of the peak.
    """
    subprocess.check_call(["punlearn", "dmstat"])
    subprocess.check_call([
        "dmstat", "infile=%s" % image,
        "centroid=no", "media=no", "sigma=no", "clip=no", "verbose=0"
    ])
    peak = subprocess.check_output([
        "pget", "dmstat", "out_max_loc"
    ]).decode("utf-8").strip()
    peak = peak.split(",")
    return (float(peak[0]), float(peak[1]))

def get_centroid(image, center, radius=50):
    """
    Calculate the centroid on image within the specified circle.

    Parameters
    ----------
    image : str
        Path to the image file.
    center : 2-float tuple
        Central (physical) coordinate of the circle.
    radius : float
        Radius (pixel) of the circle.

    Returns
    -------
    centroid : 2-float tuple
        (Physical) coordinate of the centroid.
    """
    x, y = center
    with tempfile.NamedTemporaryFile(mode="w+") as fp:
        fp.file.write("circle(%f,%f,%f)\n" % (x, y, radius))
        fp.file.flush()
        subprocess.check_call(["punlearn", "dmstat"])
        subprocess.check_call([
            "dmstat", "infile=%s[sky=region(%s)]" % (image, fp.name),
            "centroid=yes", "media=no", "sigma=no", "clip=no", "verbose=0"
        ])
    centroid = subprocess.check_output([
        "pget", "dmstat", "out_cntrd_phys"
    ]).decode("utf-8").strip()
    centroid = centroid.split(",")
    return (float(centroid[0]), float(centroid[1]))

def main():
    parser = argparse.ArgumentParser(
        description="Calculate the emission centroid within the image")
    parser.add_argument("-i", "--infile", dest="infile", required=True,
                        help="input image file (e.g., 0.7-2.0 keV)")
    parser.add_argument("-o", "--outfile", dest="outfile",
                        default="centroid.reg",
                        help="output centroid region file " +
                        "(default: centroid.reg")
    parser.add_argument("-R", "--radius1", dest="radius1",
                        type=float, default=100,
                        help="circle radius [pixel] for first phase " +
                        "centroid calculation (default: 100 pixel)")
    parser.add_argument("-r", "--radius2", dest="radius2",
                        type=float, default=50,
                        help="circle radius [pixel] for second phase " +
                        "calculation to tune centroid (default: 50 pixel)")
    parser.add_argument("-n", "--niter", dest="niter",
                        type=int, default=10,
                        help="iterations for each phase (default: 10)")
    parser.add_argument("-s", "--start", dest="start",
                        help="a region file containing a circle/point " +
                        "that specifies the starting point " +
                        "(default: using the peak of the image)")
    parser.add_argument("-V", "--view", dest="view", action="store_true",
                        help="open DS9 to view output centroid")
    parser.add_argument("-C", "--clobber", dest="clobber", action="store_true",
                        help="overwrite existing files")
    args = parser.parse_args()

    setup_pfiles(["aconvolve", "dmstat"])

    print("Smooth input image using 'aconvolve' ...", file=sys.stderr)
    img_smoothed = smooth_image(args.infile, clobber=args.clobber)

    if args.start:
        print("Get starting point from region file: %s" % args.start,
              file=sys.stderr)
        region = Regions(args.start).regions[0]
        center = (region.xc, region.yc)
    else:
        print("Use peak as the starting point ...", file=sys.stderr)
        center = get_peak(img_smoothed)
    print("Starting point: (%f, %f)" % center, file=sys.stderr)

    centroid = center
    for phase, radius in enumerate([args.radius1, args.radius2]):
        print("Calculate centroid phase %d (circle radius: %.1f)" %
              (phase+1, radius), file=sys.stderr)
        for i in range(args.niter):
            print("%d..." % (i+1), end="", flush=True, file=sys.stderr)
            centroid = get_centroid(img_smoothed, center=centroid,
                                    radius=radius)
        print("Done!", file=sys.stderr)

    with open(args.outfile, "w") as f:
        f.write("point(%f,%f)\n" % centroid)
    print("Saved centroid to file:", args.outfile, file=sys.stderr)
    if args.view:
        ds9_view(img_smoothed, regfile=args.outfile)

    # Add calculated centroid region to manifest
    manifest = get_manifest()
    key = "reg_centroid"
    manifest.setpath(key, args.outfile)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Checker for file headers
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Make sure each Python file has a correct file header
    including copyright and license information.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from __future__ import print_function

import io
import os
import re
import sys
import getopt
from os.path import join, splitext, abspath

checkers = {}

def checker(*suffixes, **kwds):
    only_pkg = kwds.pop('only_pkg', False)
    def deco(func):
        for suffix in suffixes:
            checkers.setdefault(suffix, []).append(func)
        func.only_pkg = only_pkg
        return func
    return deco

name_mail_re = r'[\w ]+(<.*?>)?'
copyright_re = re.compile(r'^    :copyright: Copyright 2006-2014 by '
                          r'the Pygments team, see AUTHORS\.$', re.UNICODE)
copyright_2_re = re.compile(r'^                %s(, %s)*[,.]$' %
                            (name_mail_re, name_mail_re), re.UNICODE)
coding_re    = re.compile(r'coding[:=]\s*([-\w.]+)')
not_ix_re    = re.compile(r'\bnot\s+\S+?\s+i[sn]\s\S+')
is_const_re  = re.compile(r'if.*?==\s+(None|False|True)\b')

misspellings = ["developement", "adress", "verificate",  # ALLOW-MISSPELLING
                "informations"]                          # ALLOW-MISSPELLING

@checker('.py')
def check_syntax(fn, lines):
    try:
        compile(''.join(lines), fn, "exec")
    except SyntaxError as err:
        yield 0, "not compilable: %s" % err

@checker('.py')
def check_style_and_encoding(fn, lines):
    encoding = 'ascii'
    for lno, line in enumerate(lines):
        if len(line) > 90:
            yield lno+1, "line too long"
        m = not_ix_re.search(line)
        if m:
            yield lno+1, '"' + m.group() + '"'
        if is_const_re.search(line):
            yield lno+1, 'using == None/True/False'
        if lno < 2:
            co = coding_re.search(line)
            if co:
                encoding = co.group(1)
        try:
            line.decode(encoding)
        except AttributeError:
            # Python 3 - encoding was already checked
            pass
        except UnicodeDecodeError as err:
            yield lno+1, "not decodable: %s\n   Line: %r" % (err, line)
        except LookupError as err:
            yield 0, "unknown encoding: %s" % encoding
            encoding = 'latin1'

@checker('.py', only_pkg=True)
def check_fileheader(fn, lines):
    # line number correction
    c = 1
    if lines[0:1] == ['#!/usr/bin/env python\n']:
        lines = lines[1:]
        c = 2

    llist = []
    docopen = False
    for lno, l in enumerate(lines):
        llist.append(l)
        if lno == 0:
            if l == '# -*- coding: rot13 -*-\n':
                # special-case pony package
                return
            elif l != '# -*- coding: utf-8 -*-\n':
                yield 1, "missing coding declaration"
        elif lno == 1:
            if l != '"""\n' and l != 'r"""\n':
                yield 2, 'missing docstring begin (""")'
            else:
                docopen = True
        elif docopen:
            if l == '"""\n':
                # end of docstring
                if lno <= 4:
                    yield lno+c, "missing module name in docstring"
                break

            if l != "\n" and l[:4] != '    ' and docopen:
                yield lno+c, "missing correct docstring indentation"

            if lno == 2:
                # if not in package, don't check the module name
                modname = fn[:-3].replace('/', '.').replace('.__init__', '')
                while modname:
                    if l.lower()[4:-1] == modname:
                        break
                    modname = '.'.join(modname.split('.')[1:])
                else:
                    yield 3, "wrong module name in docstring heading"
                modnamelen = len(l.strip())
            elif lno == 3:
                if l.strip() != modnamelen * "~":
                    yield 4, "wrong module name underline, should be ~~~...~"

    else:
        yield 0, "missing end and/or start of docstring..."

    # check for copyright and license fields
    license = llist[-2:-1]
    if license != ["    :license: BSD, see LICENSE for details.\n"]:
        yield 0, "no correct license info"

    ci = -3
    copyright = llist[ci:ci+1]
    while copyright and copyright_2_re.match(copyright[0]):
        ci -= 1
        copyright = llist[ci:ci+1]
    if not copyright or not copyright_re.match(copyright[0]):
        yield 0, "no correct copyright info"

def main(argv):
    try:
        gopts, args = getopt.getopt(argv[1:], "vi:")
    except getopt.GetoptError:
        print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
        return 2
    opts = {}
    for opt, val in gopts:
        if opt == '-i':
            val = abspath(val)
        opts.setdefault(opt, []).append(val)

    if len(args) == 0:
        path = '.'
    elif len(args) == 1:
        path = args[0]
    else:
        print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
        return 2

    verbose = '-v' in opts

    num = 0
    out = io.StringIO()

    # TODO: replace os.walk run with iteration over output of
    #       `svn list -R`.

    for root, dirs, files in os.walk(path):
        if '.hg' in dirs:
            dirs.remove('.hg')
        if '-i' in opts and abspath(root) in opts['-i']:
            del dirs[:]
            continue
        # XXX: awkward: for the Makefile call: don't check non-package
        #      files for file headers
        in_pocoo_pkg = root.startswith('./pygments')
        for fn in files:

            fn = join(root, fn)
            if fn[:2] == './': fn = fn[2:]

            if '-i' in opts and abspath(fn) in opts['-i']:
                continue

            ext = splitext(fn)[1]
            checkerlist = checkers.get(ext, None)
            if not checkerlist:
                continue

            if verbose:
                print("Checking %s..." % fn)

            try:
                f = open(fn, 'r')
                lines = list(f)
            except (IOError, OSError) as err:
                print("%s: cannot open: %s" % (fn, err))
                num += 1
                continue

            for checker in checkerlist:
                if not in_pocoo_pkg and checker.only_pkg:
                    continue
                for lno, msg in checker(fn, lines):
                    print(u"%s:%d: %s" % (fn, lno, msg), file=out)
                    num += 1
    if verbose:
        print()
    if num == 0:
        print("No errors found.")
    else:
        print(out.getvalue().rstrip('\n'))
        print("%d error%s found." % (num, num > 1 and "s" or ""))
    return int(num > 0)

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>check_sources: allow 110 line length, skip examplefiles directory<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Checker for file headers
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Make sure each Python file has a correct file header
    including copyright and license information.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from __future__ import print_function

import io
import os
import re
import sys
import getopt
from os.path import join, splitext, abspath

checkers = {}

def checker(*suffixes, **kwds):
    only_pkg = kwds.pop('only_pkg', False)

    def deco(func):
        for suffix in suffixes:
            checkers.setdefault(suffix, []).append(func)
        func.only_pkg = only_pkg
        return func
    return deco

name_mail_re = r'[\w ]+(<.*?>)?'
copyright_re = re.compile(r'^    :copyright: Copyright 2006-2014 by '
                          r'the Pygments team, see AUTHORS\.$', re.UNICODE)
copyright_2_re = re.compile(r'^                %s(, %s)*[,.]$' %
                            (name_mail_re, name_mail_re), re.UNICODE)
coding_re    = re.compile(r'coding[:=]\s*([-\w.]+)')
not_ix_re    = re.compile(r'\bnot\s+\S+?\s+i[sn]\s\S+')
is_const_re  = re.compile(r'if.*?==\s+(None|False|True)\b')

misspellings = ["developement", "adress", "verificate",  # ALLOW-MISSPELLING
                "informations"]                          # ALLOW-MISSPELLING

@checker('.py')
def check_syntax(fn, lines):
    try:
        compile(''.join(lines), fn, "exec")
    except SyntaxError as err:
        yield 0, "not compilable: %s" % err

@checker('.py')
def check_style_and_encoding(fn, lines):
    encoding = 'ascii'
    for lno, line in enumerate(lines):
        if len(line) > 110:
            yield lno+1, "line too long"
        m = not_ix_re.search(line)
        if m:
            yield lno+1, '"' + m.group() + '"'
        if is_const_re.search(line):
            yield lno+1, 'using == None/True/False'
        if lno < 2:
            co = coding_re.search(line)
            if co:
                encoding = co.group(1)
        try:
            line.decode(encoding)
        except AttributeError:
            # Python 3 - encoding was already checked
            pass
        except UnicodeDecodeError as err:
            yield lno+1, "not decodable: %s\n   Line: %r" % (err, line)
        except LookupError as err:
            yield 0, "unknown encoding: %s" % encoding
            encoding = 'latin1'

@checker('.py', only_pkg=True)
def check_fileheader(fn, lines):
    # line number correction
    c = 1
    if lines[0:1] == ['#!/usr/bin/env python\n']:
        lines = lines[1:]
        c = 2

    llist = []
    docopen = False
    for lno, l in enumerate(lines):
        llist.append(l)
        if lno == 0:
            if l == '# -*- coding: rot13 -*-\n':
                # special-case pony package
                return
            elif l != '# -*- coding: utf-8 -*-\n':
                yield 1, "missing coding declaration"
        elif lno == 1:
            if l != '"""\n' and l != 'r"""\n':
                yield 2, 'missing docstring begin (""")'
            else:
                docopen = True
        elif docopen:
            if l == '"""\n':
                # end of docstring
                if lno <= 4:
                    yield lno+c, "missing module name in docstring"
                break

            if l != "\n" and l[:4] != '    ' and docopen:
                yield lno+c, "missing correct docstring indentation"

            if lno == 2:
                # if not in package, don't check the module name
                modname = fn[:-3].replace('/', '.').replace('.__init__', '')
                while modname:
                    if l.lower()[4:-1] == modname:
                        break
                    modname = '.'.join(modname.split('.')[1:])
                else:
                    yield 3, "wrong module name in docstring heading"
                modnamelen = len(l.strip())
            elif lno == 3:
                if l.strip() != modnamelen * "~":
                    yield 4, "wrong module name underline, should be ~~~...~"

    else:
        yield 0, "missing end and/or start of docstring..."

    # check for copyright and license fields
    license = llist[-2:-1]
    if license != ["    :license: BSD, see LICENSE for details.\n"]:
        yield 0, "no correct license info"

    ci = -3
    copyright = llist[ci:ci+1]
    while copyright and copyright_2_re.match(copyright[0]):
        ci -= 1
        copyright = llist[ci:ci+1]
    if not copyright or not copyright_re.match(copyright[0]):
        yield 0, "no correct copyright info"

def main(argv):
    try:
        gopts, args = getopt.getopt(argv[1:], "vi:")
    except getopt.GetoptError:
        print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
        return 2
    opts = {}
    for opt, val in gopts:
        if opt == '-i':
            val = abspath(val)
        opts.setdefault(opt, []).append(val)

    if len(args) == 0:
        path = '.'
    elif len(args) == 1:
        path = args[0]
    else:
        print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
        return 2

    verbose = '-v' in opts

    num = 0
    out = io.StringIO()

    # TODO: replace os.walk run with iteration over output of
    #       `svn list -R`.

    for root, dirs, files in os.walk(path):
        if '.hg' in dirs:
            dirs.remove('.hg')
        if 'examplefiles' in dirs:
            dirs.remove('examplefiles')
        if '-i' in opts and abspath(root) in opts['-i']:
            del dirs[:]
            continue
        # XXX: awkward: for the Makefile call: don't check non-package
        #      files for file headers
        in_pygments_pkg = root.startswith('./pygments')
        for fn in files:

            fn = join(root, fn)
            if fn[:2] == './':
                fn = fn[2:]

            if '-i' in opts and abspath(fn) in opts['-i']:
                continue

            ext = splitext(fn)[1]
            checkerlist = checkers.get(ext, None)
            if not checkerlist:
                continue

            if verbose:
                print("Checking %s..." % fn)

            try:
                f = open(fn, 'r')
                lines = list(f)
            except (IOError, OSError) as err:
                print("%s: cannot open: %s" % (fn, err))
                num += 1
                continue

            for checker in checkerlist:
                if not in_pygments_pkg and checker.only_pkg:
                    continue
                for lno, msg in checker(fn, lines):
                    print(u"%s:%d: %s" % (fn, lno, msg), file=out)
                    num += 1
    if verbose:
        print()
    if num == 0:
        print("No errors found.")
    else:
        print(out.getvalue().rstrip('\n'))
        print("%d error%s found." % (num, num > 1 and "s" or ""))
    return int(num > 0)

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>from django.test import TestCase

<commit_msg>unittest: add an initial set of autocomplete-search tests<commit_after># -*- coding: utf-8 -*-

from django.test import TestCase
from django.core.urlresolvers import reverse
import json

class AutocompleteSearchTest(TestCase):
    fixtures = ['test_keyword', 'test_sessionkeyword', 'test_member']
    req_params = {
            u'max_results'      : 6,
            u'thumbnail_width'  : 30,
            u'thumbnail_height' : 30
    }
    autocomplete_url = reverse('votes.views.search_autocomplete')
    qpar_str = u'?query='
    search_kw_url = reverse('votes.views.search_by_keyword') + qpar_str
    search_mb_url = reverse('votes.views.search') + qpar_str

    def setUp(self):
        def query_res(name, exp_url):
            return { 'name' : name, 'url' : exp_url + name }

        def kw_res(name):
            return query_res(name, self.search_kw_url)

        def mb_res(name):
            return query_res(name, self.search_mb_url)

        def query_item(term, exp_url, exp_names):
            if type(exp_names) is not list:
                exp_names = [exp_names]

            return {
                'term'    : term,
                'exp_res' : [query_res(n, exp_url) for n in exp_names]
            }

        def kw_query(term, exp_names):
            return query_item(term, self.search_kw_url, exp_names)

        def mb_query(term, exp_names):
            return query_item(term, self.search_mb_url, exp_names)

        def mixed_query(term, exp_results):
            return {
                'term'    : term,
                'exp_res' : exp_results
            }

        self.queries = [
                # member queries returning a single result
                mb_query(u'aalto',               u'Aaltonen Markus'),
                mb_query(u'Markus ',             u'Aaltonen Markus'),
                mb_query(u' Aaltonen   Markus ', u'Aaltonen Markus'),
                mb_query(u'ala-nissilä',         [u'Ala-Nissilä Olavi']),

                # keyword queries returning a single result
                kw_query(u'aasia',               u'Aasia'),
                kw_query(u'Aasia ',              u'Aasia'),
                kw_query(u' Aasia ',             u'Aasia'),
                kw_query(u'ja rahoitus- kehittä',
                                u'Asumisen rahoitus- ja kehittämiskeskus'),

                # member queries returning multiple results
                mb_query(u'aho',                 [u'Aho Esko',
                                                  u'Aho Hannu',
                                                  u'Ahonen Esko']),

                # keyword queries returning multiple results
                kw_query('af',                   [u'Afganistan',
                                                  u'Afrikka']),

                # mixed queries returning both members and keywords
                mixed_query(u'aa',               [mb_res(u'Aaltonen Markus'),
                                                  kw_res(u'Aasia')]),

                # mixed queries returning all members and keywords limited
                # by max_results
                mixed_query(u'',                 [mb_res(u'Aaltonen Markus'),
                                                  kw_res(u'Aasia'),
                                                  kw_res(u'Adoptio'),
                                                  kw_res(u'Afganistan'),
                                                  kw_res(u'Afrikka'),
                                                  mb_res(u'Ahde Matti')]),
        ]

    def decode_res(self, res):
        dr = json.loads(res)
        # TODO: validate also the thumbnail path which is dynamically generated
        return [{'name': x[0], 'url' : x[2]} for x in dr]

    def run_query(self, term, exp_res):
        rp = self.req_params.copy()
        rp[u'name'] = term
        mymsg = 'query=' + term + ' expected=' + unicode(exp_res)
        try:
            response = self.client.get(self.autocomplete_url, rp)
        except:
            print mymsg
            raise           # Re-raise the exception we're handling

        mymsg_raw = u'(resp_cont=' + response.content + u')'
        self.assertEqual(response.status_code, 200,
                         msg=mymsg + ' ' + mymsg_raw)
        decoded_res = self.decode_res(response.content)
        mymsg = mymsg + ' got=' + unicode(decoded_res) + ' ' + mymsg_raw
        self.assertEqual(decoded_res, exp_res, msg=mymsg)

    def test_queries(self):
        for q in self.queries:
            self.run_query(q['term'], q['exp_res'])

<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
import sys

hidden_state_size = 64
input_size = 99

char_to_index = {9: 96,
                 10: 97,
                 11: 98}
index_to_char = {96: 9,
                 97: 10,
                 98: 11}

def make_onehot(char):
  o = ord(char)
  if o in char_to_index:
    index = char_to_index[o]
  elif o > (32 + 96):
    print("\n\tBad char: ", o)
    index = ord(" ") - 32
  else:
   index = o - 32
   assert 0 <= index < input_size, "ord is " + str(o)

  output = np.zeros([1,input_size], dtype=np.float32)
  output[0,index] = 1
  return output

def unonehot(mat):
  best_i = -1
  best_v = -1
  for row in mat:
    for i, v in enumerate(row):
      if v > best_v:
        best_v = v
        best_i = i

  if best_i in index_to_char:
   return index_to_char[best_i]

  return chr(best_i + 32)

# Returns new_state and output
def create_LSTM_cell(cprev_state, cprev_output, cinput):
  concat_size = hidden_state_size + input_size

  # Make the matrixes for generating weightings from prev_state/prev_output/input
  cgen_memory_weights = tf.Variable(tf.random_normal((concat_size,hidden_state_size)))
  cgen_update_weights = tf.Variable(tf.random_normal((concat_size, hidden_state_size)))
  cgen_update_vec = tf.Variable(tf.random_normal((concat_size, hidden_state_size)))
  cgen_output_weights = tf.Variable(tf.random_normal((concat_size, hidden_state_size)))

  # Biases
  cmemory_weights_bias = tf.Variable(tf.random_normal((1,hidden_state_size)))
  cupdate_weights_bias = tf.Variable(tf.random_normal((1,hidden_state_size)))
  cupdate_vec_bias = tf.Variable(tf.random_normal((1,hidden_state_size)))
  calmost_output_bias = tf.Variable(tf.random_normal((1,hidden_state_size)))

  inputandoldoutput = tf.concat(1, [cprev_output, cinput])

  cmemory_weights = tf.sigmoid(tf.add(tf.matmul(inputandoldoutput, cgen_memory_weights), cmemory_weights_bias))
  cupdate_weights = tf.sigmoid(tf.add(tf.matmul(inputandoldoutput, cgen_update_weights), cupdate_weights_bias))
  cupdate_vec = tf.tanh(tf.add(tf.matmul(inputandoldoutput, cgen_update_vec), cupdate_vec_bias))
  calmost_output_vec = tf.sigmoid(tf.add(tf.matmul(inputandoldoutput, cgen_output_weights), calmost_output_bias))

  cweighted_state = tf.mul(cprev_state, cmemory_weights)
  cweighted_update = tf.mul(cupdate_vec, cupdate_weights)
  cnewstate = tf.add(cweighted_state, cweighted_update)

  coutput_vec = tf.mul(calmost_output_vec, tf.tanh(cnewstate))

  return cnewstate, coutput_vec

def main(filename):
  print("Reading input text from ", filename)

  cprev_state = tf.placeholder(tf.float32, shape=(1, hidden_state_size))
  cprev_output = tf.placeholder(tf.float32, shape=(1, hidden_state_size))
  vinitial_state = np.zeros([1,hidden_state_size], dtype=np.float32)
  vinitial_prev_output = np.zeros([1,hidden_state_size], dtype=np.float32)
  cinput = tf.placeholder(tf.float32, shape=(1,input_size))
  cnew_state, coutput = create_LSTM_cell(cprev_state, cprev_output, cinput)

  # Gotta translate the output back into character probabilities
  cgen_realoutput = tf.Variable(tf.random_normal((hidden_state_size, input_size)))
  creal_output_bias = tf.Variable(tf.random_normal((1, input_size)))
  creal_output = tf.nn.softmax(tf.add(tf.matmul(coutput, cgen_realoutput), creal_output_bias))

  ccorrect = tf.placeholder(tf.float32, shape=(1,input_size))
  cost = -tf.reduce_mean(tf.reduce_sum(tf.mul(tf.log(creal_output),ccorrect)))
  train_step = tf.train.AdamOptimizer().minimize(cost)

  session = tf.Session()
  initializer = tf.initialize_all_variables()
  session.run([initializer])

  vnewstate = np.copy(vinitial_state)
  voutput = np.copy(vinitial_prev_output)
  count = 0
  with open(filename, 'r') as f:
    text = f.read()
    prev = text[0]
    for char in text[1:]:
      correct = make_onehot(char)
      encoded = make_onehot(prev)

      count += 1
      if count > 300:
        print()
        count = 0
        vnewstate = np.copy(vinitial_state)
        voutput = np.copy(vinitial_prev_output)

      _, vnewstate, voutput, vrealoutput =  session.run([train_step, cnew_state,coutput,creal_output], feed_dict={cinput: encoded, cprev_state: vnewstate, cprev_output: voutput, ccorrect: correct})

      print(unonehot(vrealoutput), end="")
      prev = char

if __name__ == '__main__':
  filename = sys.argv[1]
  main(filename)

<commit_msg>Add a second layer to the network :D<commit_after>import tensorflow as tf
import numpy as np
import sys

hidden_state_size = 64
input_size = 99

char_to_index = {9: 96,
                 10: 97,
                 11: 98}
index_to_char = {96: 9,
                 97: 10,
                 98: 11}

def make_onehot(char):
  o = ord(char)
  if o in char_to_index:
    index = char_to_index[o]
  elif o > (32 + 96):
    print("\n\tBad char: ", o)
    index = ord(" ") - 32
  else:
   index = o - 32
   assert 0 <= index < input_size, "ord is " + str(o)

  output = np.zeros([1,input_size], dtype=np.float32)
  output[0,index] = 1
  return output

def unonehot(mat):
  best_i = -1
  best_v = -1
  for row in mat:
    for i, v in enumerate(row):
      if v > best_v:
        best_v = v
        best_i = i

  if best_i in index_to_char:
   return index_to_char[best_i]

  return chr(best_i + 32)

# Returns new_state and output
def create_LSTM_cell1(cprev_state, cprev_output, cinput):
  return create_LSTM_cell(cprev_state, cprev_output, cinput)

# Returns new_state and output
def create_LSTM_cell2(cprev_state, cprev_output, cinput):
  return create_LSTM_cell(cprev_state, cprev_output, cinput, is_=hidden_state_size)

# Returns new_state and output
def create_LSTM_cell(cprev_state, cprev_output, cinput, hss=hidden_state_size, is_=input_size):
  concat_size = hss + is_

  # Make the matrixes for generating weightings from prev_state/prev_output/input
  cgen_memory_weights = tf.Variable(tf.random_normal((concat_size,hss)))
  cgen_update_weights = tf.Variable(tf.random_normal((concat_size, hss)))
  cgen_update_vec = tf.Variable(tf.random_normal((concat_size, hss)))
  cgen_output_weights = tf.Variable(tf.random_normal((concat_size, hss)))

  # Biases
  cmemory_weights_bias = tf.Variable(tf.random_normal((1,hss)))
  cupdate_weights_bias = tf.Variable(tf.random_normal((1,hss)))
  cupdate_vec_bias = tf.Variable(tf.random_normal((1,hss)))
  calmost_output_bias = tf.Variable(tf.random_normal((1,hss)))

  inputandoldoutput = tf.concat(1, [cprev_output, cinput])

  cmemory_weights = tf.sigmoid(tf.add(tf.matmul(inputandoldoutput, cgen_memory_weights), cmemory_weights_bias))
  cupdate_weights = tf.sigmoid(tf.add(tf.matmul(inputandoldoutput, cgen_update_weights), cupdate_weights_bias))
  cupdate_vec = tf.tanh(tf.add(tf.matmul(inputandoldoutput, cgen_update_vec), cupdate_vec_bias))
  calmost_output_vec = tf.sigmoid(tf.add(tf.matmul(inputandoldoutput, cgen_output_weights), calmost_output_bias))

  cweighted_state = tf.mul(cprev_state, cmemory_weights)
  cweighted_update = tf.mul(cupdate_vec, cupdate_weights)
  cnewstate = tf.add(cweighted_state, cweighted_update)
  coutput_vec = tf.mul(calmost_output_vec, tf.tanh(cnewstate))

  return cnewstate, coutput_vec

def main(filename):
  print("Reading input text from ", filename)

  ## Layer 1
  cprev_state1 = tf.placeholder(tf.float32, shape=(1, hidden_state_size))
  cprev_output1 = tf.placeholder(tf.float32, shape=(1, hidden_state_size))
  vinitial_state1 = np.zeros([1,hidden_state_size], dtype=np.float32)
  vinitial_prev_output1 = np.zeros([1,hidden_state_size], dtype=np.float32)
  cinput1 = tf.placeholder(tf.float32, shape=(1,input_size))
  cnew_state1, coutput1 = create_LSTM_cell1(cprev_state1, cprev_output1, cinput1)

  ## Layer 2
  cprev_state2 = tf.placeholder(tf.float32, shape=(1, hidden_state_size))
  cprev_output2 = tf.placeholder(tf.float32, shape=(1, hidden_state_size))
  vinitial_state2 = np.zeros([1,hidden_state_size], dtype=np.float32)
  vinitial_prev_output2 = np.zeros([1,hidden_state_size], dtype=np.float32)
  cnew_state2, coutput2 = create_LSTM_cell2(cprev_state2, cprev_output2, coutput1)

  # Gotta translate the output back into character probabilities
  cgen_realoutput = tf.Variable(tf.random_normal((hidden_state_size, input_size)))
  creal_output_bias = tf.Variable(tf.random_normal((1, input_size)))
  creal_output = tf.nn.softmax(tf.add(tf.matmul(coutput2, cgen_realoutput), creal_output_bias))

  ccorrect = tf.placeholder(tf.float32, shape=(1,input_size))
  cost = -tf.reduce_mean(tf.reduce_sum(tf.mul(tf.log(creal_output),ccorrect)))
  train_step = tf.train.AdamOptimizer().minimize(cost)

  session = tf.Session()
  initializer = tf.initialize_all_variables()
  session.run([initializer])

  vnewstate1 = np.copy(vinitial_state1)
  voutput1 = np.copy(vinitial_prev_output1)
  vnewstate2 = np.copy(vinitial_state2)
  voutput2 = np.copy(vinitial_prev_output2)
  count = 0
  with open(filename, 'r') as f:
    text = f.read()
    prev = text[0]
    for char in text[1:]:
      correct = make_onehot(char)
      encoded = make_onehot(prev)

      count += 1
      if count > 300:
        print()
        count = 0
        vnewstate1 = np.copy(vinitial_state1)
        voutput1 = np.copy(vinitial_prev_output1)
        vnewstate2 = np.copy(vinitial_state2)
        voutput2 = np.copy(vinitial_prev_output2)

      _, vnewstate1, voutput1, vnewstate2, voutput2, vrealoutput =  session.run([train_step, cnew_state1,coutput1, cnew_state2,coutput2,creal_output], feed_dict={cinput1: encoded, cprev_state1: vnewstate1, cprev_output1: voutput1, cprev_state2: vnewstate2, cprev_output2: voutput2, ccorrect: correct})

      print(unonehot(vrealoutput), end="")
      prev = char

if __name__ == '__main__':
  filename = sys.argv[1]
  main(filename)

<|endoftext|>


<commit_before>#!usr/bin/python
#
# (C) Legoktm 2009
# Released under the MIT License
#
# All pages should be in the wiki.Page() format
__version__ = '$Id$'

import wiki, config, catlib
import sys, re

def category(page, recurse=False):
	return catlib.Category(page, recurse=recurse)
def transclude(page):
	"""
	Returns pages that transclude a certain template
	"""
	API = wiki.API(wiki=page.getSite())
	print 'Getting references to [[%s]]...' %(page.title())
	params = {
		'action':'query',
		'list':'embeddedin',
		'eititle':page.title(),
		'eilimit':'max',
	}
	res = API.query(params)
	list = []
	for page in res['query']['embeddedin']:
		yield wiki.Page(page['title'])
	return list

def prefixindex(page):
	"""
	Returns list of pages with prefix of the page ([[Special:PrefixIndex]])
	"""
	API = wiki.API(wiki=page.getSite())
	ns = page.namespace()
	prefix = page.titlewonamespace()
	params = {
		'action':'query',
		'list':'allpages',
		'apprefix':prefix,
		'apnamespace':str(ns),
		'aplimit':'max',
	}
	res = API.query(params)['query']['allpages']
	list = []
	for page in res:
		yield wiki.Page(page['title'])

def catlinks(page):
	print 'Getting categories on %s...' %str(page)
	gen = links(page, ns=14)
	for cat in gen:
		for page in category(cat):
			yield page

def recentchanges(limit = 500, nobot = True, onlyanon = False, hidepatrolled = True, nponly = False, wiki=config.wiki):
	"""
	Returns a list of articles that were recently changed ([[Special:RecentChanges]])
	If nponly = True, returns only newpages ([[Special:NewPages]])
	"""
	rcshow = []
	if nobot:
		rcshow.append('!bot')
	if onlyanon:
		rcshow.append('anon')
#	if hidepatrolled:
#		rcshow.append('!patrolled')
	rcshowparam = ''
	if len(rcshow) != 0:
		for i in rcshow:
			if i == rcshow[len(rcshow)-1]: #meaning it is the last one..
				rcshowparam += i
			else:
				rcshowparam += i + '|'
	params = {
		'action':'query',
		'list':'recentchanges',
		'rcshow':rcshowparam,
		'rcprop':'title',
		'rclimit':limit
	}
	if nponly:
		print 'Fetching the %s newest pages' %limit
		params['rctype'] = 'new'
	else:
		print 'Fetching the %s latest edits' %limit
	API = wiki.API(qcontinue=False, wiki=wiki)
	res = API.query(params)['query']['recentchanges']
	list = []
	for page in res:
		yield wiki.Page(page['title'])

def links(page, ns=None):
	"""
	Returns links on a provided page
	Not Special:WhatLinksHere
	ns is an (optional) int() which is the only namespace returned
	"""
	API = wiki.API(wiki=page.getSite())
	params = {
		'action':'query',
		'titles':page.title(),
		'prop':'links',
		'pllimit':'max'
	}
	if ns:
		params['plnamespace'] = int(ns)
	res = API.query(params)['query']['pages']
	list = res[res.keys()[0]]['links']
	for page in list:
		yield wiki.Page(page['title'])

def whatlinkshere(page):
	"""
	[[Special:WhatLinksHere]]
	"""
	API = wiki.API(wiki=page.getSite())
	params = {
		'action':'query',
		'bltitle':page.title(),
		'list':'backlinks',
		'bllimit':'max',
	}
	res = API.query(params)['query']['backlinks']
	for i in res:
		yield wiki.Page(i['title'])

parameterHelp = """\
-cat              Work on all pages which are in a specific category.
                  Argument can also be given as "-cat:categoryname" or
                  as "-cat:categoryname|fromtitle".

-new              Work on the 60 newest pages. If given as -new:x, will work
                  on the x newest pages.

-file             Read a list of pages to treat from the named text file.
                  Page titles in the file must be enclosed with [[brackets]].
                  Argument can also be given as "-file:filename".

-links            Work on all pages that are linked from a certain page.
                  Argument can also be given as "-links:linkingpagetitle".

-catlinks        Work on all categories that are linked from a certain page.
				 Argument can also be given as "-catlinks:linkingpagetitle".

-prefixindex      Work on pages commencing with a common prefix.
"""

def handleArgs():
	"""
	Picks the generator per argument passed in command line
	Usage:
	gen = pagegen.handleArgs()
	for page in gen:
	...do something
	"""
	for arg in wiki.getArgs():
		if arg.startswith('-catlinks'):
			if len(arg) == 9:
				pg = raw_input('Which page should be operated on? ')
				return catlinks(wiki.Page(pg))
			else: #means that page has been supplied
				pg = arg[10:]
				return catlinks(wiki.Page(pg))
		if arg.startswith('-cat'):
			if len(arg) == 4:
				cat = raw_input('Which category should be operated on? ')
				if not 'category' in cat.lower():
					cat = 'Category:' + cat
				return category(wiki.Page(cat))
			else: #means that category has been supplied
				cat = arg[5:]
				if not 'category' in cat.lower():
					cat = 'Category:' + cat
				return category(wiki.Page(cat))
		if arg.startswith('-file'):
			if len(arg) == 5:
				fil = raw_input('Which file should be read? ')
			else: #means that file has been supplied
				fil = arg[6:]
			try:
				file = open(fil, 'r')
			except IOError, err:
				print err
				sys.exit(1)
			text = file.read()
			list = re.findall('\[\[(.*?)\]\]', text)
			newlist = []
			for i in list:
				newlist.append(wiki.Page(i))
			return newlist
		if arg.startswith('-new'):
			if len(arg) == 4:
				return recentchanges()
			else:
				try:
					limit = int(arg[5:])
				except:
					return recentchanges()
				return recentchanges(limit=limit)
	#nothing found
	print parameterHelp
	sys.exit(1)
<commit_msg>fix return error<commit_after>#!usr/bin/python
#
# (C) Legoktm 2009
# Released under the MIT License
#
# All pages should be in the wiki.Page() format
__version__ = '$Id$'

import wiki, config, catlib
import sys, re

def category(page, recurse=False):
	return catlib.Category(page, recurse=recurse)
def transclude(page):
	"""
	Returns pages that transclude a certain template
	"""
	API = wiki.API(wiki=page.getSite())
	print 'Getting references to [[%s]]...' %(page.title())
	params = {
		'action':'query',
		'list':'embeddedin',
		'eititle':page.title(),
		'eilimit':'max',
	}
	res = API.query(params)
	for page in res['query']['embeddedin']:
		yield wiki.Page(page['title'])

def prefixindex(page):
	"""
	Returns list of pages with prefix of the page ([[Special:PrefixIndex]])
	"""
	API = wiki.API(wiki=page.getSite())
	ns = page.namespace()
	prefix = page.titlewonamespace()
	params = {
		'action':'query',
		'list':'allpages',
		'apprefix':prefix,
		'apnamespace':str(ns),
		'aplimit':'max',
	}
	res = API.query(params)['query']['allpages']
	list = []
	for page in res:
		yield wiki.Page(page['title'])

def catlinks(page):
	print 'Getting categories on %s...' %str(page)
	gen = links(page, ns=14)
	for cat in gen:
		for page in category(cat):
			yield page

def recentchanges(limit = 500, nobot = True, onlyanon = False, hidepatrolled = True, nponly = False, wiki=config.wiki):
	"""
	Returns a list of articles that were recently changed ([[Special:RecentChanges]])
	If nponly = True, returns only newpages ([[Special:NewPages]])
	"""
	rcshow = []
	if nobot:
		rcshow.append('!bot')
	if onlyanon:
		rcshow.append('anon')
#	if hidepatrolled:
#		rcshow.append('!patrolled')
	rcshowparam = ''
	if len(rcshow) != 0:
		for i in rcshow:
			if i == rcshow[len(rcshow)-1]: #meaning it is the last one..
				rcshowparam += i
			else:
				rcshowparam += i + '|'
	params = {
		'action':'query',
		'list':'recentchanges',
		'rcshow':rcshowparam,
		'rcprop':'title',
		'rclimit':limit
	}
	if nponly:
		print 'Fetching the %s newest pages' %limit
		params['rctype'] = 'new'
	else:
		print 'Fetching the %s latest edits' %limit
	API = wiki.API(qcontinue=False, wiki=wiki)
	res = API.query(params)['query']['recentchanges']
	for page in res:
		yield wiki.Page(page['title'])

def links(page, ns=None):
	"""
	Returns links on a provided page
	Not Special:WhatLinksHere
	ns is an (optional) int() which is the only namespace returned
	"""
	API = wiki.API(wiki=page.getSite())
	params = {
		'action':'query',
		'titles':page.title(),
		'prop':'links',
		'pllimit':'max'
	}
	if ns:
		params['plnamespace'] = int(ns)
	res = API.query(params)['query']['pages']
	list = res[res.keys()[0]]['links']
	for page in list:
		yield wiki.Page(page['title'])

def whatlinkshere(page):
	"""
	[[Special:WhatLinksHere]]
	"""
	API = wiki.API(wiki=page.getSite())
	params = {
		'action':'query',
		'bltitle':page.title(),
		'list':'backlinks',
		'bllimit':'max',
	}
	res = API.query(params)['query']['backlinks']
	for i in res:
		yield wiki.Page(i['title'])

parameterHelp = """\
-cat              Work on all pages which are in a specific category.
                  Argument can also be given as "-cat:categoryname" or
                  as "-cat:categoryname|fromtitle".

-new              Work on the 60 newest pages. If given as -new:x, will work
                  on the x newest pages.

-file             Read a list of pages to treat from the named text file.
                  Page titles in the file must be enclosed with [[brackets]].
                  Argument can also be given as "-file:filename".

-links            Work on all pages that are linked from a certain page.
                  Argument can also be given as "-links:linkingpagetitle".

-catlinks        Work on all categories that are linked from a certain page.
				 Argument can also be given as "-catlinks:linkingpagetitle".

-prefixindex      Work on pages commencing with a common prefix.
"""

def handleArgs():
	"""
	Picks the generator per argument passed in command line
	Usage:
	gen = pagegen.handleArgs()
	for page in gen:
	...do something
	"""
	for arg in wiki.getArgs():
		if arg.startswith('-catlinks'):
			if len(arg) == 9:
				pg = raw_input('Which page should be operated on? ')
				return catlinks(wiki.Page(pg))
			else: #means that page has been supplied
				pg = arg[10:]
				return catlinks(wiki.Page(pg))
		if arg.startswith('-cat'):
			if len(arg) == 4:
				cat = raw_input('Which category should be operated on? ')
				if not 'category' in cat.lower():
					cat = 'Category:' + cat
				return category(wiki.Page(cat))
			else: #means that category has been supplied
				cat = arg[5:]
				if not 'category' in cat.lower():
					cat = 'Category:' + cat
				return category(wiki.Page(cat))
		if arg.startswith('-file'):
			if len(arg) == 5:
				fil = raw_input('Which file should be read? ')
			else: #means that file has been supplied
				fil = arg[6:]
			try:
				file = open(fil, 'r')
			except IOError, err:
				print err
				sys.exit(1)
			text = file.read()
			list = re.findall('\[\[(.*?)\]\]', text)
			newlist = []
			for i in list:
				newlist.append(wiki.Page(i))
			return newlist
		if arg.startswith('-new'):
			if len(arg) == 4:
				return recentchanges()
			else:
				try:
					limit = int(arg[5:])
				except:
					return recentchanges()
				return recentchanges(limit=limit)
	#nothing found
	print parameterHelp
	sys.exit(1)
<|endoftext|>