<commit_before># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Interactive UIs for the ciEditor."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
try:
    unicode
except NameError:
    unicode = str
    unichr = chr

import os
import re

import app.config
import app.controller
import app.text_buffer

def parseInt(inStr):
    if app.config.strict_debug:
        assert isinstance(inStr, unicode), type(inStr)
    i = 0
    k = 0
    if len(inStr) > i and inStr[i] in ('+', '-'):
        i += 1
    k = i
    while len(inStr) > k and inStr[k].isdigit():
        k += 1
    if k > i:
        return int(inStr[:k])
    return 0

def test_parseInt():
    assert parseInt('0') == 0
    assert parseInt('0e') == 0
    assert parseInt('text') == 0
    assert parseInt('10') == 10
    assert parseInt('+10') == 10
    assert parseInt('-10') == -10
    assert parseInt('--10') == 0
    assert parseInt('--10') == 0

class InteractivePrediction(app.controller.Controller):
    """Make a guess about what the user desires."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractivePrediction), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'prediction')

    def cancel(self):
        self.items = [(self.priorTextBuffer, self.priorTextBuffer.fullPath, '')]
        self.index = 0
        self.changeToHostWindow()

    def cursorMoveTo(self, row, col):
        if app.config.strict_debug:
            assert isinstance(row, int)
            assert isinstance(col, int)
        textBuffer = self.view.host.textBuffer
        textBuffer.cursorMoveTo(row, col)
        textBuffer.cursorScrollToMiddle()
        textBuffer.redo()

    def focus(self):
        app.log.info('InteractivePrediction.focus')
        app.controller.Controller.focus(self)
        self.priorTextBuffer = self.view.host.textBuffer
        self.index = self.buildFileList(self.view.host.textBuffer.fullPath)
        self.view.host.setTextBuffer(text_buffer.TextBuffer())
        self.commandDefault = self.view.textBuffer.insertPrintable
        self.view.host.textBuffer.lineLimitIndicator = 0
        self.view.host.textBuffer.rootGrammar = \
            self.view.program.prefs.getGrammar('_pre')

    def info(self):
        app.log.info('InteractivePrediction command set')

    def buildFileList(self, currentFile):
        if app.config.strict_debug:
            assert isinstance(currentFile, str)
        self.items = []
        bufferManager = self.view.program.bufferManager
        for i in bufferManager.buffers:
            dirty = '*' if i.isDirty() else '.'
            if i.fullPath:
                self.items.append((i, i.fullPath, dirty))
            else:
                self.items.append(
                    (i, '<new file> %s' % (i.parser.rowText(0)[:20]), dirty))
        dirPath, fileName = os.path.split(currentFile)
        fileName, ext = os.path.splitext(fileName)
        # TODO(dschuyler): rework this ignore list.
        ignoreExt = set((
            '.pyc',
            '.pyo',
            '.o',
            '.obj',
            '.tgz',
            '.zip',
            '.tar',
        ))
        try:
            contents = os.listdir(
                os.path.expandvars(os.path.expanduser(dirPath)) or '.')
        except OSError:
            contents = []
        contents.sort()
        for i in contents:
            f, e = os.path.splitext(i)
            if fileName == f and ext != e and e not in ignoreExt:
                self.items.append((None, os.path.join(dirPath, i), '='))
        if 1:
            app.log.info()
            # Chromium specific hack.
            if currentFile.endswith('-extracted.js'):
                chromiumPath = currentFile[:-len('-extracted.js')] + '.html'
                app.log.info(chromiumPath)
                if os.path.isfile(chromiumPath):
                    app.log.info()
                    self.items.append((None, chromiumPath, '='))
            elif currentFile.endswith('.html'):
                app.log.info()
                chromiumPath = currentFile[:-len('.html')] + '-extracted.js'
                if os.path.isfile(chromiumPath):
                    app.log.info()
                    self.items.append((None, chromiumPath, '='))
        # Suggest item.
        return (len(bufferManager.buffers) - 2) % len(self.items)

    def onChange(self):
        clip = []
        limit = max(5, self.view.host.cols - 10)
        for i, item in enumerate(self.items):
            prefix = '-->' if i == self.index else '   '
            suffix = ' <--' if i == self.index else ''
            clip.append(
                "%s %s %s%s" % (prefix, item[1][-limit:], item[2], suffix))
        self.view.host.textBuffer.selectionAll()
        self.view.host.textBuffer.editPasteLines(tuple(clip))
        self.cursorMoveTo(self.index, 0)

    def nextItem(self):
        self.index = (self.index + 1) % len(self.items)

    def priorItem(self):
        self.index = (self.index - 1) % len(self.items)

    def selectItem(self):
        self.changeToHostWindow()

    def unfocus(self):
        app.controller.Controller.unfocus(self)
        if self.items is None:
            return
        bufferManager = self.view.program.bufferManager
        textBuffer, fullPath = self.items[self.index][:2]
        if textBuffer is not None:
            self.view.host.setTextBuffer(
                bufferManager.getValidTextBuffer(textBuffer))
        else:
            expandedPath = os.path.abspath(os.path.expanduser(fullPath))
            textBuffer = bufferManager.loadTextBuffer(expandedPath)
            self.view.host.setTextBuffer(textBuffer)
        self.items = None

class InteractiveFind(app.controller.Controller):
    """Find text within the current document."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractiveFind), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'find')

    def findNext(self):
        self.findCmd = self.view.host.textBuffer.findNext

    def findPrior(self):
        self.findCmd = self.view.host.textBuffer.findPrior

    def focus(self):
        self.findCmd = self.view.host.textBuffer.find
        selection = self.view.host.textBuffer.getSelectedText()
        if selection:
            self.view.findLine.textBuffer.selectionAll()
            # Make a single regex line.
            selection = "\\n".join(selection)
            app.log.info(selection)
            self.view.findLine.textBuffer.insert(re.escape(selection))
        self.view.findLine.textBuffer.selectionAll()

    def onChange(self):
        self.view.findLine.textBuffer.parseScreenMaybe()
        searchFor = self.view.findLine.textBuffer.parser.rowText(0)
        try:
            self.findCmd(searchFor)
        except re.error as e:
            self.error = e.message
        self.findCmd = self.view.host.textBuffer.find

    def replaceAndNext(self):
        replaceWith = self.view.replaceLine.textBuffer.parser.rowText(0)
        self.view.host.textBuffer.replaceFound(replaceWith)
        self.findCmd = self.view.host.textBuffer.findNext

    def replaceAndPrior(self):
        replaceWith = self.view.replaceLine.textBuffer.parser.rowText(0)
        self.view.host.textBuffer.replaceFound(replaceWith)
        self.findCmd = self.view.host.textBuffer.findPrior

class InteractiveFindInput(app.controller.Controller):
    """Find text within the current document."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractiveFindInput), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'find')

    def nextFocusableWindow(self):
        self.view.parent.expandFindWindow(True)
        app.controller.Controller.nextFocusableWindow(self)

    #def priorFocusableWindow(self):
    #  if not app.controller.Controller.priorFocusableWindow(self):
    #    self.view.host.expandFindWindow(False)

    def findNext(self):
        self.parentController().findNext()

    def findPrior(self):
        self.parentController().findPrior()

    def info(self):
        app.log.info('InteractiveFind command set')

    def onChange(self):
        self.parentController().onChange()

    def replaceAndNext(self):
        self.parentController().replaceAndNext()

    def replaceAndPrior(self):
        self.parentController().replaceAndPrior()

class InteractiveGoto(app.controller.Controller):
    """Jump to a particular line number."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractiveGoto), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'goto')

    def focus(self):
        app.log.info('InteractiveGoto.focus')
        self.textBuffer.selectionAll()
        self.textBuffer.insert(str(self.view.host.textBuffer.penRow + 1))
        self.textBuffer.selectionAll()

    def info(self):
        app.log.info('InteractiveGoto command set')

    def gotoBottom(self):
        app.log.info()
        self.textBuffer.selectionAll()
        self.textBuffer.insert(str(self.view.host.textBuffer.parser.rowCount()))
        self.changeToHostWindow()

    def gotoHalfway(self):
        self.textBuffer.selectionAll()
        self.textBuffer.insert(
            str(self.view.host.textBuffer.parser.rowCount() // 2 + 1))
        self.changeToHostWindow()

    def gotoTop(self):
        self.textBuffer.selectionAll()
        self.textBuffer.insert("0")
        self.changeToHostWindow()

    def cursorMoveTo(self, row, col):
        if app.config.strict_debug:
            assert isinstance(row, int)
            assert isinstance(col, int)
        textBuffer = self.view.host.textBuffer
        textBuffer.cursorMoveTo(row, col)
        textBuffer.cursorScrollToMiddle()
        textBuffer.redo()

    def onChange(self):
        app.log.info()
        self.textBuffer.parseDocument()
        line = self.textBuffer.parser.rowText(0)
        gotoLine, gotoCol = (line.split(U',') + [U'0', U'0'])[:2]
        self.cursorMoveTo(parseInt(gotoLine) - 1, parseInt(gotoCol))

class ToggleController(app.controller.Controller):

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, ToggleController), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'toggle')

    def clearValue(self):
        category = self.view.prefCategory
        name = self.view.prefName
        prefs = self.view.program.prefs
        prefs.save(category, name, None)
        self.view.onPrefChanged(category, name)

    def toggleValue(self):
        category = self.view.prefCategory
        name = self.view.prefName
        prefs = self.view.program.prefs
        prefs.save(category, name, not prefs.category(category)[name])
        self.view.onPrefChanged(category, name)
<commit_msg>fix re.error reporting<commit_after># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Interactive UIs for the ciEditor."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
try:
    unicode
except NameError:
    unicode = str
    unichr = chr

import os
import re

import app.config
import app.controller
import app.text_buffer

def parseInt(inStr):
    if app.config.strict_debug:
        assert isinstance(inStr, unicode), type(inStr)
    i = 0
    k = 0
    if len(inStr) > i and inStr[i] in ('+', '-'):
        i += 1
    k = i
    while len(inStr) > k and inStr[k].isdigit():
        k += 1
    if k > i:
        return int(inStr[:k])
    return 0

def test_parseInt():
    assert parseInt('0') == 0
    assert parseInt('0e') == 0
    assert parseInt('text') == 0
    assert parseInt('10') == 10
    assert parseInt('+10') == 10
    assert parseInt('-10') == -10
    assert parseInt('--10') == 0
    assert parseInt('--10') == 0

class InteractivePrediction(app.controller.Controller):
    """Make a guess about what the user desires."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractivePrediction), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'prediction')

    def cancel(self):
        self.items = [(self.priorTextBuffer, self.priorTextBuffer.fullPath, '')]
        self.index = 0
        self.changeToHostWindow()

    def cursorMoveTo(self, row, col):
        if app.config.strict_debug:
            assert isinstance(row, int)
            assert isinstance(col, int)
        textBuffer = self.view.host.textBuffer
        textBuffer.cursorMoveTo(row, col)
        textBuffer.cursorScrollToMiddle()
        textBuffer.redo()

    def focus(self):
        app.log.info('InteractivePrediction.focus')
        app.controller.Controller.focus(self)
        self.priorTextBuffer = self.view.host.textBuffer
        self.index = self.buildFileList(self.view.host.textBuffer.fullPath)
        self.view.host.setTextBuffer(text_buffer.TextBuffer())
        self.commandDefault = self.view.textBuffer.insertPrintable
        self.view.host.textBuffer.lineLimitIndicator = 0
        self.view.host.textBuffer.rootGrammar = \
            self.view.program.prefs.getGrammar('_pre')

    def info(self):
        app.log.info('InteractivePrediction command set')

    def buildFileList(self, currentFile):
        if app.config.strict_debug:
            assert isinstance(currentFile, str)
        self.items = []
        bufferManager = self.view.program.bufferManager
        for i in bufferManager.buffers:
            dirty = '*' if i.isDirty() else '.'
            if i.fullPath:
                self.items.append((i, i.fullPath, dirty))
            else:
                self.items.append(
                    (i, '<new file> %s' % (i.parser.rowText(0)[:20]), dirty))
        dirPath, fileName = os.path.split(currentFile)
        fileName, ext = os.path.splitext(fileName)
        # TODO(dschuyler): rework this ignore list.
        ignoreExt = set((
            '.pyc',
            '.pyo',
            '.o',
            '.obj',
            '.tgz',
            '.zip',
            '.tar',
        ))
        try:
            contents = os.listdir(
                os.path.expandvars(os.path.expanduser(dirPath)) or '.')
        except OSError:
            contents = []
        contents.sort()
        for i in contents:
            f, e = os.path.splitext(i)
            if fileName == f and ext != e and e not in ignoreExt:
                self.items.append((None, os.path.join(dirPath, i), '='))
        if 1:
            app.log.info()
            # Chromium specific hack.
            if currentFile.endswith('-extracted.js'):
                chromiumPath = currentFile[:-len('-extracted.js')] + '.html'
                app.log.info(chromiumPath)
                if os.path.isfile(chromiumPath):
                    app.log.info()
                    self.items.append((None, chromiumPath, '='))
            elif currentFile.endswith('.html'):
                app.log.info()
                chromiumPath = currentFile[:-len('.html')] + '-extracted.js'
                if os.path.isfile(chromiumPath):
                    app.log.info()
                    self.items.append((None, chromiumPath, '='))
        # Suggest item.
        return (len(bufferManager.buffers) - 2) % len(self.items)

    def onChange(self):
        clip = []
        limit = max(5, self.view.host.cols - 10)
        for i, item in enumerate(self.items):
            prefix = '-->' if i == self.index else '   '
            suffix = ' <--' if i == self.index else ''
            clip.append(
                "%s %s %s%s" % (prefix, item[1][-limit:], item[2], suffix))
        self.view.host.textBuffer.selectionAll()
        self.view.host.textBuffer.editPasteLines(tuple(clip))
        self.cursorMoveTo(self.index, 0)

    def nextItem(self):
        self.index = (self.index + 1) % len(self.items)

    def priorItem(self):
        self.index = (self.index - 1) % len(self.items)

    def selectItem(self):
        self.changeToHostWindow()

    def unfocus(self):
        app.controller.Controller.unfocus(self)
        if self.items is None:
            return
        bufferManager = self.view.program.bufferManager
        textBuffer, fullPath = self.items[self.index][:2]
        if textBuffer is not None:
            self.view.host.setTextBuffer(
                bufferManager.getValidTextBuffer(textBuffer))
        else:
            expandedPath = os.path.abspath(os.path.expanduser(fullPath))
            textBuffer = bufferManager.loadTextBuffer(expandedPath)
            self.view.host.setTextBuffer(textBuffer)
        self.items = None

class InteractiveFind(app.controller.Controller):
    """Find text within the current document."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractiveFind), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'find')

    def findNext(self):
        self.findCmd = self.view.host.textBuffer.findNext

    def findPrior(self):
        self.findCmd = self.view.host.textBuffer.findPrior

    def focus(self):
        self.findCmd = self.view.host.textBuffer.find
        selection = self.view.host.textBuffer.getSelectedText()
        if selection:
            self.view.findLine.textBuffer.selectionAll()
            # Make a single regex line.
            selection = "\\n".join(selection)
            app.log.info(selection)
            self.view.findLine.textBuffer.insert(re.escape(selection))
        self.view.findLine.textBuffer.selectionAll()

    def onChange(self):
        self.view.findLine.textBuffer.parseScreenMaybe()
        searchFor = self.view.findLine.textBuffer.parser.rowText(0)
        try:
            self.findCmd(searchFor)
        except re.error as e:
            if hasattr(e, 'msg'):
                self.error = e.msg
            elif hasattr(e, 'message'):
                self.error = e.message
            else:
                self.error = u"invalid regex"
        self.findCmd = self.view.host.textBuffer.find

    def replaceAndNext(self):
        replaceWith = self.view.replaceLine.textBuffer.parser.rowText(0)
        self.view.host.textBuffer.replaceFound(replaceWith)
        self.findCmd = self.view.host.textBuffer.findNext

    def replaceAndPrior(self):
        replaceWith = self.view.replaceLine.textBuffer.parser.rowText(0)
        self.view.host.textBuffer.replaceFound(replaceWith)
        self.findCmd = self.view.host.textBuffer.findPrior

class InteractiveFindInput(app.controller.Controller):
    """Find text within the current document."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractiveFindInput), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'find')

    def nextFocusableWindow(self):
        self.view.parent.expandFindWindow(True)
        app.controller.Controller.nextFocusableWindow(self)

    #def priorFocusableWindow(self):
    #  if not app.controller.Controller.priorFocusableWindow(self):
    #    self.view.host.expandFindWindow(False)

    def findNext(self):
        self.parentController().findNext()

    def findPrior(self):
        self.parentController().findPrior()

    def info(self):
        app.log.info('InteractiveFind command set')

    def onChange(self):
        self.parentController().onChange()

    def replaceAndNext(self):
        self.parentController().replaceAndNext()

    def replaceAndPrior(self):
        self.parentController().replaceAndPrior()

class InteractiveGoto(app.controller.Controller):
    """Jump to a particular line number."""

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, InteractiveGoto), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'goto')

    def focus(self):
        app.log.info('InteractiveGoto.focus')
        self.textBuffer.selectionAll()
        self.textBuffer.insert(str(self.view.host.textBuffer.penRow + 1))
        self.textBuffer.selectionAll()

    def info(self):
        app.log.info('InteractiveGoto command set')

    def gotoBottom(self):
        app.log.info()
        self.textBuffer.selectionAll()
        self.textBuffer.insert(str(self.view.host.textBuffer.parser.rowCount()))
        self.changeToHostWindow()

    def gotoHalfway(self):
        self.textBuffer.selectionAll()
        self.textBuffer.insert(
            str(self.view.host.textBuffer.parser.rowCount() // 2 + 1))
        self.changeToHostWindow()

    def gotoTop(self):
        self.textBuffer.selectionAll()
        self.textBuffer.insert("0")
        self.changeToHostWindow()

    def cursorMoveTo(self, row, col):
        if app.config.strict_debug:
            assert isinstance(row, int)
            assert isinstance(col, int)
        textBuffer = self.view.host.textBuffer
        textBuffer.cursorMoveTo(row, col)
        textBuffer.cursorScrollToMiddle()
        textBuffer.redo()

    def onChange(self):
        app.log.info()
        self.textBuffer.parseDocument()
        line = self.textBuffer.parser.rowText(0)
        gotoLine, gotoCol = (line.split(U',') + [U'0', U'0'])[:2]
        self.cursorMoveTo(parseInt(gotoLine) - 1, parseInt(gotoCol))

class ToggleController(app.controller.Controller):

    def __init__(self, view):
        if app.config.strict_debug:
            assert issubclass(self.__class__, ToggleController), self
            assert issubclass(view.__class__, app.window.ViewWindow), view
        app.controller.Controller.__init__(self, view, 'toggle')

    def clearValue(self):
        category = self.view.prefCategory
        name = self.view.prefName
        prefs = self.view.program.prefs
        prefs.save(category, name, None)
        self.view.onPrefChanged(category, name)

    def toggleValue(self):
        category = self.view.prefCategory
        name = self.view.prefName
        prefs = self.view.program.prefs
        prefs.save(category, name, not prefs.category(category)[name])
        self.view.onPrefChanged(category, name)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# asciimatics documentation build configuration file, created by
# sphinx-quickstart on Fri Apr  3 17:57:45 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# import sys
# import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.ifconfig',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'asciimatics'
copyright = u'2015, Peter Brittain'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.3.0'
# The full version, including alpha/beta/rc tags.
release = '1.3.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'classic'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'asciimaticsdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
    ('index', 'asciimatics.tex', u'asciimatics Documentation',
     u'Peter Brittain', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'asciimatics', u'asciimatics Documentation',
     [u'Peter Brittain'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
    ('index', 'asciimatics', u'asciimatics Documentation',
     u'Peter Brittain', 'asciimatics', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

# Include constructor comments for classes.
autoclass_content = "both"
<commit_msg>Update conf.py<commit_after># -*- coding: utf-8 -*-
#
# asciimatics documentation build configuration file, created by
# sphinx-quickstart on Fri Apr  3 17:57:45 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# import sys
# import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.ifconfig',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'asciimatics'
copyright = u'2015, Peter Brittain'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.4.0'
# The full version, including alpha/beta/rc tags.
release = '1.4.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'classic'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'asciimaticsdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
    ('index', 'asciimatics.tex', u'asciimatics Documentation',
     u'Peter Brittain', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'asciimatics', u'asciimatics Documentation',
     [u'Peter Brittain'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
    ('index', 'asciimatics', u'asciimatics Documentation',
     u'Peter Brittain', 'asciimatics', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

# Include constructor comments for classes.
autoclass_content = "both"
<|endoftext|>


<commit_before># Copyright 2021 The Flax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Normalization modules for Flax."""

from typing import (Any, Callable, Optional, Tuple)

from jax import lax
from jax.nn import initializers
import jax.numpy as jnp

from flax.linen.module import Module, compact, merge_param

PRNGKey = Any
Array = Any
Shape = Tuple[int]
Dtype = Any  # this could be a real type?

_no_init = lambda rng, shape: ()

def _absolute_dims(rank, dims):
  return tuple([rank + dim if dim < 0 else dim for dim in dims])

class BatchNorm(Module):
  """BatchNorm Module.

  Usage Note:
  If we define a model with BatchNorm, for example::

    BN = nn.BatchNorm(use_running_average=False, momentum=0.9, epsilon=1e-5,
                      dtype=jnp.float32)

  The initialized variables dict will contain in addition to a 'params'
  collection a separate 'batch_stats' collection that will contain all the
  running statistics for all the BatchNorm layers in a model::

    vars_initialized = BN.init(key, x)  # {'params': ..., 'batch_stats': ...}

  We then update the batch_stats during training by specifying that the
  `batch_stats` collection is mutable in the `apply` method for our module.::

    vars_in = {'params': params, 'batch_stats': old_batch_stats}
    y, mutated_vars = BN.apply(vars_in, x, mutable=['batch_stats'])
    new_batch_stats = mutated_vars['batch_stats']

  During eval we would define BN with `use_running_average=True` and use the
  batch_stats collection from training to set the statistics.  In this case
  we are not mutating the batch statistics collection, and needn't mark it
  mutable::

    vars_in = {'params': params, 'batch_stats': training_batch_stats}
    y = BN.apply(vars_in, x)

  Attributes:
    use_running_average: if True, the statistics stored in batch_stats
      will be used instead of computing the batch statistics on the input.
    axis: the feature or non-batch axis of the input.
    momentum: decay rate for the exponential moving average of
      the batch statistics.
    epsilon: a small float added to variance to avoid dividing by zero.
    dtype: the dtype of the computation (default: float32).
    use_bias:  if True, bias (beta) is added.
    use_scale: if True, multiply by scale (gamma).
      When the next layer is linear (also e.g. nn.relu), this can be disabled
      since the scaling will be done by the next layer.
    bias_init: initializer for bias, by default, zero.
    scale_init: initializer for scale, by default, one.
    axis_name: the axis name used to combine batch statistics from multiple
      devices. See `jax.pmap` for a description of axis names (default: None).
    axis_index_groups: groups of axis indices within that named axis
      representing subsets of devices to reduce over (default: None). For
      example, `[[0, 1], [2, 3]]` would independently batch-normalize over
      the examples on the first two and last two devices. See `jax.lax.psum`
      for more details.
  """
  use_running_average: Optional[bool] = None
  axis: int = -1
  momentum: float = 0.99
  epsilon: float = 1e-5
  dtype: Dtype = jnp.float32
  use_bias: bool = True
  use_scale: bool = True
  bias_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.zeros
  scale_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.ones
  axis_name: Optional[str] = None
  axis_index_groups: Any = None

  @compact
  def __call__(self, x, use_running_average: Optional[bool] = None):
    """Normalizes the input using batch statistics.

    NOTE:
    During initialization (when parameters are mutable) the running average
    of the batch statistics will not be updated. Therefore, the inputs
    fed during initialization don't need to match that of the actual input
    distribution and the reduction axis (set with `axis_name`) does not have
    to exist.

    Args:
      x: the input to be normalized.
      use_running_average: if true, the statistics stored in batch_stats
        will be used instead of computing the batch statistics on the input.

    Returns:
      Normalized inputs (the same shape as inputs).
    """
    use_running_average = merge_param(
        'use_running_average', self.use_running_average, use_running_average)
    x = jnp.asarray(x, jnp.float32)
    axis = self.axis if isinstance(self.axis, tuple) else (self.axis,)
    axis = _absolute_dims(x.ndim, axis)
    feature_shape = tuple(d if i in axis else 1 for i, d in enumerate(x.shape))
    reduced_feature_shape = tuple(d for i, d in enumerate(x.shape) if i in axis)
    reduction_axis = tuple(i for i in range(x.ndim) if i not in axis)

    # see NOTE above on initialization behavior
    initializing = self.is_mutable_collection('params')

    ra_mean = self.variable('batch_stats', 'mean',
                            lambda s: jnp.zeros(s, jnp.float32),
                            reduced_feature_shape)
    ra_var = self.variable('batch_stats', 'var',
                           lambda s: jnp.ones(s, jnp.float32),
                           reduced_feature_shape)

    if use_running_average:
      mean, var = ra_mean.value, ra_var.value
    else:
      mean = jnp.mean(x, axis=reduction_axis, keepdims=False)
      mean2 = jnp.mean(lax.square(x), axis=reduction_axis, keepdims=False)
      if self.axis_name is not None and not initializing:
        concatenated_mean = jnp.concatenate([mean, mean2])
        mean, mean2 = jnp.split(
            lax.pmean(
                concatenated_mean,
                axis_name=self.axis_name,
                axis_index_groups=self.axis_index_groups), 2)
      var = mean2 - lax.square(mean)

      if not initializing:
        ra_mean.value = self.momentum * ra_mean.value + (1 - self.momentum) * mean
        ra_var.value = self.momentum * ra_var.value + (1 - self.momentum) * var

    y = x - mean.reshape(feature_shape)
    mul = lax.rsqrt(var + self.epsilon).reshape(feature_shape)
    if self.use_scale:
      scale = self.param('scale',
                         self.scale_init,
                         reduced_feature_shape).reshape(feature_shape)
      mul = mul * scale
    y = y * mul
    if self.use_bias:
      bias = self.param('bias',
                        self.bias_init,
                        reduced_feature_shape).reshape(feature_shape)
      y = y + bias
    return jnp.asarray(y, self.dtype)

class LayerNorm(Module):
  """Layer normalization (https://arxiv.org/abs/1607.06450).
  Operates on the last axis of the input data.

  It normalizes the activations of the layer for each given example in a
  batch independently, rather than across a batch like Batch Normalization.
  i.e. applies a transformation that maintains the mean activation within
  each example close to 0 and the activation standard deviation close to 1.

  Attributes:
    epsilon: A small float added to variance to avoid dividing by zero.
    dtype: the dtype of the computation (default: float32).
    use_bias:  If True, bias (beta) is added.
    use_scale: If True, multiply by scale (gamma). When the next layer is linear
      (also e.g. nn.relu), this can be disabled since the scaling will be done
      by the next layer.
    bias_init: Initializer for bias, by default, zero.
    scale_init: Initializer for scale, by default, one.
  """
  epsilon: float = 1e-6
  dtype: Any = jnp.float32
  use_bias: bool = True
  use_scale: bool = True
  bias_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.zeros
  scale_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.ones

  @compact
  def __call__(self, x):
    """Applies layer normalization on the input.

    Args:
      x: the inputs

    Returns:
      Normalized inputs (the same shape as inputs).
    """
    x = jnp.asarray(x, jnp.float32)
    features = x.shape[-1]
    feature_shape = (1,) * (x.ndim - 1) + (features,)
    mean = jnp.mean(x, axis=-1, keepdims=True)
    mean2 = jnp.mean(lax.square(x), axis=-1, keepdims=True)
    var = mean2 - lax.square(mean)
    mul = lax.rsqrt(var + self.epsilon)
    if self.use_scale:
      mul = mul * jnp.asarray(
          self.param('scale', self.scale_init, (features,)),
          self.dtype).reshape(feature_shape)
    y = (x - mean) * mul
    if self.use_bias:
      y += jnp.asarray(
          self.param('bias', self.bias_init, (features,)),
          self.dtype).reshape(feature_shape)
    return jnp.asarray(y, self.dtype)

class GroupNorm(Module):
  """Group normalization (arxiv.org/abs/1803.08494).

    This op is similar to batch normalization, but statistics are shared across
    equally-sized groups of channels and not shared across batch dimension.
    Thus, group normalization does not depend on the batch composition and does
    not require maintaining internal state for storing statistics.
    The user should either specify the total number of channel groups or the
    number of channels per group.

    Attributes:
      num_groups: the total number of channel groups. The default value of 32 is
        proposed by the original group normalization paper.
      group_size: the number of channels in a group.
      epsilon: A small float added to variance to avoid dividing by zero.
      dtype: the dtype of the computation (default: float32).
      use_bias:  If True, bias (beta) is added.
      use_scale: If True, multiply by scale (gamma). When the next layer is linear
        (also e.g. nn.relu), this can be disabled since the scaling will be done
        by the next layer.
      bias_init: Initializer for bias, by default, zero.
      scale_init: Initializer for scale, by default, one.
  """
  num_groups: int = 32
  group_size: Optional[int] = None
  epsilon: float = 1e-6
  dtype: Any = jnp.float32
  use_bias: bool = True
  use_scale: bool = True
  bias_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.zeros
  scale_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.ones

  @compact
  def __call__(self, x):
    """Applies group normalization to the input (arxiv.org/abs/1803.08494).

    Args:
      x: the input of shape N...C, where N is a batch dimension and C is a
        channels dimensions. `...` represents an arbitrary number of extra
        dimensions that are used to accumulate statistics over.

    Returns:
      Normalized inputs (the same shape as inputs).
    """
    x = jnp.asarray(x, jnp.float32)
    if ((self.num_groups is None and self.group_size is None) or
        (self.num_groups is not None and self.group_size is not None)):
      raise ValueError('Either `num_groups` or `group_size` should be '
                       'specified, but not both of them.')
    num_groups = self.num_groups

    channels = x.shape[-1]
    if self.group_size is not None:
      if channels % self.group_size != 0:
        raise ValueError('Number of channels ({}) is not multiple of the '
                         'group size ({}).'.format(channels, self.group_size))
      num_groups = channels // self.group_size

    if num_groups <= 0 or channels % num_groups != 0:
      raise ValueError('Number of groups ({}) does not divide the number'
                       ' of channels ({}).'.format(num_groups, channels))

    input_shape = x.shape
    group_shape = x.shape[:-1] + (num_groups, x.shape[-1] // num_groups)
    x = x.reshape(group_shape)

    reduction_axis = [d for d in range(1, x.ndim - 2)] + [x.ndim - 1]
    mean = jnp.mean(x, axis=reduction_axis, keepdims=True)
    mean_of_squares = jnp.mean(jnp.square(x), axis=reduction_axis,
                               keepdims=True)
    var = mean_of_squares - jnp.square(mean)
    x = (x - mean) * lax.rsqrt(var + self.epsilon)
    x = x.reshape(input_shape)

    feature_shape = tuple([1 for d in input_shape[:-1]] + [input_shape[-1]])
    if self.use_scale:
      x = x * self.param('scale', self.scale_init, feature_shape)
    if self.use_bias:
      x = x + self.param('bias', self.bias_init, feature_shape)

    return x.astype(self.dtype)
<commit_msg>Resubmit normalization refactor<commit_after># Copyright 2021 The Flax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Normalization modules for Flax."""

from typing import (Any, Callable, Optional, Tuple, Iterable, Union)

from jax import lax
from jax.nn import initializers
import jax.numpy as jnp

from flax.linen.module import Module, compact, merge_param

PRNGKey = Any
Array = Any
Shape = Tuple[int]
Dtype = Any  # this could be a real type?

Axes = Union[int, Iterable[int]]

def _canonicalize_axes(rank: int, axes: Axes) -> Iterable[int]:
  """Returns a tuple of deduplicated, sorted, and positive axes."""
  if not isinstance(axes, Iterable):
    axes = (axes,)
  return tuple(set([rank + axis if axis < 0 else axis for axis in axes]))

def _compute_stats(x: Array, axes: Axes,
                   axis_name: Optional[str] = None,
                   axis_index_groups: Any = None):
  """Computes mean and variance statistics.

  This implementation takes care of a few important details:
  - Computes in float32 precision for half precision inputs
  -  mean and variance is computable in a single XLA fusion,
    by using Var = E[x^2] - E[x]^2 instead of Var = E[(x - E[x])^2]).
  - Clips negative variances to zero which can happen due to
    roundoff errors. This avoids downstream NaNs.
  - Supports averaging across a parallel axis and subgroups of a parallel axis
    with a single `lax.pmean` call to avoid latency.
  """
  x = jnp.asarray(x, jnp.float32)
  mean = jnp.mean(x, axes)
  mean2 = jnp.mean(lax.square(x), axes)
  if axis_name is not None:
    concatenated_mean = jnp.concatenate([mean, mean2])
    mean, mean2 = jnp.split(
        lax.pmean(
            concatenated_mean,
            axis_name=axis_name,
            axis_index_groups=axis_index_groups), 2)
  # mean2 - lax.square(mean) is not guaranteed to be non-negative due
  # to floating point round-off errors.
  var = jnp.maximum(0., mean2 - lax.square(mean))
  return mean, var

def _normalize(mdl: Module, x: Array, mean: Array, var: Array,
               reduction_axes: Axes, feature_axes: Axes,
               dtype: Dtype, epsilon: float,
               use_bias: bool, use_scale: bool,
               bias_init: Callable[[PRNGKey, Shape, Dtype], Array],
               scale_init: Callable[[PRNGKey, Shape, Dtype], Array]):
  """"Normalizes the input of a normalization layer and optionally applies a learned scale and bias.

  A seperate bias and scale is learned for each feature as specified by feature_axes.
  """
  reduction_axes = _canonicalize_axes(x.ndim, reduction_axes)
  feature_axes = _canonicalize_axes(x.ndim, feature_axes)
  stats_shape = list(x.shape)
  for axis in reduction_axes:
    stats_shape[axis] = 1
  mean = mean.reshape(stats_shape)
  var = var.reshape(stats_shape)
  feature_shape = [1] * x.ndim
  reduced_feature_shape = []
  for ax in feature_axes:
    feature_shape[ax] = x.shape[ax]
    reduced_feature_shape.append(x.shape[ax])
  y = x - mean
  mul = lax.rsqrt(var + epsilon)
  if use_scale:
    scale = mdl.param('scale', scale_init, reduced_feature_shape).reshape(feature_shape)
    mul *= scale
  y *= mul
  if use_bias:
    bias = mdl.param('bias', bias_init, reduced_feature_shape).reshape(feature_shape)
    y += bias
  return jnp.asarray(y, dtype)

class BatchNorm(Module):
  """BatchNorm Module.

  Usage Note:
  If we define a model with BatchNorm, for example::

    BN = nn.BatchNorm(use_running_average=False, momentum=0.9, epsilon=1e-5,
                      dtype=jnp.float32)

  The initialized variables dict will contain in addition to a 'params'
  collection a separate 'batch_stats' collection that will contain all the
  running statistics for all the BatchNorm layers in a model::

    vars_initialized = BN.init(key, x)  # {'params': ..., 'batch_stats': ...}

  We then update the batch_stats during training by specifying that the
  `batch_stats` collection is mutable in the `apply` method for our module.::

    vars_in = {'params': params, 'batch_stats': old_batch_stats}
    y, mutated_vars = BN.apply(vars_in, x, mutable=['batch_stats'])
    new_batch_stats = mutated_vars['batch_stats']

  During eval we would define BN with `use_running_average=True` and use the
  batch_stats collection from training to set the statistics.  In this case
  we are not mutating the batch statistics collection, and needn't mark it
  mutable::

    vars_in = {'params': params, 'batch_stats': training_batch_stats}
    y = BN.apply(vars_in, x)

  Attributes:
    use_running_average: if True, the statistics stored in batch_stats
      will be used instead of computing the batch statistics on the input.
    axis: the feature or non-batch axis of the input.
    momentum: decay rate for the exponential moving average of
      the batch statistics.
    epsilon: a small float added to variance to avoid dividing by zero.
    dtype: the dtype of the computation (default: float32).
    use_bias:  if True, bias (beta) is added.
    use_scale: if True, multiply by scale (gamma).
      When the next layer is linear (also e.g. nn.relu), this can be disabled
      since the scaling will be done by the next layer.
    bias_init: initializer for bias, by default, zero.
    scale_init: initializer for scale, by default, one.
    axis_name: the axis name used to combine batch statistics from multiple
      devices. See `jax.pmap` for a description of axis names (default: None).
    axis_index_groups: groups of axis indices within that named axis
      representing subsets of devices to reduce over (default: None). For
      example, `[[0, 1], [2, 3]]` would independently batch-normalize over
      the examples on the first two and last two devices. See `jax.lax.psum`
      for more details.
  """
  use_running_average: Optional[bool] = None
  axis: int = -1
  momentum: float = 0.99
  epsilon: float = 1e-5
  dtype: Dtype = jnp.float32
  use_bias: bool = True
  use_scale: bool = True
  bias_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.zeros
  scale_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.ones
  axis_name: Optional[str] = None
  axis_index_groups: Any = None

  @compact
  def __call__(self, x, use_running_average: Optional[bool] = None):
    """Normalizes the input using batch statistics.

    NOTE:
    During initialization (when parameters are mutable) the running average
    of the batch statistics will not be updated. Therefore, the inputs
    fed during initialization don't need to match that of the actual input
    distribution and the reduction axis (set with `axis_name`) does not have
    to exist.

    Args:
      x: the input to be normalized.
      use_running_average: if true, the statistics stored in batch_stats
        will be used instead of computing the batch statistics on the input.

    Returns:
      Normalized inputs (the same shape as inputs).
    """

    use_running_average = merge_param(
        'use_running_average', self.use_running_average, use_running_average)
    feature_axes = _canonicalize_axes(x.ndim, self.axis)
    reduction_axes = tuple(i for i in range(x.ndim) if i not in feature_axes)
    feature_shape = [x.shape[ax] for ax in feature_axes]

    # see NOTE above on initialization behavior
    initializing = self.is_mutable_collection('params')

    ra_mean = self.variable('batch_stats', 'mean',
                            lambda s: jnp.zeros(s, jnp.float32),
                            feature_shape)
    ra_var = self.variable('batch_stats', 'var',
                           lambda s: jnp.ones(s, jnp.float32),
                           feature_shape)

    if use_running_average:
      mean, var = ra_mean.value, ra_var.value
    else:
      mean, var = _compute_stats(
          x, reduction_axes,
          axis_name=self.axis_name if not initializing else None,
          axis_index_groups=self.axis_index_groups)

      if not initializing:
        ra_mean.value = self.momentum * ra_mean.value + (1 - self.momentum) * mean
        ra_var.value = self.momentum * ra_var.value + (1 - self.momentum) * var

    return _normalize(
        self, x, mean, var, reduction_axes, feature_axes,
        self.dtype, self.epsilon,
        self.use_bias, self.use_scale,
        self.bias_init, self.scale_init)

class LayerNorm(Module):
  """Layer normalization (https://arxiv.org/abs/1607.06450).
  Operates on the last axis of the input data.

  It normalizes the activations of the layer for each given example in a
  batch independently, rather than across a batch like Batch Normalization.
  i.e. applies a transformation that maintains the mean activation within
  each example close to 0 and the activation standard deviation close to 1.

  Attributes:
    epsilon: A small float added to variance to avoid dividing by zero.
    dtype: the dtype of the computation (default: float32).
    use_bias:  If True, bias (beta) is added.
    use_scale: If True, multiply by scale (gamma). When the next layer is linear
      (also e.g. nn.relu), this can be disabled since the scaling will be done
      by the next layer.
    bias_init: Initializer for bias, by default, zero.
    scale_init: Initializer for scale, by default, one.
  """
  epsilon: float = 1e-6
  dtype: Any = jnp.float32
  use_bias: bool = True
  use_scale: bool = True
  bias_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.zeros
  scale_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.ones

  @compact
  def __call__(self, x):
    """Applies layer normalization on the input.

    Args:
      x: the inputs

    Returns:
      Normalized inputs (the same shape as inputs).
    """
    reduction_axes = (-1,)
    feature_axes = (-1,)

    # TODO suport axis_name for model parallelism?
    mean, var = _compute_stats(x, reduction_axes, None, None)

    return _normalize(
        self, x, mean, var, reduction_axes, feature_axes,
        self.dtype, self.epsilon,
        self.use_bias, self.use_scale,
        self.bias_init, self.scale_init)

class GroupNorm(Module):
  """Group normalization (arxiv.org/abs/1803.08494).

    This op is similar to batch normalization, but statistics are shared across
    equally-sized groups of channels and not shared across batch dimension.
    Thus, group normalization does not depend on the batch composition and does
    not require maintaining internal state for storing statistics.
    The user should either specify the total number of channel groups or the
    number of channels per group.

    Attributes:
      num_groups: the total number of channel groups. The default value of 32 is
        proposed by the original group normalization paper.
      group_size: the number of channels in a group.
      epsilon: A small float added to variance to avoid dividing by zero.
      dtype: the dtype of the computation (default: float32).
      use_bias:  If True, bias (beta) is added.
      use_scale: If True, multiply by scale (gamma). When the next layer is linear
        (also e.g. nn.relu), this can be disabled since the scaling will be done
        by the next layer.
      bias_init: Initializer for bias, by default, zero.
      scale_init: Initializer for scale, by default, one.
  """
  num_groups: int = 32
  group_size: Optional[int] = None
  epsilon: float = 1e-6
  dtype: Any = jnp.float32
  use_bias: bool = True
  use_scale: bool = True
  bias_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.zeros
  scale_init: Callable[[PRNGKey, Shape, Dtype], Array] = initializers.ones

  @compact
  def __call__(self, x):
    """Applies group normalization to the input (arxiv.org/abs/1803.08494).

    Args:
      x: the input of shape N...C, where N is a batch dimension and C is a
        channels dimensions. `...` represents an arbitrary number of extra
        dimensions that are used to accumulate statistics over.

    Returns:
      Normalized inputs (the same shape as inputs).
    """
    reduction_axes = list(range(1, x.ndim - 1)) + [-1]
    feature_axes = (-1,)

    if ((self.num_groups is None and self.group_size is None) or
        (self.num_groups is not None and self.group_size is not None)):
      raise ValueError('Either `num_groups` or `group_size` should be '
                       'specified, but not both of them.')
    num_groups = self.num_groups

    channels = x.shape[-1]
    if self.group_size is not None:
      if channels % self.group_size != 0:
        raise ValueError('Number of channels ({}) is not multiple of the '
                         'group size ({}).'.format(channels, self.group_size))
      num_groups = channels // self.group_size

    if num_groups <= 0 or channels % num_groups != 0:
      raise ValueError('Number of groups ({}) does not divide the number'
                       ' of channels ({}).'.format(num_groups, channels))

    group_size = x.shape[-1] // num_groups
    group_shape = x.shape[:-1] + (num_groups, group_size)

    def broadcast_stat(stat):
      stat = jnp.broadcast_to(stat[..., None], (x.shape[0], num_groups, group_size))
      return stat.reshape((x.shape[0], num_groups * group_size))

    # TODO suport axis_name for model parallelism?
    mean, var = _compute_stats(x.reshape(group_shape), reduction_axes, None, None)
    mean = broadcast_stat(mean)
    var = broadcast_stat(var)

    return _normalize(
        self, x, mean, var, reduction_axes[:-1], feature_axes,
        self.dtype, self.epsilon,
        self.use_bias, self.use_scale,
        self.bias_init, self.scale_init)
<|endoftext|>


<commit_before>from twisted.trial.unittest import TestCase

from axiom.store import Store
from axiom.item import Item
from axiom.attributes import integer, reference

class Referee(Item):
    schemaVersion = 1
    typeName = "test_reference_referee"

    topSecret = integer()

class SimpleReferent(Item):
    schemaVersion = 1
    typeName = "test_reference_referent"

    ref = reference()

class SomeException(Exception):
    pass

class AnotherException(SomeException):
    pass

class BadReferenceTestCase(TestCase):
    exceptions = (SomeException, AnotherException, ZeroDivisionError, ValueError)
    ntimes = 10

    def testSanity(self):
        store = Store()
        for i in xrange(self.ntimes):
            SimpleReferent(store=store, ref=Referee(store=store, topSecret=i))
            (referee,) = list(store.query(Referee))
            (referent,) = list(store.query(SimpleReferent))
            self.assertEqual(referent.ref.topSecret, referee.topSecret)
            referee.deleteFromStore()
            referent.deleteFromStore()

    def _makeReferentItem(self, whenDeleted, unique):
        class _Referent(Item):
            schemaVersion = 1
            typeName = "test_reference_referent_%r" % unique

            ref = reference(whenDeleted=whenDeleted)

        return _Referent

    def testBadReferenceRaises(self):
        store = Store()
        for (i, exc) in enumerate(self.exceptions):
            MyReferent = self._makeReferentItem(exc, i)
            referee = Referee(store=store)
            referent = MyReferent(store=store, ref=referee)
            referee.deleteFromStore()
            del referee; del referent

            (referent,) = list(store.query(MyReferent))
            self.assertRaises(exc, lambda: referent.ref)
            referent.deleteFromStore()

    def testBadReferenceNone(self):
        store = Store()
        referee = Referee(store=store, topSecret=0)
        MyReferent = self._makeReferentItem(None, None)
        referent = MyReferent(store=store, ref=referee)
        referee.deleteFromStore()
        del referee; del referent

        (referent,) = list(store.query(MyReferent))
        self.assertEqual(referent.ref, None)
<commit_msg>Fix syntax errors, add .todo since this functionality doesn't actually exist yet<commit_after>from twisted.trial.unittest import TestCase

from axiom.store import Store
from axiom.item import Item
from axiom.attributes import integer, reference

class Referee(Item):
    schemaVersion = 1
    typeName = "test_reference_referee"

    topSecret = integer()

class SimpleReferent(Item):
    schemaVersion = 1
    typeName = "test_reference_referent"

    ref = reference()

class SomeException(Exception):
    pass

class AnotherException(SomeException):
    pass

class BadReferenceTestCase(TestCase):
    exceptions = (SomeException, AnotherException, ZeroDivisionError, ValueError)
    ntimes = 10

    def testSanity(self):
        store = Store()
        for i in xrange(self.ntimes):
            SimpleReferent(store=store, ref=Referee(store=store, topSecret=i))
            (referee,) = list(store.query(Referee))
            (referent,) = list(store.query(SimpleReferent))
            self.assertEqual(referent.ref.topSecret, referee.topSecret)
            referee.deleteFromStore()
            referent.deleteFromStore()

    def _makeReferentItem(self, whenDeleted, unique):
        class _Referent(Item):
            schemaVersion = 1
            typeName = "test_reference_referent_%r" % unique

            ref = reference(whenDeleted=whenDeleted)

        return _Referent

    def testBadReferenceRaises(self):
        store = Store()
        for (i, exc) in enumerate(self.exceptions):
            MyReferent = self._makeReferentItem(exc, i)
            referee = Referee(store=store)
            referent = MyReferent(store=store, ref=referee)
            referee.deleteFromStore()
            del referee

            (referent,) = list(store.query(MyReferent))
            self.assertRaises(exc, lambda: referent.ref)
            referent.deleteFromStore()

    def testBadReferenceNone(self):
        store = Store()
        referee = Referee(store=store, topSecret=0)
        MyReferent = self._makeReferentItem(None, None)
        referent = MyReferent(store=store, ref=referee)
        referee.deleteFromStore()
        del referee

        (referent,) = list(store.query(MyReferent))
        self.assertEqual(referent.ref, None)

    testBadReferenceRaises.todo = testBadReferenceNone.todo = 'No behavior yet defined for bad references'
<|endoftext|>


<commit_before>from StringIO import StringIO
from Bio import SeqIO
from Bio import AlignIO
import glob
import numpy as np

def get_median(f):
	lengths = []
	rec_count = 0
	for seq_record in SeqIO.parse(f, "fasta"):
		lengths.append(len(seq_record.seq))
		rec_count = rec_count + 1
	median = np.median(lengths)
	return (median, rec_count)

## Get list of all new fasta files

print "\n\nGetting a list of FASTA files..."
fasta_files = glob.glob("PATH/TO/INPUT/DIRECTORY/HERE/*.fasta") # get a list of all fasta files in /fasta
file_count = len(fasta_files)
print "%s files successful found.\n" %file_count

## Ask for percentage threshold

percent = raw_input("Please enter a percentage threshold as a decimal (ie: 0.50):")

 bad_char_list = [] ## just used for bug tracking.

 for f in fasta_files: #iterate through all the files in the directory
 	print "\nChecking FASTA file %s for non-nucleotide characters..." %f
 	clean_seq_list = [] #build a new list of good sequences
 	bad_seq_count = 0 #keep track of how many sequences are discarded
 	tracker = 0 #track if a sequence is bad or not.
 	for seq_record in SeqIO.parse(f, "fasta"): #for each sequence

 		dna = seq_record.seq.lower() #assign it to a string

 		for char in dna: #for each character in the sequence
 			if char not in ('a','t','c','g','n','u','r','y','k','m','s','w','b','v','h','d','x'): #if its not an accepted base or ambiguity code
 				bad_char_list.append(char) #add it to the bad list for bug tracking
 				tracker = 1 # set the bad / good tracker to bad

 		if tracker == 1: #if bad sequence
 			bad_seq_count = bad_seq_count + 1 #increment bad counter
 		elif tracker == 0: #if good sequence
 			clean_seq_list.append(seq_record) #add the record to the good list
 		tracker = 0 #after done iterating through a sequence, reset tracker for the next sequence
 	print "Complete."
 	if bad_seq_count == 0:
 		print "There were no non-nucleotide characters in FASTA file %s." % f
 	else:
 		print "There were %s non-nucleotide containing entries found in the FASTA file. Those entries have been removed." % bad_seq_count

 	SeqIO.write(clean_seq_list, f, "fasta") #after done with all iterations, write the good seq record list to the same file we started with

for f in fasta_files: #iterate through all the files in the directory
	print "\nChecking FASTA file %s for sequences of unusual lengths..." %f
	count = 0
	new_records = []
	median, rec_count = get_median(f)
	min_len = median-(percent*median)
	max_len = median+(percent*median)
	print "Removing sequences that are shorter or longer than %s%% (%s, %s) of the median length (%s bp)..." %(percent*100, min_len, max_len, median)
	for seq_record in SeqIO.parse(f, "fasta"):
		if len(seq_record.seq) > max_len or len(seq_record.seq) < min_len:
			count = count + 1.
		else:
			new_records.append(seq_record)

	SeqIO.write(new_records, f, "fasta")
	print "%s sequences removed out of %s total sequences from file %s due to unsual lengths." %(count, rec_count, f)

print "\nFASTA clean complete."
<commit_msg>Renamed to clean-fasta.py<commit_after><|endoftext|>


<commit_before># Comet VOEvent Broker.
# Event database.

import os
import anydbm
import time
from hashlib import sha1
from threading import Lock
from collections import defaultdict
from contextlib import closing

from twisted.internet.threads import deferToThread
from twisted.internet.defer import DeferredList

import comet.log as log
from comet.utility.voevent import parse_ivorn

__all__ = ["Event_DB"]

class Event_DB(object):
    def __init__(self, root):
        self.root = root
        self.databases = defaultdict(Lock)

    @staticmethod
    def _get_event_details(event):
        auth, rsrc, local = parse_ivorn(event.attrib['ivorn'])
        db_path = os.path.join(auth, rsrc).replace(os.path.sep, "_")
        key = sha1(event.text).hexdigest()
        return db_path, key

    def check_event(self, event):
        """
        Returns True if event is unseen (and hence good to forward), False
        otherwise.
        """
        try:
            db_path, key = self._get_event_details(event)
        except Exception as e:
            log.warn("Unparseable IVORN; failing eventdb lookup");
        else:
            with self.databases[db_path]: # Acquire lock
                with closing(anydbm.open(os.path.join(self.root, db_path), 'c')) as db:
                    if not db.has_key(key):
                        db[key] = str(time.time())
                        return True
        return False

    def prune(self, expiry_time):
        """
        Remove entries with age at least expiry_time seconds from the database.
        """
        def expire_db(db_path, lock):
            remove = []
            with lock:
                db = anydbm.open(os.path.join(self.root, db_path), 'c')
                # The database returned by anydbm is guaranteed to have a
                # .keys() method, but not necessarily .(iter)items().
                for key in db.keys():
                    if int(time.time() - float(db[key])) >= expiry_time:
                        # Rounding to nearest int avoids an issue when we call
                        # call prune(0) *immediately* after an insertion and might
                        # get hit by floating point weirdness.
                        remove.append(key)
                log.info("Expiring %d events from %s" % (len(remove), db_path))
                for key in remove: del db[key]
                db.close()

        return DeferredList(
            [
                deferToThread(expire_db, db_path, lock)
                for db_path, lock in self.databases.iteritems()
            ]
        )
<commit_msg>Adjust import for Python 3.<commit_after># Comet VOEvent Broker.
# Event database.

import os
try:
    import anydbm
except ImportError:
    import dbm as anydbm
import time
from hashlib import sha1
from threading import Lock
from collections import defaultdict
from contextlib import closing

from twisted.internet.threads import deferToThread
from twisted.internet.defer import DeferredList

import comet.log as log
from comet.utility.voevent import parse_ivorn

__all__ = ["Event_DB"]

class Event_DB(object):
    def __init__(self, root):
        self.root = root
        self.databases = defaultdict(Lock)

    @staticmethod
    def _get_event_details(event):
        auth, rsrc, local = parse_ivorn(event.attrib['ivorn'])
        db_path = os.path.join(auth, rsrc).replace(os.path.sep, "_")
        key = sha1(event.text).hexdigest()
        return db_path, key

    def check_event(self, event):
        """
        Returns True if event is unseen (and hence good to forward), False
        otherwise.
        """
        try:
            db_path, key = self._get_event_details(event)
        except Exception as e:
            log.warn("Unparseable IVORN; failing eventdb lookup");
        else:
            with self.databases[db_path]: # Acquire lock
                with closing(anydbm.open(os.path.join(self.root, db_path), 'c')) as db:
                    if not db.has_key(key):
                        db[key] = str(time.time())
                        return True
        return False

    def prune(self, expiry_time):
        """
        Remove entries with age at least expiry_time seconds from the database.
        """
        def expire_db(db_path, lock):
            remove = []
            with lock:
                db = anydbm.open(os.path.join(self.root, db_path), 'c')
                # The database returned by anydbm is guaranteed to have a
                # .keys() method, but not necessarily .(iter)items().
                for key in db.keys():
                    if int(time.time() - float(db[key])) >= expiry_time:
                        # Rounding to nearest int avoids an issue when we call
                        # call prune(0) *immediately* after an insertion and might
                        # get hit by floating point weirdness.
                        remove.append(key)
                log.info("Expiring %d events from %s" % (len(remove), db_path))
                for key in remove: del db[key]
                db.close()

        return DeferredList(
            [
                deferToThread(expire_db, db_path, lock)
                for db_path, lock in self.databases.iteritems()
            ]
        )
<|endoftext|>