<commit_before>import os
import sys

__version__ = '0.1.3'

runningOnPython2 = sys.version_info[0] == 2

# if SKIP_IF_OPTIMIZED is True, the printSuggestion() and printFileSuggestion()
# functions are all no-ops and do nothing when Python is run with -O
SKIP_IF_OPTIMIZED = True

'''Suggested usage:

if FILE_NOT_FOUND:
  pydidyoumean.printFileSuggestion(FILENAME) # if no suggested file is found in the cwd, this prints nothing

if COMMAND_NOT_FOUND:
  pydidyoumean.printSuggestion(CMD_NAME, LIST_OF_ALL_CMDS) # if no suggested command is found, this prints nothing

'''

def skipBecauseOptimized():
  '''If SKIP_IF_OPTIMIZED is set to True (which it is by default), then if the
  Python interpreter is being run in optimized mode (with the -O command line
  option, it determines this by checking __debug__), the   printSuggestion()
  and printFileSuggestion() functions do nothing.

  This module is really only useful for debugging while under development, but
  in production it could cause performance slow downs without providing benefit.

  Returns True if the caller should be a no-op, returns False if the caller
  should work as normal.'''
  if SKIP_IF_OPTIMIZED and not __debug__:
    return True # no-op
  else:
    return False

def printFileSuggestion(filename, message='Did you mean %s?\n', folder='.', threshold=2, includeIdenticalFilename=False):
  '''Prints the string message to display with the closest matching filename in
  the folder. Prints nothing if there are no matches within the threshold.
  Prints to sys.stdout.

  Args:
      filename (str) - The typo'd filename the user supplied.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False.'''
  if skipBecauseOptimized(): return

  sys.stdout.write(formatFileSuggestion(filename, message, folder, threshold, includeIdenticalFilename))

def formatFileSuggestion(filename, message='Did you mean %s?\n', folder='.', threshold=2, includeIdenticalFilename=False):
  '''Returns the string message to display with the closest matching
  filename. Returns a blank string if there are no matches within the
  threshold.

  Args:
      filename (str) - The typo'd filename the user supplied.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False.'''
  suggestedFile = suggestFile(filename, folder, threshold, includeIdenticalFilename)
  if suggestedFile is not None:
    return message % (suggestedFile)
  else:
    return ''

def suggestFile(filename, folder='.', threshold=2, includeIdenticalFilename=False):
  '''Returns the closest matching filename to the filenames in the folder. If
  there are multiple files with the same edit distance, the one returned is
  undefined. Returns None if there are no suggestions within the threshold.

  Args:
      filename (str) - The typo'd filename the user supplied.
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False.'''
  try:
    genObj = suggestAllFiles(filename, folder, threshold, includeIdenticalFilename)
    if runningOnPython2:
      return genObj.next() # Python 2 code
    else:
      return genObj.__next__() # Python 3 code
  except StopIteration:
    return None

def suggestAllFiles(filename, folder='.', threshold=2, includeIdenticalFilename=False):
  '''Returns all suggestions in possibleSuggestions that are within the
  levenshtein edit distance of name, sorted by distance. If there are multiple
  files with the same edit distance, they will be returned in an undefined
  order.

  Args:
      filename (str) - The typo'd filename the user supplied.
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False.'''
  filesAndDistances = [(f, levenshtein(filename, f)) for f in os.listdir(folder) if includeIdenticalFilename or f != filename]
  filesAndDistances = [x for x in filesAndDistances if x[1] <= threshold] # remove files over the threshold
  filesAndDistances.sort(key=lambda x: x[1]) # sort by comparing the levenshtein distance in x[1]

  for fd in filesAndDistances:
    yield fd[0]

def printSuggestion(name, possibleSuggestions=None, message='Did you mean %s?\n', threshold=2, includeIdenticalName=False):
  '''Prints the string message to display with the closest matching suggestion.
  Prints nothing if there are no matches within the threshold. Prints to
  sys.stdout.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name  arg will be included in the suggestions. Default is False.'''
  if skipBecauseOptimized(): return

  sys.stdout.write(formatSuggestion(name, possibleSuggestions, message, threshold, includeIdenticalName))

def formatSuggestion(name, possibleSuggestions=None, message='Did you mean %s?\n', threshold=2, includeIdenticalName=False):
  '''Returns the string message to display with the closest matching
  suggestion. Returns a blank string if there are no matches within the
  threshold.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name arg will be included in the suggestions. Default is False.'''
  suggestedName = suggest(name, possibleSuggestions, threshold, includeIdenticalName)
  if suggestedName is not None:
    return message % (suggestedName)
  else:
    return ''

def suggest(name, possibleSuggestions=None, threshold=2, includeIdenticalName=False):
  '''Returns the closest matching name to the suggestions in
  possibleSuggestions. Pass a list of all possible matches for the
  possibleSuggestions parameter. If there are multiple names with the same edit
  distance, the one returned is undefined. Returns None if there are no
  suggestions within the threshold.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name arg will be included in the suggestions. Default is False.'''
  try:
    genObj = suggestAll(name, possibleSuggestions, threshold, includeIdenticalName)
    if runningOnPython2:
      return genObj.next() # Python 2 code
    else:
      return genObj.__next__() # Python 3 code
  except StopIteration:
    return None

def suggestAll(name, possibleSuggestions=None, threshold=2, includeIdenticalName=False):
  '''Returns all suggestions in possibleSuggestions that are within the
  levenshtein edit distance of name, sorted by distance. If there are multiple
  names with the same edit distance, they will be returned in an undefined
  order.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name arg will be included in the suggestions. Default is False.'''
  if possibleSuggestions is None:
    possibleSuggestions = []
  sugsAndDistances = [(n, levenshtein(name, n)) for n in possibleSuggestions if includeIdenticalName or n != name]
  sugsAndDistances = [x for x in sugsAndDistances if x[1] <= threshold] # remove names over the threshold
  sugsAndDistances.sort(key=lambda x: x[1]) # sort by the levenshtein distance in x[1]

  for sd in sugsAndDistances:
    yield sd[0]

def levenshtein(s1, s2):
  '''Returns the levenshtein distance (aka edit distance) between two strings.

  Args:
      s1 (str) - The first string.
      s2 (str) - The second string.'''
  # Implementation from https://code.activestate.com/recipes/576874-levenshtein-distance/
  # See also, https://en.wikipedia.org/wiki/Levenshtein_distance
  s1_length = len(s1)
  s2_length = len(s2)

  matrix = [list(range(s1_length + 1))] * (s2_length + 1)
  for zz in range(s2_length + 1):
    matrix[zz] = list(range(zz,zz + s1_length + 1))
  for zz in range(0, s2_length):
    for sz in range(0, s1_length):
      if s1[sz] == s2[zz]:
        matrix[zz+1][sz+1] = min(matrix[zz+1][sz] + 1, matrix[zz][sz+1] + 1, matrix[zz][sz])
      else:
        matrix[zz+1][sz+1] = min(matrix[zz+1][sz] + 1, matrix[zz][sz+1] + 1, matrix[zz][sz] + 1)
  return matrix[s2_length][s1_length]
<commit_msg>Formatted code with Black.<commit_after>import os
import sys

__version__ = "0.1.3"

runningOnPython2 = sys.version_info[0] == 2

# if SKIP_IF_OPTIMIZED is True, the printSuggestion() and printFileSuggestion()
# functions are all no-ops and do nothing when Python is run with -O
SKIP_IF_OPTIMIZED = True

"""Suggested usage:

if FILE_NOT_FOUND:
  pydidyoumean.printFileSuggestion(FILENAME) # if no suggested file is found in the cwd, this prints nothing

if COMMAND_NOT_FOUND:
  pydidyoumean.printSuggestion(CMD_NAME, LIST_OF_ALL_CMDS) # if no suggested command is found, this prints nothing

"""

def skipBecauseOptimized():
    """If SKIP_IF_OPTIMIZED is set to True (which it is by default), then if the
  Python interpreter is being run in optimized mode (with the -O command line
  option, it determines this by checking __debug__), the   printSuggestion()
  and printFileSuggestion() functions do nothing.

  This module is really only useful for debugging while under development, but
  in production it could cause performance slow downs without providing benefit.

  Returns True if the caller should be a no-op, returns False if the caller
  should work as normal."""
    if SKIP_IF_OPTIMIZED and not __debug__:
        return True  # no-op
    else:
        return False

def printFileSuggestion(
    filename,
    message="Did you mean %s?\n",
    folder=".",
    threshold=2,
    includeIdenticalFilename=False,
):
    """Prints the string message to display with the closest matching filename in
  the folder. Prints nothing if there are no matches within the threshold.
  Prints to sys.stdout.

  Args:
      filename (str) - The typo'd filename the user supplied.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False."""
    if skipBecauseOptimized():
        return

    sys.stdout.write(
        formatFileSuggestion(
            filename, message, folder, threshold, includeIdenticalFilename
        )
    )

def formatFileSuggestion(
    filename,
    message="Did you mean %s?\n",
    folder=".",
    threshold=2,
    includeIdenticalFilename=False,
):
    """Returns the string message to display with the closest matching
  filename. Returns a blank string if there are no matches within the
  threshold.

  Args:
      filename (str) - The typo'd filename the user supplied.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False."""
    suggestedFile = suggestFile(filename, folder, threshold, includeIdenticalFilename)
    if suggestedFile is not None:
        return message % (suggestedFile)
    else:
        return ""

def suggestFile(filename, folder=".", threshold=2, includeIdenticalFilename=False):
    """Returns the closest matching filename to the filenames in the folder. If
  there are multiple files with the same edit distance, the one returned is
  undefined. Returns None if there are no suggestions within the threshold.

  Args:
      filename (str) - The typo'd filename the user supplied.
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False."""
    try:
        genObj = suggestAllFiles(filename, folder, threshold, includeIdenticalFilename)
        if runningOnPython2:
            return genObj.next()  # Python 2 code
        else:
            return genObj.__next__()  # Python 3 code
    except StopIteration:
        return None

def suggestAllFiles(filename, folder=".", threshold=2, includeIdenticalFilename=False):
    """Returns all suggestions in possibleSuggestions that are within the
  levenshtein edit distance of name, sorted by distance. If there are multiple
  files with the same edit distance, they will be returned in an undefined
  order.

  Args:
      filename (str) - The typo'd filename the user supplied.
      folder (str, optional) - The folder with filenames the user could have
        meant. Default is the . current folder.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a file from being suggested. Default is 2.
      includeIdenticalFilename (bool, optional) - If True, a filename identical
        to the filename arg will be included in the suggestions. Default is
        False."""
    filesAndDistances = [
        (f, levenshtein(filename, f))
        for f in os.listdir(folder)
        if includeIdenticalFilename or f != filename
    ]
    filesAndDistances = [
        x for x in filesAndDistances if x[1] <= threshold
    ]  # remove files over the threshold
    filesAndDistances.sort(
        key=lambda x: x[1]
    )  # sort by comparing the levenshtein distance in x[1]

    for fd in filesAndDistances:
        yield fd[0]

def printSuggestion(
    name,
    possibleSuggestions=None,
    message="Did you mean %s?\n",
    threshold=2,
    includeIdenticalName=False,
):
    """Prints the string message to display with the closest matching suggestion.
  Prints nothing if there are no matches within the threshold. Prints to
  sys.stdout.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name  arg will be included in the suggestions. Default is False."""
    if skipBecauseOptimized():
        return

    sys.stdout.write(
        formatSuggestion(
            name, possibleSuggestions, message, threshold, includeIdenticalName
        )
    )

def formatSuggestion(
    name,
    possibleSuggestions=None,
    message="Did you mean %s?\n",
    threshold=2,
    includeIdenticalName=False,
):
    """Returns the string message to display with the closest matching
  suggestion. Returns a blank string if there are no matches within the
  threshold.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      message (str, optional) - The format of the suggestion message to print.
        A %s conversion specifier will be replaced with the suggested name.
        Default is 'Did you mean %s?\n'
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name arg will be included in the suggestions. Default is False."""
    suggestedName = suggest(name, possibleSuggestions, threshold, includeIdenticalName)
    if suggestedName is not None:
        return message % (suggestedName)
    else:
        return ""

def suggest(name, possibleSuggestions=None, threshold=2, includeIdenticalName=False):
    """Returns the closest matching name to the suggestions in
  possibleSuggestions. Pass a list of all possible matches for the
  possibleSuggestions parameter. If there are multiple names with the same edit
  distance, the one returned is undefined. Returns None if there are no
  suggestions within the threshold.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name arg will be included in the suggestions. Default is False."""
    try:
        genObj = suggestAll(name, possibleSuggestions, threshold, includeIdenticalName)
        if runningOnPython2:
            return genObj.next()  # Python 2 code
        else:
            return genObj.__next__()  # Python 3 code
    except StopIteration:
        return None

def suggestAll(name, possibleSuggestions=None, threshold=2, includeIdenticalName=False):
    """Returns all suggestions in possibleSuggestions that are within the
  levenshtein edit distance of name, sorted by distance. If there are multiple
  names with the same edit distance, they will be returned in an undefined
  order.

  Args:
      name (str) - The typo'd name the user supplied.
      possibleSuggestions (sequence of str, optional) - A sequence of strings
        for all possible suggestions that the name could match to. This
        function will pull suggestions from this list.
      threshold (int, optional) - A levenshtein edit distance above this
        threshold will exclude a name from being suggested. Default is 2.
      includeIdenticalName (bool, optional) - If True, a name identical to
        the name arg will be included in the suggestions. Default is False."""
    if possibleSuggestions is None:
        possibleSuggestions = []
    sugsAndDistances = [
        (n, levenshtein(name, n))
        for n in possibleSuggestions
        if includeIdenticalName or n != name
    ]
    sugsAndDistances = [
        x for x in sugsAndDistances if x[1] <= threshold
    ]  # remove names over the threshold
    sugsAndDistances.sort(
        key=lambda x: x[1]
    )  # sort by the levenshtein distance in x[1]

    for sd in sugsAndDistances:
        yield sd[0]

def levenshtein(s1, s2):
    """Returns the levenshtein distance (aka edit distance) between two strings.

  Args:
      s1 (str) - The first string.
      s2 (str) - The second string."""
    # Implementation from https://code.activestate.com/recipes/576874-levenshtein-distance/
    # See also, https://en.wikipedia.org/wiki/Levenshtein_distance
    s1_length = len(s1)
    s2_length = len(s2)

    matrix = [list(range(s1_length + 1))] * (s2_length + 1)
    for zz in range(s2_length + 1):
        matrix[zz] = list(range(zz, zz + s1_length + 1))
    for zz in range(0, s2_length):
        for sz in range(0, s1_length):
            if s1[sz] == s2[zz]:
                matrix[zz + 1][sz + 1] = min(
                    matrix[zz + 1][sz] + 1, matrix[zz][sz + 1] + 1, matrix[zz][sz]
                )
            else:
                matrix[zz + 1][sz + 1] = min(
                    matrix[zz + 1][sz] + 1, matrix[zz][sz + 1] + 1, matrix[zz][sz] + 1
                )
    return matrix[s2_length][s1_length]
<|endoftext|>


<commit_before>"""
===========================================================================

The code for autoregressive components

===========================================================================

This code implements the autoregressive component as a sub-class of dynamic.
Different from the dynamic component, the features in the autoReg is generated
from the data, and updated according to the data. All other features are
similar to @dynamic.

"""
from numpy import matrix
from warnings import warn
from .component import component

import numpy as np
import pydlm.base.tools as tl

class autoReg(component):
    """ The autoReg class allows user to add an autoregressive component to the dlm.
    This code implements the autoregressive component as a child class of
    component. Different from the dynamic component, the features in the
    autoReg is generated from the data, and updated according to the data.

    The latent states of autoReg are aligned in the order of
    [today - degree, today - degree + 1, ..., today - 2, today - 1]. Thus,
    when fetching the latents from autoReg component, use this order to
    correctly align the coefficients.

    (TODO: change the implementation of autoReg, so that the component uses
     filteredObs as the feature instead of the observed data. To do this, we
     might need pass the dlm as a parameter to autoReg, so that autoReg can
     fetch the filtered observation on the fly)

    Args:
        data: the time series data
        degree: the order of the autoregressive component
        discount: the discount factor
        name: the name of the trend component
        w: the value to set the prior covariance. Default to a diagonal
           matrix with 1e7 on the diagonal.

    Attributes:
        degree: the degree of autoregressive, i.e., how many days to look back
        data: the time series data used for constructing the autoregressive
              features
        discount factor: the discounting factor
        name: the name of the component
        padding: either 0 or None. The number to be padded for the first degree
                 days, as no previous data is observed to form the feature
                 matrix

    """

    def __init__(self,
                 data=None,
                 degree=2,
                 discount=0.99,
                 name='ar2',
                 w=100,
                 padding=0):

        if data is not None:
            warn('The data argument in autoReg is deprecated. Please do not use it.')

        self.componentType = 'autoReg'
        self.d = degree
        self.name = name
        self.discount = np.ones(self.d) * discount
        self.padding = padding

        # Initialize all basic quantities
        self.evaluation = None
        self.transition = None
        self.covPrior = None
        self.meanPrior = None

        # create all basic quantities
        self.createTransition()
        self.createCovPrior(scale=w)
        self.createMeanPrior()

        # record current step in case of lost
        self.step = 0

    def createEvaluation(self, step, data):
        """ The evaluation matrix for auto regressor.

        """
        if step > len(data):
            raise NameError("There is no sufficient data for creating autoregressor.")
        # We pad numbers if the step is too early
        self.evaluation = matrix([[self.padding] * (self.d - step) +
                                  data[max(0, (step - self.d)) : step]])

    def createTransition(self):
        """ Create the transition matrix.

        For the dynamic component, the transition matrix is just the identity matrix

        """
        self.transition = np.matrix(np.eye(self.d))

    def createCovPrior(self, cov = None, scale = 1e6):
        """ Create the prior covariance matrix for the latent states

        """
        if cov is None:
            self.covPrior = np.matrix(np.eye(self.d)) * scale
        else:
            self.covPrior = cov * scale

    def createMeanPrior(self, mean = None, scale = 1):
        """ Create the prior latent state

        """
        if mean is None:
            self.meanPrior = np.matrix(np.zeros((self.d, 1))) * scale
        else:
            self.meanPrior = mean * scale

    def checkDimensions(self):
        """ if user supplies their own covPrior and meanPrior, this can
        be used to check if the dimension matches

        """
        tl.checker.checkVectorDimension(self.meanPrior, self.covPrior)
        print('The dimesnion looks good!')

    def updateEvaluation(self, date, data):
        self.createEvaluation(step=date, data=data)
<commit_msg>Fixing python 3 range issue. (Hopefully)<commit_after>"""
===========================================================================

The code for autoregressive components

===========================================================================

This code implements the autoregressive component as a sub-class of dynamic.
Different from the dynamic component, the features in the autoReg is generated
from the data, and updated according to the data. All other features are
similar to @dynamic.

"""
from numpy import matrix
from warnings import warn
from .component import component

import numpy as np
import pydlm.base.tools as tl

class autoReg(component):
    """ The autoReg class allows user to add an autoregressive component to the dlm.
    This code implements the autoregressive component as a child class of
    component. Different from the dynamic component, the features in the
    autoReg is generated from the data, and updated according to the data.

    The latent states of autoReg are aligned in the order of
    [today - degree, today - degree + 1, ..., today - 2, today - 1]. Thus,
    when fetching the latents from autoReg component, use this order to
    correctly align the coefficients.

    (TODO: change the implementation of autoReg, so that the component uses
     filteredObs as the feature instead of the observed data. To do this, we
     might need pass the dlm as a parameter to autoReg, so that autoReg can
     fetch the filtered observation on the fly)

    Args:
        data: the time series data
        degree: the order of the autoregressive component
        discount: the discount factor
        name: the name of the trend component
        w: the value to set the prior covariance. Default to a diagonal
           matrix with 1e7 on the diagonal.

    Attributes:
        degree: the degree of autoregressive, i.e., how many days to look back
        data: the time series data used for constructing the autoregressive
              features
        discount factor: the discounting factor
        name: the name of the component
        padding: either 0 or None. The number to be padded for the first degree
                 days, as no previous data is observed to form the feature
                 matrix

    """

    def __init__(self,
                 data=None,
                 degree=2,
                 discount=0.99,
                 name='ar2',
                 w=100,
                 padding=0):

        if data is not None:
            warn('The data argument in autoReg is deprecated. Please do not use it.')

        self.componentType = 'autoReg'
        self.d = degree
        self.name = name
        self.discount = np.ones(self.d) * discount
        self.padding = padding

        # Initialize all basic quantities
        self.evaluation = None
        self.transition = None
        self.covPrior = None
        self.meanPrior = None

        # create all basic quantities
        self.createTransition()
        self.createCovPrior(scale=w)
        self.createMeanPrior()

        # record current step in case of lost
        self.step = 0

    def createEvaluation(self, step, data):
        """ The evaluation matrix for auto regressor.

        """
        if step > len(data):
            raise NameError("There is no sufficient data for creating autoregressor.")
        # We pad numbers if the step is too early
        self.evaluation = matrix([[self.padding] * (self.d - step) +
                                  list(data[max(0, (step - self.d)) : step])])

    def createTransition(self):
        """ Create the transition matrix.

        For the dynamic component, the transition matrix is just the identity matrix

        """
        self.transition = np.matrix(np.eye(self.d))

    def createCovPrior(self, cov = None, scale = 1e6):
        """ Create the prior covariance matrix for the latent states

        """
        if cov is None:
            self.covPrior = np.matrix(np.eye(self.d)) * scale
        else:
            self.covPrior = cov * scale

    def createMeanPrior(self, mean = None, scale = 1):
        """ Create the prior latent state

        """
        if mean is None:
            self.meanPrior = np.matrix(np.zeros((self.d, 1))) * scale
        else:
            self.meanPrior = mean * scale

    def checkDimensions(self):
        """ if user supplies their own covPrior and meanPrior, this can
        be used to check if the dimension matches

        """
        tl.checker.checkVectorDimension(self.meanPrior, self.covPrior)
        print('The dimesnion looks good!')

    def updateEvaluation(self, date, data):
        self.createEvaluation(step=date, data=data)
<|endoftext|>


<commit_before>#Import dependencies
#Load OS functions from the standard library
import os

os.chdir('/home/israel/Development/Python_Exercises/python-octo-wookie/head1stpython/Chapter3')
#Change path for the current directory

data = read('sketch.txt)
#Start iteration over the text file
for each_line in data:
	(role, line_spoken) = each_line.split(':', 1)
	print(role, end = '')
	print(' said: ', end = '')
	print(line_spoken, end = '')
<commit_msg>Added exception handling for sketch.py<commit_after>#Import dependencies
#Load OS functions from the standard library
import os

os.chdir('/home/israel/Development/Python_Exercises/python-octo-wookie/head1stpython/Chapter3')
#Change path for the current directory

data = open('sketch.txt')

#Start iteration over the text file
for each_line in data:
            try:
                (role, line_spoken) = each_line.split(':', 1)
                print(role, end = '')
                print(' said: ', end = '')
                print(line_spoken, end = '')
            except:
                pass

data.close()

<|endoftext|>


<commit_before><commit_msg>remove pomegranate p1<commit_after><|endoftext|>


<commit_before><commit_msg>gravy in ensemble plot helper<commit_after><|endoftext|>


<commit_before><commit_msg>fix in Pst.rename_obs()<commit_after><|endoftext|>


<commit_before>"""

Module :mod:`pyesgf.search.results`
===================================

Search results are retrieved through the :class:`ResultSet` class.  This class
hides paging of large result sets behind a client-side cache.  Subclasses of
:class:`Result` represent results of different SOLr record type.

"""

from collections import Sequence, defaultdict
import re

from .consts import (DEFAULT_BATCH_SIZE, TYPE_DATASET, TYPE_FILE,
                     TYPE_AGGREGATION)
from .exceptions import EsgfSearchException

class ResultSet(Sequence):
    """
    :ivar context: The search context object used to generate this resultset
    :property batch_size: The number of results that will be requested
        from esgf-search as one call.  This must be set on creation and
        cannot change.

    """
    def __init__(self, context, batch_size=DEFAULT_BATCH_SIZE, eager=True):
        """
        :param context: The search context object used to generate this resultset
        :param batch_size: The number of results that will be requested from
            esgf-search as one call.
        :param eager: Boolean specifying whether to retrieve the first batch on
            instantiation.

        """
        self.context = context
        self.__batch_size = batch_size
        self.__batch_cache = [None] * ((len(self) / batch_size) + 1)
        if eager and len(self)>0:
            self.__batch_cache[0] = self.__get_batch(0)

    def __getitem__(self, index):
        batch_i = index / self.batch_size
        offset = index % self.batch_size
        if self.__batch_cache[batch_i] is None:
            self.__batch_cache[batch_i] = self.__get_batch(batch_i)
        batch = self.__batch_cache[batch_i]

        search_type = self.context.search_type
        ResultClass = _result_classes[search_type]

        #!TODO: should probably wrap the json inside self.__batch_cache
        return ResultClass(batch[offset], self.context)

    def __len__(self):
        return self.context.hit_count

    @property
    def batch_size(self):
        return self.__batch_size

    def _build_result(self, result):
        """
        Construct a result object from the raw json.

        This method is designed to be overridden in subclasses if desired.
        The default implementation simply returns the json.

        """
        return result

    def __get_batch(self, batch_i):
        offset = self.batch_size * batch_i
        limit = self.batch_size

        query_dict = self.context._build_query()
        response = self.context.connection.send_search(query_dict, limit=limit,
                                                       offset=offset,
                                                       shards=self.context.shards)

        #!TODO: strip out results
        return response['response']['docs']

class BaseResult(object):
    """
    Base class for results.

    Subclasses represent different search types such as File and Dataset.

    :ivar json: The oroginial json representation of the result.
    :ivar context: The SearchContext which generated this result.
    :property urls: a dictionary of the form {service: [(url, mime_type), ...], ...}
    :property opendap_url: The url of an OPeNDAP endpoint for this result if available
    :property download_url: The url for downloading the result by HTTP if available
    :property index_node: The index node from where the metadata is stored.
        Calls to *_context() will optimise queries to only address this node.

    """
    def __init__(self, json, context):
        self.json = json
        self.context = context

    @property
    def urls(self):
        url_dict = defaultdict(list)
        for encoded in self.json['url']:
            url, mime_type, service = encoded.split('|')
            url_dict[service].append((url, mime_type))

        return url_dict

    @property
    def opendap_url(self):
        try:
            url, mime = self.urls['OPENDAP'][0]
        except (KeyError, IndexError):
            return None

        url = re.sub(r'.html$', '', url)

        return url

    @property
    def download_url(self):
        try:
            url, mime = self.urls['HTTPServer'][0]
        except (KeyError, IndexError):
            return None

        return url

    @property
    def index_node(self):
        try:
            index_node = self.json['index_node']
        except KeyError:
            return None

        return index_node

class DatasetResult(BaseResult):
    """
    A result object for ESGF datasets.

    :property dataset_id: The solr dataset_id which is unique throughout the system.

    """

    @property
    def dataset_id(self):
        #!TODO: should we decode this into a tuple?  self.json['id'].split('|')
        return self.json['id']

    def file_context(self):
        """
        Return a SearchContext for searching for files within this dataset.
        """
        from .context import FileSearchContext

        if self.context.connection.distrib:
            # If the index node is in the available shards for this connection then
            # restrict shards to that node.  Otherwise do nothing to handle the case
            # when the shard is replicated
            available_shards = self.context.connection.get_shard_list().keys()
            if self.index_node in available_shards:
                shards = [self.index_node]
            else:
                shards = None
        else:
            shards = None

        files_context = FileSearchContext(
            connection=self.context.connection,
            constraints={'dataset_id': self.dataset_id},
            shards=shards,
            )
        return files_context

    def aggregation_context(self):
        """
        Return a SearchContext for searching for aggregations within this dataset.
        """
        from .context import AggregationSearchContext

        if self.context.connection.distrib:
            shards=[self.index_node]
        else:
            shards=None

        agg_context = AggregationSearchContext(
            connection=self.context.connection,
            constraints={'dataset_id': self.dataset_id},
            shards=shards,
            )
        return agg_context

class FileResult(BaseResult):
    """
    A result object for ESGF files.  Properties from :class:`BaseResult` are inherited.

    :property file_id: The identifier for the file
    :property checksum: The checksum of the file
    :property checksum_type: The algorithm used for generating the checksum
    :property filename: The filename
    :proprty size: The file size in bytes

    """
    @property
    def file_id(self):
        return self.json['id']

    @property
    def checksum(self):
        return self.json['checksum'][0]

    @property
    def checksum_type(self):
        self.json['checksum_type'][0]

    @property
    def filename(self):
        return self.json['title']

    @property
    def size(self):
        return int(self.json['size'])

class AggregationResult(BaseResult):
    """
    A result object for ESGF aggregations.  Properties from :class:`BaseResult` are inherited.

    :property aggregation_id: The aggregation id
    """
    @property
    def aggregation_id(self):
        return self.json['id']

_result_classes = {
    TYPE_DATASET: DatasetResult,
    TYPE_FILE: FileResult,
    TYPE_AGGREGATION: AggregationResult,
    }
<commit_msg>workaround for shard error added<commit_after>"""

Module :mod:`pyesgf.search.results`
===================================

Search results are retrieved through the :class:`ResultSet` class.  This class
hides paging of large result sets behind a client-side cache.  Subclasses of
:class:`Result` represent results of different SOLr record type.

"""

from collections import Sequence, defaultdict
import re

from .consts import (DEFAULT_BATCH_SIZE, TYPE_DATASET, TYPE_FILE,
                     TYPE_AGGREGATION)
from .exceptions import EsgfSearchException

class ResultSet(Sequence):
    """
    :ivar context: The search context object used to generate this resultset
    :property batch_size: The number of results that will be requested
        from esgf-search as one call.  This must be set on creation and
        cannot change.

    """
    def __init__(self, context, batch_size=DEFAULT_BATCH_SIZE, eager=True):
        """
        :param context: The search context object used to generate this resultset
        :param batch_size: The number of results that will be requested from
            esgf-search as one call.
        :param eager: Boolean specifying whether to retrieve the first batch on
            instantiation.

        """
        self.context = context
        self.__batch_size = batch_size
        self.__batch_cache = [None] * ((len(self) / batch_size) + 1)
        if eager and len(self)>0:
            self.__batch_cache[0] = self.__get_batch(0)

    def __getitem__(self, index):
        batch_i = index / self.batch_size
        offset = index % self.batch_size
        if self.__batch_cache[batch_i] is None:
            self.__batch_cache[batch_i] = self.__get_batch(batch_i)
        batch = self.__batch_cache[batch_i]

        search_type = self.context.search_type
        ResultClass = _result_classes[search_type]

        #!TODO: should probably wrap the json inside self.__batch_cache
        return ResultClass(batch[offset], self.context)

    def __len__(self):
        return self.context.hit_count

    @property
    def batch_size(self):
        return self.__batch_size

    def _build_result(self, result):
        """
        Construct a result object from the raw json.

        This method is designed to be overridden in subclasses if desired.
        The default implementation simply returns the json.

        """
        return result

    def __get_batch(self, batch_i):
        offset = self.batch_size * batch_i
        limit = self.batch_size

        query_dict = self.context._build_query()
        response = self.context.connection.send_search(query_dict, limit=limit,
                                                       offset=offset,
                                                       #shards=self.context.shards)
                                                       shards=None)

        #!TODO: strip out results
        return response['response']['docs']

class BaseResult(object):
    """
    Base class for results.

    Subclasses represent different search types such as File and Dataset.

    :ivar json: The oroginial json representation of the result.
    :ivar context: The SearchContext which generated this result.
    :property urls: a dictionary of the form {service: [(url, mime_type), ...], ...}
    :property opendap_url: The url of an OPeNDAP endpoint for this result if available
    :property download_url: The url for downloading the result by HTTP if available
    :property index_node: The index node from where the metadata is stored.
        Calls to *_context() will optimise queries to only address this node.

    """
    def __init__(self, json, context):
        self.json = json
        self.context = context

    @property
    def urls(self):
        url_dict = defaultdict(list)
        for encoded in self.json['url']:
            url, mime_type, service = encoded.split('|')
            url_dict[service].append((url, mime_type))

        return url_dict

    @property
    def opendap_url(self):
        try:
            url, mime = self.urls['OPENDAP'][0]
        except (KeyError, IndexError):
            return None

        url = re.sub(r'.html$', '', url)

        return url

    @property
    def download_url(self):
        try:
            url, mime = self.urls['HTTPServer'][0]
        except (KeyError, IndexError):
            return None

        return url

    @property
    def index_node(self):
        try:
            index_node = self.json['index_node']
        except KeyError:
            return None

        return index_node

class DatasetResult(BaseResult):
    """
    A result object for ESGF datasets.

    :property dataset_id: The solr dataset_id which is unique throughout the system.

    """

    @property
    def dataset_id(self):
        #!TODO: should we decode this into a tuple?  self.json['id'].split('|')
        return self.json['id']

    def file_context(self):
        """
        Return a SearchContext for searching for files within this dataset.
        """
        from .context import FileSearchContext

        if self.context.connection.distrib:
            # If the index node is in the available shards for this connection then
            # restrict shards to that node.  Otherwise do nothing to handle the case
            # when the shard is replicated
            available_shards = self.context.connection.get_shard_list().keys()
            if self.index_node in available_shards:
                shards = [self.index_node]
            else:
                shards = None
        else:
            shards = None

        files_context = FileSearchContext(
            connection=self.context.connection,
            constraints={'dataset_id': self.dataset_id},
            shards=shards,
            )
        return files_context

    def aggregation_context(self):
        """
        Return a SearchContext for searching for aggregations within this dataset.
        """
        from .context import AggregationSearchContext

        if self.context.connection.distrib:
            shards=[self.index_node]
        else:
            shards=None

        agg_context = AggregationSearchContext(
            connection=self.context.connection,
            constraints={'dataset_id': self.dataset_id},
            shards=shards,
            )
        return agg_context

class FileResult(BaseResult):
    """
    A result object for ESGF files.  Properties from :class:`BaseResult` are inherited.

    :property file_id: The identifier for the file
    :property checksum: The checksum of the file
    :property checksum_type: The algorithm used for generating the checksum
    :property filename: The filename
    :proprty size: The file size in bytes

    """
    @property
    def file_id(self):
        return self.json['id']

    @property
    def checksum(self):
        return self.json['checksum'][0]

    @property
    def checksum_type(self):
        self.json['checksum_type'][0]

    @property
    def filename(self):
        return self.json['title']

    @property
    def size(self):
        return int(self.json['size'])

class AggregationResult(BaseResult):
    """
    A result object for ESGF aggregations.  Properties from :class:`BaseResult` are inherited.

    :property aggregation_id: The aggregation id
    """
    @property
    def aggregation_id(self):
        return self.json['id']

_result_classes = {
    TYPE_DATASET: DatasetResult,
    TYPE_FILE: FileResult,
    TYPE_AGGREGATION: AggregationResult,
    }
<|endoftext|>


<commit_before>
from specialproducts.models import SpecialProductsWidget
from profileactions.models import ProfileActionsWidget
from quickcart.models import QuickCartWidget
from productcatalog.models import ProductCatalogWidget
from productlist.models import ProductListWidget
from productcatalog.models import ProductCatalogWidget
from storeactions.models import StoreActionsWidget
from storecontact.models import StoreContactWidget
#from storesearch.models import StoreSearchWidget
<commit_msg>cleanup widget init<commit_after><|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Utility
=======

General utility which are not view or tool specific
"""

from httplib import BAD_REQUEST

try:
    from json import dumps as _dumps
except ImportError: # pragma: no cover
    from simplejson import dumps as _dumps

from werkzeug.datastructures import ImmutableDict
from flask import Response
from pyfarm.core.enums import APIError
from pyfarm.master.application import app

PRETTY_JSON = app.config["PYFARM_JSON_PRETTY"]
COLUMN_CACHE = {}

def dumps(*args, **kwargs):
    """
    Wrapper around :func:`._dumps` which does some work to respect any
    application settings.
    """
    if PRETTY_JSON:
        kwargs.setdefault("indent", 4)
    return _dumps(*args, **kwargs)

def get_column_sets(model):
    """
    returns a tuple of two sets containing all columns and required columns
    for the model provided
    """
    all_columns = set()
    required_columns = set()

    for name, column in model.__table__.c.items():
        # skip autoincremented primary keys
        if column.primary_key and column.autoincrement:
            continue

        all_columns.add(name)

        # column is non-nullable without a default
        if not column.nullable and column.default is None:
            required_columns.add(name)

    return all_columns, required_columns

class JSONResponse(Response):
    """
    Wrapper around :class:`.Response` which will set the proper content type
    and serialize any input
    """
    content_type = "application/json"

    def __init__(self, data=None, **kwargs):
        if isinstance(data, ReducibleDictionary):
            data.reduce()

        if data is not None:
            super(JSONResponse, self).__init__(dumps(data), **kwargs)
        else:
            super(JSONResponse, self).__init__(**kwargs)

class ReducibleDictionary(dict):
    """
    Adds a :meth:`.reduce` method to :class:`dict` class
    which will remove empty values from a dictionary.

    >>> data = ReducibleDictionary({"foo": True, "bar": None})
    >>> data.reduce()
    >>> print data
    {'foo': True}
    """
    def reduce(self):
        for key, value in self.copy().iteritems():
            if value is None:
                self.pop(key)

class TemplateDictionary(ImmutableDict):
    """
    Simple dictionary which subclasses werkzeug's :class:`.ImmutableDict`
    but allows for new instanced to be produced using :meth:`.__call__`.

    >>> template = TemplateDictionary({"foo": None})
    >>> data = template()
    >>> data["foo"] = True
    >>> data["bar"] = False
    >>> print data, template
    {'foo': True, 'bar': False} TemplateDictionary({'foo': None})

    This class is mainly meant for simple REST responses, other considerations
    should be taken for more complex structures.
    """
    def __call__(self, reducible=True):
        class_ = ReducibleDictionary if reducible else dict
        return class_(self.copy())

def json_from_request(request, all_keys=None, required_keys=None,
                      disallowed_keys=None):
    """
    Returns the json data from the request or a :class:`.JSONResponse` object
    on failure.

    :keyword set all_keys:
        a set of all possible keys which may be present in the json request

    :keyword set required_keys:
        a set of keys which must be present in the json request

    :keyword set disallowed_keys:
        a set of keys which cannot be part of the request
    """
    try:
        data = request.get_json()

    except ValueError, e:
        errorno, msg = APIError.JSON_DECODE_FAILED
        msg += ": %s" % e
        return JSONResponse((errorno, msg), status=BAD_REQUEST)

    if isinstance(data, dict) and (all_keys or required_keys or disallowed_keys):
        request_keys = set(data)

        # make sure that we don't have more request keys
        # than there are total keys
        if all_keys is not None and not request_keys.issubset(all_keys):
            errorno, msg = APIError.EXTRA_FIELDS_ERROR
            msg += ".  Extra fields were: %s" % list(request_keys - all_keys)
            return JSONResponse((errorno, msg), status=BAD_REQUEST)

        # if required keys were provided, make sure the
        # request has at least those fields
        if required_keys is not None and not \
            request_keys.issuperset(required_keys):
            missing_keys = list(required_keys-request_keys)
            errorno, msg = APIError.MISSING_FIELDS
            msg += ".  Missing fields are: %s" % missing_keys
            return JSONResponse((errorno, msg), status=BAD_REQUEST)

        if disallowed_keys is not None and \
            request_keys.issuperset(disallowed_keys):
            errorno, msg = APIError.EXTRA_FIELDS_ERROR
            disallowed_extras = list(request_keys.intersection(disallowed_keys))
            msg += ".  Extra fields were: %s" % disallowed_extras
            return JSONResponse((errorno, msg), status=BAD_REQUEST)

    return data
<commit_msg>properly setting content_type<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Utility
=======

General utility which are not view or tool specific
"""

from httplib import BAD_REQUEST

try:
    from json import dumps as _dumps
except ImportError: # pragma: no cover
    from simplejson import dumps as _dumps

from werkzeug.datastructures import ImmutableDict
from flask import Response
from pyfarm.core.enums import APIError
from pyfarm.master.application import app

PRETTY_JSON = app.config["PYFARM_JSON_PRETTY"]
COLUMN_CACHE = {}

def dumps(*args, **kwargs):
    """
    Wrapper around :func:`._dumps` which does some work to respect any
    application settings.
    """
    if PRETTY_JSON:
        kwargs.setdefault("indent", 4)
    return _dumps(*args, **kwargs)

def get_column_sets(model):
    """
    returns a tuple of two sets containing all columns and required columns
    for the model provided
    """
    all_columns = set()
    required_columns = set()

    for name, column in model.__table__.c.items():
        # skip autoincremented primary keys
        if column.primary_key and column.autoincrement:
            continue

        all_columns.add(name)

        # column is non-nullable without a default
        if not column.nullable and column.default is None:
            required_columns.add(name)

    return all_columns, required_columns

class JSONResponse(Response):
    """
    Wrapper around :class:`.Response` which will set the proper content type
    and serialize any input
    """
    def __init__(self, data=None, **kwargs):
        if isinstance(data, ReducibleDictionary):
            data.reduce()

        kwargs.setdefault("content_type", "application/json")

        if data is not None:
            super(JSONResponse, self).__init__(dumps(data), **kwargs)
        else:
            super(JSONResponse, self).__init__(**kwargs)

class ReducibleDictionary(dict):
    """
    Adds a :meth:`.reduce` method to :class:`dict` class
    which will remove empty values from a dictionary.

    >>> data = ReducibleDictionary({"foo": True, "bar": None})
    >>> data.reduce()
    >>> print data
    {'foo': True}
    """
    def reduce(self):
        for key, value in self.copy().iteritems():
            if value is None:
                self.pop(key)

class TemplateDictionary(ImmutableDict):
    """
    Simple dictionary which subclasses werkzeug's :class:`.ImmutableDict`
    but allows for new instanced to be produced using :meth:`.__call__`.

    >>> template = TemplateDictionary({"foo": None})
    >>> data = template()
    >>> data["foo"] = True
    >>> data["bar"] = False
    >>> print data, template
    {'foo': True, 'bar': False} TemplateDictionary({'foo': None})

    This class is mainly meant for simple REST responses, other considerations
    should be taken for more complex structures.
    """
    def __call__(self, reducible=True):
        class_ = ReducibleDictionary if reducible else dict
        return class_(self.copy())

def json_from_request(request, all_keys=None, required_keys=None,
                      disallowed_keys=None):
    """
    Returns the json data from the request or a :class:`.JSONResponse` object
    on failure.

    :keyword set all_keys:
        a set of all possible keys which may be present in the json request

    :keyword set required_keys:
        a set of keys which must be present in the json request

    :keyword set disallowed_keys:
        a set of keys which cannot be part of the request
    """
    try:
        data = request.get_json()

    except ValueError, e:
        errorno, msg = APIError.JSON_DECODE_FAILED
        msg += ": %s" % e
        return JSONResponse((errorno, msg), status=BAD_REQUEST)

    if isinstance(data, dict) and (all_keys or required_keys or disallowed_keys):
        request_keys = set(data)

        # make sure that we don't have more request keys
        # than there are total keys
        if all_keys is not None and not request_keys.issubset(all_keys):
            errorno, msg = APIError.EXTRA_FIELDS_ERROR
            msg += ".  Extra fields were: %s" % list(request_keys - all_keys)
            return JSONResponse((errorno, msg), status=BAD_REQUEST)

        # if required keys were provided, make sure the
        # request has at least those fields
        if required_keys is not None and not \
            request_keys.issuperset(required_keys):
            missing_keys = list(required_keys-request_keys)
            errorno, msg = APIError.MISSING_FIELDS
            msg += ".  Missing fields are: %s" % missing_keys
            return JSONResponse((errorno, msg), status=BAD_REQUEST)

        if disallowed_keys is not None and \
            request_keys.issuperset(disallowed_keys):
            errorno, msg = APIError.EXTRA_FIELDS_ERROR
            disallowed_extras = list(request_keys.intersection(disallowed_keys))
            msg += ".  Extra fields were: %s" % disallowed_extras
            return JSONResponse((errorno, msg), status=BAD_REQUEST)

    return data
<|endoftext|>


<commit_before>#!/usr/bin/python

import networkx as nx, sys, os, time, random, numpy as np
sys.path.append("../comet")
from permute_matrix import bipartite_edge_swap
from networkx.algorithms import bipartite
import permute as P

def check_degrees(G, H):
    if len(set(G.nodes()) ^ set(H.nodes())) != 0:
        print set(G.nodes()) - set(H.nodes())
        print set(H.nodes()) - set(G.nodes())
    for i, n in enumerate(G.nodes()):
        if H.degree(n) != G.degree(n):
            print i, n, H.degree(n), G.degree(n)
            return False
    return True

def call_bipartite_edge_swap( G, xs, ys, Q ):
    # Compute the desired pieces of the graph structure
    A = np.array(bipartite.biadjacency_matrix(G, row_order=xs, column_order=ys, dtype=np.int32))

    # Set up and call the permute matrix function
    max_tries = 2**31-1
    seed      = random.randint(0, 2**31-1)
    nswap     = len(G.edges()) * Q
    B = bipartite_edge_swap(A, nswap, max_tries, seed=seed, verbose=True)
    H = nx.Graph()
    H.add_edges_from([ (xs[u], ys[v]) for u, v in zip(*np.where(B == 1)) ])
    return H

def test_random_bipartite_graph(numXs, numYs, numEdges ):
    # Create a random bipartite graph
    G = nx.Graph()
    genes = ["G{}".format(i) for i in range(1, numXs+1) ]
    patients = ["TCGA-{}".format(i) for i in range(1, numYs+1) ]
    assert(numEdges < numXs * numYs)
    xs, ys = set(), set()
    for i in range(numEdges):
        x = random.choice(genes)
        y = random.choice(patients)
        G.add_edge(x,y)
        xs.add(x)
        ys.add(y)
    return list(xs), list(ys), G

if __name__ == "__main__":
    Q = 100
    for i, (numXs, numYs, numEdges) in enumerate([(300, 400, 20000)]):
        # Create a test graph
        numXs, numYs, numEdges = 400, 400, 20000
        xs, ys, G = test_random_bipartite_graph(numXs, numYs, numEdges)
        print "Test {}: {} genes x {} samples, {} mutations".format(i+1, len(xs), len(ys), numEdges)

        # Run and time in Fortran
        start = time.time()
        H = call_bipartite_edge_swap( G, xs, ys, Q=Q )
        print "\tFortran:", time.time() - start, 'secs'
        worked = check_degrees(G, H)
        if not worked:
            raise ValueError("Degrees in permuted graph are different than original graph.")
        sys.exit()

        # Run and time in Python
        start = time.time()
        H = P.bipartite_double_edge_swap( G, xs, ys, nswap=numEdges * Q)
        print "\tPython:", time.time() - start, 'secs'
        check_degrees(G, H)
        if not worked:
            raise ValueError("Degrees in permuted graph are different than original graph.")

        print

<commit_msg>Added paths to test code<commit_after>#!/usr/bin/python

import networkx as nx, sys, os, time, random, numpy as np
sys.path.append("../comet/")
sys.path.append("../comet/src/fortran")
from permute_matrix import bipartite_edge_swap
from networkx.algorithms import bipartite
import permute as P

def check_degrees(G, H):
    if len(set(G.nodes()) ^ set(H.nodes())) != 0:
        print set(G.nodes()) - set(H.nodes())
        print set(H.nodes()) - set(G.nodes())
    for i, n in enumerate(G.nodes()):
        if H.degree(n) != G.degree(n):
            print i, n, H.degree(n), G.degree(n)
            return False
    return True

def call_bipartite_edge_swap( G, xs, ys, Q ):
    # Compute the desired pieces of the graph structure
    A = np.array(bipartite.biadjacency_matrix(G, row_order=xs, column_order=ys, dtype=np.int32))

    # Set up and call the permute matrix function
    max_tries = 2**31-1
    seed      = random.randint(0, 2**31-1)
    nswap     = len(G.edges()) * Q
    B = bipartite_edge_swap(A, nswap, max_tries, seed=seed, verbose=True)
    H = nx.Graph()
    H.add_edges_from([ (xs[u], ys[v]) for u, v in zip(*np.where(B == 1)) ])
    return H

def test_random_bipartite_graph(numXs, numYs, numEdges ):
    # Create a random bipartite graph
    G = nx.Graph()
    genes = ["G{}".format(i) for i in range(1, numXs+1) ]
    patients = ["TCGA-{}".format(i) for i in range(1, numYs+1) ]
    assert(numEdges < numXs * numYs)
    xs, ys = set(), set()
    for i in range(numEdges):
        x = random.choice(genes)
        y = random.choice(patients)
        G.add_edge(x,y)
        xs.add(x)
        ys.add(y)
    return list(xs), list(ys), G

if __name__ == "__main__":
    Q = 100
    for i, (numXs, numYs, numEdges) in enumerate([(300, 400, 20000)]):
        # Create a test graph
        numXs, numYs, numEdges = 400, 400, 20000
        xs, ys, G = test_random_bipartite_graph(numXs, numYs, numEdges)
        print "Test {}: {} genes x {} samples, {} mutations".format(i+1, len(xs), len(ys), numEdges)

        # Run and time in Fortran
        start = time.time()
        H = call_bipartite_edge_swap( G, xs, ys, Q=Q )
        print "\tFortran:", time.time() - start, 'secs'
        worked = check_degrees(G, H)
        if not worked:
            raise ValueError("Degrees in permuted graph are different than original graph.")

        # Run and time in Python
        start = time.time()
        H = P.bipartite_double_edge_swap( G, xs, ys, nswap=numEdges * Q)
        print "\tPython:", time.time() - start, 'secs'
        check_degrees(G, H)
        if not worked:
            raise ValueError("Degrees in permuted graph are different than original graph.")

        print

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import itertools

import numpy as np
import sympy as sy

from pyfr.bases.base import BasisBase
from pyfr.quad_points import points_for_rule
from pyfr.util import ndrange

def cart_prod_points(points, ndim, compact=True):
    """Performs a cartesian product extension of *points* into *ndim*

    For idiosyncratic reason the counting order of indices is from
    first to last, i.e, it is the first index that counts quickest,
    followed by the second index and so on.

    **Example**
    >>> cart_prod_points([-1, 0, 1], 2)
    array([[-1., -1.],
       [ 0., -1.],
       [ 1., -1.],
       [-1.,  0.],
       [ 0.,  0.],
       [ 1.,  0.],
       [-1.,  1.],
       [ 0.,  1.],
       [ 1.,  1.]])
    """
    npoints = len(points)

    cprodpts = np.empty((npoints,)*ndim + (ndim,))
    for i,ax in enumerate(np.ix_(*(points,)*ndim)):
        # -i-1 ensures we count first-to-last
        cprodpts[...,-i-1] = ax

    # Compact into an array of ndim component tuples
    if compact:
        return cprodpts.reshape(-1, ndim)
    else:
        return cprodpts

def _nodal_basis1d(points, sym):
    """Generates a basis of polynomials, :math:`l_i(x)`, such that
    .. math::
       l_i(x) = \delta^x_{p_i}
    where :math:`p_i` is the i'th entry in *points* and :math:`x \in p`.
    """
    n = len(points)

    return [sy.interpolating_poly(n, sym, points, (0,)*i + (1,) + (0,)*(n-i))
            for i in xrange(n)]

def nodal_basis(points, dims, compact=True):
    """Generates a nodal basis for *points* over *dims*

    .. note::
      This function adcfg the same first-to-last counting order as
      :func:`cart_prod_points` with the first index varying quickest.

    **Example**
    >>> import sympy as sy
    >>> nb = nodal_basis([-1, 1], sy.symbols('p q'))
    >>> nb[0]
    (-p/2 + 1/2)*(-q/2 + 1/2)
    >>> nb[0].subs(dict(p=-1, q=-1))
    1
    >>> nb[0].subs(dict(p=1, q=-1))
    0
    """
    p = list(points)

    # Evaluate the basis function in terms of each dimension
    basis = [_nodal_basis1d(p, d) for d in reversed(dims)]

    # Take the cartesian product of these and multiply the resulting tuples
    cpbasis = np.array([np.prod(b) for b in itertools.product(*basis)])

    return cpbasis if compact else cpbasis.reshape((len(p),)*len(dims))

# Cube map face rotation scheme to go from face 1 -> 0..5
_cube_map_rots_sy = [sy.rot_axis2(-sy.pi)*sy.rot_axis1(sy.pi/2),
                     sy.eye(3),
                     sy.rot_axis3(sy.pi/2),
                     sy.rot_axis3(sy.pi),
                     sy.rot_axis3(-sy.pi/2),
                     sy.rot_axis1(-sy.pi/2)]

# Rotation scheme as numpy arrays
_cube_map_rots_np = [np.asanyarray(sy.matrix2numpy(r), dtype=np.float64)
                     for r in _cube_map_rots_sy]

def cube_map_face(fpoints):
    """Given a matrix of points (p,q,r) corresponding to face one of
    `the cube' this method maps these points onto the remaining faces

    On a cube parameterized by (p,q,r) -> (-1,-1,-1) × (1,1,1) face one
    is defined by (-1,-1,-1) × (1,-1,1)."""
    mfpoints = np.empty((6,) + fpoints.shape)

    for i,frot in enumerate(_cube_map_rots_np):
        mfpoints[i,...] = np.dot(fpoints, frot)

    return mfpoints

def diff_vcjh_correctionfn(k, eta, sym):
    # Expand shorthand forms of eta_k for common schemes
    etacommon = dict(dg='0', sd='k/(k+1)', hu='(k+1)/k')

    eta_k = sy.S(etacommon.get(eta, eta), locals=dict(k=k))

    lkm1, lk, lkp1 = [sy.legendre_poly(m, sym) for m in [k-1, k, k+1]]

    # Correction function derivatives, Eq. 3.46 and 3.47
    diffgr = (sy.S(1)/2 * (lk + (eta_k*lkm1 + lkp1)/(1 + eta_k))).diff()
    diffgl = -diffgr.subs(sym, -sym)

    return diffgl, diffgr

class TensorProdBasis(object):
    def _get_pts1d(self):
        rule = self._cfg.get('mesh-elements', 'quad-rule')
        return points_for_rule(rule, self._order + 1)

    def get_upts(self):
        pts1d = self._get_pts1d()

        pts = cart_prod_points(pts1d, 3)
        basis = nodal_basis(pts1d, self._dims)

        return pts, basis

    def get_spts(self, nspts):
        dims, ndims = self._dims, len(self._dims)

        # Root the number of shape points to get the # in each dim
        nsptsord = sy.S(nspts)**(sy.S(1)/len(dims))

        if not nsptsord.is_Number:
            raise ValueError('Invalid number of shape points for {} dims'\
                             .format(len(dims)))

        # Generate nsptsord equispaced points from (-1, 1)
        pts1d = np.linspace(-1, 1, nsptsord)

        pts = cart_prod_points(pts1d, len(dims))
        basis = nodal_basis(pts1d, dims)

        return pts, basis

    def get_nfpts(self):
        return [(self._order + 1)**(self.ndims - 1)] * (2*self.ndims)

class HexBasis(TensorProdBasis, BasisBase):
    name = 'hex'
    ndims = 3

    def __init__(self, *args, **kwargs):
        super(HexBasis, self).__init__(*args, **kwargs)

        k = self._order + 1

        # Pre-compute all possible flux point rotation schemes
        self._rschemes = rs = np.empty((6, 5), dtype=np.object)
        for face,rtag in ndrange(*rs.shape):
            fpts = np.arange(face*k*k, (face+1)*k*k).reshape(k,k)

            if rtag == 0:
                pass
            elif rtag == 1:
                fpts = np.fliplr(fpts)
            elif rtag == 2:
                fpts = np.fliplr(fpts)[::-1]
            elif rtag == 3:
                fpts = np.transpose(fpts)
            elif rtag == 4:
                fpts = np.transpose(fpts)[::-1]

            rs[face,rtag] = fpts.ravel()

    def get_fpts(self):
        # Get the 1D points
        pts1d = self._get_pts1d()

        # Perform a 2D extension to get the (p,r) points of face one
        pts2d = cart_prod_points(pts1d, 2, compact=False)

        # 3D points are just (p,-1,r) for face one
        fonepts = np.empty(pts2d.shape[:-1] + (3,))
        fonepts[...,(0,2)] = pts2d
        fonepts[...,1] = -1

        # Cube map face one to get faces zero through five
        fpts = cube_map_face(fonepts)

        # Allocate space for the flux points basis
        fbasis = np.empty(fpts.shape[:-1], dtype=np.object)

        # Pair up opposite faces with their associated (normal) dimension
        for fpair,sym in zip([(4,2), (1,3), (0,5)], self._dims):
            nbdims = [d for d in self._dims if d is not sym]
            fbasis[fpair,...] = nodal_basis(pts1d, nbdims, compact=False)

            eta = self._cfg.get('mesh-elements', 'vcjh-eta')
            diffcorfn = diff_vcjh_correctionfn(self._order, eta, sym)

            for p,gfn in zip(fpair, diffcorfn):
                if p in (0,3,4):
                    fbasis[p] = np.fliplr(fbasis[p])
                fbasis[p,...] *= gfn

        # Correct faces with negative normals
        fbasis[(4,1,0),...] *= -1

        return fpts.reshape(-1,3), fbasis.reshape(-1)

    def get_norm_fpts(self):
        # Normals for face one are (0,-1,0)
        fonenorms = np.zeros([self._order + 1]*2 + [3])
        fonenorms[...,1] = -1

        # Cube map to get the remaining face normals
        return cube_map_face(fonenorms).reshape(-1, 3)

    def get_fpts_for_face(self, face, rtag):
        return self._rschemes[face, rtag]
<commit_msg>Improve element matrix generation performance.<commit_after># -*- coding: utf-8 -*-

import itertools

import numpy as np
import sympy as sy

from pyfr.bases.base import BasisBase
from pyfr.quad_points import points_for_rule
from pyfr.util import ndrange

def cart_prod_points(points, ndim, compact=True):
    """Performs a cartesian product extension of *points* into *ndim*

    For idiosyncratic reason the counting order of indices is from
    first to last, i.e, it is the first index that counts quickest,
    followed by the second index and so on.

    **Example**
    >>> cart_prod_points([-1, 0, 1], 2)
    array([[-1., -1.],
       [ 0., -1.],
       [ 1., -1.],
       [-1.,  0.],
       [ 0.,  0.],
       [ 1.,  0.],
       [-1.,  1.],
       [ 0.,  1.],
       [ 1.,  1.]])
    """
    npoints = len(points)

    cprodpts = np.empty((npoints,)*ndim + (ndim,))
    for i,ax in enumerate(np.ix_(*(points,)*ndim)):
        # -i-1 ensures we count first-to-last
        cprodpts[...,-i-1] = ax

    # Compact into an array of ndim component tuples
    if compact:
        return cprodpts.reshape(-1, ndim)
    else:
        return cprodpts

def _nodal_basis1d(points, sym):
    """Generates a basis of polynomials, :math:`l_i(x)`, such that
    .. math::
       l_i(x) = \delta^x_{p_i}
    where :math:`p_i` is the i'th entry in *points* and :math:`x \in p`.
    """
    n = len(points)
    lagrange_poly = sy.interpolating_poly

    return [lagrange_poly(n, sym, points, (0,)*i + (1,) + (0,)*(n-i)).expand()
            for i in xrange(n)]

def nodal_basis(points, dims, compact=True):
    """Generates a nodal basis for *points* over *dims*

    .. note::
      This function adcfg the same first-to-last counting order as
      :func:`cart_prod_points` with the first index varying quickest.

    **Example**
    >>> import sympy as sy
    >>> nb = nodal_basis([-1, 1], sy.symbols('p q'))
    >>> nb[0]
    (-p/2 + 1/2)*(-q/2 + 1/2)
    >>> nb[0].subs(dict(p=-1, q=-1))
    1
    >>> nb[0].subs(dict(p=1, q=-1))
    0
    """
    p = list(points)

    # Evaluate the basis function in terms of each dimension
    basis = [_nodal_basis1d(p, d) for d in reversed(dims)]

    # Take the cartesian product of these and multiply the resulting tuples
    cpbasis = np.array([np.prod(b) for b in itertools.product(*basis)])

    return cpbasis if compact else cpbasis.reshape((len(p),)*len(dims))

# Cube map face rotation scheme to go from face 1 -> 0..5
_cube_map_rots_sy = [sy.rot_axis2(-sy.pi)*sy.rot_axis1(sy.pi/2),
                     sy.eye(3),
                     sy.rot_axis3(sy.pi/2),
                     sy.rot_axis3(sy.pi),
                     sy.rot_axis3(-sy.pi/2),
                     sy.rot_axis1(-sy.pi/2)]

# Rotation scheme as numpy arrays
_cube_map_rots_np = [np.asanyarray(sy.matrix2numpy(r), dtype=np.float64)
                     for r in _cube_map_rots_sy]

def cube_map_face(fpoints):
    """Given a matrix of points (p,q,r) corresponding to face one of
    `the cube' this method maps these points onto the remaining faces

    On a cube parameterized by (p,q,r) -> (-1,-1,-1) × (1,1,1) face one
    is defined by (-1,-1,-1) × (1,-1,1)."""
    mfpoints = np.empty((6,) + fpoints.shape)

    for i,frot in enumerate(_cube_map_rots_np):
        mfpoints[i,...] = np.dot(fpoints, frot)

    return mfpoints

def diff_vcjh_correctionfn(k, eta, sym):
    # Expand shorthand forms of eta_k for common schemes
    etacommon = dict(dg='0', sd='k/(k+1)', hu='(k+1)/k')

    eta_k = sy.S(etacommon.get(eta, eta), locals=dict(k=k))

    lkm1, lk, lkp1 = [sy.legendre_poly(m, sym) for m in [k-1, k, k+1]]

    # Correction function derivatives, Eq. 3.46 and 3.47
    diffgr = (sy.S(1)/2 * (lk + (eta_k*lkm1 + lkp1)/(1 + eta_k))).diff()
    diffgl = -diffgr.subs(sym, -sym)

    return diffgl, diffgr

class TensorProdBasis(object):
    def _get_pts1d(self):
        rule = self._cfg.get('mesh-elements', 'quad-rule')
        return points_for_rule(rule, self._order + 1)

    def get_upts(self):
        pts1d = self._get_pts1d()

        pts = cart_prod_points(pts1d, 3)
        basis = nodal_basis(pts1d, self._dims)

        return pts, basis

    def get_spts(self, nspts):
        dims, ndims = self._dims, len(self._dims)

        # Root the number of shape points to get the # in each dim
        nsptsord = sy.S(nspts)**(sy.S(1)/len(dims))

        if not nsptsord.is_Number:
            raise ValueError('Invalid number of shape points for {} dims'\
                             .format(len(dims)))

        # Generate nsptsord equispaced points from (-1, 1)
        pts1d = np.linspace(-1, 1, nsptsord)

        pts = cart_prod_points(pts1d, len(dims))
        basis = nodal_basis(pts1d, dims)

        return pts, basis

    def get_nfpts(self):
        return [(self._order + 1)**(self.ndims - 1)] * (2*self.ndims)

class HexBasis(TensorProdBasis, BasisBase):
    name = 'hex'
    ndims = 3

    def __init__(self, *args, **kwargs):
        super(HexBasis, self).__init__(*args, **kwargs)

        k = self._order + 1

        # Pre-compute all possible flux point rotation schemes
        self._rschemes = rs = np.empty((6, 5), dtype=np.object)
        for face,rtag in ndrange(*rs.shape):
            fpts = np.arange(face*k*k, (face+1)*k*k).reshape(k,k)

            if rtag == 0:
                pass
            elif rtag == 1:
                fpts = np.fliplr(fpts)
            elif rtag == 2:
                fpts = np.fliplr(fpts)[::-1]
            elif rtag == 3:
                fpts = np.transpose(fpts)
            elif rtag == 4:
                fpts = np.transpose(fpts)[::-1]

            rs[face,rtag] = fpts.ravel()

    def get_fpts(self):
        # Get the 1D points
        pts1d = self._get_pts1d()

        # Perform a 2D extension to get the (p,r) points of face one
        pts2d = cart_prod_points(pts1d, 2, compact=False)

        # 3D points are just (p,-1,r) for face one
        fonepts = np.empty(pts2d.shape[:-1] + (3,))
        fonepts[...,(0,2)] = pts2d
        fonepts[...,1] = -1

        # Cube map face one to get faces zero through five
        fpts = cube_map_face(fonepts)

        # Allocate space for the flux points basis
        fbasis = np.empty(fpts.shape[:-1], dtype=np.object)

        # Pair up opposite faces with their associated (normal) dimension
        for fpair,sym in zip([(4,2), (1,3), (0,5)], self._dims):
            nbdims = [d for d in self._dims if d is not sym]
            fbasis[fpair,...] = nodal_basis(pts1d, nbdims, compact=False)

            eta = self._cfg.get('mesh-elements', 'vcjh-eta')
            diffcorfn = diff_vcjh_correctionfn(self._order, eta, sym)

            for p,gfn in zip(fpair, diffcorfn):
                if p in (0,3,4):
                    fbasis[p] = np.fliplr(fbasis[p])
                fbasis[p,...] *= gfn

        # Correct faces with negative normals
        fbasis[(4,1,0),...] *= -1

        return fpts.reshape(-1,3), fbasis.reshape(-1)

    def get_norm_fpts(self):
        # Normals for face one are (0,-1,0)
        fonenorms = np.zeros([self._order + 1]*2 + [3])
        fonenorms[...,1] = -1

        # Cube map to get the remaining face normals
        return cube_map_face(fonenorms).reshape(-1, 3)

    def get_fpts_for_face(self, face, rtag):
        return self._rschemes[face, rtag]
<|endoftext|>