<commit_before>import numpy as np
import constants as c
from utils import normalize_frames, get_test_batch
from glob import glob
from scipy.misc import imread, imsave
import os

def save_batch(batch, num_rec_out):
    # TEST
    for clip_num, clip in enumerate(batch):
        for frame_num in xrange(c.HIST_LEN + num_rec_out):
            imsave(c.get_dir('TEST/' + str(clip_num) + '/') + str(frame_num) + '.png',
                   clip[:, :, frame_num * 3:(frame_num + 1) * 3])

def get_full_clips(data_dir, num_clips, num_rec_out=1):
    """
    Loads a batch of random clips from the unprocessed train or test data.

    @param data_dir: The directory of the data to read. Should be either c.TRAIN_DIR or c.TEST_DIR.
    @param num_clips: The number of clips to read.
    @param num_rec_out: The number of outputs to predict. Outputs > 1 are computed recursively,
                        using the previously-generated frames as input. Default = 1.

    @return: An array of shape
             [num_clips, c.TRAIN_HEIGHT, c.TRAIN_WIDTH, (3 * (c.HIST_LEN + num_rec_out))].
             A batch of frame sequences with values normalized in range [-1, 1].
    """
    clips = np.empty([num_clips,
                      c.FULL_HEIGHT,
                      c.FULL_WIDTH,
                      (3 * (c.HIST_LEN + num_rec_out))])

    # get num_clips random episodes
    ep_dirs = np.random.choice(glob(data_dir + '*'), num_clips)
    print ep_dirs

    # get a random clip of length HIST_LEN + num_rec_out from each episode
    for clip_num, ep_dir in enumerate(ep_dirs):
        ep_frame_paths = glob(os.path.join(ep_dir, '*'))
        start_index = np.random.choice(len(ep_frame_paths) - (c.HIST_LEN + num_rec_out - 1))
        clip_frame_paths = ep_frame_paths[start_index:start_index + (c.HIST_LEN + num_rec_out)]
        print clip_num
        print clip_frame_paths

        # read in frames
        for frame_num, frame_path in enumerate(clip_frame_paths):
            frame = imread(frame_path, mode='RGB')
            norm_frame = normalize_frames(frame)

            clips[clip_num, :, :, frame_num * 3:(frame_num + 1) * 3] = norm_frame

    # TEST
    save_batch(clips, num_rec_out)

    return clips

get_full_clips('../Data/Ms_Pacman/Test/', 1, num_rec_out=1)

# def test():
#     """
#     Runs one test step on the generator network.
#     """
#     batch = get_test_batch(c.BATCH_SIZE, num_rec_out=2)
#     save_batch(batch, 2)
#
#     # self.g_model.test_batch(
#     #     batch, self.global_step, num_rec_out=2)
#
# test()
<commit_msg>updated test<commit_after>import numpy as np
import constants as c
from utils import normalize_frames, get_test_batch
from glob import glob
from scipy.misc import imread, imsave
import os

def save_batch(batch, num_rec_out):
    # TEST
    for clip_num, clip in enumerate(batch):
        for frame_num in xrange(c.HIST_LEN + num_rec_out):
            imsave(c.get_dir('TEST/' + str(clip_num) + '/') + str(frame_num) + '.png',
                   clip[:, :, frame_num * 3:(frame_num + 1) * 3])

def get_full_clips(data_dir, num_clips, num_rec_out=1):
    """
    Loads a batch of random clips from the unprocessed train or test data.

    @param data_dir: The directory of the data to read. Should be either c.TRAIN_DIR or c.TEST_DIR.
    @param num_clips: The number of clips to read.
    @param num_rec_out: The number of outputs to predict. Outputs > 1 are computed recursively,
                        using the previously-generated frames as input. Default = 1.

    @return: An array of shape
             [num_clips, c.TRAIN_HEIGHT, c.TRAIN_WIDTH, (3 * (c.HIST_LEN + num_rec_out))].
             A batch of frame sequences with values normalized in range [-1, 1].
    """
    clips = np.empty([num_clips,
                      c.FULL_HEIGHT,
                      c.FULL_WIDTH,
                      (3 * (c.HIST_LEN + num_rec_out))])

    # get num_clips random episodes
    ep_dirs = np.random.choice(glob(data_dir + '*'), num_clips)
    print ep_dirs

    # get a random clip of length HIST_LEN + num_rec_out from each episode
    for clip_num, ep_dir in enumerate(ep_dirs):
        ep_frame_paths = sorted(glob(os.path.join(ep_dir, '*')))
        start_index = np.random.choice(len(ep_frame_paths) - (c.HIST_LEN + num_rec_out - 1))
        clip_frame_paths = ep_frame_paths[start_index:start_index + (c.HIST_LEN + num_rec_out)]
        print clip_num
        print clip_frame_paths

        # read in frames
        for frame_num, frame_path in enumerate(clip_frame_paths):
            frame = imread(frame_path, mode='RGB')
            norm_frame = normalize_frames(frame)

            clips[clip_num, :, :, frame_num * 3:(frame_num + 1) * 3] = norm_frame

    # TEST
    save_batch(clips, num_rec_out)

    return clips

get_full_clips('../Data/Ms_Pacman/Test/', 1, num_rec_out=1)

# def test():
#     """
#     Runs one test step on the generator network.
#     """
#     batch = get_test_batch(c.BATCH_SIZE, num_rec_out=2)
#     save_batch(batch, 2)
#
#     # self.g_model.test_batch(
#     #     batch, self.global_step, num_rec_out=2)
#
# test()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Swap edges in a graph.
"""
#    Copyright (C) 2004-2015 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
import math
import random
import networkx as nx
__author__ = "\n".join(['Aric Hagberg (hagberg@lanl.gov)',
                        'Pieter Swart (swart@lanl.gov)',
                        'Dan Schult (dschult@colgate.edu)'
                        'Joel Miller (joel.c.miller.research@gmail.com)'
                        'Ben Edwards'])

__all__ = ['double_edge_swap',
           'connected_double_edge_swap']

def double_edge_swap(G, nswap=1, max_tries=100):
    """Swap two edges in the graph while keeping the node degrees fixed.

    A double-edge swap removes two randomly chosen edges u-v and x-y
    and creates the new edges u-x and v-y::

     u--v            u  v
            becomes  |  |
     x--y            x  y

    If either the edge u-x or v-y already exist no swap is performed
    and another attempt is made to find a suitable edge pair.

    Parameters
    ----------
    G : graph
       An undirected graph

    nswap : integer (optional, default=1)
       Number of double-edge swaps to perform

    max_tries : integer (optional)
       Maximum number of attempts to swap edges

    Returns
    -------
    G : graph
       The graph after double edge swaps.

    Notes
    -----
    Does not enforce any connectivity constraints.

    The graph G is modified in place.
    """
    if G.is_directed():
        raise nx.NetworkXError(\
            "double_edge_swap() not defined for directed graphs.")
    if nswap>max_tries:
        raise nx.NetworkXError("Number of swaps > number of tries allowed.")
    if len(G) < 4:
        raise nx.NetworkXError("Graph has less than four nodes.")
    # Instead of choosing uniformly at random from a generated edge list,
    # this algorithm chooses nonuniformly from the set of nodes with
    # probability weighted by degree.
    n=0
    swapcount=0
    keys,degrees=zip(*G.degree().items()) # keys, degree
    cdf=nx.utils.cumulative_distribution(degrees)  # cdf of degree
    while swapcount < nswap:
#        if random.random() < 0.5: continue # trick to avoid periodicities?
        # pick two random edges without creating edge list
        # choose source node indices from discrete distribution
        (ui,xi)=nx.utils.discrete_sequence(2,cdistribution=cdf)
        if ui==xi:
            continue # same source, skip
        u=keys[ui] # convert index to label
        x=keys[xi]
        # choose target uniformly from neighbors
        v=random.choice(list(G[u]))
        y=random.choice(list(G[x]))
        if v==y:
            continue # same target, skip
        if (x not in G[u]) and (y not in G[v]): # don't create parallel edges
            G.add_edge(u,x)
            G.add_edge(v,y)
            G.remove_edge(u,v)
            G.remove_edge(x,y)
            swapcount+=1
        if n >= max_tries:
            e=('Maximum number of swap attempts (%s) exceeded '%n +
            'before desired swaps achieved (%s).'%nswap)
            raise nx.NetworkXAlgorithmError(e)
        n+=1
    return G

def connected_double_edge_swap(G, nswap=1):
    """Attempt nswap double-edge swaps in the graph G.

    A double-edge swap removes two randomly chosen edges u-v and x-y
    and creates the new edges u-x and v-y::

     u--v            u  v
            becomes  |  |
     x--y            x  y

    If either the edge u-x or v-y already exist no swap is performed so
    the actual count of swapped edges is always <= nswap

    Parameters
    ----------
    G : graph
       An undirected graph

    nswap : integer (optional, default=1)
       Number of double-edge swaps to perform

    Returns
    -------
    G : int
       The number of successful swaps

    Notes
    -----
    The initial graph G must be connected, and the resulting graph is connected.
    The graph G is modified in place.

    References
    ----------
    .. [1] C. Gkantsidis and M. Mihail and E. Zegura,
           The Markov chain simulation method for generating connected
           power law random graphs, 2003.
           http://citeseer.ist.psu.edu/gkantsidis03markov.html
    """
    import math
    if not nx.is_connected(G):
       raise nx.NetworkXError("Graph not connected")
    if len(G) < 4:
        raise nx.NetworkXError("Graph has less than four nodes.")
    n=0
    swapcount=0
    deg=G.degree()
    dk=list(deg.keys()) # Label key for nodes
    cdf=nx.utils.cumulative_distribution(list(G.degree().values()))
    window=1
    while n < nswap:
        wcount=0
        swapped=[]
        while wcount < window and n < nswap:
            # Pick two random edges without creating edge list
            # Choose source nodes from discrete degree distribution
            (ui,xi)=nx.utils.discrete_sequence(2,cdistribution=cdf)
            if ui==xi:
                continue # same source, skip
            u=dk[ui] # convert index to label
            x=dk[xi]
            # Choose targets uniformly from neighbors
            v=random.choice(G.neighbors(u))
            y=random.choice(G.neighbors(x)) #
            if v==y: continue # same target, skip
            if (not G.has_edge(u,x)) and (not G.has_edge(v,y)):
                G.remove_edge(u,v)
                G.remove_edge(x,y)
                G.add_edge(u,x)
                G.add_edge(v,y)
                swapped.append((u,v,x,y))
                swapcount+=1
            n+=1
            wcount+=1
        if nx.is_connected(G):
            window+=1
        else:
            # not connected, undo changes from previous window, decrease window
            while swapped:
                (u,v,x,y)=swapped.pop()
                G.add_edge(u,v)
                G.add_edge(x,y)
                G.remove_edge(u,x)
                G.remove_edge(v,y)
                swapcount-=1
            window = int(math.ceil(float(window)/2))
    return swapcount

<commit_msg>Optimization for connected double edge swap.<commit_after># -*- coding: utf-8 -*-
"""Swap edges in a graph.
"""
#    Copyright (C) 2004-2015 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
from __future__ import division

import math
import random

import networkx as nx

__author__ = "\n".join(['Aric Hagberg (hagberg@lanl.gov)',
                        'Pieter Swart (swart@lanl.gov)',
                        'Dan Schult (dschult@colgate.edu)',
                        'Joel Miller (joel.c.miller.research@gmail.com)',
                        'Ben Edwards'])

__all__ = ['double_edge_swap',
           'connected_double_edge_swap']

def double_edge_swap(G, nswap=1, max_tries=100):
    """Swap two edges in the graph while keeping the node degrees fixed.

    A double-edge swap removes two randomly chosen edges u-v and x-y
    and creates the new edges u-x and v-y::

     u--v            u  v
            becomes  |  |
     x--y            x  y

    If either the edge u-x or v-y already exist no swap is performed
    and another attempt is made to find a suitable edge pair.

    Parameters
    ----------
    G : graph
       An undirected graph

    nswap : integer (optional, default=1)
       Number of double-edge swaps to perform

    max_tries : integer (optional)
       Maximum number of attempts to swap edges

    Returns
    -------
    G : graph
       The graph after double edge swaps.

    Notes
    -----
    Does not enforce any connectivity constraints.

    The graph G is modified in place.
    """
    if G.is_directed():
        raise nx.NetworkXError(\
            "double_edge_swap() not defined for directed graphs.")
    if nswap>max_tries:
        raise nx.NetworkXError("Number of swaps > number of tries allowed.")
    if len(G) < 4:
        raise nx.NetworkXError("Graph has less than four nodes.")
    # Instead of choosing uniformly at random from a generated edge list,
    # this algorithm chooses nonuniformly from the set of nodes with
    # probability weighted by degree.
    n=0
    swapcount=0
    keys,degrees=zip(*G.degree().items()) # keys, degree
    cdf=nx.utils.cumulative_distribution(degrees)  # cdf of degree
    while swapcount < nswap:
#        if random.random() < 0.5: continue # trick to avoid periodicities?
        # pick two random edges without creating edge list
        # choose source node indices from discrete distribution
        (ui,xi)=nx.utils.discrete_sequence(2,cdistribution=cdf)
        if ui==xi:
            continue # same source, skip
        u=keys[ui] # convert index to label
        x=keys[xi]
        # choose target uniformly from neighbors
        v=random.choice(list(G[u]))
        y=random.choice(list(G[x]))
        if v==y:
            continue # same target, skip
        if (x not in G[u]) and (y not in G[v]): # don't create parallel edges
            G.add_edge(u,x)
            G.add_edge(v,y)
            G.remove_edge(u,v)
            G.remove_edge(x,y)
            swapcount+=1
        if n >= max_tries:
            e=('Maximum number of swap attempts (%s) exceeded '%n +
            'before desired swaps achieved (%s).'%nswap)
            raise nx.NetworkXAlgorithmError(e)
        n+=1
    return G

def connected_double_edge_swap(G, nswap=1, _window_threshold=3):
    """Attempts the specified number of double-edge swaps in the graph ``G``.

    A double-edge swap removes two randomly chosen edges ``(u, v)`` and ``(x,
    y)`` and creates the new edges ``(u, x)`` and ``(v, y)``::

     u--v            u  v
            becomes  |  |
     x--y            x  y

    If either ``(u, x)`` or ``(v, y)`` already exist, then no swap is performed
    so the actual number of swapped edges is always *at most* ``nswap``.

    Parameters
    ----------
    G : graph
       An undirected graph

    nswap : integer (optional, default=1)
       Number of double-edge swaps to perform

    _window_threshold : integer

       The window size below which connectedness of the graph will be checked
       after each swap.

       The "window" in this function is a dynamically updated integer that
       represents the number of swap attempts to make before checking if the
       graph remains connected. It is an optimization used to decrease the
       running time of the algorithm in exchange for increased complexity of
       implementation.

       If the window size is below this threshold, then the algorithm checks
       after each swap if the graph remains connected by checking if there is a
       path joining the two nodes whose edge was just removed. If the window
       size is above this threshold, then the algorithm performs do all the
       swaps in the window and only then check if the graph is still connected.

    Returns
    -------
    int
       The number of successful swaps

    Raises
    ------

    NetworkXError

       If the input graph is not connected, or if the graph has fewer than four
       nodes.

    Notes
    -----

    The initial graph ``G`` must be connected, and the resulting graph is
    connected. The graph ``G`` is modified in place.

    References
    ----------
    .. [1] C. Gkantsidis and M. Mihail and E. Zegura,
           The Markov chain simulation method for generating connected
           power law random graphs, 2003.
           http://citeseer.ist.psu.edu/gkantsidis03markov.html
    """
    if not nx.is_connected(G):
        raise nx.NetworkXError("Graph not connected")
    if len(G) < 4:
        raise nx.NetworkXError("Graph has less than four nodes.")
    n = 0
    swapcount = 0
    deg = G.degree()
    # Label key for nodes
    dk = list(deg.keys())
    cdf = nx.utils.cumulative_distribution(list(G.degree().values()))
    window = 1
    while n < nswap:
        wcount = 0
        swapped = []
        # If the window is small, we just check each time whether the graph is
        # connected by checking if the nodes that were just separated are still
        # connected.
        if window < _window_threshold:
            # This Boolean keeps track of whether there was a failure or not.
            fail = False
            while wcount < window and n < nswap:
                # Pick two random edges without creating the edge list. Choose
                # source nodes from the discrete degree distribution.
                (ui, xi) = nx.utils.discrete_sequence(2, cdistribution=cdf)
                # If the source nodes are the same, skip this pair.
                if ui == xi:
                    continue
                # Convert an index to a node label.
                u = dk[ui]
                x = dk[xi]
                # Choose targets uniformly from neighbors.
                v = random.choice(G.neighbors(u))
                y = random.choice(G.neighbors(x))
                # If the target nodes are the same, skip this pair.
                if v == y:
                    continue
                if x not in G[u] and y not in G[v]:
                    G.remove_edge(u, v)
                    G.remove_edge(x, y)
                    G.add_edge(u, x)
                    G.add_edge(v, y)
                    swapped.append((u, v, x, y))
                    swapcount += 1
                n += 1
                # If G remains connected...
                if nx.has_path(G, u, v):
                    wcount += 1
                # Otherwise, undo the changes.
                else:
                    G.add_edge(u, v)
                    G.add_edge(x, y)
                    G.remove_edge(u, x)
                    G.remove_edge(v, y)
                    swapcount -= 1
                    fail = True
            # If one of the swaps failed, reduce the window size.
            if fail:
                window = int(math.ceil(window / 2))
            else:
                window += 1
        # If the window is large, then there is a good chance that a bunch of
        # swaps will work. It's quicker to do all those swaps first and then
        # check if the graph remains connected.
        else:
            while wcount < window and n < nswap:
                # Pick two random edges without creating the edge list. Choose
                # source nodes from the discrete degree distribution.
                (ui, xi) = nx.utils.discrete_sequence(2, cdistribution=cdf)
                # If the source nodes are the same, skip this pair.
                if ui == xi:
                    continue
                # Convert an index to a node label.
                u = dk[ui]
                x = dk[xi]
                # Choose targets uniformly from neighbors.
                v = random.choice(G.neighbors(u))
                y = random.choice(G.neighbors(x))
                # If the target nodes are the same, skip this pair.
                if v == y:
                    continue
                if x not in G[u] and y not in G[v]:
                    G.remove_edge(u, v)
                    G.remove_edge(x, y)
                    G.add_edge(u, x)
                    G.add_edge(v, y)
                    swapped.append((u, v, x, y))
                    swapcount += 1
                n += 1
                wcount += 1
            # If the graph remains connected, increase the window size.
            if nx.is_connected(G):
                window += 1
            # Otherwise, undo the changes from the previous window and decrease
            # the window size.
            else:
                while swapped:
                    (u, v, x, y) = swapped.pop()
                    G.add_edge(u, v)
                    G.add_edge(x, y)
                    G.remove_edge(u, x)
                    G.remove_edge(v, y)
                    swapcount -= 1
                window = int(math.ceil(window / 2))
    return swapcount
<|endoftext|>


<commit_before>import theano
import theano.tensor as T
import numpy as np

from neupy.utils import asfloat
from neupy.core.properties import ProperFractionProperty, NumberProperty
from .base import MinibatchGradientDescent

__all__ = ('Adam',)

class Adam(MinibatchGradientDescent):
    """
    Adam algorithm.

    Parameters
    ----------
    beta1 : float
        Decay rate. Value need to be between ``0`` and ``1``.
        Defaults to ``0.95``.

    beta2 : float
        Decay rate. Value need to be between ``0`` and ``1``.
        Defaults to ``0.95``.

    epsilon : float
        Value need to be greater than ``0``. Defaults to ``1e-5``.

    step : float
        Learning rate, defaults to ``0.001``.

    {MinibatchGradientDescent.batch_size}

    {GradientDescent.addons}

    {ConstructibleNetwork.connection}

    {ConstructibleNetwork.error}

    {BaseNetwork.show_epoch}

    {BaseNetwork.shuffle_data}

    {BaseNetwork.epoch_end_signal}

    {BaseNetwork.train_end_signal}

    {Verbose.verbose}

    Attributes
    ----------
    {MinibatchGradientDescent.Attributes}

    Methods
    -------
    {MinibatchGradientDescent.Methods}

    Examples
    --------
    >>> import numpy as np
    >>> from neupy import algorithms
    >>>
    >>> x_train = np.array([[1, 2], [3, 4]])
    >>> y_train = np.array([[1], [0]])
    >>>
    >>> mnet = algorithms.Adam((2, 3, 1))
    >>> mnet.train(x_train, y_train)
    """
    step = NumberProperty(default=0.001, minval=0)
    beta1 = ProperFractionProperty(default=0.9)
    beta2 = ProperFractionProperty(default=0.999)
    epsilon = NumberProperty(default=1e-7, minval=0)

    def init_param_updates(self, layer, parameter):
        epoch = self.variables.epoch

        parameter_shape = T.shape(parameter).eval()
        prev_first_moment = theano.shared(
            name="{}/prev-first-moment".format(parameter.name),
            value=asfloat(np.zeros(parameter_shape)),
        )
        prev_second_moment = theano.shared(
            name="{}/prev-second-moment".format(parameter.name),
            value=asfloat(np.zeros(parameter_shape)),
        )

        step = self.variables.step
        gradient = T.grad(self.variables.error_func, wrt=parameter)

        first_moment = (
            self.beta1 * prev_first_moment +
            (1. - self.beta1) * gradient)
        second_moment = (
            self.beta2 * prev_second_moment +
            (1. - self.beta2) * gradient ** 2
        )

        first_moment_bias_corrected = first_moment / (1. - self.beta1 ** epoch)
        second_moment_bias_corrected = second_moment / (1. - self.beta2 ** epoch)

        parameter_delta = first_moment_bias_corrected * (
            T.sqrt(second_moment_bias_corrected) + self.epsilon
        )

        return [
            (prev_first_moment, first_moment),
            (prev_second_moment, second_moment),
            (parameter, parameter - step * parameter_delta),
        ]
<commit_msg>fixed pep8 error<commit_after>import theano
import theano.tensor as T
import numpy as np

from neupy.utils import asfloat
from neupy.core.properties import ProperFractionProperty, NumberProperty
from .base import MinibatchGradientDescent

__all__ = ('Adam',)

class Adam(MinibatchGradientDescent):
    """
    Adam algorithm.

    Parameters
    ----------
    beta1 : float
        Decay rate. Value need to be between ``0`` and ``1``.
        Defaults to ``0.95``.

    beta2 : float
        Decay rate. Value need to be between ``0`` and ``1``.
        Defaults to ``0.95``.

    epsilon : float
        Value need to be greater than ``0``. Defaults to ``1e-5``.

    step : float
        Learning rate, defaults to ``0.001``.

    {MinibatchGradientDescent.batch_size}

    {GradientDescent.addons}

    {ConstructibleNetwork.connection}

    {ConstructibleNetwork.error}

    {BaseNetwork.show_epoch}

    {BaseNetwork.shuffle_data}

    {BaseNetwork.epoch_end_signal}

    {BaseNetwork.train_end_signal}

    {Verbose.verbose}

    Attributes
    ----------
    {MinibatchGradientDescent.Attributes}

    Methods
    -------
    {MinibatchGradientDescent.Methods}

    Examples
    --------
    >>> import numpy as np
    >>> from neupy import algorithms
    >>>
    >>> x_train = np.array([[1, 2], [3, 4]])
    >>> y_train = np.array([[1], [0]])
    >>>
    >>> mnet = algorithms.Adam((2, 3, 1))
    >>> mnet.train(x_train, y_train)
    """
    step = NumberProperty(default=0.001, minval=0)
    beta1 = ProperFractionProperty(default=0.9)
    beta2 = ProperFractionProperty(default=0.999)
    epsilon = NumberProperty(default=1e-7, minval=0)

    def init_param_updates(self, layer, parameter):
        epoch = self.variables.epoch

        parameter_shape = T.shape(parameter).eval()
        prev_first_moment = theano.shared(
            name="{}/prev-first-moment".format(parameter.name),
            value=asfloat(np.zeros(parameter_shape)),
        )
        prev_second_moment = theano.shared(
            name="{}/prev-second-moment".format(parameter.name),
            value=asfloat(np.zeros(parameter_shape)),
        )

        step = self.variables.step
        gradient = T.grad(self.variables.error_func, wrt=parameter)

        first_moment = (
            self.beta1 * prev_first_moment +
            (1. - self.beta1) * gradient)
        second_moment = (
            self.beta2 * prev_second_moment +
            (1. - self.beta2) * gradient ** 2
        )

        first_moment_bias_corrected = first_moment / (1. - self.beta1 ** epoch)
        second_moment_bias_corrected = (
            second_moment / (1. - self.beta2 ** epoch))

        parameter_delta = first_moment_bias_corrected * (
            T.sqrt(second_moment_bias_corrected) + self.epsilon
        )

        return [
            (prev_first_moment, first_moment),
            (prev_second_moment, second_moment),
            (parameter, parameter - step * parameter_delta),
        ]
<|endoftext|>


<commit_before>
import numpy as np
from neurosynth.base import imageutils
import stats
from scipy import sparse
from scipy.stats import norm

def analyze_features(dataset, features, image_type=None, threshold=0.001, q=0.05, save=None):
  """ Generate meta-analysis images for a set of features.

  Args:
    dataset: A Dataset instance containing feature and activation data.
    features: A list of named features to generate meta-analysis maps for.
    image_type: The type of image to return. Specify one of the extensions
      generated by the MetaAnalysis procedure--e.g., pSgA_z, pAgS, etc. By
      default, will use pSgA_z (i.e., z-scores reflecting the probability
      that a Mappable has a feature given that activation is present). If
      None, set to pSgA_z.
    threshold: The threshold for determining whether or not a Mappable has
      a feature. By default, this is 0.001, which is only sensible in the
      case of term-based features (so be sure to specify it for other kinds).
    q: The FDR rate to use for multiple comparisons correction (default = 0.05).
    save: An optional path to save all meta-analysis images to. Images will be
      prepended with the entire path plus the feature name; e.g., passing
      save='test' would result in images called "test_feature1_*" for
      a feature named feature1. If none, returns all the data as a matrix.

  Returns:
    If save is None, an n_voxels x n_features 2D numpy array.
  """
  if image_type is None:
    image_type = 'pSgA_z'
  if save is None:
    result = np.zeros((dataset.volume.num_vox_in_mask, len(features)))

  for i, f in enumerate(features):
    ids = dataset.get_ids_by_features(f, threshold=threshold)
    ma = MetaAnalysis(dataset, ids, q=q)
    if save is None:
      result[:,i] = ma.images[image_type]
    else:
      ma.save_results('%s_%s', (save, f))
  return result

class MetaAnalysis:

  """ Meta-analysis of a Dataset. Currently contrasts two subsets of
  studies within a Dataset and saves a bunch of statistical images.
  Only one list of study IDs (ids) needs to be passed; the Universe will
  be bisected into studies that are and are not included in the
  list, and the contrast is then performed across these two groups.
  If a seond optional second study list is provided (ids2), the Dataset
  is first constrained to the union of ids1 and ids2, and the standard
  contrast is then performed."""

  # DESPERATELY NEEDS REFACTORING!!!

  def __init__(self, dataset, ids, ids2=None, **kwargs):

    """ Initiaize a new MetaAnalysis instance and run an analysis.
    Args:
      dataset: A Dataset instance.
      ids: A list of Mappable IDs to include in the meta-analysis.
      ids2: Optional second list of Mappable IDs. If passed, the set of studies will
        be restricted to the union of ids and ids2 before performing the meta-analysis.
        This is useful for meta-analytic contrasts, as the resulting images will in
        effect identify regions that are reported/activated more frequently in one
        list than in the other.
      kwargs: Additional optional arguments. Currently implemented:
        q: The FDR threshold to use when correcting for multiple comparisons. Set to
          .01 by default.
    """

    self.dataset = dataset
    mt = dataset.image_table
    self.selected_ids = list(set(mt.ids) & set(ids))
    self.selected_id_indices = np.in1d(mt.ids, ids)

    # Calculate different count variables
    # print "Calculating counts..."
    n_mappables = len(mt.ids)
    n_selected = len(self.selected_ids)
    n_unselected = n_mappables - n_selected

    # If ids2 is provided, we only use mappables explicitly in either ids or ids2.
    # Otherwise, all mappables not in the ids list are used as the control condition.
    unselected_id_indices = ~self.selected_id_indices if ids2 == None else np.in1d(mt.ids, ids2)

    n_selected_active_voxels = mt.data.dot(self.selected_id_indices)
    n_unselected_active_voxels = mt.data.dot(unselected_id_indices)

    # Nomenclature for variables below: p = probability, S = selected, g = given,
    # U = unselected, A = activation. So, e.g., pAgS = p(A|S) = probability of activation
    # in voxel given that the mappable is selected (i.e., is included in the ids list
    # passed to the constructor).
    pS = (n_selected+1.0)/(n_mappables+2)

    # Conditional probabilities, with Laplace smoothing
    # print "Calculating conditional probabilities..."
    # pA = np.asarray(sparse.spmatrix.mean(mt.data, 1)) + 1.0/n_mappables
    pA = (n_selected_active_voxels+1.0) / (n_mappables+2)
    pAgS = (n_selected_active_voxels+1.0)/(n_selected+2)
    pAgU = (n_unselected_active_voxels+1.0)/(n_unselected+2)
    pSgA = pAgS * pS / pA

    # Recompute conditionals with uniform prior
    # print "Recomputing with uniform priors..."
    prior_pS = kwargs.get('prior', 0.5)
    pAgS_unif = prior_pS * pAgS + (1-prior_pS) * pAgU
    pSgA_unif = pAgS * prior_pS / pAgS_unif

    # Set voxel-wise FDR to .05 unless explicitly specified
    q = kwargs.get('q', 0.05)

    # One-way chi-square test for consistency of activation
    p_vals = stats.one_way(np.squeeze(n_selected_active_voxels), n_selected)
    p_vals[p_vals < 1e-240] = 1e-240  # prevents overflow due to loss of precision
    z_sign = np.sign(n_selected_active_voxels - np.mean(n_selected_active_voxels)).ravel()
    pAgS_z = np.abs(norm.ppf(p_vals/2)) * z_sign

    fdr_thresh = stats.fdr(p_vals, q)
    pAgS_z_FDR = imageutils.threshold_img(pAgS_z, fdr_thresh, p_vals, mask_out='above')

    # Two-way chi-square for specificity of activation
    cells = np.squeeze(np.array([[n_selected_active_voxels, n_unselected_active_voxels],
      [n_selected-n_selected_active_voxels, n_unselected-n_unselected_active_voxels]]).T)

    p_vals = stats.two_way(cells)
    p_vals[p_vals < 1e-240] = 1e-240  # prevents overflow
    z_sign = np.sign(pAgS - pAgU).ravel()
    pSgA_z = np.abs(norm.ppf(p_vals/2)) * z_sign
    fdr_thresh = stats.fdr(p_vals, q)
    pSgA_z_FDR = imageutils.threshold_img(pSgA_z, fdr_thresh, p_vals, mask_out='above')

    # Retain any images we may want to save or access later
    self.images = { 'pAgS': pAgS,
            'pSgA': pSgA,
            'pAgS_unif': pAgS_unif,
            'pSgA_unif': pSgA_unif,
            'pAgS_z': pAgS_z,
            'pSgA_z': pSgA_z,
            ('pAgS_z_FDR_%s' % q): pAgS_z_FDR,
            ('pSgA_z_FDR_%s' % q): pSgA_z_FDR }

  def save_results(self, outroot, image_list=None):
    """ Write out any images generated by the meta-analysis. The outroot argument is prepended
    to all file names. Optionally, a restricted list of images to save can be passed; otherwise,
    all images currently stored in self.images will be saved. """
    print "Saving results..."
    if image_list == None: image_list = self.images.keys()
    for suffix, img in self.images.items():
      if suffix in image_list:
        imageutils.save_img(img, '%s_%s.nii.gz' % (outroot, suffix), self.dataset.volume)
<commit_msg>added null definition for result when save is defined<commit_after>
import numpy as np
from neurosynth.base import imageutils
import stats
from scipy import sparse
from scipy.stats import norm

def analyze_features(dataset, features, image_type=None, threshold=0.001, q=0.05, save=None):
  """ Generate meta-analysis images for a set of features.

  Args:
    dataset: A Dataset instance containing feature and activation data.
    features: A list of named features to generate meta-analysis maps for.
    image_type: The type of image to return. Specify one of the extensions
      generated by the MetaAnalysis procedure--e.g., pSgA_z, pAgS, etc. By
      default, will use pSgA_z (i.e., z-scores reflecting the probability
      that a Mappable has a feature given that activation is present). If
      None, set to pSgA_z.
    threshold: The threshold for determining whether or not a Mappable has
      a feature. By default, this is 0.001, which is only sensible in the
      case of term-based features (so be sure to specify it for other kinds).
    q: The FDR rate to use for multiple comparisons correction (default = 0.05).
    save: An optional path to save all meta-analysis images to. Images will be
      prepended with the entire path plus the feature name; e.g., passing
      save='test' would result in images called "test_feature1_*" for
      a feature named feature1. If none, returns all the data as a matrix.

  Returns:
    If save is None, an n_voxels x n_features 2D numpy array.
  """
  if image_type is None:
    image_type = 'pSgA_z'
  if save is None:
    result = np.zeros((dataset.volume.num_vox_in_mask, len(features)))
  else:
    result=[]

  for i, f in enumerate(features):
    ids = dataset.get_ids_by_features(f, threshold=threshold)
    ma = MetaAnalysis(dataset, ids, q=q)
    if save is None:
      result[:,i] = ma.images[image_type]
    else:
      ma.save_results('%s_%s', (save, f))
  return result

class MetaAnalysis:

  """ Meta-analysis of a Dataset. Currently contrasts two subsets of
  studies within a Dataset and saves a bunch of statistical images.
  Only one list of study IDs (ids) needs to be passed; the Universe will
  be bisected into studies that are and are not included in the
  list, and the contrast is then performed across these two groups.
  If a seond optional second study list is provided (ids2), the Dataset
  is first constrained to the union of ids1 and ids2, and the standard
  contrast is then performed."""

  # DESPERATELY NEEDS REFACTORING!!!

  def __init__(self, dataset, ids, ids2=None, **kwargs):

    """ Initiaize a new MetaAnalysis instance and run an analysis.
    Args:
      dataset: A Dataset instance.
      ids: A list of Mappable IDs to include in the meta-analysis.
      ids2: Optional second list of Mappable IDs. If passed, the set of studies will
        be restricted to the union of ids and ids2 before performing the meta-analysis.
        This is useful for meta-analytic contrasts, as the resulting images will in
        effect identify regions that are reported/activated more frequently in one
        list than in the other.
      kwargs: Additional optional arguments. Currently implemented:
        q: The FDR threshold to use when correcting for multiple comparisons. Set to
          .01 by default.
    """

    self.dataset = dataset
    mt = dataset.image_table
    self.selected_ids = list(set(mt.ids) & set(ids))
    self.selected_id_indices = np.in1d(mt.ids, ids)

    # Calculate different count variables
    # print "Calculating counts..."
    n_mappables = len(mt.ids)
    n_selected = len(self.selected_ids)
    n_unselected = n_mappables - n_selected

    # If ids2 is provided, we only use mappables explicitly in either ids or ids2.
    # Otherwise, all mappables not in the ids list are used as the control condition.
    unselected_id_indices = ~self.selected_id_indices if ids2 == None else np.in1d(mt.ids, ids2)

    n_selected_active_voxels = mt.data.dot(self.selected_id_indices)
    n_unselected_active_voxels = mt.data.dot(unselected_id_indices)

    # Nomenclature for variables below: p = probability, S = selected, g = given,
    # U = unselected, A = activation. So, e.g., pAgS = p(A|S) = probability of activation
    # in voxel given that the mappable is selected (i.e., is included in the ids list
    # passed to the constructor).
    pS = (n_selected+1.0)/(n_mappables+2)

    # Conditional probabilities, with Laplace smoothing
    # print "Calculating conditional probabilities..."
    # pA = np.asarray(sparse.spmatrix.mean(mt.data, 1)) + 1.0/n_mappables
    pA = (n_selected_active_voxels+1.0) / (n_mappables+2)
    pAgS = (n_selected_active_voxels+1.0)/(n_selected+2)
    pAgU = (n_unselected_active_voxels+1.0)/(n_unselected+2)
    pSgA = pAgS * pS / pA

    # Recompute conditionals with uniform prior
    # print "Recomputing with uniform priors..."
    prior_pS = kwargs.get('prior', 0.5)
    pAgS_unif = prior_pS * pAgS + (1-prior_pS) * pAgU
    pSgA_unif = pAgS * prior_pS / pAgS_unif

    # Set voxel-wise FDR to .05 unless explicitly specified
    q = kwargs.get('q', 0.05)

    # One-way chi-square test for consistency of activation
    p_vals = stats.one_way(np.squeeze(n_selected_active_voxels), n_selected)
    p_vals[p_vals < 1e-240] = 1e-240  # prevents overflow due to loss of precision
    z_sign = np.sign(n_selected_active_voxels - np.mean(n_selected_active_voxels)).ravel()
    pAgS_z = np.abs(norm.ppf(p_vals/2)) * z_sign

    fdr_thresh = stats.fdr(p_vals, q)
    pAgS_z_FDR = imageutils.threshold_img(pAgS_z, fdr_thresh, p_vals, mask_out='above')

    # Two-way chi-square for specificity of activation
    cells = np.squeeze(np.array([[n_selected_active_voxels, n_unselected_active_voxels],
      [n_selected-n_selected_active_voxels, n_unselected-n_unselected_active_voxels]]).T)

    p_vals = stats.two_way(cells)
    p_vals[p_vals < 1e-240] = 1e-240  # prevents overflow
    z_sign = np.sign(pAgS - pAgU).ravel()
    pSgA_z = np.abs(norm.ppf(p_vals/2)) * z_sign
    fdr_thresh = stats.fdr(p_vals, q)
    pSgA_z_FDR = imageutils.threshold_img(pSgA_z, fdr_thresh, p_vals, mask_out='above')

    # Retain any images we may want to save or access later
    self.images = { 'pAgS': pAgS,
            'pSgA': pSgA,
            'pAgS_unif': pAgS_unif,
            'pSgA_unif': pSgA_unif,
            'pAgS_z': pAgS_z,
            'pSgA_z': pSgA_z,
            ('pAgS_z_FDR_%s' % q): pAgS_z_FDR,
            ('pSgA_z_FDR_%s' % q): pSgA_z_FDR }

  def save_results(self, outroot, image_list=None):
    """ Write out any images generated by the meta-analysis. The outroot argument is prepended
    to all file names. Optionally, a restricted list of images to save can be passed; otherwise,
    all images currently stored in self.images will be saved. """
    print "Saving results..."
    if image_list == None: image_list = self.images.keys()
    for suffix, img in self.images.items():
      if suffix in image_list:
        imageutils.save_img(img, '%s_%s.nii.gz' % (outroot, suffix), self.dataset.volume)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Holds misc. utility methods which prove to be
useful throughout this library.
"""
__title__ = 'newspaper'
__author__ = 'Lucas Ou-Yang'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014, Lucas Ou-Yang'

import time
import hashlib
import re
import os
import random
import codecs
import threading
import sys
import pickle
import logging
import string

from hashlib import sha1
from .encoding import (
    smart_unicode, smart_str, DjangoUnicodeDecodeError)
from .. import settings

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)

class FileHelper(object):
    @classmethod
    def loadResourceFile(self, filename):
        if not os.path.isabs(filename):
            # _PARENT_DIR = os.path.join(_TEST_DIR, '../..') # packages/goose
            # dirpath = os.path.dirname(goose.__file__)
            dirpath = os.path.abspath(os.path.dirname(__file__)) # goose/utils
            path = os.path.join(dirpath, '../resources', filename)
        else:
            path = filename
        try:
            f = codecs.open(path, 'r', 'utf-8')
            content = f.read()
            f.close()
            return content
        except IOError:
            raise IOError("Couldn't open file %s" % path)

class ParsingCandidate(object):

    def __init__(self, urlString, link_hash):
        self.urlString = self.url = urlString
        self.link_hash = link_hash

class RawHelper(object):
    @classmethod
    def get_parsing_candidate(self, url, raw_html):
        if isinstance(raw_html, unicode):
            raw_html = raw_html.encode('utf-8')
        link_hash = '%s.%s' % (hashlib.md5(raw_html).hexdigest(), time.time())
        return ParsingCandidate(url, link_hash)

class URLHelper(object):
    @classmethod
    def get_parsing_candidate(self, url_to_crawl):
        # replace shebang in urls
        final_url = url_to_crawl.replace('#!', '?_escaped_fragment_=') \
                    if '#!' in url_to_crawl else url_to_crawl
        link_hash = '%s.%s' % (hashlib.md5(final_url).hexdigest(), time.time())
        return ParsingCandidate(final_url, link_hash)

class StringSplitter(object):
    """
    """
    def __init__(self, pattern):
        self.pattern = re.compile(pattern)

    def split(self, string):
        if not string:
            return []
        return self.pattern.split(string)

class StringReplacement(object):
    def __init__(self, pattern, replaceWith):
        self.pattern = pattern
        self.replaceWith = replaceWith

    def replaceAll(self, string):
        if not string:
            return u''
        return string.replace(self.pattern, self.replaceWith)

class ReplaceSequence(object):
    def __init__(self):
        self.replacements = []

    #@classmethod
    def create(self, firstPattern, replaceWith=None):
        result = StringReplacement(firstPattern, replaceWith or u'')
        self.replacements.append(result)
        return self

    def append(self, pattern, replaceWith=None):
        return self.create(pattern, replaceWith)

    def replaceAll(self, string):
        if not string:
            return u''

        mutatedString = string

        for rp in self.replacements:
            mutatedString = rp.replaceAll(mutatedString)
        return mutatedString

class TimeoutError(Exception):
    pass

def timelimit(timeout):
    """
    Borrowed from web.py, rip Aaron Swartz.
    """
    def _1(function):
        def _2(*args, **kw):
            class Dispatch(threading.Thread):
                def __init__(self):
                    threading.Thread.__init__(self)
                    self.result = None
                    self.error = None

                    self.setDaemon(True)
                    self.start()

                def run(self):
                    try:
                        self.result = function(*args, **kw)
                    except:
                        self.error = sys.exc_info()

            c = Dispatch()
            c.join(timeout)
            if c.isAlive():
                raise TimeoutError()
            if c.error:
                raise c.error[0], c.error[1]
            return c.result
        return _2
    return _1

def domain_to_filename(domain):
    """
    All '/' are turned into '-', no trailing. schema's
    are gone, only the raw domain + ".txt" remains.
    """
    filename =  domain.replace('/', '-')
    if filename[-1] == '-':
        filename = filename[:-1]
    filename += ".txt"
    return filename

def filename_to_domain(filename):
    """
    [:-4] for the .txt at end.
    """
    return filename.replace('-', '/')[:-4]

def is_ascii(word):
    """
    True if a word is only ascii chars.
    """
    def onlyascii(char):
        if ord(char) > 127:
            return ''
        else:
            return char
    for c in word:
        if not onlyascii(c):
            return False
    return True

def to_valid_filename(s):
    """
    Converts arbitrary string (for us domain name)
    into a valid file name for caching.
    """
    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
    return ''.join(c for c in s if c in valid_chars)

def cache_disk(seconds=(86400*5), cache_folder="/tmp"):
    """
    Caching extracting category locations & rss feeds for 5 days.
    """
    def do_cache(function):
        def inner_function(*args, **kwargs):
            """
            Calculate a cache key based on the decorated method signature
            args[1] indicates the domain of the inputs, we hash on domain!
            """
            key = sha1(str(args[1]) + str(kwargs)).hexdigest()
            filepath = os.path.join(cache_folder, key)

            # verify that the cached object exists and is less than X seconds old
            if os.path.exists(filepath):
                modified = os.path.getmtime(filepath)
                age_seconds = time.time() - modified
                if age_seconds < seconds:
                    return pickle.load(open(filepath, "rb"))

            # call the decorated function...
            result = function(*args, **kwargs)
            # ... and save the cached object for next time
            pickle.dump(result, open(filepath, "wb"))
            return result

        return inner_function
    return do_cache

def print_duration(method):
    """
    Prints out the runtime duration of a method in seconds.
    """
    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()
        print '%r %2.2f sec' % (method.__name__, te-ts)
        return result

    return timed

def chunks(l, n):
    """
    Yield n successive chunks from l.
    """
    newn = int(len(l) / n)
    for i in xrange(0, n-1):
        yield l[i*newn:i*newn+newn]
    yield l[n*newn-newn:]

def purge(fn, pattern):
    """
    Delete files in a dir matching pattern.
    """
    import os, re
    for f in os.listdir(fn):
        if re.search(pattern, f):
            os.remove(os.path.join(fn, f))

def clear_memo_cache(source):
    """
    Clears the memoization cache for this specific news domain.
    """
    d_pth = os.path.join(settings.MEMO_DIR, domain_to_filename(source.domain))
    if os.path.exists(d_pth):
        os.remove(d_pth)
    else:
        print 'memo file for', source.domain, 'has already been deleted!'

def encodeValue(value):
    """
    """
    if value is None:
        return u''
    string_org = value
    try:
        value = smart_unicode(value)
    except (UnicodeEncodeError, DjangoUnicodeDecodeError):
        value = smart_str(value)
    except:
        value = string_org
    return value.strip()

def memoize_articles(source, articles):
    """
    When we parse the <a> links in an <html> page, on the 2nd run
    and later, check the <a> links of previous runs. If they match,
    it means the link must not be an article, because article urls
    change as time passes. This method also uniquifies articles.
    """
    source_domain = source.domain
    config = source.config

    if len(articles) == 0:
        return []

    cur_articles = { article.url:article for article in articles }
    memo = {}
    # print '******current article urls!', cur_articles.keys()[:10]

    d_pth = os.path.join(settings.MEMO_DIR, domain_to_filename(source_domain))

    if os.path.exists(d_pth):
        f = codecs.open(d_pth, 'r', 'utf8')
        urls = f.readlines() # list of urls, unicode
        f.close()
        urls = [u.strip() for u in urls]

        memo = { url:True for url in urls }
        # prev_length = len(memo)
        for url, article in cur_articles.items():
            if memo.get(url):
                del cur_articles[url]

        valid_urls = memo.keys() + cur_articles.keys()

        memo_text = u'\r\n'.join(
            [encodeValue(href.strip()) for href in (valid_urls)])
    # Our first run with memoization, save every url as valid
    else:
        memo_text = u'\r\n'.join(
            [encodeValue(href.strip()) for href in cur_articles.keys()])

    # new_length = len(cur_articles)

    if len(memo) > config.MAX_FILE_MEMO:
        # We still keep current batch of articles though!
        log.critical('memo overflow, dumping')
        memo_text = ''

    # TODO if source: source.write_upload_times(prev_length, new_length)
    ff = codecs.open(d_pth, 'w', 'utf-8')
    ff.write(memo_text)
    ff.close()
    # print '***** final article urls', cur_articles.keys()[:10]
    return cur_articles.values() # articles returned

def get_useragent():
    """
    Uses generator to return next useragent in saved file.
    """
    with open(settings.USERAGENTS, 'r') as f:
        agents = f.readlines()
        selection = random.randint(0, len(agents)-1)
        agent = agents[selection]
        return agent.strip()

def get_available_languages():
    """
    Returns a list of available languages and their 2 char input codes.
    """
    stopword_files = os.listdir(os.path.join(settings.STOPWORDS_DIR))
    two_dig_codes = [f.split('-')[1].split('.')[0] for f in stopword_files]
    for d in two_dig_codes:
        assert len(d) == 2
    return two_dig_codes

def print_available_languages():
    """
    Prints available languages with their full names
    """
    language_dict = {
        'ar':   'Arabic',
        'ru':   'Russian',
        'nl':   'Dutch',
        'de':   'German',
        'en':   'English',
        'es':   'Spanish',
        'fr':   'French',
        'it':   'Italian',
        'ko':   'Korean',
        'no':   'Norwegian',
        'pt':   'Portuguese',
        'sv':   'Swedish',
        'hu':   'Hungarian',
        'fi':   'Finnish',
        'da':   'Danish',
        'zh':   'Chinese'
    }

    codes = get_available_languages()
    print '\nYour available langauges are:'
    print '\ninput code\t\tfull name'
    for code in codes:
        print '  %s\t\t\t  %s' % (code, language_dict[code])
    print

def extend_config(config, config_items):
    """
    We are handling config value setting like this for a cleaner api.
    Users just need to pass in a named param to this source and we can
    dynamically generate a config object for it.
    """
    for key, val in config_items.items():
        if hasattr(config, key):
            setattr(config, key, val)

    return config

<commit_msg>Fix small typo when calling newspaper.languages()<commit_after># -*- coding: utf-8 -*-
"""
Holds misc. utility methods which prove to be
useful throughout this library.
"""
__title__ = 'newspaper'
__author__ = 'Lucas Ou-Yang'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014, Lucas Ou-Yang'

import time
import hashlib
import re
import os
import random
import codecs
import threading
import sys
import pickle
import logging
import string

from hashlib import sha1
from .encoding import (
    smart_unicode, smart_str, DjangoUnicodeDecodeError)
from .. import settings

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)

class FileHelper(object):
    @classmethod
    def loadResourceFile(self, filename):
        if not os.path.isabs(filename):
            # _PARENT_DIR = os.path.join(_TEST_DIR, '../..') # packages/goose
            # dirpath = os.path.dirname(goose.__file__)
            dirpath = os.path.abspath(os.path.dirname(__file__)) # goose/utils
            path = os.path.join(dirpath, '../resources', filename)
        else:
            path = filename
        try:
            f = codecs.open(path, 'r', 'utf-8')
            content = f.read()
            f.close()
            return content
        except IOError:
            raise IOError("Couldn't open file %s" % path)

class ParsingCandidate(object):

    def __init__(self, urlString, link_hash):
        self.urlString = self.url = urlString
        self.link_hash = link_hash

class RawHelper(object):
    @classmethod
    def get_parsing_candidate(self, url, raw_html):
        if isinstance(raw_html, unicode):
            raw_html = raw_html.encode('utf-8')
        link_hash = '%s.%s' % (hashlib.md5(raw_html).hexdigest(), time.time())
        return ParsingCandidate(url, link_hash)

class URLHelper(object):
    @classmethod
    def get_parsing_candidate(self, url_to_crawl):
        # replace shebang in urls
        final_url = url_to_crawl.replace('#!', '?_escaped_fragment_=') \
                    if '#!' in url_to_crawl else url_to_crawl
        link_hash = '%s.%s' % (hashlib.md5(final_url).hexdigest(), time.time())
        return ParsingCandidate(final_url, link_hash)

class StringSplitter(object):
    """
    """
    def __init__(self, pattern):
        self.pattern = re.compile(pattern)

    def split(self, string):
        if not string:
            return []
        return self.pattern.split(string)

class StringReplacement(object):
    def __init__(self, pattern, replaceWith):
        self.pattern = pattern
        self.replaceWith = replaceWith

    def replaceAll(self, string):
        if not string:
            return u''
        return string.replace(self.pattern, self.replaceWith)

class ReplaceSequence(object):
    def __init__(self):
        self.replacements = []

    #@classmethod
    def create(self, firstPattern, replaceWith=None):
        result = StringReplacement(firstPattern, replaceWith or u'')
        self.replacements.append(result)
        return self

    def append(self, pattern, replaceWith=None):
        return self.create(pattern, replaceWith)

    def replaceAll(self, string):
        if not string:
            return u''

        mutatedString = string

        for rp in self.replacements:
            mutatedString = rp.replaceAll(mutatedString)
        return mutatedString

class TimeoutError(Exception):
    pass

def timelimit(timeout):
    """
    Borrowed from web.py, rip Aaron Swartz.
    """
    def _1(function):
        def _2(*args, **kw):
            class Dispatch(threading.Thread):
                def __init__(self):
                    threading.Thread.__init__(self)
                    self.result = None
                    self.error = None

                    self.setDaemon(True)
                    self.start()

                def run(self):
                    try:
                        self.result = function(*args, **kw)
                    except:
                        self.error = sys.exc_info()

            c = Dispatch()
            c.join(timeout)
            if c.isAlive():
                raise TimeoutError()
            if c.error:
                raise c.error[0], c.error[1]
            return c.result
        return _2
    return _1

def domain_to_filename(domain):
    """
    All '/' are turned into '-', no trailing. schema's
    are gone, only the raw domain + ".txt" remains.
    """
    filename =  domain.replace('/', '-')
    if filename[-1] == '-':
        filename = filename[:-1]
    filename += ".txt"
    return filename

def filename_to_domain(filename):
    """
    [:-4] for the .txt at end.
    """
    return filename.replace('-', '/')[:-4]

def is_ascii(word):
    """
    True if a word is only ascii chars.
    """
    def onlyascii(char):
        if ord(char) > 127:
            return ''
        else:
            return char
    for c in word:
        if not onlyascii(c):
            return False
    return True

def to_valid_filename(s):
    """
    Converts arbitrary string (for us domain name)
    into a valid file name for caching.
    """
    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
    return ''.join(c for c in s if c in valid_chars)

def cache_disk(seconds=(86400*5), cache_folder="/tmp"):
    """
    Caching extracting category locations & rss feeds for 5 days.
    """
    def do_cache(function):
        def inner_function(*args, **kwargs):
            """
            Calculate a cache key based on the decorated method signature
            args[1] indicates the domain of the inputs, we hash on domain!
            """
            key = sha1(str(args[1]) + str(kwargs)).hexdigest()
            filepath = os.path.join(cache_folder, key)

            # verify that the cached object exists and is less than X seconds old
            if os.path.exists(filepath):
                modified = os.path.getmtime(filepath)
                age_seconds = time.time() - modified
                if age_seconds < seconds:
                    return pickle.load(open(filepath, "rb"))

            # call the decorated function...
            result = function(*args, **kwargs)
            # ... and save the cached object for next time
            pickle.dump(result, open(filepath, "wb"))
            return result

        return inner_function
    return do_cache

def print_duration(method):
    """
    Prints out the runtime duration of a method in seconds.
    """
    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()
        print '%r %2.2f sec' % (method.__name__, te-ts)
        return result

    return timed

def chunks(l, n):
    """
    Yield n successive chunks from l.
    """
    newn = int(len(l) / n)
    for i in xrange(0, n-1):
        yield l[i*newn:i*newn+newn]
    yield l[n*newn-newn:]

def purge(fn, pattern):
    """
    Delete files in a dir matching pattern.
    """
    import os, re
    for f in os.listdir(fn):
        if re.search(pattern, f):
            os.remove(os.path.join(fn, f))

def clear_memo_cache(source):
    """
    Clears the memoization cache for this specific news domain.
    """
    d_pth = os.path.join(settings.MEMO_DIR, domain_to_filename(source.domain))
    if os.path.exists(d_pth):
        os.remove(d_pth)
    else:
        print 'memo file for', source.domain, 'has already been deleted!'

def encodeValue(value):
    """
    """
    if value is None:
        return u''
    string_org = value
    try:
        value = smart_unicode(value)
    except (UnicodeEncodeError, DjangoUnicodeDecodeError):
        value = smart_str(value)
    except:
        value = string_org
    return value.strip()

def memoize_articles(source, articles):
    """
    When we parse the <a> links in an <html> page, on the 2nd run
    and later, check the <a> links of previous runs. If they match,
    it means the link must not be an article, because article urls
    change as time passes. This method also uniquifies articles.
    """
    source_domain = source.domain
    config = source.config

    if len(articles) == 0:
        return []

    cur_articles = { article.url:article for article in articles }
    memo = {}
    # print '******current article urls!', cur_articles.keys()[:10]

    d_pth = os.path.join(settings.MEMO_DIR, domain_to_filename(source_domain))

    if os.path.exists(d_pth):
        f = codecs.open(d_pth, 'r', 'utf8')
        urls = f.readlines() # list of urls, unicode
        f.close()
        urls = [u.strip() for u in urls]

        memo = { url:True for url in urls }
        # prev_length = len(memo)
        for url, article in cur_articles.items():
            if memo.get(url):
                del cur_articles[url]

        valid_urls = memo.keys() + cur_articles.keys()

        memo_text = u'\r\n'.join(
            [encodeValue(href.strip()) for href in (valid_urls)])
    # Our first run with memoization, save every url as valid
    else:
        memo_text = u'\r\n'.join(
            [encodeValue(href.strip()) for href in cur_articles.keys()])

    # new_length = len(cur_articles)

    if len(memo) > config.MAX_FILE_MEMO:
        # We still keep current batch of articles though!
        log.critical('memo overflow, dumping')
        memo_text = ''

    # TODO if source: source.write_upload_times(prev_length, new_length)
    ff = codecs.open(d_pth, 'w', 'utf-8')
    ff.write(memo_text)
    ff.close()
    # print '***** final article urls', cur_articles.keys()[:10]
    return cur_articles.values() # articles returned

def get_useragent():
    """
    Uses generator to return next useragent in saved file.
    """
    with open(settings.USERAGENTS, 'r') as f:
        agents = f.readlines()
        selection = random.randint(0, len(agents)-1)
        agent = agents[selection]
        return agent.strip()

def get_available_languages():
    """
    Returns a list of available languages and their 2 char input codes.
    """
    stopword_files = os.listdir(os.path.join(settings.STOPWORDS_DIR))
    two_dig_codes = [f.split('-')[1].split('.')[0] for f in stopword_files]
    for d in two_dig_codes:
        assert len(d) == 2
    return two_dig_codes

def print_available_languages():
    """
    Prints available languages with their full names
    """
    language_dict = {
        'ar':   'Arabic',
        'ru':   'Russian',
        'nl':   'Dutch',
        'de':   'German',
        'en':   'English',
        'es':   'Spanish',
        'fr':   'French',
        'it':   'Italian',
        'ko':   'Korean',
        'no':   'Norwegian',
        'pt':   'Portuguese',
        'sv':   'Swedish',
        'hu':   'Hungarian',
        'fi':   'Finnish',
        'da':   'Danish',
        'zh':   'Chinese'
    }

    codes = get_available_languages()
    print '\nYour available languages are:'
    print '\ninput code\t\tfull name'
    for code in codes:
        print '  %s\t\t\t  %s' % (code, language_dict[code])
    print

def extend_config(config, config_items):
    """
    We are handling config value setting like this for a cleaner api.
    Users just need to pass in a named param to this source and we can
    dynamically generate a config object for it.
    """
    for key, val in config_items.items():
        if hasattr(config, key):
            setattr(config, key, val)

    return config

<|endoftext|>


<commit_before>""" General matlab interface code """

# Stdlib imports
import os
import re
import tempfile
import numpy as np
from nipype.interfaces.base import CommandLine, InterfaceResult, Bunch
from copy import deepcopy

class MatlabCommandLine(CommandLine):
    """Object that sets up Matlab specific tools and interfaces

    >>> matlab.MatlabCommandLine.matlab_cmd = "matlab.2009a -nodesktop -nosplash"
    >>> mcmd = matlab.MatlabCommandLine()
    >>> mcmd.inputs.script_lines = "which('who')"
    >>> out = mcmd.run()
    """
    _cmdline = None
    _cmdline_inputs = None
    @property
    def cmdline(self):
        # This is currently a very inefficient hash! We can become more
        # efficient once we decide on our logic
        if self._cmdline is None or self._cmdline_inputs != self.inputs:
            self._compile_command()
            self_cmdline_inputs = self.inputs

        return self._cmdline

    matlab_cmd = 'matlab -nodesktop -nosplash'
    def __init__(self, matlab_cmd=None,**inputs):
        """initializes interface to matlab
        (default 'matlab -nodesktop -nosplash'
        """
        super(MatlabCommandLine,self).__init__(**inputs)
        self.cmdline2 = None
        if matlab_cmd is not None:
            self.matlab_cmd = matlab_cmd

    def set_matlabcmd(self, cmd):
        """reset the base matlab command
        """
        self.matlab_cmd = cmd

    def inputs_help(self):
        doc = """
            Optional Parameters
            -------------------
            (all default to None and are unset)

            script_lines : string
                matlab_script or function name or matlab code to run
            cwd : string
                working directory for command
            """
        print doc

    def _populate_inputs(self):
        self.inputs = Bunch(script_lines='',
                            script_name='pyscript',
                            cwd='.')

    def run(self):
        results = self._runner()
        if  'MatlabScriptException' in err:
            results.runtime.returncode = 1
        return results

    def _compile_command(self, mfile=True):
        '''Generate necessary Matlab files and cmdline

        Note that this could be called before accessing .cmdline, and we won't
        regenerate anything (unless the inputs have changed)
        '''
        self._cmdline = self.gen_matlab_command(
                script_lines=self.inputs.script_lines,
                script_name=self.inputs.script_name,
                mfile=mfile)
        return self._cmdline

    def gen_matlab_command(self,script_lines='',script_name='pyscript',
            cwd=None,mfile=True):
        """ Put multiline matlab script into script file and run
        Arguments
        ---------

        self :
        script_lines :
        script_name :
        cwd :
            Note that unlike calls to Popen, cwd=None will still check
            self.inputs.cwd!  Use an alternative like '.' if you need it
        """
        if cwd is None:
            cwd = self.inputs.get('cwd', None)
        prescript  = ''
        if mfile:
            prescript += "diary(sprintf('%s.log',mfilename));\n"
            prescript += "fprintf(1,'Executing %s at %s:\\n',mfilename,datestr(now));\n"
        else:
            prescript += "fprintf(1,'Executing code at %s:\\n',datestr(now));\n"
        prescript += "ver,\n"
        prescript += "try,\n"
        postscript  = ''
        postscript += "\n,catch ME,\n"
        if mfile:
            postscript += "diary off;\n"
            postscript += "diary(sprintf('%s_error.log',mfilename));\n"
        postscript += "ME,\n"
        postscript += "ME.stack,\n"
        postscript += "fprintf('%s\\n',ME.message);\n"
        postscript += "fprintf(2,'<MatlabScriptException>');\n"
        postscript += "fprintf(2,'%s\\n',ME.message);\n"
        postscript += "fprintf(2,'File:%s\\nName:%s\\nLine:%d\\n',ME.stack.file,ME.stack.name,ME.stack.line);\n"
        postscript += "fprintf(2,'</MatlabScriptException>');\n"
        if mfile:
            postscript += "diary off,\n"
        postscript += "end;\n"
        script_lines = prescript+script_lines+postscript
        if mfile:
            mfile = file(os.path.join(cwd,script_name + '.m'), 'wt')
            mfile.write(script_lines)
            mfile.close()
        else:
            script_name = ''.join(script_lines.split('\n'))
        return '%s -r \"%s;exit\" ' % (self.matlab_cmd, script_name)

# Useful Functions for working with matlab

def fltcols(vals):
    ''' Trivial little function to make 1xN float vector '''
    return np.atleast_2d(np.array(vals, dtype=float))

def mlab_tempfile(dir=None):
    """Returns a temporary file-like object with valid matlab name.

    The file name is accessible as the .name attribute of the returned object.
    The caller is responsible for closing the returned object, at which time
    the underlying file gets deleted from the filesystem.

    Parameters
    ----------

      dir : str
        A path to use as the starting directory.  Note that this directory must
        already exist, it is NOT created if it doesn't (in that case, OSError
        is raised instead).

    Returns
    -------
      f : A file-like object.

    Examples
    --------

    >>> f = mlab_tempfile()
    >>> '-' not in f.name
    True
    >>> f.close()
    """
    valid_name = re.compile(r'^\w+$')

    # Make temp files until we get one whose name is a valid matlab identifier,
    # since matlab imposes that constraint.  Since the temp file routines may
    # return names that aren't valid matlab names, but we can't control that
    # directly, we just keep trying until we get a valid name.  To avoid an
    # infinite loop for some strange reason, we only try 100 times.
    for n in range(100):
        f = tempfile.NamedTemporaryFile(suffix='.m',prefix='tmp_matlab_',
                                        dir=dir)
        # Check the file name for matlab compilance
        fname =  os.path.splitext(os.path.basename(f.name))[0]
        if valid_name.match(fname):
            break
        # Close the temp file we just made if its name is not valid; the
        # tempfile module then takes care of deleting the actual file on disk.
        f.close()
    else:
        raise ValueError("Could not make temp file after 100 tries")

    return f
<commit_msg>BUG: fix reference to undefined variable in MatlabCommandLine.<commit_after>""" General matlab interface code """

# Stdlib imports
import os
import re
import tempfile
import numpy as np
from nipype.interfaces.base import CommandLine, InterfaceResult, Bunch
from copy import deepcopy

class MatlabCommandLine(CommandLine):
    """Object that sets up Matlab specific tools and interfaces

    >>> matlab.MatlabCommandLine.matlab_cmd = "matlab.2009a -nodesktop -nosplash"
    >>> mcmd = matlab.MatlabCommandLine()
    >>> mcmd.inputs.script_lines = "which('who')"
    >>> out = mcmd.run()
    """
    _cmdline = None
    _cmdline_inputs = None
    @property
    def cmdline(self):
        # This is currently a very inefficient hash! We can become more
        # efficient once we decide on our logic
        if self._cmdline is None or self._cmdline_inputs != self.inputs:
            self._compile_command()
            self_cmdline_inputs = self.inputs

        return self._cmdline

    matlab_cmd = 'matlab -nodesktop -nosplash'
    def __init__(self, matlab_cmd=None,**inputs):
        """initializes interface to matlab
        (default 'matlab -nodesktop -nosplash'
        """
        super(MatlabCommandLine,self).__init__(**inputs)
        self.cmdline2 = None
        if matlab_cmd is not None:
            self.matlab_cmd = matlab_cmd

    def set_matlabcmd(self, cmd):
        """reset the base matlab command
        """
        self.matlab_cmd = cmd

    def inputs_help(self):
        doc = """
            Optional Parameters
            -------------------
            (all default to None and are unset)

            script_lines : string
                matlab_script or function name or matlab code to run
            cwd : string
                working directory for command
            """
        print doc

    def _populate_inputs(self):
        self.inputs = Bunch(script_lines='',
                            script_name='pyscript',
                            cwd='.')

    def run(self):
        results = self._runner()
        if  'MatlabScriptException' in results.runtime.stderr:
            results.runtime.returncode = 1
        return results

    def _compile_command(self, mfile=True):
        '''Generate necessary Matlab files and cmdline

        Note that this could be called before accessing .cmdline, and we won't
        regenerate anything (unless the inputs have changed)
        '''
        self._cmdline = self.gen_matlab_command(
                script_lines=self.inputs.script_lines,
                script_name=self.inputs.script_name,
                mfile=mfile)
        return self._cmdline

    def gen_matlab_command(self,script_lines='',script_name='pyscript',
            cwd=None,mfile=True):
        """ Put multiline matlab script into script file and run
        Arguments
        ---------

        self :
        script_lines :
        script_name :
        cwd :
            Note that unlike calls to Popen, cwd=None will still check
            self.inputs.cwd!  Use an alternative like '.' if you need it
        """
        if cwd is None:
            cwd = self.inputs.get('cwd', None)
        prescript  = ''
        if mfile:
            prescript += "diary(sprintf('%s.log',mfilename));\n"
            prescript += "fprintf(1,'Executing %s at %s:\\n',mfilename,datestr(now));\n"
        else:
            prescript += "fprintf(1,'Executing code at %s:\\n',datestr(now));\n"
        prescript += "ver,\n"
        prescript += "try,\n"
        postscript  = ''
        postscript += "\n,catch ME,\n"
        if mfile:
            postscript += "diary off;\n"
            postscript += "diary(sprintf('%s_error.log',mfilename));\n"
        postscript += "ME,\n"
        postscript += "ME.stack,\n"
        postscript += "fprintf('%s\\n',ME.message);\n"
        postscript += "fprintf(2,'<MatlabScriptException>');\n"
        postscript += "fprintf(2,'%s\\n',ME.message);\n"
        postscript += "fprintf(2,'File:%s\\nName:%s\\nLine:%d\\n',ME.stack.file,ME.stack.name,ME.stack.line);\n"
        postscript += "fprintf(2,'</MatlabScriptException>');\n"
        if mfile:
            postscript += "diary off,\n"
        postscript += "end;\n"
        script_lines = prescript+script_lines+postscript
        if mfile:
            mfile = file(os.path.join(cwd,script_name + '.m'), 'wt')
            mfile.write(script_lines)
            mfile.close()
        else:
            script_name = ''.join(script_lines.split('\n'))
        return '%s -r \"%s;exit\" ' % (self.matlab_cmd, script_name)

# Useful Functions for working with matlab

def fltcols(vals):
    ''' Trivial little function to make 1xN float vector '''
    return np.atleast_2d(np.array(vals, dtype=float))

def mlab_tempfile(dir=None):
    """Returns a temporary file-like object with valid matlab name.

    The file name is accessible as the .name attribute of the returned object.
    The caller is responsible for closing the returned object, at which time
    the underlying file gets deleted from the filesystem.

    Parameters
    ----------

      dir : str
        A path to use as the starting directory.  Note that this directory must
        already exist, it is NOT created if it doesn't (in that case, OSError
        is raised instead).

    Returns
    -------
      f : A file-like object.

    Examples
    --------

    >>> f = mlab_tempfile()
    >>> '-' not in f.name
    True
    >>> f.close()
    """
    valid_name = re.compile(r'^\w+$')

    # Make temp files until we get one whose name is a valid matlab identifier,
    # since matlab imposes that constraint.  Since the temp file routines may
    # return names that aren't valid matlab names, but we can't control that
    # directly, we just keep trying until we get a valid name.  To avoid an
    # infinite loop for some strange reason, we only try 100 times.
    for n in range(100):
        f = tempfile.NamedTemporaryFile(suffix='.m',prefix='tmp_matlab_',
                                        dir=dir)
        # Check the file name for matlab compilance
        fname =  os.path.splitext(os.path.basename(f.name))[0]
        if valid_name.match(fname):
            break
        # Close the temp file we just made if its name is not valid; the
        # tempfile module then takes care of deleting the actual file on disk.
        f.close()
    else:
        raise ValueError("Could not make temp file after 100 tries")

    return f
<|endoftext|>


<commit_before>"""
Overview
========

This module provides data structures for representing first-order
models. A model is a pair M{<D,V>}, where M{D} is a domain of discourse and
M{V} is a valuation function for the non-logical constants of a
first-order language. We assume that the language is based on the
lambda calculus, in the style of Montague grammar.

We assume that non-logical constants are either individual constants
or functors. In particular, rather than interpreting a one-place
predicate M{P} as a set M{S}, we interpret it as the corresponding
characteristic function M{f}, where M{f(a) = True} iff M{a} is in
M{S}. For example, instead of interpreting 'dog' as the set of
individuals M{{'d1', 'd2', 'd3'}}, we interpret it as the function
which maps 'd1', 'd2' and 'd3' to M{True} and every other entity to
M{False}.

Thus, as a first approximation, non-logical constants are interpreted as
follows (note that M{e} is the type of I{entities} and M{t} is the
type of truth values):

  - if M{alpha} is an individual constant, then M{V(alpha)}
    is an element of M{D}.
  - If M{gamma} is a functor of type (M{e} x ... x M{e}) -> M{t}, then
    M{V(gamma)} is a function M{f} from  M{D} x ... x M{D} to M{{True, False}}.

However, since we are basing our language on the lambda calculus (see
L{logic}), a binary relation such as 'like' will not in fact be
associated with the type (M{e} x M{e}) -> M{t}, but rather the type
(M{e} -> (M{e} -> M{t}); i.e., a function from entities to a function
from entities to truth values. In other words, functors are assigned
'Curried' functions as their values. We should also point out that
expressions of our languge are not explicitly typed.  We leave it to
the grammar writer to assign 'sensible' values to expressions rather
than enforcing any type-to-denotation consistency.

Characteristic Functions
========================

Within C{models}, Curried characteristic functions are implemented as
a subclass of dictionaries, using the C{CharFun()} constructor.

   >>> cf = CharFun({'d1' : {'d2': True}, 'd2' : {'d1': True}})
   >>> cf['d1']
   {'d2': True}

Values of a C{CharFun} are accessed by indexing in the usual way:

   >>> cf['d1']['d2']
   True
   >>> cf['not in domain']
   False

In practise, it may be more convenient to specify interpretations as
n-ary relations (i.e., sets of n-tuples) rather than as n-ary
functions. C{CharFun} provides a C{parse()} method which will convert
such relations into Curried characteristic functions:

   >>> s = set([('d1', 'd2'), ('d2', 'd1')])
   >>> cf = CharFun()
   >>> cf.parse(s)
   >>> cf
   {'d2': {'d1': True}, 'd1': {'d2': True}}

C{parse()} will raise an exception if the set is not in fact a
relation (i.e., contains tuples of different lengths:

  >>> wrong = set([('d1', 'd2'), ('d2', 'd1', 'd3')])
  >>> cf.parse(wrong)
  Traceback (most recent call last):
  ...
  ValueError: Set contains sequences of different lengths

However, unary relations can be parsed to characteristic functions.

  >>> unary = set(['d1', 'd2'])
  >>> cf.parse(unary)
  >>> cf
  {'d2': True, 'd1': True}

The function C{flatten} returns a set of the entities used as keys in
a C{CharFun} instance. The same information can be accessed via the
C{domain} attribute of C{CharFun}.

   >>> cf = CharFun({'d1' : {'d2': True}, 'd2' : {'d1': True}})
   >>> flatten(cf)
   set(['d2', 'd1'])
   >>> cf.domain
   set(['d2', 'd1'])

Valuations and Models
=====================

A I{Valuation} is a mapping from non-logical constants to appropriate semantic
values in the model. Valuations are created using the C{Valuation} constructor.

   >>> val = Valuation({'Fido' : 'd1', 'dog' : {'d1' : True, 'd2' : True}})
   >>> val
   {'Fido': 'd1', 'dog': {'d2': True, 'd1': True}}

As with C{CharFun}, C{Valuation} will also parse valuations using
relations rather than characteristic functions as interpretations.

   >>> setval = [('adam', 'b1'), ('betty', 'g1'),\
   ('girl', set(['g2', 'g1'])), ('boy', set(['b1', 'b2'])),\
   ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
   >>> val = models.Valuation()
   >>> val.parse(setval)
   >>> print val
   {'adam': 'b1',
   'betty': 'g1',
   'boy': {'b1': True, 'b2': True},
   'girl': {'g2': True, 'g1': True},
   'love': {'b1': {'g1': True},
            'b2': {'g2': True},
            'g1': {'b1': True},
            'g2': {'b1': True}}}

Valuations have a C{domain} attribute, like C{CharFun}, and also a C{symbols}
attribute.

   >>> val.domain
   set(['g1', 'g2', 'b2', 'b1'])
   >>> v.symbols
   ['boy', 'girl', 'love', 'adam', 'betty']

The C{Model} constructor takes two parameters, a C{set} and a C{Valuation}.

   >>> m = Model(val.domain, val)

"""

import logic
from pprint import pformat

class CharFun(dict):
    """
    A dictionary which represents a Curried characteristic function.
    """

    def __init__(self, charfun=None):
        dict.__init__(self)
        if charfun:
            self.update(charfun)

    def __getitem__(self, key):
        if key in self:
            return dict.__getitem__(self, key)
        else:
            return False

    def _isrel(self, s):
        """check whether a set represents a relation (of any arity)"""

        assert isinstance(s, set), "Argument is not a set"
        if not isinstance(max(s),tuple) or len(max(s))==len(min(s)):
            return True
        else:
            raise ValueError, "Set contains sequences of different lengths"

    def _item2dict(self, item):
        """
        @return: A characteristic function corresponding to the input.
        @rtype: C{CharFun}
        @param item: a literal or a tuple
        """

        chf = {}
        if isinstance(item, tuple):
            if len(item)==1:
                chf[item[0]] = True
            elif len(item) > 1:
                chf[item[0]] = self._item2dict(item[1:])
        else:
            chf[item] = True
        return chf

    def _merge(self, chf1, chf2):
        k = chf2.keys()[0]
        if k not in chf1:
            chf1.update(chf2)
        else:
            self._merge(chf1[k], chf2[k])
        return chf1

    def parse(self, s):
        """
        Convert an n-ary relation into its corresponding characteristic function.
        @return: C{CharFun}
        @type s: set
        """

        assert self._isrel(s)

        charfuns = []
        for item in s:
            charfuns.append(self._item2dict(item))

        chf = reduce(self._merge, charfuns)
        self.update(chf)

    def _getDomain(self):
        return flatten(self)

    domain = property(_getDomain, doc='Set-theoretic domain of a curried function')

def flatten(value):
    """
    @return: The set of keys of a C{CharFun} instance.
    @rtype: set
    @type value: dict
    """

    flat = []
    try:
        flat.extend(value.keys())
        for v in value.values():
            if isinstance(v, dict):
                flat.extend(flatten(v))
            else:
                flat.append(v)
    except AttributeError:
        flat.append(value)

    result = set(flat)
    result.discard(True)
    return result

class Valuation(dict):
    """
    A dictionary which represents a model-theoretic Valuation of non-logical constants.
    """

    def __init__(self, valuation=None):
        dict.__init__(self)
        if valuation:
            self.update(valuation)

    def parse(self, seq):
        """
        Parse a list such as  C{[('j', 'b1'), ('girl', set(['g1', 'g2']))]} into a C{Valuation}.
        @rtype: C{Valuation}
        @param seq: A list of tuples of the form (I{constant}, I{relation}), where I{relation} is a set of tuples.
        """
        d = dict(seq)
        for k in d.keys():
            val = d[k]
            if isinstance(val, str):
                pass
            else:
                cf = CharFun()
                cf.parse(d[k])
                d[k] = cf
        self.update(d)

    def __str__(self):
        return pformat(self)

    def _getDomain(self):
        dom = set()
        for v in self.values():
            flat = flatten(v)
            dom = dom.union(flat)
        return dom

    domain = property(_getDomain,
             doc='Set-theoretic domain of the value-space of a Valuation')

    def _getSymbols(self):
        return self.keys()

    symbols = property(_getSymbols,
              doc='The non-logical constants which the Valuation applies to')

class Model:
    """A first order model is a domain M{D} of discourse and a valuation M{V}.

    A domain M{D} is a set, and a valuation M{V} is a map that associates
    expressions with values in the model.
    The domain of M{V} should be a subset of M{D}."""

    def __init__(self, domain, valuation):
  ##       assert isinstance(domain, set)
##         assert isinstance(valuation, Valuation)
        self.domain = domain
        self.valuation = valuation

    def __repr__(self):
        return "(%s, %s)" % (self.domain, self.valuation)

    def __str__(self):
        return "Domain = %s,\nValuation = \n%s" % (self.domain, pformat(self.valuation))

    def i(self, phi):
        """
        Recursive interpretation function for an expression of first-order logic.

        @return: Returns a truth value or a function
        @param phi: An expression of

        """
        if isinstance(phi, logic.ApplicationExpression):
            fun = phi.first.name()
            repr(fun)
            arg = phi.second.name()
            repr(arg)
            funsem = self.valuation[fun]
            argsem = self.valuation[arg]
            result = funsem[argsem]
        else: print 'failed to parse'
        print result

def demo():
    """Trivial example of a model."""
    v = [('j', 'b1'), ('m', 'g1'),\
         ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])),\
         ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1'),]))]
    val = Valuation()
    val.parse(v)
    dom = val.domain
    m = Model(dom, val)
    print m
    s = logic.Parser(r'(boy b)').next()
##     print s, s.__class__.__name__
    m.i(s)

## if __name__ == "__main__":
##     demo()

v = [('j', 'b1'), ('m', 'g1'),\
     ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])),\
     ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1'),]))]
val = Valuation()
val.parse(v)
dom = val.domain
m = Model(dom, val)
print m
s = logic.Parser(r'(boy j)').next()
## print s, s.__class__.__name__
fun = s.first.name()
arg = s.second.name()
funsem = m.valuation[fun]
argsem = m.valuation[arg]
m.i(s)

<commit_msg>minor corrections<commit_after>"""
Overview
========

This module provides data structures for representing first-order
models. A model is a pair M{<D,V>}, where M{D} is a domain of discourse and
M{V} is a valuation function for the non-logical constants of a
first-order language. We assume that the language is based on the
lambda calculus, in the style of Montague grammar.

We assume that non-logical constants are either individual constants
or functors. In particular, rather than interpreting a one-place
predicate M{P} as a set M{S}, we interpret it as the corresponding
characteristic function M{f}, where M{f(a) = True} iff M{a} is in
M{S}. For example, instead of interpreting 'dog' as the set of
individuals M{{'d1', 'd2', 'd3'}}, we interpret it as the function
which maps 'd1', 'd2' and 'd3' to M{True} and every other entity to
M{False}.

Thus, as a first approximation, non-logical constants are interpreted as
follows (note that M{e} is the type of I{entities} and M{t} is the
type of truth values):

  - if M{alpha} is an individual constant, then M{V(alpha)}
    is an element of M{D}.
  - If M{gamma} is a functor of type (M{e} x ... x M{e}) -> M{t}, then
    M{V(gamma)} is a function M{f} from  M{D} x ... x M{D} to M{{True, False}}.

However, since we are basing our language on the lambda calculus (see
L{logic}), a binary relation such as 'like' will not in fact be
associated with the type (M{e} x M{e}) -> M{t}, but rather the type
(M{e} -> (M{e} -> M{t}); i.e., a function from entities to a function
from entities to truth values. In other words, functors are assigned
'Curried' functions as their values. We should also point out that
expressions of our languge are not explicitly typed.  We leave it to
the grammar writer to assign 'sensible' values to expressions rather
than enforcing any type-to-denotation consistency.

Characteristic Functions
========================

Within C{models}, Curried characteristic functions are implemented as
a subclass of dictionaries, using the C{CharFun()} constructor.

   >>> cf = CharFun({'d1' : {'d2': True}, 'd2' : {'d1': True}})
   >>> cf['d1']
   {'d2': True}

Values of a C{CharFun} are accessed by indexing in the usual way:

   >>> cf['d1']['d2']
   True
   >>> cf['not in domain']
   False

In practise, it may be more convenient to specify interpretations as
n-ary relations (i.e., sets of n-tuples) rather than as n-ary
functions. C{CharFun} provides a C{parse()} method which will convert
such relations into Curried characteristic functions:

   >>> s = set([('d1', 'd2'), ('d2', 'd1')])
   >>> cf = CharFun()
   >>> cf.parse(s)
   >>> cf
   {'d2': {'d1': True}, 'd1': {'d2': True}}

C{parse()} will raise an exception if the set is not in fact a
relation (i.e., contains tuples of different lengths:

  >>> wrong = set([('d1', 'd2'), ('d2', 'd1', 'd3')])
  >>> cf.parse(wrong)
  Traceback (most recent call last):
  ...
  ValueError: Set contains sequences of different lengths

However, unary relations can be parsed to characteristic functions.

  >>> unary = set(['d1', 'd2'])
  >>> cf.parse(unary)
  >>> cf
  {'d2': True, 'd1': True}

The function C{flatten} returns a set of the entities used as keys in
a C{CharFun} instance. The same information can be accessed via the
C{domain} attribute of C{CharFun}.

   >>> cf = CharFun({'d1' : {'d2': True}, 'd2' : {'d1': True}})
   >>> flatten(cf)
   set(['d2', 'd1'])
   >>> cf.domain
   set(['d2', 'd1'])

Valuations and Models
=====================

A I{Valuation} is a mapping from non-logical constants to appropriate semantic
values in the model. Valuations are created using the C{Valuation} constructor.

   >>> val = Valuation({'Fido' : 'd1', 'dog' : {'d1' : True, 'd2' : True}})
   >>> val
   {'Fido': 'd1', 'dog': {'d2': True, 'd1': True}}

As with C{CharFun}, C{Valuation} will also parse valuations using
relations rather than characteristic functions as interpretations.

   >>> setval = [('adam', 'b1'), ('betty', 'g1'),\
   ('girl', set(['g2', 'g1'])), ('boy', set(['b1', 'b2'])),\
   ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
   >>> val = models.Valuation()
   >>> val.parse(setval)
   >>> print val
   {'adam': 'b1',
   'betty': 'g1',
   'boy': {'b1': True, 'b2': True},
   'girl': {'g2': True, 'g1': True},
   'love': {'b1': {'g1': True},
            'b2': {'g2': True},
            'g1': {'b1': True},
            'g2': {'b1': True}}}

Valuations have a C{domain} attribute, like C{CharFun}, and also a C{symbols}
attribute.

   >>> val.domain
   set(['g1', 'g2', 'b2', 'b1'])
   >>> v.symbols
   ['boy', 'girl', 'love', 'adam', 'betty']

The C{Model} constructor takes two parameters, a C{set} and a C{Valuation}.

   >>> m = Model(val.domain, val)

"""

import logic
from pprint import pformat

class CharFun(dict):
    """
    A dictionary which represents a Curried characteristic function.
    """

    def __init__(self, charfun=None):
        dict.__init__(self)
        if charfun:
            self.update(charfun)

    def __getitem__(self, key):
        if key in self:
            return dict.__getitem__(self, key)
        else:
            return False

    def _isrel(self, s):
        """check whether a set represents a relation (of any arity)"""

        assert isinstance(s, set), "Argument is not a set"
        if not isinstance(max(s),tuple) or len(max(s))==len(min(s)):
            return True
        else:
            raise ValueError, "Set contains sequences of different lengths"

    def _item2dict(self, item):
        """
        @return: A characteristic function corresponding to the input.
        @rtype: C{CharFun}
        @param item: a literal or a tuple
        """

        chf = {}
        if isinstance(item, tuple):
            if len(item)==1:
                chf[item[0]] = True
            elif len(item) > 1:
                chf[item[0]] = self._item2dict(item[1:])
        else:
            chf[item] = True
        return chf

    def _merge(self, chf1, chf2):
        k = chf2.keys()[0]
        if k not in chf1:
            chf1.update(chf2)
        else:
            self._merge(chf1[k], chf2[k])
        return chf1

    def parse(self, s):
        """
        Convert an n-ary relation into its corresponding characteristic function.
        @return: C{CharFun}
        @type s: set
        """

        assert self._isrel(s)

        charfuns = []
        for item in s:
            charfuns.append(self._item2dict(item))

        chf = reduce(self._merge, charfuns)
        self.update(chf)

    def _getDomain(self):
        return flatten(self)

    domain = property(_getDomain, doc='Set-theoretic domain of a curried function')

def flatten(value):
    """
    @return: The set of keys of a C{CharFun} instance.
    @rtype: set
    @type value: dict
    """

    flat = []
    try:
        flat.extend(value.keys())
        for v in value.values():
            if isinstance(v, dict):
                flat.extend(flatten(v))
            else:
                flat.append(v)
    except AttributeError:
        flat.append(value)

    result = set(flat)
    result.discard(True)
    return result

class Valuation(dict):
    """
    A dictionary which represents a model-theoretic Valuation of non-logical constants.
    """

    def __init__(self, valuation=None):
        dict.__init__(self)
        if valuation:
            self.update(valuation)

    def parse(self, seq):
        """
        Parse a list such as  C{[('j', 'b1'), ('girl', set(['g1', 'g2']))]} into a C{Valuation}.
        @rtype: C{Valuation}
        @param seq: A list of tuples of the form (I{constant}, I{relation}), where I{relation} is a set of tuples.
        """
        d = dict(seq)
        for k in d.keys():
            val = d[k]
            if isinstance(val, str):
                pass
            else:
                cf = CharFun()
                cf.parse(d[k])
                d[k] = cf
        self.update(d)

    def __str__(self):
        return pformat(self)

    def _getDomain(self):
        dom = set()
        for v in self.values():
            flat = flatten(v)
            dom = dom.union(flat)
        return dom

    domain = property(_getDomain,
             doc='Set-theoretic domain of the value-space of a Valuation')

    def _getSymbols(self):
        return self.keys()

    symbols = property(_getSymbols,
              doc='The non-logical constants which the Valuation applies to')

class Model:
    """A first order model is a domain M{D} of discourse and a valuation M{V}.

    A domain M{D} is a set, and a valuation M{V} is a map that associates
    expressions with values in the model.
    The domain of M{V} should be a subset of M{D}."""

    def __init__(self, domain, valuation):
  ##       assert isinstance(domain, set)
##         assert isinstance(valuation, Valuation)
        self.domain = domain
        self.valuation = valuation

    def __repr__(self):
        return "(%r, %r)" % (self.domain, self.valuation)

    def __str__(self):
        return "Domain = %s,\nValuation = \n%s" % (self.domain, self.valuation)

    def i(self, phi):
        """
        Recursive interpretation function for an expression of first-order logic.

        @return: Returns a truth value or a function
        @param phi: An expression of

        """
        if isinstance(phi, logic.ApplicationExpression):
            fun = phi.first.name()
            repr(fun)
            arg = phi.second.name()
            repr(arg)
            funsem = self.valuation[fun]
            argsem = self.valuation[arg]
            result = funsem[argsem]
        else: print 'failed to parse'
        print result

def demo():
    """Trivial example of a model."""
    v = [('j', 'b1'), ('m', 'g1'),\
         ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])),\
         ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1'),]))]
    val = Valuation()
    val.parse(v)
    dom = val.domain
    m = Model(dom, val)
    print m
    s = logic.Parser(r'(boy b)').next()
    m.i(s)

if __name__ == "__main__":
    demo()

<|endoftext|>