<commit_before>import numpy as np

__all__ = ['Tractography']

class Tractography:
    r"""
    Class to represent a tractography dataset

    Parameters
    ----------
    tracts : list of float array :math:`N_i\times 3`
        Each element of the list is a tract represented as point array,
        the length of the i-th tract is :math:`N_i`
    tracts_data : dict of <data name>= list of float array of :math:`N_i\times M`
        Each element in the list corresponds to a tract,
        :math:`N_i` is the length of the i-th tract and M is the
        number of components of that data type.
    validate : bool
        Check that tracts and tracts_data are valid
    """

    def __init__(self, tracts=None, tracts_data=None, validate=True):
        if tracts is not None and tracts_data is None:
            tracts_data = {}
        self._tracts = []
        self._quantity_of_points_per_tract = None

        self._tract_map = None
        self._subsampled_tracts = None
        self._subsampled_data = None

        if tracts is not None:
            self.append(tracts, tracts_data, validate=validate)

    def append(self, tracts, tracts_data=None, validate=True):
        r"""
        Append tracts and corresponding data to the current set

        Parameters
        ----------
        tracts : list of float array :math:`N_i\times 3`
            Each element of the list is a tract represented as point array,
            the length of the i-th tract is :math:`N_i`
        tracts_data : dict of <data name>= list of float array of :math:`N_i\times M`
            Each element in the list corresponds to a tract,
            :math:`N_i` is the length of the i-th tract and M is the
            number of components of that data type.
        validate : bool
            Check that tracts and tracts_data are valid
        """
        if tracts_data is None:
            tracts_data = dict()

        if len(self._tracts) == 0:
            self._tracts = tracts
            self._tracts_data = tracts_data
            appending = False
        else:
            appending = True

        if validate:
            if tracts is not None:
                try:
                    if any(
                        not (
                            t.ndim == 2 and
                            t.shape[1] == 3
                        ) for t in tracts
                    ):
                        raise ValueError(
                            'First argument is not a list of tracts')
                except AttributeError:
                    raise ValueError('First argument is not a list of tracts')

                if tracts_data is not None and hasattr(tracts_data, 'iteritems'):
                    for k, v in tracts_data.iteritems():
                        if len(v) != len(tracts):
                            raise ValueError(
                                'Number of elements in attribute %s must '
                                'be the same as the number of tracts' % k
                            )
                        _, M = v[0].shape
                        for i, tract_v in enumerate(v):
                            N, tract_M = tract_v.shape
                            if (
                                (N != len(tracts[i])) or
                                (tract_M != M)
                            ):
                                raise ValueError(
                                    "Data for tract %s: %d is inconsistent" % (
                                        k, i)
                                )
        if appending:
            if tracts_data.keys() != self._tracts_data.keys():
                raise ValueError("Tract data to append not compatible")
            if any(
                self._tracts_data[k][0].shape[1] != v[0].shape[1]
                for k, v in tracts_data.iteritems()
            ):
                raise ValueError("Tract data to append not compatible")

            for k, v in tracts_data.iteritems():
                self._tracts_data[k] += v

            self._tracts += tracts

            if self.are_tracts_subsampled():
                self.subsample_tracts(self._quantity_of_points_per_tract)
            if self.are_tracts_filtered():
                self.filter_tracts(self._criterium)

    def unsubsample_tracts(self):
        r"""
        Reset any subsampling applied to the tracts
        """
        self._subsampled_tracts = []
        self._subsampled_data = []

    def unfilter_tracts(self):
        r"""
        Reset any filtering applied to the tracts
        """

        self._tract_map = []

    def subsample_tracts(self, points_per_tract):
        r"""
        Subsample the tracts in the dataset to a maximum number of
        points per tract

        Parameters
        ----------
        points_per_tract: int
            Maximum number of points per tract after the operation
            is executed
        """
        self._quantity_of_points_per_tract = points_per_tract
        self._subsampled_tracts = []
        self._subsampled_data = {}

        for k in self._tracts_data:
            self._subsampled_data[k] = []

        for i in xrange(len(self._tracts)):
            f = self._tracts[i]
            s = np.linspace(
                0,
                f.shape[0] - 1,
                min(f.shape[0], self._quantity_of_points_per_tract)
            ).round().astype(int)

            self._subsampled_tracts.append(f[s, :])

            for k in self._tracts_data:
                self._subsampled_data[k].append(self._tracts_data[k][i][s])

        self._interpolated = False

    def filter_tracts(self, criterium):
        r"""
        Filter the tracts in the set according to a criterium function

        Parameters
        ----------

        criterium : function of array :math:`N\times 3` -> Bool
            A function taking a tract as an array of
            3D points and returning True or False with
            specifying if it should be included
        """
        if len(self._subsampled_tracts) > 0:
            tracts = self._subsampled_tracts
            data = self._subsampled_data
        else:
            tracts = self._tracts
            data = self._data

        self._tract_map = filter(
            lambda i: criterium(tracts),
            xrange(len(tracts))
        )

        self._filtered_tracts = [tracts[i] for i in self._tract_map]
        self._filtered_data = {}
        for k, v in data.iteritems():
            self._filtered_data[k] = [
                v[i] for i in self._tract_map
            ]

        self._criterium = criterium

    def are_tracts_filtered(self):
        return self._tract_map is not None

    def are_tracts_subsampled(self):
        return self._subsampled_tracts is not None

    def original_tracts(self):
        r"""
        Tract set used to original construct this
        tractography object, no subsampling or filtering
        applied

        Returns
        -------
        tracts : list of float array :math:`N_i\times3`
            Each element of the list is a tract represented as point array,
            the length of the i-th tract is :math:`N_i`
        """
        return self._tracts

    def original_tracts_data(self):
        r"""
        Tract data contained of the original dataset of this tractography object

        Returns
        -------
        tract data : dict of <data name>= list of float array of :math:`N_i\times M`
                     Each element in the list corresponds to a tract,
                     :math:`N_i` is the length of the i-th tract and M is the
                     number of components of that data type.
        """
        return self._tracts_data

    def filtered_tracts_map(self):
        r"""
        Tract indices included after the filtering

        Returns
        -------
        List of tract indices included after the filtering
        """
        return self._tract_map

    def tracts(self):
        r"""
        Tracts contained in this tractography object after filtering and
        subsampling if these operations have been applied

        Returns
        -------
        tracts : list of float array :math:`N_i\times 3`
            Each element of the list is a tract represented as point array,
            the length of the i-th tract is :math:`N_i`
        """
        if self._tract_map is not None:
            return self._filtered_tracts
        elif self._subsampled_tracts is not None:
            return self._subsampled_tracts
        else:
            return self._tracts

    def tracts_data(self):
        r"""
        Tract data contained in this tractography object after filtering and
        subsampling if these operations have been applied

        Returns
        -------
        tract data : dict of <data name>= list of float array of :math:`N_i\times M`
                     Each element in the list corresponds to a tract,
                     :math:`N_i` is the length of the i-th tract and M is the
                     number of components of that data type.
        """
        if self._tract_map is not None:
            return self._filtered_data
        elif self._subsampled_data is not None:
            return self._subsampled_data
        else:
            return self._tracts_data
<commit_msg>ENH: Improved and documneted tractogrpahy support class<commit_after>import numpy as np

__all__ = ['Tractography']

class Tractography:
    r"""
    Class to represent a tractography dataset

    Parameters
    ----------
    tracts : list of float array :math:`N_i\times 3`
        Each element of the list is a tract represented as point array,
        the length of the i-th tract is :math:`N_i`
    tracts_data : dict of <data name>= list of float array of :math:`N_i\times M`
        Each element in the list corresponds to a tract,
        :math:`N_i` is the length of the i-th tract and M is the
        number of components of that data type.
    validate : bool
        Check that tracts and tracts_data are valid
    """

    def __init__(self, tracts=None, tracts_data=None, validate=True):
        if tracts is not None and tracts_data is None:
            tracts_data = {}
        self._tracts = []
        self._quantity_of_points_per_tract = None

        self._tract_map = None
        self._subsampled_tracts = None
        self._subsampled_data = None

        if tracts is not None:
            self.append(tracts, tracts_data, validate=validate)

    def append(self, tracts, tracts_data=None, validate=True):
        r"""
        Append tracts and corresponding data to the current set

        Parameters
        ----------
        tracts : list of float array :math:`N_i\times 3`
            Each element of the list is a tract represented as point array,
            the length of the i-th tract is :math:`N_i`
        tracts_data : dict of <data name>= list of float array of :math:`N_i\times M`
            Each element in the list corresponds to a tract,
            :math:`N_i` is the length of the i-th tract and M is the
            number of components of that data type.
        validate : bool
            Check that tracts and tracts_data are valid
        """
        if tracts_data is None:
            tracts_data = {}

        if len(self._tracts) == 0:
            self._tracts = tracts
            self._tracts_data = tracts_data
            appending = False
        else:
            appending = True

        if validate:
            if tracts is not None:
                try:
                    if any(
                        not (
                            t.ndim == 2 and
                            t.shape[1] == 3
                        ) for t in tracts
                    ):
                        raise ValueError(
                            'First argument is not a list of tracts')
                except AttributeError:
                    raise ValueError('First argument is not a list of tracts')

                if tracts_data is not None and hasattr(tracts_data, 'iteritems'):
                    for k, v in tracts_data.iteritems():
                        if len(v) != len(tracts):
                            raise ValueError(
                                'Number of elements in attribute %s must '
                                'be the same as the number of tracts' % k
                            )
                        _, M = v[0].shape
                        for i, tract_v in enumerate(v):
                            N, tract_M = tract_v.shape
                            if (
                                (N != len(tracts[i])) or
                                (tract_M != M)
                            ):
                                raise ValueError(
                                    "Data for tract %s: %d is inconsistent" % (
                                        k, i)
                                )
        if appending:
            if tracts_data.keys() != self._tracts_data.keys():
                raise ValueError("Tract data to append not compatible")
            if any(
                self._tracts_data[k][0].shape[1] != v[0].shape[1]
                for k, v in tracts_data.iteritems()
            ):
                raise ValueError("Tract data to append not compatible")

            for k, v in tracts_data.iteritems():
                self._tracts_data[k] += v

            self._tracts += tracts

            if self.are_tracts_subsampled():
                self.subsample_tracts(self._quantity_of_points_per_tract)
            if self.are_tracts_filtered():
                self.filter_tracts(self._criterium)

    def unsubsample_tracts(self):
        r"""
        Reset any subsampling applied to the tracts
        """
        self._subsampled_tracts = None
        self._subsampled_data = None

    def unfilter_tracts(self):
        r"""
        Reset any filtering applied to the tracts
        """

        self._tract_map = None

    def subsample_tracts(self, points_per_tract):
        r"""
        Subsample the tracts in the dataset to a maximum number of
        points per tract

        Parameters
        ----------
        points_per_tract: int
            Maximum number of points per tract after the operation
            is executed
        """
        self._quantity_of_points_per_tract = points_per_tract
        self._subsampled_tracts = []
        self._subsampled_data = {}

        for k in self._tracts_data:
            self._subsampled_data[k] = []

        for i in xrange(len(self._tracts)):
            f = self._tracts[i]
            s = np.linspace(
                0,
                f.shape[0] - 1,
                min(f.shape[0], self._quantity_of_points_per_tract)
            ).round().astype(int)

            self._subsampled_tracts.append(f[s, :])

            for k in self._tracts_data:
                self._subsampled_data[k].append(self._tracts_data[k][i][s])

        self._interpolated = False

    def filter_tracts(self, criterium):
        r"""
        Filter the tracts in the set according to a criterium function

        Parameters
        ----------

        criterium : function of array :math:`N\times 3` -> Bool
            A function taking a tract as an array of
            3D points and returning True or False with
            specifying if it should be included
        """
        if len(self._subsampled_tracts) > 0:
            tracts = self._subsampled_tracts
            data = self._subsampled_data
        else:
            tracts = self._tracts
            data = self._data

        self._tract_map = filter(
            lambda i: criterium(tracts),
            xrange(len(tracts))
        )

        self._filtered_tracts = [tracts[i] for i in self._tract_map]
        self._filtered_data = {}
        for k, v in data.iteritems():
            self._filtered_data[k] = [
                v[i] for i in self._tract_map
            ]

        self._criterium = criterium

    def are_tracts_filtered(self):
        return self._tract_map is not None

    def are_tracts_subsampled(self):
        return self._subsampled_tracts is not None

    def original_tracts(self):
        r"""
        Tract set used to original construct this
        tractography object, no subsampling or filtering
        applied

        Returns
        -------
        tracts : list of float array :math:`N_i\times3`
            Each element of the list is a tract represented as point array,
            the length of the i-th tract is :math:`N_i`
        """
        return self._tracts

    def original_tracts_data(self):
        r"""
        Tract data contained of the original dataset of this tractography object

        Returns
        -------
        tract data : dict of <data name>= list of float array of :math:`N_i\times M`
                     Each element in the list corresponds to a tract,
                     :math:`N_i` is the length of the i-th tract and M is the
                     number of components of that data type.
        """
        return self._tracts_data

    def filtered_tracts_map(self):
        r"""
        Tract indices included after the filtering

        Returns
        -------
        List of tract indices included after the filtering
        """
        return self._tract_map

    def tracts(self):
        r"""
        Tracts contained in this tractography object after filtering and
        subsampling if these operations have been applied

        Returns
        -------
        tracts : list of float array :math:`N_i\times 3`
            Each element of the list is a tract represented as point array,
            the length of the i-th tract is :math:`N_i`
        """
        if self._tract_map is not None:
            return self._filtered_tracts
        elif self._subsampled_tracts is not None:
            return self._subsampled_tracts
        else:
            return self._tracts

    def tracts_data(self):
        r"""
        Tract data contained in this tractography object after filtering and
        subsampling if these operations have been applied

        Returns
        -------
        tract data : dict of <data name>= list of float array of :math:`N_i\times M`
                     Each element in the list corresponds to a tract,
                     :math:`N_i` is the length of the i-th tract and M is the
                     number of components of that data type.
        """
        if self._tract_map is not None:
            return self._filtered_data
        elif self._subsampled_data is not None:
            return self._subsampled_data
        else:
            return self._tracts_data

    def add_tract_data_from_array(self, name, array):
        r"""
        Add a new data element reproducing a constant data
        value for each of the :math:`$M$` tracts.

        After execution, the tract data will have a new set
        original_tracts_data()[name][i][:] == array[i]

        Parameters
        ----------
        name : str
            Name of the new data element
        array : array of length :math:`$M$`
            Data value for each tract
       """
        data = [
            np.ones((len(self.original_tracts()[i]), 1)) * array[i]
            for i in xrange(len(self.tracts()))
        ]

        self.original_tracts_data()[name] = data

        if self._subsampled_tracts is not None:
            self.subsample_tracts(self._quantity_of_points_per_tract)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import json
import math
import time

import bayeslite.core as core
import bayeslite.stats as stats

from bayeslite.sqlite3_util import sqlite3_quote_name

from bayeslite.util import casefold
from bayeslite.util import unique
from bayeslite.util import unique_indices

def bayesdb_install_bql(db, cookie):
    def function(name, nargs, fn):
        db.create_function(name, nargs,
            lambda *args: bayesdb_bql(fn, cookie, *args))
    function("bql_column_correlation", 3, bql_column_correlation)
    function("bql_column_dependence_probability", 4,
        bql_column_dependence_probability)
    function("bql_column_mutual_information", 5, bql_column_mutual_information)
    function("bql_column_typicality", 3, bql_column_typicality)
    function("bql_column_value_probability", 4, bql_column_value_probability)
    function("bql_row_similarity", -1, bql_row_similarity)
    function("bql_row_typicality", 3, bql_row_typicality)
    function("bql_row_column_predictive_probability", 4,
        bql_row_column_predictive_probability)
    function("bql_predict", 5, bql_predict)
    function("bql_predict_confidence", 4, bql_predict_confidence)
    function("bql_json_get", 2, bql_json_get)

# XXX XXX XXX Temporary debugging kludge!
import sys
import traceback

def bayesdb_bql(fn, cookie, *args):
    try:
        return fn(cookie, *args)
    except Exception as e:
        print >>sys.stderr, traceback.format_exc()
        raise e

### BayesDB column functions

# Two-column function:  CORRELATION [OF <col0> WITH <col1>]
def bql_column_correlation(bdb, generator_id, colno0, colno1):
    st0 = core.bayesdb_generator_column_stattype(bdb, generator_id, colno0)
    st1 = core.bayesdb_generator_column_stattype(bdb, generator_id, colno1)
    if (st0, st1) not in correlation_methods:
        raise NotImplementedError('No correlation method for %s/%s.' %
            (st0, st1))
    table_name = core.bayesdb_generator_table(bdb, generator_id)
    qt = sqlite3_quote_name(table_name)
    colname0 = core.bayesdb_generator_column_name(bdb, generator_id, colno0)
    colname1 = core.bayesdb_generator_column_name(bdb, generator_id, colno1)
    qcn0 = sqlite3_quote_name(colname0)
    qcn1 = sqlite3_quote_name(colname1)
    data_sql = '''
        SELECT %s, %s FROM %s WHERE %s IS NOT NULL AND %s IS NOT NULL
    ''' % (qcn0, qcn1, qt, qcn0, qcn1)
    data = list(bdb.sql_execute(data_sql))
    data0 = [row[0] for row in data]
    data1 = [row[1] for row in data]
    return correlation_methods[st0, st1](data0, data1)

def correlation_pearsonr2(data0, data1):
    return stats.pearsonr(data0, data1)**2

def correlation_cramerphi(data0, data1):
    n = len(data0)
    assert n == len(data1)
    if n == 0:
        return float('NaN')
    unique0 = unique_indices(data0)
    unique1 = unique_indices(data1)
    n0 = len(unique0)
    n1 = len(unique1)
    if n0 == 1 and n1 == 1:
        return 1.
    min_levels = min(n0, n1)
    if min_levels == 1:
        return 0.
    ct = [0] * n0
    for i0, j0 in enumerate(unique0):
        ct[i0] = [0] * n1
        for i1, j1 in enumerate(unique1):
            c = 0
            for i in range(n):
                if data0[i] == data0[j0] and data1[i] == data1[j1]:
                    c += 1
            ct[i0][i1] = c
    chisq = stats.chi2_contingency(ct, correction=False)
    return math.sqrt(chisq / (n * (min_levels - 1)))

def correlation_anovar2(data_group, data_y):
    n = len(data_group)
    assert n == len(data_y)
    group_values = unique(data_group)
    n_groups = len(group_values)
    if n_groups == n or n_groups == 0:
        return float('NaN')
    if n_groups == 1:
        if all(data_y[i] == data_y[0] for i in xrange(1, len(data_y))):
            return 1.
        else:
            return 0.
    samples = []
    for v in group_values:
        sample = []
        for i in range(n):
            if data_group[i] == v:
                sample.append(data_y[i])
        samples.append(sample)
    F = stats.f_oneway(samples)
    return 1 - 1/(1 + F*(float(n_groups - 1) / float(n - n_groups)))

def correlation_anovar2_dc(discrete_data, continuous_data):
    return correlation_anovar2(discrete_data, continuous_data)
def correlation_anovar2_cd(continuous_data, discrete_data):
    return correlation_anovar2(discrete_data, continuous_data)

correlation_methods = {}

def define_correlation(stattype0, stattype1, method):
    assert casefold(stattype0) == stattype0
    assert casefold(stattype1) == stattype1
    assert (stattype0, stattype1) not in correlation_methods
    correlation_methods[stattype0, stattype1] = method

define_correlation('categorical', 'categorical', correlation_cramerphi)
define_correlation('categorical', 'numerical', correlation_anovar2_dc)
define_correlation('numerical', 'categorical', correlation_anovar2_cd)
define_correlation('numerical', 'numerical', correlation_pearsonr2)

# Two-column function:  DEPENDENCE PROBABILITY [OF <col0> WITH <col1>]
def bql_column_dependence_probability(bdb, generator_id, modelno, colno0,
        colno1):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_dependence_probability(bdb, generator_id, modelno,
        colno0, colno1)

# Two-column function:  MUTUAL INFORMATION [OF <col0> WITH <col1>]
def bql_column_mutual_information(bdb, generator_id, modelno, colno0, colno1,
        numsamples=None):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_mutual_information(bdb, generator_id, modelno,
        colno0, colno1, numsamples=numsamples)

# One-column function:  TYPICALITY OF <col>
def bql_column_typicality(bdb, generator_id, modelno, colno):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_typicality(bdb, generator_id, modelno, colno)

# One-column function:  PROBABILITY OF <col>=<value>
def bql_column_value_probability(bdb, generator_id, modelno, colno, value):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_value_probability(bdb, generator_id, modelno,
        colno, value)

### BayesDB row functions

# Row function:  SIMILARITY TO <target_row> [WITH RESPECT TO <columns>]
def bql_row_similarity(bdb, generator_id, modelno, rowid, target_rowid,
        *colnos):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    if len(colnos) == 0:
        colnos = core.bayesdb_generator_column_numbers(bdb, generator_id)
    return metamodel.row_similarity(bdb, generator_id, modelno, rowid,
        target_rowid, colnos)

# Row function:  TYPICALITY
def bql_row_typicality(bdb, generator_id, modelno, rowid):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.row_typicality(bdb, generator_id, modelno, rowid)

# Row function:  PREDICTIVE PROBABILITY OF <column>
def bql_row_column_predictive_probability(bdb, generator_id, modelno, rowid,
        colno):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.row_column_predictive_probability(bdb, generator_id,
        modelno, rowid, colno)

### Predict and simulate

def bql_predict(bdb, generator_id, modelno, colno, rowid, threshold,
        numsamples=None):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.predict(bdb, generator_id, modelno, colno, rowid,
        threshold, numsamples=numsamples)

def bql_predict_confidence(bdb, generator_id, modelno, colno, rowid,
        numsamples=None):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    value, confidence = metamodel.predict_confidence(bdb, generator_id,
        modelno, colno, rowid, numsamples=numsamples)
    # XXX Whattakludge!
    return json.dumps({'value': value, 'confidence': confidence})

# XXX Whattakludge!
def bql_json_get(bdb, blob, key):
    return json.loads(blob)[key]

def bayesdb_simulate(bdb, generator_id, constraints, colnos,
        modelno=None, numpredictions=1):
    """Simulate rows from a generative model, subject to constraints.

    Returns a list of `numpredictions` tuples, with a value for each
    column specified in the list `colnos`, conditioned on the
    constraints in the list `constraints` of tuples ``(colno,
    value)``.
    """
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.simulate(bdb, generator_id, modelno, constraints, colnos,
        numpredictions=numpredictions)

def bayesdb_insert(bdb, generator_id, row):
    """Notify a generator that a row has been inserted into its table."""
    bayesdb_insertmany(bdb, generator_id, [row])

def bayesdb_insertmany(bdb, generator_id, rows):
    """Notify a generator that rows have been inserted into its table."""
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    metamodel.insertmany(bdb, generator_id, rows)
<commit_msg>Make correlation_anovar2 run in O(x + y) time, not O(x + xy) time.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import json
import math
import time

import bayeslite.core as core
import bayeslite.stats as stats

from bayeslite.sqlite3_util import sqlite3_quote_name

from bayeslite.util import casefold
from bayeslite.util import unique
from bayeslite.util import unique_indices

def bayesdb_install_bql(db, cookie):
    def function(name, nargs, fn):
        db.create_function(name, nargs,
            lambda *args: bayesdb_bql(fn, cookie, *args))
    function("bql_column_correlation", 3, bql_column_correlation)
    function("bql_column_dependence_probability", 4,
        bql_column_dependence_probability)
    function("bql_column_mutual_information", 5, bql_column_mutual_information)
    function("bql_column_typicality", 3, bql_column_typicality)
    function("bql_column_value_probability", 4, bql_column_value_probability)
    function("bql_row_similarity", -1, bql_row_similarity)
    function("bql_row_typicality", 3, bql_row_typicality)
    function("bql_row_column_predictive_probability", 4,
        bql_row_column_predictive_probability)
    function("bql_predict", 5, bql_predict)
    function("bql_predict_confidence", 4, bql_predict_confidence)
    function("bql_json_get", 2, bql_json_get)

# XXX XXX XXX Temporary debugging kludge!
import sys
import traceback

def bayesdb_bql(fn, cookie, *args):
    try:
        return fn(cookie, *args)
    except Exception as e:
        print >>sys.stderr, traceback.format_exc()
        raise e

### BayesDB column functions

# Two-column function:  CORRELATION [OF <col0> WITH <col1>]
def bql_column_correlation(bdb, generator_id, colno0, colno1):
    st0 = core.bayesdb_generator_column_stattype(bdb, generator_id, colno0)
    st1 = core.bayesdb_generator_column_stattype(bdb, generator_id, colno1)
    if (st0, st1) not in correlation_methods:
        raise NotImplementedError('No correlation method for %s/%s.' %
            (st0, st1))
    table_name = core.bayesdb_generator_table(bdb, generator_id)
    qt = sqlite3_quote_name(table_name)
    colname0 = core.bayesdb_generator_column_name(bdb, generator_id, colno0)
    colname1 = core.bayesdb_generator_column_name(bdb, generator_id, colno1)
    qcn0 = sqlite3_quote_name(colname0)
    qcn1 = sqlite3_quote_name(colname1)
    data_sql = '''
        SELECT %s, %s FROM %s WHERE %s IS NOT NULL AND %s IS NOT NULL
    ''' % (qcn0, qcn1, qt, qcn0, qcn1)
    data = list(bdb.sql_execute(data_sql))
    data0 = [row[0] for row in data]
    data1 = [row[1] for row in data]
    return correlation_methods[st0, st1](data0, data1)

def correlation_pearsonr2(data0, data1):
    return stats.pearsonr(data0, data1)**2

def correlation_cramerphi(data0, data1):
    n = len(data0)
    assert n == len(data1)
    if n == 0:
        return float('NaN')
    unique0 = unique_indices(data0)
    unique1 = unique_indices(data1)
    n0 = len(unique0)
    n1 = len(unique1)
    if n0 == 1 and n1 == 1:
        return 1.
    min_levels = min(n0, n1)
    if min_levels == 1:
        return 0.
    ct = [0] * n0
    for i0, j0 in enumerate(unique0):
        ct[i0] = [0] * n1
        for i1, j1 in enumerate(unique1):
            c = 0
            for i in range(n):
                if data0[i] == data0[j0] and data1[i] == data1[j1]:
                    c += 1
            ct[i0][i1] = c
    chisq = stats.chi2_contingency(ct, correction=False)
    return math.sqrt(chisq / (n * (min_levels - 1)))

def correlation_anovar2(data_group, data_y):
    n = len(data_group)
    assert n == len(data_y)
    group_index = {}
    for x in data_group:
        if x not in group_index:
            group_index[x] = len(group_index)
    n_groups = len(group_index)
    if n_groups == n or n_groups == 0:
        return float('NaN')
    if n_groups == 1:
        if all(data_y[i] == data_y[0] for i in xrange(1, len(data_y))):
            return 1.
        else:
            return 0.
    groups = [None] * n_groups
    for i in xrange(n_groups):
        groups[i] = []
    for x, y in zip(data_group, data_y):
        groups[group_index[x]].append(y)
    F = stats.f_oneway(groups)
    return 1 - 1/(1 + F*(float(n_groups - 1) / float(n - n_groups)))

def correlation_anovar2_dc(discrete_data, continuous_data):
    return correlation_anovar2(discrete_data, continuous_data)
def correlation_anovar2_cd(continuous_data, discrete_data):
    return correlation_anovar2(discrete_data, continuous_data)

correlation_methods = {}

def define_correlation(stattype0, stattype1, method):
    assert casefold(stattype0) == stattype0
    assert casefold(stattype1) == stattype1
    assert (stattype0, stattype1) not in correlation_methods
    correlation_methods[stattype0, stattype1] = method

define_correlation('categorical', 'categorical', correlation_cramerphi)
define_correlation('categorical', 'numerical', correlation_anovar2_dc)
define_correlation('numerical', 'categorical', correlation_anovar2_cd)
define_correlation('numerical', 'numerical', correlation_pearsonr2)

# Two-column function:  DEPENDENCE PROBABILITY [OF <col0> WITH <col1>]
def bql_column_dependence_probability(bdb, generator_id, modelno, colno0,
        colno1):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_dependence_probability(bdb, generator_id, modelno,
        colno0, colno1)

# Two-column function:  MUTUAL INFORMATION [OF <col0> WITH <col1>]
def bql_column_mutual_information(bdb, generator_id, modelno, colno0, colno1,
        numsamples=None):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_mutual_information(bdb, generator_id, modelno,
        colno0, colno1, numsamples=numsamples)

# One-column function:  TYPICALITY OF <col>
def bql_column_typicality(bdb, generator_id, modelno, colno):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_typicality(bdb, generator_id, modelno, colno)

# One-column function:  PROBABILITY OF <col>=<value>
def bql_column_value_probability(bdb, generator_id, modelno, colno, value):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.column_value_probability(bdb, generator_id, modelno,
        colno, value)

### BayesDB row functions

# Row function:  SIMILARITY TO <target_row> [WITH RESPECT TO <columns>]
def bql_row_similarity(bdb, generator_id, modelno, rowid, target_rowid,
        *colnos):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    if len(colnos) == 0:
        colnos = core.bayesdb_generator_column_numbers(bdb, generator_id)
    return metamodel.row_similarity(bdb, generator_id, modelno, rowid,
        target_rowid, colnos)

# Row function:  TYPICALITY
def bql_row_typicality(bdb, generator_id, modelno, rowid):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.row_typicality(bdb, generator_id, modelno, rowid)

# Row function:  PREDICTIVE PROBABILITY OF <column>
def bql_row_column_predictive_probability(bdb, generator_id, modelno, rowid,
        colno):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.row_column_predictive_probability(bdb, generator_id,
        modelno, rowid, colno)

### Predict and simulate

def bql_predict(bdb, generator_id, modelno, colno, rowid, threshold,
        numsamples=None):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.predict(bdb, generator_id, modelno, colno, rowid,
        threshold, numsamples=numsamples)

def bql_predict_confidence(bdb, generator_id, modelno, colno, rowid,
        numsamples=None):
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    value, confidence = metamodel.predict_confidence(bdb, generator_id,
        modelno, colno, rowid, numsamples=numsamples)
    # XXX Whattakludge!
    return json.dumps({'value': value, 'confidence': confidence})

# XXX Whattakludge!
def bql_json_get(bdb, blob, key):
    return json.loads(blob)[key]

def bayesdb_simulate(bdb, generator_id, constraints, colnos,
        modelno=None, numpredictions=1):
    """Simulate rows from a generative model, subject to constraints.

    Returns a list of `numpredictions` tuples, with a value for each
    column specified in the list `colnos`, conditioned on the
    constraints in the list `constraints` of tuples ``(colno,
    value)``.
    """
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    return metamodel.simulate(bdb, generator_id, modelno, constraints, colnos,
        numpredictions=numpredictions)

def bayesdb_insert(bdb, generator_id, row):
    """Notify a generator that a row has been inserted into its table."""
    bayesdb_insertmany(bdb, generator_id, [row])

def bayesdb_insertmany(bdb, generator_id, rows):
    """Notify a generator that rows have been inserted into its table."""
    metamodel = core.bayesdb_generator_metamodel(bdb, generator_id)
    metamodel.insertmany(bdb, generator_id, rows)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
from pyramid.config import Configurator
from pyramid.view import view_config
from waitress import serve
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()

class Entry(Base):
    __tablename__ = 'entries'

    id = sa.Column(sa.Integer, primary_key=True)
    title = sa.Column(sa.String(128))
    text = sa.Column(sa.String, nullable=False)
    created = sa.Column(sa.DateTime, nullable=False,
                        default=datetime.date.utcnow)

    def __repr__(self):
        return "<Entry(title='%s', created='%s')>" % (
            self.title, self.created)

@view_config(route_name='home', renderer='string')
def home(request):
    return "Hello World"

def main():
    """Create a configured wsgi app"""
    settings = {}
    debug = os.environ.get('DEBUG', True)
    settings['reload_all'] = debug
    settings['debug_all'] = debug
    # configuration setup
    config = Configurator(
        settings=settings
        )
    config.add_route('home', '/')
    config.scan()
    app = config.make_wsgi_app()
    return app

if __name__ == '__main__':
    app = main()
    port = os.environ.get('PORT', 5000)
    serve(app, host='0.0.0.0', port=port)
<commit_msg>add db init method<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
from pyramid.config import Configurator
from pyramid.view import view_config
from waitress import serve
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
import datetime
from sqlalchemy import create_engine

Base = declarative_base()

class Entry(Base):
    __tablename__ = 'entries'

    id = sa.Column(sa.Integer, primary_key=True)
    title = sa.Column(sa.String(128))
    text = sa.Column(sa.String, nullable=False)
    created = sa.Column(sa.DateTime, nullable=False,
                        default=datetime.datetime.utcnow)

    def __repr__(self):
        return "<Entry(title='%s', created='%s')>" % (
            self.title, self.created)

def init_db():
    engine = create_engine(
        'postgresql://meslater@localhost:5432/learning-journal')
    Base.metadata.create_all(engine)

@view_config(route_name='home', renderer='string')
def home(request):
    return "Hello World"

def main():
    """Create a configured wsgi app"""
    settings = {}
    debug = os.environ.get('DEBUG', True)
    settings['reload_all'] = debug
    settings['debug_all'] = debug
    # configuration setup
    config = Configurator(
        settings=settings
        )
    config.add_route('home', '/')
    config.scan()
    app = config.make_wsgi_app()
    return app

if __name__ == '__main__':
    app = main()
    port = os.environ.get('PORT', 5000)
    serve(app, host='0.0.0.0', port=port)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Flask
from flask import g
from flask import render_template
from flask import abort
from flask import request
from flask import url_for
from flask import redirect
from flask import session
from passlib.hash import pbkdf2_sha256
from contextlib import closing
import os
import psycopg2
import datetime
import markdown

# _____SQL SCRIPTS_____
DB_SCHEMA = """
DROP TABLE IF EXISTS entries;
CREATE TABLE entries (
    id serial PRIMARY KEY,
    title VARCHAR (127) NOT NULL,
    text TEXT NOT NULL,
    created TIMESTAMP NOT NULL
)
"""

DB_ENTRY_INSERT = """
INSERT INTO entries (title, text, created) VALUES (%s, %s, %s)
"""

DB_ENTRIES_LIST = """
SELECT id, title, text, created FROM entries ORDER BY created DESC
"""

DB_ENTRY_GET = """
SELECT id, title, text, created FROM entries WHERE id = %s
"""

DB_UPDATE_ENTRY = """
UPDATE entries SET title = %s, text = %s WHERE id = %s
"""

# _____database functions_____
app = Flask(__name__)
app.config['DATABASE'] = os.environ.get(
    'DATABASE_URL', 'dbname=learning_journal'
)
app.config['ADMIN_USERNAME'] = os.environ.get(
    'ADMIN_USERNAME', 'admin'
)
app.config['ADMIN_PASSWORD'] = os.environ.get(
    'ADMIN_PASSWORD', pbkdf2_sha256.encrypt('admin')
)
app.config['SECRET_KEY'] = os.environ.get(
    'FLASK_SECRET_KEY', 'sooperseekritvaluenooneshouldknow'
)

def connect_db():
    """Return a connection to the configured database"""
    return psycopg2.connect(app.config['DATABASE'])

def init_db():
    """Initialize the database using DB_SCHEMA

    WARNING: executing this function will drop existing tables.
    """
    with closing(connect_db()) as db:
        db.cursor().execute(DB_SCHEMA)
        db.commit()

def get_database_connection():
    db = getattr(g, 'db', None)
    if db is None:
        g.db = db = connect_db()
    return db

@app.teardown_request
def teardown_request(exception):
    db = getattr(g, 'db', None)
    if db is not None:
        if exception and isinstance(exception, psycopg2.Error):
            # if there was a problem with the database, rollback any
            # existing transaction
            db.rollback()
        else:
            # otherwise, commit
            db.commit()
        db.close()

# _____journal methods_____
def write_entry(title, text):
    if not title or not text:
        raise ValueError("Title and text required for writing an entry")
    con = get_database_connection()
    cur = con.cursor()
    now = datetime.datetime.utcnow()
    cur.execute(DB_ENTRY_INSERT, [title, text, now])

def get_all_entries():
    """return a list of all entries as dicts"""
    con = get_database_connection()
    cur = con.cursor()
    cur.execute(DB_ENTRIES_LIST)
    keys = ('id', 'title', 'text', 'created')
    myEntries = [dict(zip(keys, row)) for row in cur.fetchall()]
    for record in myEntries:
        record['text'] = markdown_text(record['text'])
    return myEntries

def get_one_entry(entry_id):
    """return the selected entry for editing"""
    con = get_database_connection()
    cur = con.cursor()
    cur.execute(DB_ENTRY_GET, [entry_id])
    return cur.fetchone()

def update_entry(title, text, entry_id):
    """update the entry selected for editing"""
    if session.get('logged_in', False):
        con = get_database_connection()
        cur = con.cursor()
        cur.execute(DB_UPDATE_ENTRY, [title, text, entry_id])

def do_login(username='', passwd=''):
    if username != app.config['ADMIN_USERNAME']:
        raise ValueError
    if not pbkdf2_sha256.verify(passwd, app.config['ADMIN_PASSWORD']):
        raise ValueError
    session['logged_in'] = True

def markdown_text(user_input):
    return markdown.markdown(user_input, extensions=['codehilite'])

# _____app methods_____
@app.route('/')
def show_entries():
    entries = get_all_entries()
    return render_template('list_entries.html', entries=entries)

@app.route('/add', methods=['POST'])
def add_entry():
    # if session.get('logged_in', False):
    try:
        write_entry(request.form['title'], request.form['text'])
    except psycopg2.Error:
        # this will catch any errors generated by the database
        abort(500)
    return redirect(url_for('show_entries'))

@app.route('/edit/<int:entry_id>', methods=['GET'])
def edit_entry(entry_id):
    if session.get('logged_in', False):
        try:
            edit_this = get_one_entry(entry_id)
        except psycopg2.Error:
            abort(500)
        else:
            return render_template('edit_entry.html', entries=edit_this)
    return redirect(url_for('show_entries'))

@app.route('/update_entry/<int:entry_id>', methods=['POST'])
def update(entry_id):
    if session.get('logged_in', False):
        try:
            title = request.form['title']
            text = request.form['text']
            update_entry(title, text, entry_id)
        except psycopg2.Error:
            abort(500)
    return redirect(url_for('show_entries'))

@app.route('/login', methods=['GET', 'POST'])
def login():
    error = None
    if request.method == 'POST':
        try:
            do_login(request.form['username'].encode('utf-8'),
                     request.form['password'].encode('utf-8'))
        except ValueError:
            error = "Login Failed"
        else:
            return redirect(url_for('show_entries'))
    return render_template('login.html', error=error)

@app.route('/logout')
def logout():
    session.pop('logged_in', None)
    return redirect(url_for('show_entries'))

if __name__ == '__main__':
    app.run(debug=True)
<commit_msg>Re-add security to /add view<commit_after># -*- coding: utf-8 -*-
from flask import Flask
from flask import g
from flask import render_template
from flask import abort
from flask import request
from flask import url_for
from flask import redirect
from flask import session
from passlib.hash import pbkdf2_sha256
from contextlib import closing
import os
import psycopg2
import datetime
import markdown

# _____SQL SCRIPTS_____
DB_SCHEMA = """
DROP TABLE IF EXISTS entries;
CREATE TABLE entries (
    id serial PRIMARY KEY,
    title VARCHAR (127) NOT NULL,
    text TEXT NOT NULL,
    created TIMESTAMP NOT NULL
)
"""

DB_ENTRY_INSERT = """
INSERT INTO entries (title, text, created) VALUES (%s, %s, %s)
"""

DB_ENTRIES_LIST = """
SELECT id, title, text, created FROM entries ORDER BY created DESC
"""

DB_ENTRY_GET = """
SELECT id, title, text, created FROM entries WHERE id = %s
"""

DB_UPDATE_ENTRY = """
UPDATE entries SET title = %s, text = %s WHERE id = %s
"""

# _____database functions_____
app = Flask(__name__)
app.config['DATABASE'] = os.environ.get(
    'DATABASE_URL', 'dbname=learning_journal'
)
app.config['ADMIN_USERNAME'] = os.environ.get(
    'ADMIN_USERNAME', 'admin'
)
app.config['ADMIN_PASSWORD'] = os.environ.get(
    'ADMIN_PASSWORD', pbkdf2_sha256.encrypt('admin')
)
app.config['SECRET_KEY'] = os.environ.get(
    'FLASK_SECRET_KEY', 'sooperseekritvaluenooneshouldknow'
)

def connect_db():
    """Return a connection to the configured database"""
    return psycopg2.connect(app.config['DATABASE'])

def init_db():
    """Initialize the database using DB_SCHEMA

    WARNING: executing this function will drop existing tables.
    """
    with closing(connect_db()) as db:
        db.cursor().execute(DB_SCHEMA)
        db.commit()

def get_database_connection():
    db = getattr(g, 'db', None)
    if db is None:
        g.db = db = connect_db()
    return db

@app.teardown_request
def teardown_request(exception):
    db = getattr(g, 'db', None)
    if db is not None:
        if exception and isinstance(exception, psycopg2.Error):
            # if there was a problem with the database, rollback any
            # existing transaction
            db.rollback()
        else:
            # otherwise, commit
            db.commit()
        db.close()

# _____journal methods_____
def write_entry(title, text):
    if not title or not text:
        raise ValueError("Title and text required for writing an entry")
    con = get_database_connection()
    cur = con.cursor()
    now = datetime.datetime.utcnow()
    cur.execute(DB_ENTRY_INSERT, [title, text, now])

def get_all_entries():
    """return a list of all entries as dicts"""
    con = get_database_connection()
    cur = con.cursor()
    cur.execute(DB_ENTRIES_LIST)
    keys = ('id', 'title', 'text', 'created')
    myEntries = [dict(zip(keys, row)) for row in cur.fetchall()]
    for record in myEntries:
        record['text'] = markdown_text(record['text'])
    return myEntries

def get_one_entry(entry_id):
    """return the selected entry for editing"""
    con = get_database_connection()
    cur = con.cursor()
    cur.execute(DB_ENTRY_GET, [entry_id])
    return cur.fetchone()

def update_entry(title, text, entry_id):
    """update the entry selected for editing"""
    if session.get('logged_in', False):
        con = get_database_connection()
        cur = con.cursor()
        cur.execute(DB_UPDATE_ENTRY, [title, text, entry_id])

def do_login(username='', passwd=''):
    if username != app.config['ADMIN_USERNAME']:
        raise ValueError
    if not pbkdf2_sha256.verify(passwd, app.config['ADMIN_PASSWORD']):
        raise ValueError
    session['logged_in'] = True

def markdown_text(user_input):
    return markdown.markdown(user_input, extensions=['codehilite'])

# _____app methods_____
@app.route('/')
def show_entries():
    entries = get_all_entries()
    return render_template('list_entries.html', entries=entries)

@app.route('/add', methods=['POST'])
def add_entry():
    if session.get('logged_in', False):
        try:
            write_entry(request.form['title'], request.form['text'])
        except psycopg2.Error:
            # this will catch any errors generated by the database
                abort(500)
    return redirect(url_for('show_entries'))

@app.route('/edit/<int:entry_id>', methods=['GET'])
def edit_entry(entry_id):
    if session.get('logged_in', False):
        try:
            edit_this = get_one_entry(entry_id)
        except psycopg2.Error:
            abort(500)
        else:
            return render_template('edit_entry.html', entries=edit_this)
    return redirect(url_for('show_entries'))

@app.route('/update_entry/<int:entry_id>', methods=['POST'])
def update(entry_id):
    if session.get('logged_in', False):
        try:
            title = request.form['title']
            text = request.form['text']
            update_entry(title, text, entry_id)
        except psycopg2.Error:
            abort(500)
    return redirect(url_for('show_entries'))

@app.route('/login', methods=['GET', 'POST'])
def login():
    error = None
    if request.method == 'POST':
        try:
            do_login(request.form['username'].encode('utf-8'),
                     request.form['password'].encode('utf-8'))
        except ValueError:
            error = "Login Failed"
        else:
            return redirect(url_for('show_entries'))
    return render_template('login.html', error=error)

@app.route('/logout')
def logout():
    session.pop('logged_in', None)
    return redirect(url_for('show_entries'))

if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import logging
import psycopg2
import datetime

from pyramid.config import Configurator
from pyramid.session import SignedCookieSessionFactory
from pyramid.view import view_config
from waitress import serve
from contextlib import closing
from pyramid.events import NewRequest, subscriber

DB_SCHEMA = """
CREATE TABLE IF NOT EXISTS entries (
    id serial PRIMARY KEY,
    title VARCHAR (127) NOT NULL,
    text TEXT NOT NULL,
    created TIMESTAMP NOT NULL
)
"""
INSERT_ENTRY = """
INSERT INTO entries (title, text, created) VALUES (%s, %s, %s)
"""

logging.basicConfig()
log = logging.getLogger(__file__)

@view_config(route_name='home', renderer='string')
def home(request):
    return "Hello World"

def connect_db(settings):
    """Return a connection to the configured database"""
    return psycopg2.connect(settings['db'])

def init_db():
    """Create database dables defined by DB_SCHEMA

    Warning: This function will not update existing table definitions
    """
    settings = {}
    settings['db'] = os.environ.get(
        'DATABASE_URL', 'dbname=learning_journal user=jakeanderson'
    )
    with closing(connect_db(settings)) as db:
        db.cursor().execute(DB_SCHEMA)
        db.commit()

@subscriber(NewRequest)
def open_connection(event):
    request = event.request
    settings = request.registry.settings
    request.db = connect_db(settings)
    request.add_finished_callback(close_connection)

def close_connection(request):
    """close the database connection for this request

    If there has been an error in the processing of the request, abort any
    open transactions.
    """
    db = getattr(request, 'db', None)
    if db is not None:
        if request.exception is not None:
            db.rollback()
        else:
            db.commit()
        request.db.close()

def write_entry(request):
    """write a single entry to the database"""
    title = request.params.get('title', None)
    text = request.params.get('text', None)
    created = datetime.datetime.utcnow()
    request.db.cursor().execute(INSERT_ENTRY, [title, text, created])

def main():
    """Create a configured wsgi app"""
    settings = {}
    settings['reload_all'] = os.environ.get('DEBUG', True)
    settings['debug_all'] = os.environ.get('DEBUG', True)
    settings['db'] = os.environ.get(
        'DATABASE_URL', 'dbname=learning_journal user=jakeanderson'
    )
    # secret value for session signing:
    secret = os.environ.get('JOURNAL_SESSION_SECRET', 'itsaseekrit')
    session_factory = SignedCookieSessionFactory(secret)
    # configuration setup
    config = Configurator(
        settings=settings,
        session_factory=session_factory
    )
    config.add_route('home', '/')
    config.scan()
    app = config.make_wsgi_app()
    return app

if __name__ == '__main__':
    app = main()
    port = os.environ.get('PORT', 5000)
    serve(app, host='0.0.0.0', port=port)
<commit_msg>updated requirements so heroku knows about jinja2<commit_after># -*- coding: utf-8 -*-
import os
import logging
from pyramid.config import Configurator
from pyramid.session import SignedCookieSessionFactory
from pyramid.view import view_config
from waitress import serve
import psycopg2
from contextlib import closing
from pyramid.events import NewRequest, subscriber
import datetime

DB_SCHEMA = """
CREATE TABLE IF NOT EXISTS entries (
    id serial PRIMARY KEY,
    title VARCHAR (127) NOT NULL,
    text TEXT NOT NULL,
    created TIMESTAMP NOT NULL
)
"""

INSERT_ENTRY = """
INSERT INTO entries (title, text, created) VALUES (%s, %s, %s)
"""

READ_ENTRIES = """
SELECT id, title, text, created FROM entries
"""

logging.basicConfig()
log = logging.getLogger(__file__)

@view_config(route_name='home', renderer='string')
def home(request):
    return "Hello World"

def connect_db(settings):
    """Return a connection to the configured database"""
    return psycopg2.connect(settings['db'])

def init_db():
    """Create database dables defined by DB_SCHEMA
    Warning: This function will not update existing table definitions
    """
    settings = {}
    settings['db'] = os.environ.get(
        'DATABASE_URL', 'dbname=learning_journal user=chatzis'
    )
    with closing(connect_db(settings)) as db:
        db.cursor().execute(DB_SCHEMA)
        db.commit()

@subscriber(NewRequest)
def open_connection(event):
    request = event.request
    settings = request.registry.settings
    request.db = connect_db(settings)
    request.add_finished_callback(close_connection)

def close_connection(request):
    """close the database connection for this request
    If there has been an error in the processing of the request, abort any
    open transactions.
    """
    db = getattr(request, 'db', None)
    if db is not None:
        if request.exception is not None:
            db.rollback()
        else:
            db.commit()
        request.db.close()

def main():
    """Create a configured wsgi app"""
    settings = {}
    settings['reload_all'] = os.environ.get('DEBUG', True)
    settings['debug_all'] = os.environ.get('DEBUG', True)
    # secret value for session signing:
    secret = os.environ.get('JOURNAL_SESSION_SECRET', 'itsaseekrit')
    session_factory = SignedCookieSessionFactory(secret)
    # configuration setup
    config = Configurator(
        settings=settings,
        session_factory=session_factory
    )
    config.include('pyramid_jinja2')
    config.add_route('home', '/')
    config.scan()
    app = config.make_wsgi_app()
    return app

    settings['reload_all'] = os.environ.get('DEBUG', True)  # <- THERE NOW
    settings['debug_all'] = os.environ.get('DEBUG', True)  # <- THERE NOW
    # ADD THIS  vvv
    settings['db'] = os.environ.get(
        'DATABASE_URL', 'dbname=learning_journal user=chatzis'
    )

def write_entry(request):
    """Add an entry to the database"""
    title = request.params.get('title')
    text = request.params.get('text')
    created = datetime.datetime.utcnow()
    request.db.cursor().execute(INSERT_ENTRY, [title, text, created])

def read_entries(request):
    """Read entries in the database"""
    cursor = request.db.cursor()
    cursor.execute(READ_ENTRIES)
    results = cursor.fetchall()
    keys = ('id', 'title', 'text', 'created')
    entries = [dict(zip(keys, item)) for item in results]
    return {'entries': entries}

if __name__ == '__main__':
    app = main()
    port = os.environ.get('PORT', 5000)
    serve(app, host='0.0.0.0', port=port)
<|endoftext|>


<commit_before>from sys import stderr, exit
import shutil
import os
from resources import read_template, get_resource_root, get_kernel, get_kernel_inc
from project import Project
from knightos import prepare_environment

def copytree(src, dst, symlinks=False, ignore=None): # shutil.copytree doesn't let you merge dirs
    if not os.path.exists(dst):
        os.makedirs(dst)
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            copytree(s, d, symlinks, ignore)
        else:
            if not os.path.exists(d) or os.stat(src).st_mtime - os.stat(dst).st_mtime > 1:
                shutil.copy2(s, d)

def execute(project_name, root=None):
    if root == None: root = os.getcwd()
    prepare_environment()
    setup_root(root)
    proj = Project(root)
    proj.open(os.path.join(root, ".gitignore"), "w+").write(read_template("gitignore", project_name))
    proj.open(os.path.join(root, "main.asm"), "w+").write(read_template("main.asm", project_name))
    proj.open(os.path.join(root, "Makefile"), "w+").write(read_template("Makefile", project_name))
    proj.open(os.path.join(root, ".knightos", "sdk.make"), "w+").write(read_template("sdk.make", project_name))
    proj.open(os.path.join(root, ".knightos", "variables.make"), "w+").write(read_template("variables.make", project_name))
    shutil.copyfile(get_kernel(), os.path.join(root, ".knightos", "kernel.rom"))
    shutil.copyfile(get_kernel_inc(), os.path.join(root, ".knightos", "include", "kernel.inc"))
    default_packages = ["base", "corelib"]
    # Temporary - install a couple of required packages manually
    for p in default_packages:
        copytree(os.path.join(get_resource_root(), "templates", "temp", p), os.path.join(root, ".knightos", "pkgroot"))
    if os.path.isdir(os.path.join(root, ".knightos", "pkgroot", "include")):
        copytree(os.path.join(root, ".knightos", "pkgroot", "include"), os.path.join(root, ".knightos"))
    # End temporary

def setup_root(root):
    os.makedirs(root, mode=0o755, exist_ok=True)
    if len(os.listdir(root)) > 0:
        stderr.write("{path} not empty. Aborting.\n".format(path=os.path.relpath(root)))
        exit(1)
    os.makedirs(os.path.join(root, ".knightos"), mode=0o755)
    os.makedirs(os.path.join(root, ".knightos", "include"), mode=0o755)
    os.makedirs(os.path.join(root, ".knightos", "pkgroot"), mode=0o755)
<commit_msg>Fix copying of include files<commit_after>from sys import stderr, exit
import shutil
import os
from resources import read_template, get_resource_root, get_kernel, get_kernel_inc
from project import Project
from knightos import prepare_environment

def copytree(src, dst, symlinks=False, ignore=None): # shutil.copytree doesn't let you merge dirs
    if not os.path.exists(dst):
        os.makedirs(dst)
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            copytree(s, d, symlinks, ignore)
        else:
            if not os.path.exists(d) or os.stat(src).st_mtime - os.stat(dst).st_mtime > 1:
                shutil.copy2(s, d)

def execute(project_name, root=None):
    if root == None: root = os.getcwd()
    prepare_environment()
    setup_root(root)
    proj = Project(root)
    proj.open(os.path.join(root, ".gitignore"), "w+").write(read_template("gitignore", project_name))
    proj.open(os.path.join(root, "main.asm"), "w+").write(read_template("main.asm", project_name))
    proj.open(os.path.join(root, "Makefile"), "w+").write(read_template("Makefile", project_name))
    proj.open(os.path.join(root, ".knightos", "sdk.make"), "w+").write(read_template("sdk.make", project_name))
    proj.open(os.path.join(root, ".knightos", "variables.make"), "w+").write(read_template("variables.make", project_name))
    shutil.copyfile(get_kernel(), os.path.join(root, ".knightos", "kernel.rom"))
    shutil.copyfile(get_kernel_inc(), os.path.join(root, ".knightos", "include", "kernel.inc"))
    default_packages = ["base", "corelib"]
    # Temporary - install a couple of required packages manually
    for p in default_packages:
        copytree(os.path.join(get_resource_root(), "templates", "temp", p), os.path.join(root, ".knightos", "pkgroot"))
    if os.path.isdir(os.path.join(root, ".knightos", "pkgroot", "include")):
        copytree(os.path.join(root, ".knightos", "pkgroot", "include"), os.path.join(root, ".knightos", "include"))
    # End temporary

def setup_root(root):
    os.makedirs(root, mode=0o755, exist_ok=True)
    if len(os.listdir(root)) > 0:
        stderr.write("{path} not empty. Aborting.\n".format(path=os.path.relpath(root)))
        exit(1)
    os.makedirs(os.path.join(root, ".knightos"), mode=0o755)
    os.makedirs(os.path.join(root, ".knightos", "include"), mode=0o755)
    os.makedirs(os.path.join(root, ".knightos", "pkgroot"), mode=0o755)
<|endoftext|>


<commit_before>import sys

if sys.version_info.major == 3:
    import xmlrpc.client
    from xmlrpc.client import _Method
    import urllib.request, urllib.error, urllib.parse
else:
    import xmlrpclib
    from xmlrpclib import _Method
    import urllib2

import base64
import struct
import time
import hmac
import hashlib

def getOTP(shared_secret):
    key = base64.b32decode(shared_secret, True)
    msg = struct.pack(">Q", int(time.time())//30)
    h = hmac.new(key, msg, hashlib.sha1).digest()
    if sys.version_info.major == 3:
        o = h[19] & 15
    else:
        o = ord(h[19]) & 15
    h = (struct.unpack(">I", h[o:o+4])[0] & 0x7fffffff) % 1000000
    return h

class domrobot ():
    def __init__ (self, address, debug = False):
        self.url = address
        self.debug = debug
        self.cookie = None
        self.version = "1.0"

    def __getattr__(self, name):
        return _Method(self.__request, name)

    def __request (self, methodname, params):
        tuple_params = tuple([params[0]])
        if sys.version_info.major == 3:
            requestContent = xmlrpc.client.dumps(tuple_params, methodname)
        else:
            requestContent = xmlrpclib.dumps(tuple_params, methodname)
        if(self.debug == True):
            print(("Anfrage: "+str(requestContent).replace("\n", "")))
        headers = { 'User-Agent' : 'DomRobot/'+self.version+' Python-v2.7', 'Content-Type': 'text/xml','content-length': str(len(requestContent))}
        if(self.cookie!=None):
            headers['Cookie'] = self.cookie

        if sys.version_info.major == 3:
            req = urllib.request.Request(self.url, bytearray(requestContent, 'ascii'), headers)
            response = urllib.request.urlopen(req)
        else:
            req = urllib2.Request(self.url, bytearray(requestContent, 'ascii'), headers)
            response = urllib2.urlopen(req)

        responseContent = response.read()

        if sys.version_info.major == 3:
            cookies = response.getheader('Set-Cookie')
        else:
            cookies = response.info().getheader('Set-Cookie')

        if(self.debug == True):
            print(("Antwort: "+str(responseContent).replace("\n", "")))
        if sys.version_info.major == 3:
            apiReturn = xmlrpc.client.loads(responseContent)
        else:
            apiReturn = xmlrpclib.loads(responseContent)
        apiReturn = apiReturn[0][0]
        if(apiReturn["code"]!=1000):
            raise NameError('There was a problem: %s (Error code %s)' % (apiReturn['msg'], apiReturn['code']), apiReturn)
            return False

        if(cookies!=None):
                if sys.version_info.major == 3:
                    cookies = response.getheader('Set-Cookie')
                else:
                    cookies = response.info().getheader('Set-Cookie')
                self.cookie = cookies
                if(self.debug == True):
                    print(("Cookie:" + self.cookie))
        return apiReturn

class prettyprint (object):
    """
    This object is just a collection of prettyprint helper functions for the output of the XML-API.
    """

    @staticmethod
    def contacts(contacts):
        """
        iterable contacts:  The list of contacts to be printed.
        """
	if("resData" in contacts):
            contacts = contacts['resData']

        output = "\nCurrently you have %i contacts set up for your account at InterNetworX:\n\n" % len(contacts['contact'])
        for contact in contacts['contact']:
            output += "ID: %s\nType: %s\n%s\n%s\n%s %s\n%s\n%s\nTel: %s\n------\n" % (contact['id'], contact['type'], contact['name'], contact['street'], contact['pc'], contact['city'], contact['cc'], contact['email'], contact['voice'])
        return output

    @staticmethod
    def domains(domains):
        """
        list domains:  The list of domains to be pretty printed.
        """
	if("resData" in domains):
	    domains = domains['resData']

        output = "\n%i domains:\n" % len(domains['domain'])
        for domain in domains['domain']:
            output += "Domain: %s (Status: %s)\n" % (domain['domain'], domain['status'])
        return output

    @staticmethod
    def nameserversets(nameserversets):
        """
        list namerserversets:  The list of nameserversets to be pretty printed.
        """
	if("resData" in nameserversets):
	    nameserversets = nameserversets['resData']

        count, total = 0, len(nameserversets['nsset'])
        output = "\n%i nameserversets:\n" % total
        for nameserverset in nameserversets['nsset']:
            count += 1
            output += "%i of %i - ID: %i consisting of [%s]\n" % (count, total, nameserverset['id'], ", ".join(nameserverset['ns']))
        return output

    @staticmethod
    def domain_log(logs):
        """
        list logs:  The list of nameserversets to be pretty printed.
        """
        if("resData" in logs):
            logs = logs['resData']

        count, total = 0, len(logs['domain'])
        output = "\n%i log entries:\n" % total
        for log in logs['domain']:
            count += 1
            output += "%i of %i - %s status: '%s' price: %.2f invoice: %s date: %s remote address: %s\n" % (count, total, log['domain'], log['status'], log['price'], log['invoice'], log['date'], log['remoteAddr'])
            output += "           user text: '%s'\n" % log['userText'].replace("\n",'\n           ')
        return output

    @staticmethod
    def domain_check(checks):
        """
        list checks:  The list of domain checks to be pretty printed.
        """
	if("resData" in checks):
		checks = checks['resData']

        count, total = 0, len(checks)
        output = "\n%i domain check(s):\n" % total
        for check in checks['domain']:
            count += 1
            output += "%s = %s" % (check['domain'], check['status'])
        return output
<commit_msg>do not mix tabs and spaces<commit_after>import sys

if sys.version_info.major == 3:
    import xmlrpc.client
    from xmlrpc.client import _Method
    import urllib.request, urllib.error, urllib.parse
else:
    import xmlrpclib
    from xmlrpclib import _Method
    import urllib2

import base64
import struct
import time
import hmac
import hashlib

def getOTP(shared_secret):
    key = base64.b32decode(shared_secret, True)
    msg = struct.pack(">Q", int(time.time())//30)
    h = hmac.new(key, msg, hashlib.sha1).digest()
    if sys.version_info.major == 3:
        o = h[19] & 15
    else:
        o = ord(h[19]) & 15
    h = (struct.unpack(">I", h[o:o+4])[0] & 0x7fffffff) % 1000000
    return h

class domrobot ():
    def __init__ (self, address, debug = False):
        self.url = address
        self.debug = debug
        self.cookie = None
        self.version = "1.0"

    def __getattr__(self, name):
        return _Method(self.__request, name)

    def __request (self, methodname, params):
        tuple_params = tuple([params[0]])
        if sys.version_info.major == 3:
            requestContent = xmlrpc.client.dumps(tuple_params, methodname)
        else:
            requestContent = xmlrpclib.dumps(tuple_params, methodname)
        if(self.debug == True):
            print(("Anfrage: "+str(requestContent).replace("\n", "")))
        headers = { 'User-Agent' : 'DomRobot/'+self.version+' Python-v2.7', 'Content-Type': 'text/xml','content-length': str(len(requestContent))}
        if(self.cookie!=None):
            headers['Cookie'] = self.cookie

        if sys.version_info.major == 3:
            req = urllib.request.Request(self.url, bytearray(requestContent, 'ascii'), headers)
            response = urllib.request.urlopen(req)
        else:
            req = urllib2.Request(self.url, bytearray(requestContent, 'ascii'), headers)
            response = urllib2.urlopen(req)

        responseContent = response.read()

        if sys.version_info.major == 3:
            cookies = response.getheader('Set-Cookie')
        else:
            cookies = response.info().getheader('Set-Cookie')

        if(self.debug == True):
            print(("Antwort: "+str(responseContent).replace("\n", "")))
        if sys.version_info.major == 3:
            apiReturn = xmlrpc.client.loads(responseContent)
        else:
            apiReturn = xmlrpclib.loads(responseContent)
        apiReturn = apiReturn[0][0]
        if(apiReturn["code"]!=1000):
            raise NameError('There was a problem: %s (Error code %s)' % (apiReturn['msg'], apiReturn['code']), apiReturn)
            return False

        if(cookies!=None):
                if sys.version_info.major == 3:
                    cookies = response.getheader('Set-Cookie')
                else:
                    cookies = response.info().getheader('Set-Cookie')
                self.cookie = cookies
                if(self.debug == True):
                    print(("Cookie:" + self.cookie))
        return apiReturn

class prettyprint (object):
    """
    This object is just a collection of prettyprint helper functions for the output of the XML-API.
    """

    @staticmethod
    def contacts(contacts):
        """
        iterable contacts:  The list of contacts to be printed.
        """
        if("resData" in contacts):
            contacts = contacts['resData']

        output = "\nCurrently you have %i contacts set up for your account at InterNetworX:\n\n" % len(contacts['contact'])
        for contact in contacts['contact']:
            output += "ID: %s\nType: %s\n%s\n%s\n%s %s\n%s\n%s\nTel: %s\n------\n" % (contact['id'], contact['type'], contact['name'], contact['street'], contact['pc'], contact['city'], contact['cc'], contact['email'], contact['voice'])
        return output

    @staticmethod
    def domains(domains):
        """
        list domains:  The list of domains to be pretty printed.
        """
        if("resData" in domains):
            domains = domains['resData']

        output = "\n%i domains:\n" % len(domains['domain'])
        for domain in domains['domain']:
            output += "Domain: %s (Status: %s)\n" % (domain['domain'], domain['status'])
        return output

    @staticmethod
    def nameserversets(nameserversets):
        """
        list namerserversets:  The list of nameserversets to be pretty printed.
        """
        if("resData" in nameserversets):
            nameserversets = nameserversets['resData']

        count, total = 0, len(nameserversets['nsset'])
        output = "\n%i nameserversets:\n" % total
        for nameserverset in nameserversets['nsset']:
            count += 1
            output += "%i of %i - ID: %i consisting of [%s]\n" % (count, total, nameserverset['id'], ", ".join(nameserverset['ns']))
        return output

    @staticmethod
    def domain_log(logs):
        """
        list logs:  The list of nameserversets to be pretty printed.
        """
        if("resData" in logs):
            logs = logs['resData']

        count, total = 0, len(logs['domain'])
        output = "\n%i log entries:\n" % total
        for log in logs['domain']:
            count += 1
            output += "%i of %i - %s status: '%s' price: %.2f invoice: %s date: %s remote address: %s\n" % (count, total, log['domain'], log['status'], log['price'], log['invoice'], log['date'], log['remoteAddr'])
            output += "           user text: '%s'\n" % log['userText'].replace("\n",'\n           ')
        return output

    @staticmethod
    def domain_check(checks):
        """
        list checks:  The list of domain checks to be pretty printed.
        """
        if("resData" in checks):
                checks = checks['resData']

        count, total = 0, len(checks)
        output = "\n%i domain check(s):\n" % total
        for check in checks['domain']:
            count += 1
            output += "%s = %s" % (check['domain'], check['status'])
        return output
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import ast
from collections import defaultdict, deque

from funcy import *
from astpp import dump # use astor?

# TODO: distinguish python versions
import __builtin__
BUILTINS = set(dir(__builtin__))

class Scope(object):
    def __init__(self, parent, node):
        self.parent = parent
        if parent:
            parent.children.append(self)
        self.children = []

        self.node = node
        self.names = defaultdict(list)
        self.unscoped_names = defaultdict(list)
        self.global_names = set()

    @property
    def is_module(self):
        return isinstance(self.node, ast.Module)

    @cached_property
    def module(self):
        scope = self
        while not scope.is_module:
            scope = scope.parent
        return scope

    # Names

    def add(self, name, node):
        # Params are always in current scope.
        # Other names could be made global or passed to parent scope upon exit,
        # unless we are in a module.
        is_param = isinstance(node, ast.Name) and isinstance(node.ctx, ast.Param)
        if is_param or name in self.names or self.is_module:
            self.names[name].append(node)
        else:
            self.unscoped_names[name].append(node)

    def make_global(self, names):
        self.global_names.update(names)

    def resolve(self):
        # print 'RESOVE', self.node, self.unscoped_names.keys()

        # Extract global names to module scope
        for name in self.global_names:
            nodes = self.unscoped_names.pop(name, [])
            self.module.names[name].extend(nodes)

        # TODO: add nonlocal support here

        # Detect local names
        for name, nodes in list(self.unscoped_names.items()):
            if self.is_module or any(is_write, nodes):
                self.names[name].extend(nodes)
                self.unscoped_names.pop(name)

    def pass_unscoped(self, other_scope):
        # print "Passing unscoped", self.unscoped_names.keys(), "from", self.node, "to", other_scope.node
        other_scope.unscoped_names.update(self.unscoped_names)
        self.unscoped_names = empty(self.unscoped_names)

    def walk(self):
        for name, nodes in self.names.items():
            yield self, name, nodes

        for child in self.children:
            for scope, name, nodes in child.walk():
                yield scope, name, nodes

    # Stringification

    def dump(self, indent=''):
        name = self.node.__class__.__name__
        if hasattr(self.node, 'name'):
            name += ' ' + self.node.name
        if hasattr(self.node, 'lineno'):
            name += ' a line %d' % self.node.lineno

        title = indent + 'Scope ' + name
        names = '\n'.join(indent + '  %s = %s' % (name, nodes)
                          for name, nodes in sorted(self.names.items()))
        unscoped = '\n'.join(indent + '  unscoped %s = %s' % (name, nodes)
                             for name, nodes in sorted(self.unscoped_names.items()))
        children = ''.join(c.dump(indent + '  ') for c in self.children)

        return '\n'.join([title, names, unscoped, children])

    def __str__(self):
        return self.dump()

def is_write(node):
    return isinstance(node, (ast.Import, ast.ImportFrom, ast.FunctionDef)) \
        or isinstance(node.ctx, (ast.Store, ast.Del, ast.Param))

def is_use(node):
    return isinstance(node, ast.Name) \
       and isinstance(node.ctx, (ast.Load, ast.Del))

class ScopeBuilder(ast.NodeVisitor):
    def __init__(self):
        self.scopes = deque()

    def visit_all(self, *node_lists):
        for node in icat(node_lists):
            self.visit(node)

    # Scope mechanics
    @property
    def scope(self):
        if self.scopes:
            return self.scopes[-1]
        else:
            return None

    def push_scope(self, node):
        self.scopes.append(Scope(self.scope, node))

    def pop_scope(self):
        current = self.scope
        current.resolve()
        self.scopes.pop()
        if self.scope:
            current.pass_unscoped(self.scope)
        return current

    # Visiting
    def visit_Module(self, node):
        self.push_scope(node)
        self.generic_visit(node)
        return self.pop_scope()

    def visit_Import(self, node):
        for alias in node.names:
            self.scope.add(alias.asname or alias.name, node)

    def visit_ImportFrom(self, node):
        if node.module != '__future__':
            self.visit_Import(node)

    def visit_ClassDef(self, node):
        self.scope.add(node.name, node)
        # TODO: handle python 3 style metaclass
        self.visit_all(node.decorator_list, node.bases)

        self.push_scope(node)
        self.visit_all(node.body)
        return self.pop_scope()

    def visit_FunctionDef(self, node):
        # print 'visit_FunctionDef'
        self.scope.add(node.name, node)
        self.visit_all(node.decorator_list, node.args.defaults)

        self.push_scope(node)
        # TODO: handle varargs, kwargs, and kwonlyargs
        self.visit_all(node.args.args, node.body)
        # print 'exit visit_FunctionDef', self.scope.unscoped_names
        return self.pop_scope()

    def visit_Name(self, node):
        # print 'Name', node.id, node.ctx
        # TODO: respect assignments to these or make it a separate error
        if node.id not in BUILTINS:
            self.scope.add(node.id, node)

    def visit_Global(self, node):
        self.scope.make_global(node.names)

def slurp(filename):
    with open(filename) as f:
        return f.read()

def name_class(node):
    if isinstance(node, (ast.Import, ast.ImportFrom)):
        return 'import'
    elif isinstance(node, ast.FunctionDef):
        return 'function'
    elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Param):
        return 'param'
    else:
        return 'variable'

def main():
    filename = sys.argv[1]
    print '> Analyzing %s...' % filename

    source = slurp(filename)
    tree = ast.parse(source, filename=filename)
    # print dump(tree)

    sb = ScopeBuilder()
    top = sb.visit(tree)
    # print top

    for scope, name, nodes in top.walk():
        node = nodes[0]
        if all(is_use, nodes):
            print 'Undefined variable %s at %d:%d' % (name, node.lineno, node.col_offset)
        if all(is_write, nodes):
            print '%s %s is never used at %d:%d' % \
                  (name_class(node).title(), name, node.lineno, node.col_offset)

if __name__ == '__main__':
    main()
<commit_msg>Make class def a write<commit_after>#!/usr/bin/env python
import sys
import ast
from collections import defaultdict, deque

from funcy import *
from astpp import dump # use astor?

# TODO: distinguish python versions
import __builtin__
BUILTINS = set(dir(__builtin__))

class Scope(object):
    def __init__(self, parent, node):
        self.parent = parent
        if parent:
            parent.children.append(self)
        self.children = []

        self.node = node
        self.names = defaultdict(list)
        self.unscoped_names = defaultdict(list)
        self.global_names = set()

    @property
    def is_module(self):
        return isinstance(self.node, ast.Module)

    @cached_property
    def module(self):
        scope = self
        while not scope.is_module:
            scope = scope.parent
        return scope

    # Names

    def add(self, name, node):
        # Params are always in current scope.
        # Other names could be made global or passed to parent scope upon exit,
        # unless we are in a module.
        is_param = isinstance(node, ast.Name) and isinstance(node.ctx, ast.Param)
        if is_param or name in self.names or self.is_module:
            self.names[name].append(node)
        else:
            self.unscoped_names[name].append(node)

    def make_global(self, names):
        self.global_names.update(names)

    def resolve(self):
        # print 'RESOVE', self.node, self.unscoped_names.keys()

        # Extract global names to module scope
        for name in self.global_names:
            nodes = self.unscoped_names.pop(name, [])
            self.module.names[name].extend(nodes)

        # TODO: add nonlocal support here

        # Detect local names
        for name, nodes in list(self.unscoped_names.items()):
            if self.is_module or any(is_write, nodes):
                self.names[name].extend(nodes)
                self.unscoped_names.pop(name)

    def pass_unscoped(self, other_scope):
        # print "Passing unscoped", self.unscoped_names.keys(), "from", self.node, "to", other_scope.node
        other_scope.unscoped_names.update(self.unscoped_names)
        self.unscoped_names = empty(self.unscoped_names)

    def walk(self):
        for name, nodes in self.names.items():
            yield self, name, nodes

        for child in self.children:
            for scope, name, nodes in child.walk():
                yield scope, name, nodes

    # Stringification

    def dump(self, indent=''):
        name = self.node.__class__.__name__
        if hasattr(self.node, 'name'):
            name += ' ' + self.node.name
        if hasattr(self.node, 'lineno'):
            name += ' a line %d' % self.node.lineno

        title = indent + 'Scope ' + name
        names = '\n'.join(indent + '  %s = %s' % (name, nodes)
                          for name, nodes in sorted(self.names.items()))
        unscoped = '\n'.join(indent + '  unscoped %s = %s' % (name, nodes)
                             for name, nodes in sorted(self.unscoped_names.items()))
        children = ''.join(c.dump(indent + '  ') for c in self.children)

        return '\n'.join([title, names, unscoped, children])

    def __str__(self):
        return self.dump()

def is_write(node):
    return isinstance(node, (ast.Import, ast.ImportFrom, ast.FunctionDef, ast.ClassDef)) \
        or isinstance(node.ctx, (ast.Store, ast.Del, ast.Param))

def is_use(node):
    return isinstance(node, ast.Name) \
       and isinstance(node.ctx, (ast.Load, ast.Del))

class ScopeBuilder(ast.NodeVisitor):
    def __init__(self):
        self.scopes = deque()

    def visit_all(self, *node_lists):
        for node in icat(node_lists):
            self.visit(node)

    # Scope mechanics
    @property
    def scope(self):
        if self.scopes:
            return self.scopes[-1]
        else:
            return None

    def push_scope(self, node):
        self.scopes.append(Scope(self.scope, node))

    def pop_scope(self):
        current = self.scope
        current.resolve()
        self.scopes.pop()
        if self.scope:
            current.pass_unscoped(self.scope)
        return current

    # Visiting
    def visit_Module(self, node):
        self.push_scope(node)
        self.generic_visit(node)
        return self.pop_scope()

    def visit_Import(self, node):
        for alias in node.names:
            self.scope.add(alias.asname or alias.name, node)

    def visit_ImportFrom(self, node):
        if node.module != '__future__':
            self.visit_Import(node)

    def visit_ClassDef(self, node):
        self.scope.add(node.name, node)
        # TODO: handle python 3 style metaclass
        self.visit_all(node.decorator_list, node.bases)

        self.push_scope(node)
        self.visit_all(node.body)
        return self.pop_scope()

    def visit_FunctionDef(self, node):
        # print 'visit_FunctionDef'
        self.scope.add(node.name, node)
        self.visit_all(node.decorator_list, node.args.defaults)

        self.push_scope(node)
        # TODO: handle varargs, kwargs, and kwonlyargs
        self.visit_all(node.args.args, node.body)
        # print 'exit visit_FunctionDef', self.scope.unscoped_names
        return self.pop_scope()

    def visit_Name(self, node):
        # print 'Name', node.id, node.ctx
        # TODO: respect assignments to these or make it a separate error
        if node.id not in BUILTINS:
            self.scope.add(node.id, node)

    def visit_Global(self, node):
        self.scope.make_global(node.names)

def slurp(filename):
    with open(filename) as f:
        return f.read()

def name_class(node):
    if isinstance(node, (ast.Import, ast.ImportFrom)):
        return 'import'
    elif isinstance(node, ast.FunctionDef):
        return 'function'
    elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Param):
        return 'param'
    else:
        return 'variable'

def main():
    filename = sys.argv[1]
    print '> Analyzing %s...' % filename

    source = slurp(filename)
    tree = ast.parse(source, filename=filename)
    # print dump(tree)

    sb = ScopeBuilder()
    top = sb.visit(tree)
    # print top

    for scope, name, nodes in top.walk():
        node = nodes[0]
        if all(is_use, nodes):
            print 'Undefined variable %s at %d:%d' % (name, node.lineno, node.col_offset)
        if all(is_write, nodes):
            print '%s %s is never used at %d:%d' % \
                  (name_class(node).title(), name, node.lineno, node.col_offset)

if __name__ == '__main__':
    main()
<|endoftext|>