<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from bayeslite.core import *
import bayeslite.metamodel
import bayeslite.bqlfn

import bdbcontrib
from bdbcontrib.foreign.orbital_mech import OrbitalMechanics
from bdbcontrib.foreign.random_forest import RandomForest

class Composer(bayeslite.metamodel.IBayesDBMetamodel):

    """A metamodel which composes foreign predictors with CrossCat. The
    dependency of the variables must form a directed acyclic graph.
    """

    def __init__(self):
        self.cc_id = None
        self.cc = None

    def name(self):
        """Return the name of the metamodel as a str."""
        return 'fp_composer'

    def register(self, bdb):
        # XXX Figure out what to do.
        # XXX Causes serialization problem, should have v0 of schema which
        # is only ever before the first time this metamodel is registered
        # in the history of the bdb.
        bdb.sql_execute("""
            INSERT INTO bayesdb_metamodel (name, version)
                VALUES ('fp_composer', 1);
            """)
        bdb.sql_execute("""
            CREATE TABLE bayesdb_fp_composer_cc_lookup (
                generator_id INTEGER NOT NULL PRIMARY KEY
                        REFERENCES bayesdb_generator(id),
                crosscat_generator_id INTEGER NOT NULL
                        REFERENCES bayesdb_generator(id)
            );
            """)

    def create_generator(self, bdb, table, schema, instantiate):
        # TODO: register foreign predictors externally.
        # TODO: Serialize the mapping from FP names to objects into database.
        self.fp_constructors = {'random_forest': RandomForest,
            'orbital_mechanics' : OrbitalMechanics}

        # TODO: Parse the local, foreign, and condition columns
        # and (foreign predictors) all from schema.
        schema = [
            ('Country_of_Operator', 'CATEGORICAL'),
            ('Operator_Owner', 'CATEGORICAL'),
            ('Users', 'CATEGORICAL'),
            ('Purpose', 'CATEGORICAL'),
            ('Type_of_Orbit', 'CATEGORICAL'),
            ('Perigee_km', 'NUMERICAL'),
            ('Apogee_km', 'NUMERICAL'),
            ('Eccentricity', 'NUMERICAL'),
            ('Launch_Mass_kg', 'NUMERICAL'),
            ('Dry_Mass_kg', 'NUMERICAL'),
            ('Power_watts', 'NUMERICAL'),
            ('Date_of_Launch', 'NUMERICAL'),
            ('Anticipated_Lifetime', 'NUMERICAL'),
            ('Contractor', 'CATEGORICAL'),
            ('Country_of_Contractor', 'CATEGORICAL'),
            ('Launch_Site', 'CATEGORICAL'),
            ('Launch_Vehicle', 'CATEGORICAL'),
            ('Source_Used_for_Orbital_Data', 'CATEGORICAL'),
            ('longitude_radians_of_geo', 'NUMERICAL'),
            ('Inclination_radians', 'NUMERICAL'),
            ('Class_of_orbit', 'CATEGORICAL'),
            ('Period_minutes', 'NUMERICAL')
        ]

        # Maps target FP targets to their conditions columns.
        foreign = {
            # Orbital Mechanics
            'Period_minutes' : ['Apogee_km', 'Perigee_km'],
            # Random Forest
            'Type_of_Orbit' : ['Apogee_km', 'Perigee_km', 'Eccentricity',
                'Period_minutes', 'Launch_Mass_kg', 'Power_watts',
                'Anticipated_Lifetime', 'Class_of_orbit']
        }
        # Maps target FP columns to their FP, currently these are strings
        # but will be mapped to actual instances when the initialize_models
        # is called.
        foreign_lookup = {
            'Period_minutes' : 'orbital_mechanics',
            'Type_of_Orbit' : 'random_forest',
        }

        # All other columns shall be modeled by CrossCat.
        local = [(col,stat) for (col,stat) in schema if col not in foreign]

        # Instantiate **this** generator.
        generator_id, columns = instantiate(schema)

        # Convert foriegn cols from strings to BayesDB column numbers.
        self.foreign = {}
        self.foreign_lookup = {}
        for f in foreign:
            f_col = bayesdb_generator_column_number(bdb, generator_id, f)
            self.foreign[f_col] = [bayesdb_generator_column_number(bdb,
                generator_id, col) for col in foreign[f]]
            self.foreign_lookup[f_col] = foreign_lookup[f]

        # Convert local cols from strings to BayesDB column numbers.
        self.local = [bayesdb_generator_column_number(bdb, generator_id, col[0])
            for col in local]

        # Sort foriegn cols in topological order.
        self.topo = self.topolgical_sort(self.foreign)

        # Create the internal crosscat generator.
        self.cc_name = bayeslite.core.bayesdb_generator_name(bdb,
            generator_id) + '_cc'
        ignore = ','.join(['{} IGNORE'.format(pair[0]) for pair in schema
            if pair[0] in foreign])
        cc = ','.join(['{} {}'.format(pair[0], pair[1]) for pair in local])
        bql = """
            CREATE GENERATOR {} FOR satellites USING crosscat(
                GUESS(*),
                {} ,
                {}
            );
        """.format(self.cc_name, ignore, cc)
        bdb.execute(bql)

        # Obtain the generator_id of internal cc, and the metamodel object.
        # Should be stored in the database table bayesdb_fp_composer_cc_lookup
        self.cc_id = bayesdb_get_generator(bdb, self.cc_name)
        self.cc = bayesdb_generator_metamodel(bdb, self.cc_id)

    def drop_generator(self, bdb, generator_id):
        # Drop the internal crosscat instance.
        bdb.execute('''DROP GENERATOR {}'''.format(self.cc_name))
        # XXX TODO: Additional clean up operations.

    def initialize_models(self, bdb, generator_id, modelnos, model_config):
        # Initializeinternal Crosscat.
        bql = """
            INITIALIZE {} MODELS FOR {};
            """.format(len(modelnos), self.cc_name)
        bdb.execute(bql)

        # Obtain the dataframe for foreign predictors.
        df = bdbcontrib.cursor_to_df(bdb.execute('''
            SELECT * FROM {}
            '''.format(bayesdb_generator_table(bdb, generator_id))))

        # Initialize the foriegn predictors.
        for f_target, f_conditions in self.foreign.items():
            # Convert column numbers to names.
            targets = \
                [(bayesdb_generator_column_name(bdb, generator_id, f_target),
                bayesdb_generator_column_stattype(bdb, generator_id,f_target))]
            conditions = \
            [(bayesdb_generator_column_name(bdb, generator_id, f_c),
                bayesdb_generator_column_stattype(bdb, generator_id, f_c))
                for f_c in f_conditions]

            # Replace the string with an actual trained FP instance.
            self.foreign_lookup[f_target] = \
                self.fp_constructors[self.foreign_lookup[f_target]](df, targets,
                conditions)

    def drop_models(self, bdb, generator_id, modelnos=None):
        # Delegate
        raise NotImplementedError()

    def analyze_models(self, bdb, generator_id, modelnos=None, iterations=1,
                max_seconds=None, ckpt_iterations=None, ckpt_seconds=None):
        # Delegate
        self.cc.analyze_models(bdb, self.cc_id, modelnos=modelnos,
            iterations=iterations, max_seconds=max_seconds,
            ckpt_iterations=ckpt_iterations, ckpt_seconds=ckpt_seconds)

    def column_dependence_probability(self, bdb, generator_id, modelno, colno0,
                colno1, depth=0):
        if depth == 2:
            # We are looping too much, set a trace to explore.
            import ipdb; ipdb.set_trace()

        # Determine which object modeled colno0, colno1
        colno0_foreign = colno0 in self.unmodeled
        colno1_foreign = colno1 in self.unmodeled

        # If neither col is foreign, delegate to CrossCat
        if not (colno0_foreign or colno1_foreign):
            return self.cc.column_dependence_probability(bdb, self.cc_id,
                modelno,
                self._get_internal_cc_colno(bdb, generator_id, colno0),
                self._get_internal_cc_colno(bdb, generator_id, colno1))

        # If (colno0, colno1) form a (target, given) pair, then we explicitly
        # modeled them as dependent!
        # TODO: What if an FP determines it is not dependent on one of its
        # conditions? (ie regression with zero coefficient.)
        # -- check for RF
        elif ((colno0 == self.sat_rf_target and colno1 in self.sat_rf_conditions)
                or (colno0 in self.sat_rf_conditions and colno1 ==
                    self.sat_rf_target)):
            return 1
        # -- check for OM
        elif ((colno0 == self.sat_om_target and colno1 in self.sat_om_conditions)
                or (colno0 in self.sat_om_conditions and colno1 ==
                    self.sat_om_target)):
            return 1

        # If one col is an FP target and another is modeled by CC, return
        # the average dependence probability of the CC col with FP's targets.
        # XXX TODO: Write out the mathematics of the theoretical justification
        # for this heuristic (not polished).
        elif (colno0_foreign and not colno1_foreign) or (colno1_foreign and not
                colno0_foreign):
            # Obtain foreign column, and its condition columns.
            foreign_col = colno0 if colno0_foreign else colno1
            cc_col = colno1 if colno0_foreign else colno0

            deps = 0.
            for fc in self._get_condition_cols(foreign_col):
                # If fc is modeled by CC we are golden.
                if fc not in self.unmodeled:
                    deps += self.cc.column_dependence_probability(bdb,
                        self.cc_id, modelno,
                        self._get_internal_cc_colno(bdb, generator_id, cc_col),
                        self._get_internal_cc_colno(bdb, generator_id, fc))
                # Otherwise fc is itself a target of the GPM, do the recursive
                # call. Will terminate iff the DAG of GPMs is acyclic.
                # TODO: Prove.
                else:
                    depth += 1  # For debugging.
                    deps += self.column_dependence_probability(bdb,
                        generator_id, modelno, cc_col, fc, depth=depth)
            return deps / len(self._get_condition_cols(foreign_col))

        # If both colno0 and colno1 are FP targets AND neither is a condition
        # of the other, we compute the average pairwise dependence probability
        # of all their targets.
        elif colno0_foreign and colno1_foreign:
            assert self._get_condition_cols(colno0) != self.get

    def column_mutual_information(self, bdb, generator_id, modelno, colno0,
                colno1, numsamples=100):
        # Delegate
        cc_colnos = self._get_internal_cc_colnos(bdb, generator_id,
            [colno0, colno1])
        return self.cc.column_mutual_information(bdb, self.cc_id, modelno,
            cc_colnos[0], cc_colnos[1], numsamples=numsamples)

    def column_value_probability(self, bdb, generator_id, modelno, colno,
                value, constraints):
        # Delegate
        cc_colno = self._get_internal_cc_colno(bdb, generator_id, colno)
        return self.cc.column_value_probability(bdb, self.cc_id, modelno,
            cc_colno, value, constraints)

    def row_similarity(self, bdb, generator_id, modelno, rowid, target_rowid,
                colnos):
        # Delegate
        cc_colnos = self._get_internal_cc_colnos(bdb, generator_id, colnos)
        return self.cc.row_similarity(bdb, self.cc_id, modelno, rowid,
            target_rowid, cc_colnos)

    def row_column_predictive_probability(self, bdb, generator_id, modelno,
                rowid, colno):
        # Delegate
        cc_colno = self._get_internal_cc_colno(bdb, generator_id, colno)
        return self.cc.row_column_predictive_probability(bdb, self.cc_id,
            modelno, rowid, cc_colno)

    def predict_confidence(self, bdb, generator_id, modelno, colno, rowid,
                numsamples=None):
        # Delegate
        cc_colno = self._get_internal_cc_colno(bdb, generator_id, colno)
        return self.predict_confidence(bdb, self.cc_id, modelno, cc_colno,
            rowid, numsamples=numsamples)

    def simulate(self, bdb, generator_id, modelno, constraints, colnos,
                numpredictions=1):
        # Delegate
        cc_colnos = self._get_internal_cc_colnos(bdb, generator_id, colnos)
        return self.cc.simulate(bdb, self.cc_id, modelno, constraints, cc_colnos,
            numpredictions=numpredictions)

    # TODO: Conver to SQL table queries on bayesdb_fp_composer_cc_lookup.
    def _get_internal_cc_colno(self, bdb, generator_id, colno):
        return self._get_internal_cc_colnos(bdb, generator_id, colno)[0]

    def _get_internal_cc_colnos(self, bdb, generator_id, colnos):
        # First get the names from this generator.
        colnames = [bayeslite.core.bayesdb_generator_column_name(bdb,
            generator_id, no) for no in colnos]
        # Now do the inverse mapping.
        return [bayeslite.core.bayesdb_generator_column_number(bdb, self.cc_id,
            name) for name in colnames]

    # TODO: Move this to somewhere reasonable?
    def topolgical_sort(self, graph):
        """Topologically sort a graph represented as an adjacency list.

        Parameters
        ----------
        graph : dict or list
            Adjacency list or dict representing the graph, for example
                graph_d = {10: [8, 7], 5: [8, 7, 9, 10, 11, 13, 15]}
                graph_l = [(10, [8, 7]), (5, [8, 7, 9, 10, 11, 13, 15])]

        Returns
        -------
        graph_sorted : list
            The original graph, in topologically sorted order.
        """
        graph_sorted = []
        graph = dict(graph)

        # Run until the unsorted graph is empty.
        while graph:
            acyclic = False
            for node, edges in graph.items():
                for edge in edges:
                    if edge in graph:
                        break
                else:
                    acyclic = True
                    del graph[node]
                    graph_sorted.append((node, edges))

            if not acyclic:
                raise RuntimeError('A cyclic dependency occurred in '
                    'topological_sort.')

        return graph_sorted
<commit_msg>Begin implementing dependence probability.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from bayeslite.core import *
import bayeslite.metamodel
import bayeslite.bqlfn

import bdbcontrib
from bdbcontrib.foreign.orbital_mech import OrbitalMechanics
from bdbcontrib.foreign.random_forest import RandomForest

class Composer(bayeslite.metamodel.IBayesDBMetamodel):

    """A metamodel which composes foreign predictors with CrossCat. The
    dependency of the variables must form a directed acyclic graph.
    """

    def __init__(self):
        self.cc_id = None
        self.cc = None

    def name(self):
        """Return the name of the metamodel as a str."""
        return 'fp_composer'

    def register(self, bdb):
        # XXX Figure out what to do.
        # XXX Causes serialization problem, should have v0 of schema which
        # is only ever before the first time this metamodel is registered
        # in the history of the bdb.
        bdb.sql_execute("""
            INSERT INTO bayesdb_metamodel (name, version)
                VALUES ('fp_composer', 1);
            """)
        bdb.sql_execute("""
            CREATE TABLE bayesdb_fp_composer_cc_lookup (
                generator_id INTEGER NOT NULL PRIMARY KEY
                        REFERENCES bayesdb_generator(id),
                crosscat_generator_id INTEGER NOT NULL
                        REFERENCES bayesdb_generator(id)
            );
            """)

    def create_generator(self, bdb, table, schema, instantiate):
        # TODO: register foreign predictors externally.
        # TODO: Serialize the mapping from FP names to objects into database.
        self.fp_constructors = {'random_forest': RandomForest,
            'orbital_mechanics' : OrbitalMechanics}

        # TODO: Parse the local, foreign, and condition columns
        # and (foreign predictors) all from schema.
        schema = [
            ('Country_of_Operator', 'CATEGORICAL'),
            ('Operator_Owner', 'CATEGORICAL'),
            ('Users', 'CATEGORICAL'),
            ('Purpose', 'CATEGORICAL'),
            ('Type_of_Orbit', 'CATEGORICAL'),
            ('Perigee_km', 'NUMERICAL'),
            ('Apogee_km', 'NUMERICAL'),
            ('Eccentricity', 'NUMERICAL'),
            ('Launch_Mass_kg', 'NUMERICAL'),
            ('Dry_Mass_kg', 'NUMERICAL'),
            ('Power_watts', 'NUMERICAL'),
            ('Date_of_Launch', 'NUMERICAL'),
            ('Anticipated_Lifetime', 'NUMERICAL'),
            ('Contractor', 'CATEGORICAL'),
            ('Country_of_Contractor', 'CATEGORICAL'),
            ('Launch_Site', 'CATEGORICAL'),
            ('Launch_Vehicle', 'CATEGORICAL'),
            ('Source_Used_for_Orbital_Data', 'CATEGORICAL'),
            ('longitude_radians_of_geo', 'NUMERICAL'),
            ('Inclination_radians', 'NUMERICAL'),
            ('Class_of_orbit', 'CATEGORICAL'),
            ('Period_minutes', 'NUMERICAL')
        ]

        # Maps FP target columns (fcol) to their conditions columns .
        # Currenlty strings, but will be mapped to columns after instantiate.
        fcol_to_cond = {
            # Orbital Mechanics
            'Period_minutes' : ['Apogee_km', 'Perigee_km'],
            # Random Forest
            'Type_of_Orbit' : ['Apogee_km', 'Perigee_km', 'Eccentricity',
                'Period_minutes', 'Launch_Mass_kg', 'Power_watts',
                'Anticipated_Lifetime', 'Class_of_orbit']
        }
        # Maps target FP columns to FP object. Currently these are strings
        # but will be mapped to actual instances when the initialize_models
        # is called.
        fcol_to_pred = {
            'Period_minutes' : 'orbital_mechanics',
            'Type_of_Orbit' : 'random_forest',
        }

        # locls = local columns modeled by CrossCat.
        lcols = [(col,stat) for (col,stat) in schema if col not in foreign]

        # Instantiate **this** generator.
        generator_id, columns = instantiate(schema)

        # Convert foriegn cols from strings to BayesDB column numbers.
        self.fcol_to_cond = {}
        self.fcol_to_pred = {}
        for f in foreign:
            f_col = bayesdb_generator_column_number(bdb, generator_id, f)
            self.fcol_to_cond[f_col] = [bayesdb_generator_column_number(bdb,
                generator_id, col) for col in foreign[f]]
            self.fcol_to_pred[f_col] = fcol_to_pred[f]

        # Convert lcols cols from strings to BayesDB column numbers.
        self.lcols = [bayesdb_generator_column_number(bdb, generator_id, col[0])
            for col in lcols]

        # Sort foriegn cols in topological order.
        self.topo = self.topolgical_sort(self.fcol_to_cond)

        # Create the internal crosscat generator.
        self.cc_name = bayeslite.core.bayesdb_generator_name(bdb,
            generator_id) + '_cc'
        ignore = ','.join(['{} IGNORE'.format(pair[0]) for pair in schema
            if pair[0] in foreign])
        cc = ','.join(['{} {}'.format(pair[0], pair[1]) for pair in lcols])
        bql = """
            CREATE GENERATOR {} FOR satellites USING crosscat(
                GUESS(*),
                {} ,
                {}
            );
        """.format(self.cc_name, ignore, cc)
        bdb.execute(bql)

        # Obtain the generator_id of internal cc, and the metamodel object.
        # Should be stored in the database table bayesdb_fp_composer_cc_lookup
        self.cc_id = bayesdb_get_generator(bdb, self.cc_name)
        self.cc = bayesdb_generator_metamodel(bdb, self.cc_id)

    def drop_generator(self, bdb, generator_id):
        # Drop the internal crosscat instance.
        bdb.execute('''DROP GENERATOR {}'''.format(self.cc_name))
        # XXX TODO: Additional clean up operations.

    def initialize_models(self, bdb, generator_id, modelnos, model_config):
        # Initialize internal Crosscat. If k models of Composer are instantiated
        # then k internal CC models will be created, with a 1-1 mapping.
        bql = """
            INITIALIZE {} MODELS FOR {};
            """.format(len(modelnos), self.cc_name)
        bdb.execute(bql)

        # Obtain the dataframe for foreign predictors.
        df = bdbcontrib.cursor_to_df(bdb.execute('''
            SELECT * FROM {}
            '''.format(bayesdb_generator_table(bdb, generator_id))))

        # Initialize the foriegn predictors.
        for f_target, f_conditions in self.fcol_to_cond.items():
            # Convert column numbers to names.
            targets = \
                [(bayesdb_generator_column_name(bdb, generator_id, f_target),
                bayesdb_generator_column_stattype(bdb, generator_id,f_target))]
            conditions = \
            [(bayesdb_generator_column_name(bdb, generator_id, f_c),
                bayesdb_generator_column_stattype(bdb, generator_id, f_c))
                for f_c in f_conditions]

            # Replace the string with an actual trained FP instance.
            self.fcol_to_pred[f_target] = \
                self.fp_constructors[self.fcol_to_pred[f_target]](df, targets,
                conditions)

    def drop_models(self, bdb, generator_id, modelnos=None):
        # Delegate
        raise NotImplementedError()

    def analyze_models(self, bdb, generator_id, modelnos=None, iterations=1,
                max_seconds=None, ckpt_iterations=None, ckpt_seconds=None):
        # Delegate
        self.cc.analyze_models(bdb, self.cc_id, modelnos=modelnos,
            iterations=iterations, max_seconds=max_seconds,
            ckpt_iterations=ckpt_iterations, ckpt_seconds=ckpt_seconds)

    def column_dependence_probability(self, bdb, generator_id, modelno, colno0,
                colno1,):
        # Determine which object modeled colno0, colno1
        c0_foreign = colno0 not in self.fcol_to_cond
        c1_foreign = colno1 not in self.fcol_to_cond

        # If neither col is foreign, delegate to CrossCat
        if not (c0_foreign or c1_foreign):
            return self.cc.column_dependence_probability(bdb, self.cc_id,
                modelno,
                self._internal_cc_colno(bdb, generator_id, colno0),
                self._internal_cc_colno(bdb, generator_id, colno1))

        # We know at least one column is foreign.

        # If (colno0, colno1) form a (target, given) pair, then we explicitly
        # modeled them as dependent by assumption.
        # TODO: Strong assumption, what if FP determines it is not dependent
        # on one of its conditions? (ie regression w/ 0 coefficient.)
        if colno0 in self.fcol_to_cond.get(colno1) or \
                colno1 in self.fcol_to_cond.get(colno0):
            return 1

        # If one col is an FP target and another is modeled by CC, return
        # the average dependence probability of the CC col with FP's targets.
        # XXX TODO: Write out the mathematics of the theoretical justification
        # for this heuristic (not polished).
        elif (colno0_foreign and not colno1_foreign) or (colno1_foreign and not
                colno0_foreign):
            # Obtain foreign column, and its condition columns.
            foreign_col = colno0 if colno0_foreign else colno1
            cc_col = colno1 if colno0_foreign else colno0

            deps = 0.
            for fc in self._get_condition_cols(foreign_col):
                # If fc is modeled by CC we are golden.
                if fc not in self.unmodeled:
                    deps += self.cc.column_dependence_probability(bdb,
                        self.cc_id, modelno,
                        self._internal_cc_colno(bdb, generator_id, cc_col),
                        self._internal_cc_colno(bdb, generator_id, fc))
                # Otherwise fc is itself a target of the GPM, do the recursive
                # call. Will terminate iff the DAG of GPMs is acyclic.
                # TODO: Prove.
                else:
                    depth += 1  # For debugging.
                    deps += self.column_dependence_probability(bdb,
                        generator_id, modelno, cc_col, fc)
            return deps / len(self._get_condition_cols(foreign_col))

        # If both colno0 and colno1 are FP targets AND neither is a condition
        # of the other, we compute the average pairwise dependence probability
        # of all their targets.
        elif colno0_foreign and colno1_foreign:
            assert self._get_condition_cols(colno0) != self.get

    def column_mutual_information(self, bdb, generator_id, modelno, colno0,
                colno1, numsamples=100):
        # Delegate
        cc_colnos = self._internal_cc_colnos(bdb, generator_id,
            [colno0, colno1])
        return self.cc.column_mutual_information(bdb, self.cc_id, modelno,
            cc_colnos[0], cc_colnos[1], numsamples=numsamples)

    def column_value_probability(self, bdb, generator_id, modelno, colno,
                value, constraints):
        # Delegate
        cc_colno = self._internal_cc_colno(bdb, generator_id, colno)
        return self.cc.column_value_probability(bdb, self.cc_id, modelno,
            cc_colno, value, constraints)

    def row_similarity(self, bdb, generator_id, modelno, rowid, target_rowid,
                colnos):
        # Delegate
        cc_colnos = self._internal_cc_colnos(bdb, generator_id, colnos)
        return self.cc.row_similarity(bdb, self.cc_id, modelno, rowid,
            target_rowid, cc_colnos)

    def row_column_predictive_probability(self, bdb, generator_id, modelno,
                rowid, colno):
        # Delegate
        cc_colno = self._internal_cc_colno(bdb, generator_id, colno)
        return self.cc.row_column_predictive_probability(bdb, self.cc_id,
            modelno, rowid, cc_colno)

    def predict_confidence(self, bdb, generator_id, modelno, colno, rowid,
                numsamples=None):
        # Delegate
        cc_colno = self._internal_cc_colno(bdb, generator_id, colno)
        return self.predict_confidence(bdb, self.cc_id, modelno, cc_colno,
            rowid, numsamples=numsamples)

    def simulate(self, bdb, generator_id, modelno, constraints, colnos,
                numpredictions=1):
        # Delegate
        cc_colnos = self._internal_cc_colnos(bdb, generator_id, colnos)
        return self.cc.simulate(bdb, self.cc_id, modelno, constraints, cc_colnos,
            numpredictions=numpredictions)

    # TODO: Conver to SQL table queries on bayesdb_fp_composer_cc_lookup.
    def _internal_cc_colno(self, bdb, generator_id, colno):
        return self._internal_cc_colnos(bdb, generator_id, colno)[0]

    def _internal_cc_colnos(self, bdb, generator_id, colnos):
        # First get the names from this generator.
        colnames = [bayeslite.core.bayesdb_generator_column_name(bdb,
            generator_id, no) for no in colnos]
        # Now do the inverse mapping.
        return [bayeslite.core.bayesdb_generator_column_number(bdb, self.cc_id,
            name) for name in colnames]

    # TODO: Move this to somewhere reasonable?
    def topolgical_sort(self, graph):
        """Topologically sort a graph represented as an adjacency list.

        Parameters
        ----------
        graph : dict or list
            Adjacency list or dict representing the graph, for example
                graph_d = {10: [8, 7], 5: [8, 7, 9, 10, 11, 13, 15]}
                graph_l = [(10, [8, 7]), (5, [8, 7, 9, 10, 11, 13, 15])]

        Returns
        -------
        graph_sorted : list
            The original graph, in topologically sorted order.
        """
        graph_sorted = []
        graph = dict(graph)

        # Run until the unsorted graph is empty.
        while graph:
            acyclic = False
            for node, edges in graph.items():
                for edge in edges:
                    if edge in graph:
                        break
                else:
                    acyclic = True
                    del graph[node]
                    graph_sorted.append((node, edges))

            if not acyclic:
                raise RuntimeError('A cyclic dependency occurred in '
                    'topological_sort.')

        return graph_sorted
<|endoftext|>


<commit_before>#author L. Tibaldo (ltibaldo@slac.stanford.edu)

from threeML.models.spatialmodel import SpatialModel
from threeML.models.Parameter import Parameter, SpatialParameter
import numpy as np
import scipy.stats
from angsep import angsep

import collections

class SimpleGaussian(SpatialModel):

    def setup(self):
        self.functionName        = "Gaussian"
        self.formula             = r'$f({\rm RA, Dec}) = \left(\frac{180^\circ}{\pi}\right)^2 \frac{1}{2\pi \sigma^2} \, {\rm exp}\left(-\frac{{\rm angsep}^2 ({\rm RA, Dec, RA_0, Dec_0})}{2\sigma^2} \right) $'
        self.parameters          = collections.OrderedDict()
        self.parameters['RA0']     = Parameter('RA0',1.,0,360,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['Dec0']     = Parameter('Dec0',1.,-90,90,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['sigma'] = SpatialParameter('sigma',0.1,0,20,0.01,fixed=False,nuisance=False,dataset=None)

        self.ncalls              = 0

    def __call__(self,RA,Dec,energy):
        self.ncalls             += 1
        RA0                         = self.parameters['RA0'].value
        Dec0                        = self.parameters['Dec0'].value
        sigma                       = self.parameters['sigma'].getValue(energy)

        return np.maximum( np.power(180/np.pi,2)*1./(2 * np.pi * sigma**2) * np.exp(-0.5 * np.power(angsep(RA,Dec,RA0,Dec0),2)/sigma**2), 1e-30)

    def integratedFlux(self,energy):

        return 1.

class MultiVariateGaussian(SpatialModel):

    def setup(self):
        self.functionName        = "Multivariate Gaussian"
        self.formula             = r'$f(\vec{x}) = \left(\frac{180^\circ}{\pi}\right)^2 \frac{1}{2\pi \sqrt{\det{\Sigma}}} \, {\rm exp}\left( -\frac{1}{2} (\vec{x}-\vec{x}_0)^\intercal \cdot \Sigma^{-1}\cdot (\vec{x}-\vec{x}_0)\right) \\ \vec{x}_0 = ({\rm RA}_0,{\rm Dec}_0)\\ \Lambda = \left( \begin{array}{cc} \sigma^2 & 0 \\ 0 & \sigma^2 (1-e^2) \end{array}\right) \\ U = \left( \begin{array}{cc} \cos \theta & -\sin \theta \\ \sin \theta & cos \theta \end{array}\right) \\\Sigma = U\Lambda U^\intercal$'
        self.parameters          = collections.OrderedDict()
        self.parameters['RA0']     = Parameter('RA0',1.,0,360,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['Dec0']     = Parameter('Dec0',1.,-90,90,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['sigma'] = SpatialParameter('sigma',0.1,0,10,0.01,fixed=False,nuisance=False,dataset=None)
        self.parameters['eccentricity'] = SpatialParameter('eccentricity',0.7,0,1,0.01,fixed=False,nuisance=False,dataset=None)
        self.parameters['angle'] = SpatialParameter('angle',0.,0,180,1.,fixed=False,nuisance=False,dataset=None)

        self.ncalls              = 0

    def __call__(self,RA,Dec,energy):
        self.ncalls             += 1
        RA0                         = self.parameters['RA0'].value
        Dec0                        = self.parameters['Dec0'].value
        sigma                       = self.parameters['sigma'].getValue(energy)
        eccentricity                = self.parameters['eccentricity'].getValue(energy)
        angle                       = np.deg2rad(self.parameters['angle'].getValue(energy))

        sigma1=sigma**2
        sigma2=sigma1*(1-eccentricity**2)
        rot=np.array([[np.cos(angle),-np.sin(angle)],[np.sin(angle),np.cos(angle)]])
        cov=np.dot(rot,np.dot(np.array([[sigma1,0],[0,sigma2]]),rot.T))
        return np.maximum(np.power(180/np.pi,2)*scipy.stats.multivariate_normal.pdf(np.array([Dec,RA]).T, mean=np.array([RA0,Dec0]), cov=cov), 1e-30)

    def integratedFlux(self,energy):

        return 1.

<commit_msg>Initial implementation of getBoundaries<commit_after>#author L. Tibaldo (ltibaldo@slac.stanford.edu)

from threeML.models.spatialmodel import SpatialModel
from threeML.models.Parameter import Parameter, SpatialParameter
import numpy as np
import scipy.stats
from angsep import angsep

import collections

class SimpleGaussian(SpatialModel):

    def setup(self):
        self.functionName        = "Gaussian"
        self.formula             = r'$f({\rm RA, Dec}) = \left(\frac{180^\circ}{\pi}\right)^2 \frac{1}{2\pi \sigma^2} \, {\rm exp}\left(-\frac{{\rm angsep}^2 ({\rm RA, Dec, RA_0, Dec_0})}{2\sigma^2} \right) $'
        self.parameters          = collections.OrderedDict()
        self.parameters['RA0']     = Parameter('RA0',1.,0,360,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['Dec0']     = Parameter('Dec0',1.,-90,90,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['sigma'] = SpatialParameter('sigma',0.1,0,20,0.01,fixed=False,nuisance=False,dataset=None)

        self.ncalls              = 0

    def __call__(self,RA,Dec,energy):
        self.ncalls             += 1
        RA0                         = self.parameters['RA0'].value
        Dec0                        = self.parameters['Dec0'].value
        sigma                       = self.parameters['sigma'].getValue(energy)

        return np.maximum( np.power(180/np.pi,2)*1./(2 * np.pi * sigma**2) * np.exp(-0.5 * np.power(angsep(RA,Dec,RA0,Dec0),2)/sigma**2), 1e-30)

    def integratedFlux(self,energy):

        return 1.

    def getBoundaries(self):

        #Truncate the gaussian at 2 times the max of sigma allowed

        maxSigma = self.parameters['sigma'].maxValue

        minRa = self.parameters['RA0'].value - 2 * maxSigma
        maxRa = self.parameters['RA0'].value + 2 * maxSigma

        minDec = self.parameters['Dec0'].value - 2 * maxSigma
        maxDec = self.parameters['Dec0'].value + 2 * maxSigma

        return minRa, maxRa, minDec, maxDec

class MultiVariateGaussian(SpatialModel):

    def setup(self):
        self.functionName        = "Multivariate Gaussian"
        self.formula             = r'$f(\vec{x}) = \left(\frac{180^\circ}{\pi}\right)^2 \frac{1}{2\pi \sqrt{\det{\Sigma}}} \, {\rm exp}\left( -\frac{1}{2} (\vec{x}-\vec{x}_0)^\intercal \cdot \Sigma^{-1}\cdot (\vec{x}-\vec{x}_0)\right) \\ \vec{x}_0 = ({\rm RA}_0,{\rm Dec}_0)\\ \Lambda = \left( \begin{array}{cc} \sigma^2 & 0 \\ 0 & \sigma^2 (1-e^2) \end{array}\right) \\ U = \left( \begin{array}{cc} \cos \theta & -\sin \theta \\ \sin \theta & cos \theta \end{array}\right) \\\Sigma = U\Lambda U^\intercal$'
        self.parameters          = collections.OrderedDict()
        self.parameters['RA0']     = Parameter('RA0',1.,0,360,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['Dec0']     = Parameter('Dec0',1.,-90,90,0.1,fixed=False,nuisance=False,dataset=None)
        self.parameters['sigma'] = SpatialParameter('sigma',0.1,0,10,0.01,fixed=False,nuisance=False,dataset=None)
        self.parameters['eccentricity'] = SpatialParameter('eccentricity',0.7,0,1,0.01,fixed=False,nuisance=False,dataset=None)
        self.parameters['angle'] = SpatialParameter('angle',0.,0,180,1.,fixed=False,nuisance=False,dataset=None)

        self.ncalls              = 0

    def __call__(self,RA,Dec,energy):
        self.ncalls             += 1
        RA0                         = self.parameters['RA0'].value
        Dec0                        = self.parameters['Dec0'].value
        sigma                       = self.parameters['sigma'].getValue(energy)
        eccentricity                = self.parameters['eccentricity'].getValue(energy)
        angle                       = np.deg2rad(self.parameters['angle'].getValue(energy))

        sigma1=sigma**2
        sigma2=sigma1*(1-eccentricity**2)
        rot=np.array([[np.cos(angle),-np.sin(angle)],[np.sin(angle),np.cos(angle)]])
        cov=np.dot(rot,np.dot(np.array([[sigma1,0],[0,sigma2]]),rot.T))
        return np.maximum(np.power(180/np.pi,2)*scipy.stats.multivariate_normal.pdf(np.array([Dec,RA]).T, mean=np.array([RA0,Dec0]), cov=cov), 1e-30)

    def integratedFlux(self,energy):

        return 1.

<|endoftext|>


<commit_before>"""The Chi-squared distribution."""
import numpy as np
from distribution import Distribution
from scipy.special import erf, erfinv, gamma, gammainc
from scipy.stats import chi2
RECURRENCE_PDF_SAMPLES = 8000

class Chisquared(Distribution):
    """
    The class defines a Chi-squared object. It is the child of Distribution.

    :param int dofs:
		Degrees of freedom for the chi-squared distribution.
    """
    def __init__(self, dofs):
        self.dofs = dofs
        if self.dofs == 1:
            self.bounds = np.array([1e-15, np.inf])
        else:
            self.bounds = np.array([0.0, np.inf])
        if self.dofs >= 1:
            self.mean = float(self.dofs)
            self.variance = 2 * self.mean
            self.skewness = np.sqrt(8.0 / self.mean)
            self.kurtosis = 12.0/self.mean + 3.0
            self.x_range_for_pdf = np.linspace(0.0, 10.0*self.mean,RECURRENCE_PDF_SAMPLES)

    def getDescription(self):
        """
        A description of the Chi-squared distribution.

        :param Chi-squared self:
            An instance of the Chi-squared class.
        :return:
            A string describing the Chi-squared distribution.
        """
        text = "A Chi-squared distribution is characterised by its degrees of freedom, which here is"+str(self.dofs)+"."
        return text

    def getPDF(self, points=None):
        """
        A Chi-squared  probability density function.

        :param Chi-squared  self:
            An instance of the Chi-squared  class.
        :param points:
            Matrix of points for defining the probability density function.
        :return:
            An array of N equidistant values over the support of the Chi-squared distribution.
        :return:
            Probability density values along the support of the Chi-squared distribution.
        """
        if points is not None:
            return chi2.pdf(points, self.dofs, loc=0.0, scale=1.0)
        else:
            raise(ValueError, 'Please digit an input for getPDF method')

    def getCDF(self, points=None):
        """
        A Chi-squared cumulative density function.

        :param Chi-squared self:
            An instance of the Chi-squared class.
        :param matrix points:
            Matrix of points for defining the cumulative density function.
        :return:
            An array of N equidistant values over the support of the Chi-squared distribution.
        :return:
            Cumulative density values along the support of the Chi-squared distribution.
        """
        if points is not None:
            return chi2.cdf(points, self.dofs,loc=0.0, scale=1.0)
        else:
            raise(ValueError, 'Please digit an input for getCDF method')

    def getiCDF(self, xx):
        """
        A Chi-squared inverse cumulative density function.

        :param Chi2:
            An instance of Chi-squared class
        :param matrix xx:
            A matrix of points at which the inverse cumulative density function need to be evaluated.
        :return:
            Inverse cumulative density function values of the Chi-squared distribution.
        """
        return chi2.ppf(xx, self.dofs, loc=0.0, scale=1.0)

    def getSamples(self, m=None):
        """
        Generates samples from the Chi-squared distribution.

        :param chi2 self:
            An instance of Chi-squared class
        :param integer m:
            Number of random samples. If no value is provided, a default of 5e05 is assumed.
        :return:
            A N-by-1 vector that contains the samples.
        """
        if m is not None:
            number = m
        else:
            number = 500000
        return chi2.rvs(self.dofs,loc=0.0, scale=1.0, size= number, random_state=None)

<commit_msg>Delete chisquared.py<commit_after><|endoftext|>


<commit_before>#   BSD LICENSE
#   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
#   All rights reserved.
#
#   Redistribution and use in source and binary forms, with or without
#   modification, are permitted provided that the following conditions
#   are met:
#
#   * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#   * Neither the name of Intel Corporation nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import subprocess
from sphinx.highlighting import PygmentsBridge
from pygments.formatters.latex import LatexFormatter

project = 'DPDK'

html_show_copyright = False

version = subprocess.check_output(['make', '-sRrC', '../../', 'showversion']).decode('utf-8')
release = version

master_doc = 'index'

latex_documents = [
    ('index',
     'doc.tex',
     '',
     '',
     'manual')
]

# Latex directives to be included directly in the latex/pdf docs.
latex_preamble = r"""
\usepackage[utf8]{inputenc}
\usepackage{DejaVuSansMono}
\usepackage[T1]{fontenc}
\usepackage{helvet}
\renewcommand{\familydefault}{\sfdefault}
\RecustomVerbatimEnvironment{Verbatim}{Verbatim}{xleftmargin=5mm}
"""

# Configuration for the latex/pdf docs.
latex_elements = {
    'papersize': 'a4paper',
    'pointsize': '11pt',
    # customize Latex formatting
    'preamble': latex_preamble
}

# Override the default Latex formatter in order to modify the
# code/verbatim blocks.
class CustomLatexFormatter(LatexFormatter):
    def __init__(self, **options):
        super(CustomLatexFormatter, self).__init__(**options)
        # Use the second smallest font size for code/verbatim blocks.
        self.verboptions = r'formatcom=\footnotesize'

# Replace the default latex formatter.
PygmentsBridge.latex_formatter = CustomLatexFormatter
<commit_msg>doc: remove blank pages in pdf<commit_after>#   BSD LICENSE
#   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
#   All rights reserved.
#
#   Redistribution and use in source and binary forms, with or without
#   modification, are permitted provided that the following conditions
#   are met:
#
#   * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#   * Neither the name of Intel Corporation nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import subprocess
from sphinx.highlighting import PygmentsBridge
from pygments.formatters.latex import LatexFormatter

project = 'DPDK'

html_show_copyright = False

version = subprocess.check_output(['make', '-sRrC', '../../', 'showversion']).decode('utf-8')
release = version

master_doc = 'index'

latex_documents = [
    ('index',
     'doc.tex',
     '',
     '',
     'manual')
]

# Latex directives to be included directly in the latex/pdf docs.
latex_preamble = r"""
\usepackage[utf8]{inputenc}
\usepackage{DejaVuSansMono}
\usepackage[T1]{fontenc}
\usepackage{helvet}
\renewcommand{\familydefault}{\sfdefault}
\RecustomVerbatimEnvironment{Verbatim}{Verbatim}{xleftmargin=5mm}
"""

# Configuration for the latex/pdf docs.
latex_elements = {
    'papersize': 'a4paper',
    'pointsize': '11pt',
    # remove blank pages
    'classoptions': ',openany,oneside',
    'babel': '\\usepackage[english]{babel}',
    # customize Latex formatting
    'preamble': latex_preamble
}

# Override the default Latex formatter in order to modify the
# code/verbatim blocks.
class CustomLatexFormatter(LatexFormatter):
    def __init__(self, **options):
        super(CustomLatexFormatter, self).__init__(**options)
        # Use the second smallest font size for code/verbatim blocks.
        self.verboptions = r'formatcom=\footnotesize'

# Replace the default latex formatter.
PygmentsBridge.latex_formatter = CustomLatexFormatter
<|endoftext|>


<commit_before>"""Restricted execution facilities.

The class RExec exports methods r_exec(), r_eval(), r_execfile(), and
r_import(), which correspond roughly to the built-in operations
exec, eval(), execfile() and import, but executing the code in an
environment that only exposes those built-in operations that are
deemed safe.  To this end, a modest collection of 'fake' modules is
created which mimics the standard modules by the same names.  It is a
policy decision which built-in modules and operations are made
available; this module provides a reasonable default, but derived
classes can change the policies e.g. by overriding or extending class
variables like ok_builtin_modules or methods like make_sys().

XXX To do:
- r_open should allow writing tmp dir
- r_exec etc. with explicit globals/locals? (Use rexec("exec ... in ...")?)

"""

import sys
import __builtin__
import os
import ihooks

__all__ = ["RExec"]

class FileBase:

    ok_file_methods = ('fileno', 'flush', 'isatty', 'read', 'readline',
            'readlines', 'seek', 'tell', 'write', 'writelines')

class FileWrapper(FileBase):

    # XXX This is just like a Bastion -- should use that!

    def __init__(self, f):
        self.f = f
        for m in self.ok_file_methods:
            if not hasattr(self, m) and hasattr(f, m):
                setattr(self, m, getattr(f, m))

    def close(self):
        self.flush()

TEMPLATE = """
def %s(self, *args):
        return apply(getattr(self.mod, self.name).%s, args)
"""

class FileDelegate(FileBase):

    def __init__(self, mod, name):
        self.mod = mod
        self.name = name

    for m in FileBase.ok_file_methods + ('close',):
        exec TEMPLATE % (m, m)

class RHooks(ihooks.Hooks):

    def __init__(self, *args):
        # Hacks to support both old and new interfaces:
        # old interface was RHooks(rexec[, verbose])
        # new interface is RHooks([verbose])
        verbose = 0
        rexec = None
        if args and type(args[-1]) == type(0):
            verbose = args[-1]
            args = args[:-1]
        if args and hasattr(args[0], '__class__'):
            rexec = args[0]
            args = args[1:]
        if args:
            raise TypeError, "too many arguments"
        ihooks.Hooks.__init__(self, verbose)
        self.rexec = rexec

    def set_rexec(self, rexec):
        # Called by RExec instance to complete initialization
        self.rexec = rexec

    def is_builtin(self, name):
        return self.rexec.is_builtin(name)

    def init_builtin(self, name):
        m = __import__(name)
        return self.rexec.copy_except(m, ())

    def init_frozen(self, name): raise SystemError, "don't use this"
    def load_source(self, *args): raise SystemError, "don't use this"
    def load_compiled(self, *args): raise SystemError, "don't use this"
    def load_package(self, *args): raise SystemError, "don't use this"

    def load_dynamic(self, name, filename, file):
        return self.rexec.load_dynamic(name, filename, file)

    def add_module(self, name):
        return self.rexec.add_module(name)

    def modules_dict(self):
        return self.rexec.modules

    def default_path(self):
        return self.rexec.modules['sys'].path

# XXX Backwards compatibility
RModuleLoader = ihooks.FancyModuleLoader
RModuleImporter = ihooks.ModuleImporter

class RExec(ihooks._Verbose):

    """Restricted Execution environment."""

    ok_path = tuple(sys.path)           # That's a policy decision

    ok_builtin_modules = ('audioop', 'array', 'binascii',
                          'cmath', 'errno', 'imageop',
                          'marshal', 'math', 'md5', 'operator',
                          'parser', 'regex', 'pcre', 'rotor', 'select',
                          'sha', '_sre', 'strop', 'struct', 'time')

    ok_posix_names = ('error', 'fstat', 'listdir', 'lstat', 'readlink',
                      'stat', 'times', 'uname', 'getpid', 'getppid',
                      'getcwd', 'getuid', 'getgid', 'geteuid', 'getegid')

    ok_sys_names = ('ps1', 'ps2', 'copyright', 'version',
                    'platform', 'exit', 'maxint')

    nok_builtin_names = ('open', 'reload', '__import__')

    def __init__(self, hooks = None, verbose = 0):
        ihooks._Verbose.__init__(self, verbose)
        # XXX There's a circular reference here:
        self.hooks = hooks or RHooks(verbose)
        self.hooks.set_rexec(self)
        self.modules = {}
        self.ok_dynamic_modules = self.ok_builtin_modules
        list = []
        for mname in self.ok_builtin_modules:
            if mname in sys.builtin_module_names:
                list.append(mname)
        self.ok_builtin_modules = tuple(list)
        self.set_trusted_path()
        self.make_builtin()
        self.make_initial_modules()
        # make_sys must be last because it adds the already created
        # modules to its builtin_module_names
        self.make_sys()
        self.loader = RModuleLoader(self.hooks, verbose)
        self.importer = RModuleImporter(self.loader, verbose)

    def set_trusted_path(self):
        # Set the path from which dynamic modules may be loaded.
        # Those dynamic modules must also occur in ok_builtin_modules
        self.trusted_path = filter(os.path.isabs, sys.path)

    def load_dynamic(self, name, filename, file):
        if name not in self.ok_dynamic_modules:
            raise ImportError, "untrusted dynamic module: %s" % name
        if sys.modules.has_key(name):
            src = sys.modules[name]
        else:
            import imp
            src = imp.load_dynamic(name, filename, file)
        dst = self.copy_except(src, [])
        return dst

    def make_initial_modules(self):
        self.make_main()
        self.make_osname()

    # Helpers for RHooks

    def is_builtin(self, mname):
        return mname in self.ok_builtin_modules

    # The make_* methods create specific built-in modules

    def make_builtin(self):
        m = self.copy_except(__builtin__, self.nok_builtin_names)
        m.__import__ = self.r_import
        m.reload = self.r_reload
        m.open = self.r_open

    def make_main(self):
        m = self.add_module('__main__')

    def make_osname(self):
        osname = os.name
        src = __import__(osname)
        dst = self.copy_only(src, self.ok_posix_names)
        dst.environ = e = {}
        for key, value in os.environ.items():
            e[key] = value

    def make_sys(self):
        m = self.copy_only(sys, self.ok_sys_names)
        m.modules = self.modules
        m.argv = ['RESTRICTED']
        m.path = map(None, self.ok_path)
        m.exc_info = self.r_exc_info
        m = self.modules['sys']
        l = self.modules.keys() + list(self.ok_builtin_modules)
        l.sort()
        m.builtin_module_names = tuple(l)

    # The copy_* methods copy existing modules with some changes

    def copy_except(self, src, exceptions):
        dst = self.copy_none(src)
        for name in dir(src):
            setattr(dst, name, getattr(src, name))
        for name in exceptions:
            try:
                delattr(dst, name)
            except AttributeError:
                pass
        return dst

    def copy_only(self, src, names):
        dst = self.copy_none(src)
        for name in names:
            try:
                value = getattr(src, name)
            except AttributeError:
                continue
            setattr(dst, name, value)
        return dst

    def copy_none(self, src):
        m = self.add_module(src.__name__)
        m.__doc__ = src.__doc__
        return m

    # Add a module -- return an existing module or create one

    def add_module(self, mname):
        if self.modules.has_key(mname):
            return self.modules[mname]
        self.modules[mname] = m = self.hooks.new_module(mname)
        m.__builtins__ = self.modules['__builtin__']
        return m

    # The r* methods are public interfaces

    def r_exec(self, code):
        m = self.add_module('__main__')
        exec code in m.__dict__

    def r_eval(self, code):
        m = self.add_module('__main__')
        return eval(code, m.__dict__)

    def r_execfile(self, file):
        m = self.add_module('__main__')
        return execfile(file, m.__dict__)

    def r_import(self, mname, globals={}, locals={}, fromlist=[]):
        return self.importer.import_module(mname, globals, locals, fromlist)

    def r_reload(self, m):
        return self.importer.reload(m)

    def r_unload(self, m):
        return self.importer.unload(m)

    # The s_* methods are similar but also swap std{in,out,err}

    def make_delegate_files(self):
        s = self.modules['sys']
        self.delegate_stdin = FileDelegate(s, 'stdin')
        self.delegate_stdout = FileDelegate(s, 'stdout')
        self.delegate_stderr = FileDelegate(s, 'stderr')
        self.restricted_stdin = FileWrapper(sys.stdin)
        self.restricted_stdout = FileWrapper(sys.stdout)
        self.restricted_stderr = FileWrapper(sys.stderr)

    def set_files(self):
        if not hasattr(self, 'save_stdin'):
            self.save_files()
        if not hasattr(self, 'delegate_stdin'):
            self.make_delegate_files()
        s = self.modules['sys']
        s.stdin = self.restricted_stdin
        s.stdout = self.restricted_stdout
        s.stderr = self.restricted_stderr
        sys.stdin = self.delegate_stdin
        sys.stdout = self.delegate_stdout
        sys.stderr = self.delegate_stderr

    def reset_files(self):
        self.restore_files()
        s = self.modules['sys']
        self.restricted_stdin = s.stdin
        self.restricted_stdout = s.stdout
        self.restricted_stderr = s.stderr

    def save_files(self):
        self.save_stdin = sys.stdin
        self.save_stdout = sys.stdout
        self.save_stderr = sys.stderr

    def restore_files(self):
        sys.stdin = self.save_stdin
        sys.stdout = self.save_stdout
        sys.stderr = self.save_stderr

    def s_apply(self, func, args=(), kw=None):
        self.save_files()
        try:
            self.set_files()
            if kw:
                r = apply(func, args, kw)
            else:
                r = apply(func, args)
        finally:
            self.restore_files()
        return r

    def s_exec(self, *args):
        return self.s_apply(self.r_exec, args)

    def s_eval(self, *args):
        return self.s_apply(self.r_eval, args)

    def s_execfile(self, *args):
        return self.s_apply(self.r_execfile, args)

    def s_import(self, *args):
        return self.s_apply(self.r_import, args)

    def s_reload(self, *args):
        return self.s_apply(self.r_reload, args)

    def s_unload(self, *args):
        return self.s_apply(self.r_unload, args)

    # Restricted open(...)

    def r_open(self, file, mode='r', buf=-1):
        if mode not in ('r', 'rb'):
            raise IOError, "can't open files for writing in restricted mode"
        return open(file, mode, buf)

    # Restricted version of sys.exc_info()

    def r_exc_info(self):
        ty, va, tr = sys.exc_info()
        tr = None
        return ty, va, tr

def test():
    import sys, getopt, traceback
    opts, args = getopt.getopt(sys.argv[1:], 'vt:')
    verbose = 0
    trusted = []
    for o, a in opts:
        if o == '-v':
            verbose = verbose+1
        if o == '-t':
            trusted.append(a)
    r = RExec(verbose=verbose)
    if trusted:
        r.ok_builtin_modules = r.ok_builtin_modules + tuple(trusted)
    if args:
        r.modules['sys'].argv = args
        r.modules['sys'].path.insert(0, os.path.dirname(args[0]))
    else:
        r.modules['sys'].path.insert(0, "")
    fp = sys.stdin
    if args and args[0] != '-':
        try:
            fp = open(args[0])
        except IOError, msg:
            print "%s: can't open file %s" % (sys.argv[0], `args[0]`)
            return 1
    if fp.isatty():
        print "*** RESTRICTED *** Python", sys.version
        print sys.copyright
        while 1:
            try:
                try:
                    s = raw_input('>>> ')
                except EOFError:
                    print
                    break
                if s and s[0] != '#':
                    s = s + '\n'
                    c = compile(s, '<stdin>', 'single')
                    r.s_exec(c)
            except SystemExit, n:
                return n
            except:
                traceback.print_exc()
    else:
        text = fp.read()
        fp.close()
        c = compile(text, fp.name, 'exec')
        try:
            r.s_exec(c)
        except SystemExit, n:
            return n
        except:
            traceback.print_exc()
            return 1

if __name__ == '__main__':
    sys.exit(test())
<commit_msg>Remove redundant import (PyChecker). Update greeting message to avoid the long copyright notice.<commit_after>"""Restricted execution facilities.

The class RExec exports methods r_exec(), r_eval(), r_execfile(), and
r_import(), which correspond roughly to the built-in operations
exec, eval(), execfile() and import, but executing the code in an
environment that only exposes those built-in operations that are
deemed safe.  To this end, a modest collection of 'fake' modules is
created which mimics the standard modules by the same names.  It is a
policy decision which built-in modules and operations are made
available; this module provides a reasonable default, but derived
classes can change the policies e.g. by overriding or extending class
variables like ok_builtin_modules or methods like make_sys().

XXX To do:
- r_open should allow writing tmp dir
- r_exec etc. with explicit globals/locals? (Use rexec("exec ... in ...")?)

"""

import sys
import __builtin__
import os
import ihooks

__all__ = ["RExec"]

class FileBase:

    ok_file_methods = ('fileno', 'flush', 'isatty', 'read', 'readline',
            'readlines', 'seek', 'tell', 'write', 'writelines')

class FileWrapper(FileBase):

    # XXX This is just like a Bastion -- should use that!

    def __init__(self, f):
        self.f = f
        for m in self.ok_file_methods:
            if not hasattr(self, m) and hasattr(f, m):
                setattr(self, m, getattr(f, m))

    def close(self):
        self.flush()

TEMPLATE = """
def %s(self, *args):
        return apply(getattr(self.mod, self.name).%s, args)
"""

class FileDelegate(FileBase):

    def __init__(self, mod, name):
        self.mod = mod
        self.name = name

    for m in FileBase.ok_file_methods + ('close',):
        exec TEMPLATE % (m, m)

class RHooks(ihooks.Hooks):

    def __init__(self, *args):
        # Hacks to support both old and new interfaces:
        # old interface was RHooks(rexec[, verbose])
        # new interface is RHooks([verbose])
        verbose = 0
        rexec = None
        if args and type(args[-1]) == type(0):
            verbose = args[-1]
            args = args[:-1]
        if args and hasattr(args[0], '__class__'):
            rexec = args[0]
            args = args[1:]
        if args:
            raise TypeError, "too many arguments"
        ihooks.Hooks.__init__(self, verbose)
        self.rexec = rexec

    def set_rexec(self, rexec):
        # Called by RExec instance to complete initialization
        self.rexec = rexec

    def is_builtin(self, name):
        return self.rexec.is_builtin(name)

    def init_builtin(self, name):
        m = __import__(name)
        return self.rexec.copy_except(m, ())

    def init_frozen(self, name): raise SystemError, "don't use this"
    def load_source(self, *args): raise SystemError, "don't use this"
    def load_compiled(self, *args): raise SystemError, "don't use this"
    def load_package(self, *args): raise SystemError, "don't use this"

    def load_dynamic(self, name, filename, file):
        return self.rexec.load_dynamic(name, filename, file)

    def add_module(self, name):
        return self.rexec.add_module(name)

    def modules_dict(self):
        return self.rexec.modules

    def default_path(self):
        return self.rexec.modules['sys'].path

# XXX Backwards compatibility
RModuleLoader = ihooks.FancyModuleLoader
RModuleImporter = ihooks.ModuleImporter

class RExec(ihooks._Verbose):

    """Restricted Execution environment."""

    ok_path = tuple(sys.path)           # That's a policy decision

    ok_builtin_modules = ('audioop', 'array', 'binascii',
                          'cmath', 'errno', 'imageop',
                          'marshal', 'math', 'md5', 'operator',
                          'parser', 'regex', 'pcre', 'rotor', 'select',
                          'sha', '_sre', 'strop', 'struct', 'time')

    ok_posix_names = ('error', 'fstat', 'listdir', 'lstat', 'readlink',
                      'stat', 'times', 'uname', 'getpid', 'getppid',
                      'getcwd', 'getuid', 'getgid', 'geteuid', 'getegid')

    ok_sys_names = ('ps1', 'ps2', 'copyright', 'version',
                    'platform', 'exit', 'maxint')

    nok_builtin_names = ('open', 'reload', '__import__')

    def __init__(self, hooks = None, verbose = 0):
        ihooks._Verbose.__init__(self, verbose)
        # XXX There's a circular reference here:
        self.hooks = hooks or RHooks(verbose)
        self.hooks.set_rexec(self)
        self.modules = {}
        self.ok_dynamic_modules = self.ok_builtin_modules
        list = []
        for mname in self.ok_builtin_modules:
            if mname in sys.builtin_module_names:
                list.append(mname)
        self.ok_builtin_modules = tuple(list)
        self.set_trusted_path()
        self.make_builtin()
        self.make_initial_modules()
        # make_sys must be last because it adds the already created
        # modules to its builtin_module_names
        self.make_sys()
        self.loader = RModuleLoader(self.hooks, verbose)
        self.importer = RModuleImporter(self.loader, verbose)

    def set_trusted_path(self):
        # Set the path from which dynamic modules may be loaded.
        # Those dynamic modules must also occur in ok_builtin_modules
        self.trusted_path = filter(os.path.isabs, sys.path)

    def load_dynamic(self, name, filename, file):
        if name not in self.ok_dynamic_modules:
            raise ImportError, "untrusted dynamic module: %s" % name
        if sys.modules.has_key(name):
            src = sys.modules[name]
        else:
            import imp
            src = imp.load_dynamic(name, filename, file)
        dst = self.copy_except(src, [])
        return dst

    def make_initial_modules(self):
        self.make_main()
        self.make_osname()

    # Helpers for RHooks

    def is_builtin(self, mname):
        return mname in self.ok_builtin_modules

    # The make_* methods create specific built-in modules

    def make_builtin(self):
        m = self.copy_except(__builtin__, self.nok_builtin_names)
        m.__import__ = self.r_import
        m.reload = self.r_reload
        m.open = self.r_open

    def make_main(self):
        m = self.add_module('__main__')

    def make_osname(self):
        osname = os.name
        src = __import__(osname)
        dst = self.copy_only(src, self.ok_posix_names)
        dst.environ = e = {}
        for key, value in os.environ.items():
            e[key] = value

    def make_sys(self):
        m = self.copy_only(sys, self.ok_sys_names)
        m.modules = self.modules
        m.argv = ['RESTRICTED']
        m.path = map(None, self.ok_path)
        m.exc_info = self.r_exc_info
        m = self.modules['sys']
        l = self.modules.keys() + list(self.ok_builtin_modules)
        l.sort()
        m.builtin_module_names = tuple(l)

    # The copy_* methods copy existing modules with some changes

    def copy_except(self, src, exceptions):
        dst = self.copy_none(src)
        for name in dir(src):
            setattr(dst, name, getattr(src, name))
        for name in exceptions:
            try:
                delattr(dst, name)
            except AttributeError:
                pass
        return dst

    def copy_only(self, src, names):
        dst = self.copy_none(src)
        for name in names:
            try:
                value = getattr(src, name)
            except AttributeError:
                continue
            setattr(dst, name, value)
        return dst

    def copy_none(self, src):
        m = self.add_module(src.__name__)
        m.__doc__ = src.__doc__
        return m

    # Add a module -- return an existing module or create one

    def add_module(self, mname):
        if self.modules.has_key(mname):
            return self.modules[mname]
        self.modules[mname] = m = self.hooks.new_module(mname)
        m.__builtins__ = self.modules['__builtin__']
        return m

    # The r* methods are public interfaces

    def r_exec(self, code):
        m = self.add_module('__main__')
        exec code in m.__dict__

    def r_eval(self, code):
        m = self.add_module('__main__')
        return eval(code, m.__dict__)

    def r_execfile(self, file):
        m = self.add_module('__main__')
        return execfile(file, m.__dict__)

    def r_import(self, mname, globals={}, locals={}, fromlist=[]):
        return self.importer.import_module(mname, globals, locals, fromlist)

    def r_reload(self, m):
        return self.importer.reload(m)

    def r_unload(self, m):
        return self.importer.unload(m)

    # The s_* methods are similar but also swap std{in,out,err}

    def make_delegate_files(self):
        s = self.modules['sys']
        self.delegate_stdin = FileDelegate(s, 'stdin')
        self.delegate_stdout = FileDelegate(s, 'stdout')
        self.delegate_stderr = FileDelegate(s, 'stderr')
        self.restricted_stdin = FileWrapper(sys.stdin)
        self.restricted_stdout = FileWrapper(sys.stdout)
        self.restricted_stderr = FileWrapper(sys.stderr)

    def set_files(self):
        if not hasattr(self, 'save_stdin'):
            self.save_files()
        if not hasattr(self, 'delegate_stdin'):
            self.make_delegate_files()
        s = self.modules['sys']
        s.stdin = self.restricted_stdin
        s.stdout = self.restricted_stdout
        s.stderr = self.restricted_stderr
        sys.stdin = self.delegate_stdin
        sys.stdout = self.delegate_stdout
        sys.stderr = self.delegate_stderr

    def reset_files(self):
        self.restore_files()
        s = self.modules['sys']
        self.restricted_stdin = s.stdin
        self.restricted_stdout = s.stdout
        self.restricted_stderr = s.stderr

    def save_files(self):
        self.save_stdin = sys.stdin
        self.save_stdout = sys.stdout
        self.save_stderr = sys.stderr

    def restore_files(self):
        sys.stdin = self.save_stdin
        sys.stdout = self.save_stdout
        sys.stderr = self.save_stderr

    def s_apply(self, func, args=(), kw=None):
        self.save_files()
        try:
            self.set_files()
            if kw:
                r = apply(func, args, kw)
            else:
                r = apply(func, args)
        finally:
            self.restore_files()
        return r

    def s_exec(self, *args):
        return self.s_apply(self.r_exec, args)

    def s_eval(self, *args):
        return self.s_apply(self.r_eval, args)

    def s_execfile(self, *args):
        return self.s_apply(self.r_execfile, args)

    def s_import(self, *args):
        return self.s_apply(self.r_import, args)

    def s_reload(self, *args):
        return self.s_apply(self.r_reload, args)

    def s_unload(self, *args):
        return self.s_apply(self.r_unload, args)

    # Restricted open(...)

    def r_open(self, file, mode='r', buf=-1):
        if mode not in ('r', 'rb'):
            raise IOError, "can't open files for writing in restricted mode"
        return open(file, mode, buf)

    # Restricted version of sys.exc_info()

    def r_exc_info(self):
        ty, va, tr = sys.exc_info()
        tr = None
        return ty, va, tr

def test():
    import getopt, traceback
    opts, args = getopt.getopt(sys.argv[1:], 'vt:')
    verbose = 0
    trusted = []
    for o, a in opts:
        if o == '-v':
            verbose = verbose+1
        if o == '-t':
            trusted.append(a)
    r = RExec(verbose=verbose)
    if trusted:
        r.ok_builtin_modules = r.ok_builtin_modules + tuple(trusted)
    if args:
        r.modules['sys'].argv = args
        r.modules['sys'].path.insert(0, os.path.dirname(args[0]))
    else:
        r.modules['sys'].path.insert(0, "")
    fp = sys.stdin
    if args and args[0] != '-':
        try:
            fp = open(args[0])
        except IOError, msg:
            print "%s: can't open file %s" % (sys.argv[0], `args[0]`)
            return 1
    if fp.isatty():
        print "*** RESTRICTED *** Python", sys.version
        print 'Type "help", "copyright", "credits" or "license" ' \
              'for more information.'

        while 1:
            try:
                try:
                    s = raw_input('>>> ')
                except EOFError:
                    print
                    break
                if s and s[0] != '#':
                    s = s + '\n'
                    c = compile(s, '<stdin>', 'single')
                    r.s_exec(c)
            except SystemExit, n:
                return n
            except:
                traceback.print_exc()
    else:
        text = fp.read()
        fp.close()
        c = compile(text, fp.name, 'exec')
        try:
            r.s_exec(c)
        except SystemExit, n:
            return n
        except:
            traceback.print_exc()
            return 1

if __name__ == '__main__':
    sys.exit(test())
<|endoftext|>


<commit_before>"""
/u/PlayStoreLinks__Bot

A Reddit Bot made by /u/cris9696

General workflow:

* login
* get comments
* analyze comments
* reply to valid comments
* shutdown

"""

#reddit
import praw
#general
import sys
import re
import cPickle as pickle
#web
import urllib
import HTMLParser
#mine
import Config
from PlayStore import PlayStore

def stopBot():
    logger.info("Shutting down")
    logger.debug("Dumping already_done")
    pickle.dump( already_done, open( "done.p", "wb" ) )
    logger.debug("Really shutting down")
    sys.exit(0)

def removeRedditFormatting(text):
    return text.replace("*", "").replace("~", "").replace("^", "").replace(">","").replace("[","").replace("]","").replace("(","").replace(")","")

def isDone(comment):
    #TODO check if in the database
    if comment.id in already_done:
        logger.debug("Already replied to '" + comment.id + "'")
        return True

    already_done.append(comment.id)
    return False

def generateReply(link_me_requests):
    my_reply = ""

    nOfRequestedApps = 0
    nOfFoundApps = 0
    for link_me_request in link_me_requests:    #for each linkme command
        requested_apps = link_me_request.split(",") #split the apps by ,

        for app_name in requested_apps:
            app_name = app_name.strip()

            if len(app_name) > 0:
                app_name = HTMLParser.HTMLParser().unescape(app_name)  #html encoding to normal encoding
                nOfRequestedApps += 1

                if nOfRequestedApps <= Config.maxAppsPerComment:
                    app = findApp(app_name)

                    if app:
                        nOfFoundApps += 1
                        my_reply += "[**" + app.name + "**](" + app.link + ") - " + ("Free" if app.free else "Paid") + " " + (" with IAP -" if app.IAP else " - ") + " Rating: " + app.rating + "/100 - "
                        my_reply += "Search for '" + app_name + "' on the [**Play Store**](https://play.google.com/store/search?q=" + urllib.quote_plus(app_name.encode("utf-8")) + ")\n\n"

                        logger.info("'" + app_name + "' found. Name: " + app.name)
                    else:
                        my_reply +="I am sorry, I can't find any app named '" + app_name + "'.\n\n"
                        logger.info("Can't find any app named '" + app_name + "'")

    if nOfRequestedApps > Config.maxAppsPerComment:
        my_reply = "You requested more than " + str(Config.maxAppsPerComment) + " apps. I will only link to the first " + str(Config.maxAppsPerComment) + " apps.\n\n" + my_reply

    my_reply += Config.closingFormula

    if nOfFoundApps == 0:   #return None because we don't want to answer
        my_reply = None

    return my_reply

def findApp(app_name):
    logger.debug("Searching for '" + app_name + "'")
    app_name = app_name.lower()
    app = None

    if len(app_name)>0:
        #app = searchInDatabase(app_name)
        #if app:
        #     return app
        # else:
        try:
            app = PlayStore.search(app_name)
            return app
        except PlayStore.AppNotFoundException as e:
            return None
    else:
        return None

def doReply(comment,myReply):
    logger.debug("Replying to '" + comment.id + "'")

    tryAgain = True
    while tryAgain:
        tryAgain = False
        try:
            # "#	\n\n###	\n\n#####	\n"
            comment.reply(myReply)
            logger.info("Successfully replied to comment '" + comment.id + "'\n")
            break
        except praw.errors.RateLimitExceeded as timeError:
            logger.warning("Doing too much, sleeping for " + str(timeError.sleep_time))
            time.sleep(timeError.sleep_time)
            tryAgain = True
        except Exception as e:
            logger.error("Exception '" + str(e) + "' occured while replying to '" + comment.id + "'!")
            stopBot()

#building the logger
import logging
logger = logging.getLogger('LinkMeBot')
logger.setLevel(Config.loggingLevel)
fh = logging.FileHandler(Config.logFile)
fh.setLevel(Config.loggingLevel)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

#main method
if __name__ == "__main__":

    logger.info("Starting up")

    logger.debug("Loading already_done")
    try:
        already_done = pickle.load( open( "done.p", "rb" ) )
    except IOError:
        already_done = []

    logger.debug("Logging in")

    try:
        r = praw.Reddit("/u/PlayStoreLinks__Bot by /u/cris9696 V2.0")
        r.login(Config.username, Config.password)
        logger.info("Successfully logged in")

    except praw.errors.RateLimitExceeded as error:
        logger.error("The Bot is doing too much! Sleeping for " + str(error.sleep_time) + " and then shutting down!")
        time.sleep(error.sleep_time)
        stopBot()

    except Exception as e:
        logger.error("Exception '" + str(e) + "' occured on login!")
        stopBot()

    subreddits = r.get_subreddit("+".join(Config.subreddits))

    link_me_regex = re.compile("\\blink[\s]*medebug[\s]*:[\s]*(.*?)(?:\.|;|$)", re.M | re.I)

    try:
        logger.debug("Getting the comments")
        comments = subreddits.get_comments()
        logger.info("Comments successfully downloaded")
    except Exception as e:
        logger.error("Exception '" + str(e) + "' occured while getting comments!")
        stopBot()

    for comment in comments:

        #to avoid injection of stuff
        clean_comment = removeRedditFormatting(comment.body)

        #match the request
        link_me_requests = link_me_regex.findall(clean_comment)
        #if it matches
        if len(link_me_requests) > 0:
            if not isDone(comment): #we check if we have not already answered to the comment
                logger.debug("Generating reply to '" + comment.id + "'")
                reply = generateReply(link_me_requests)
                if reply is not None:
                    doReply(comment,reply)
                else:
                    logger.info("No Apps found for comment '" + comment.id + "'. Ignoring reply.")
    stopBot()
<commit_msg>Implemented simple lock to prevent 2 copies running at the same time<commit_after>"""
/u/PlayStoreLinks__Bot

A Reddit Bot made by /u/cris9696

General workflow:

* login
* get comments
* analyze comments
* reply to valid comments
* shutdown

"""

#reddit
import praw
#general
import sys
import os
import re
import cPickle as pickle
#web
import urllib
import HTMLParser
#mine
import Config
from PlayStore import PlayStore

def stopBot(doSave = True):
    logger.info("Shutting down")

    if os.path.isfile("/tmp/botRunning"):
        logger.debug("Deleting lock file")
        os.remove('/tmp/botRunning')

    if doSave:
        logger.debug("Dumping already_done")
        pickle.dump( already_done, open( "done.p", "wb" ) )
        logger.debug("Really shutting down")

    sys.exit(0)

def removeRedditFormatting(text):
    return text.replace("*", "").replace("~", "").replace("^", "").replace(">","").replace("[","").replace("]","").replace("(","").replace(")","")

def isDone(comment):
    #TODO check if in the database
    if comment.id in already_done:
        logger.debug("Already replied to '" + comment.id + "'")
        return True

    already_done.append(comment.id)
    return False

def generateReply(link_me_requests):
    my_reply = ""

    nOfRequestedApps = 0
    nOfFoundApps = 0
    for link_me_request in link_me_requests:    #for each linkme command
        requested_apps = link_me_request.split(",") #split the apps by ,

        for app_name in requested_apps:
            app_name = app_name.strip()

            if len(app_name) > 0:
                app_name = HTMLParser.HTMLParser().unescape(app_name)  #html encoding to normal encoding
                nOfRequestedApps += 1

                if nOfRequestedApps <= Config.maxAppsPerComment:
                    app = findApp(app_name)

                    if app:
                        nOfFoundApps += 1
                        my_reply += "[**" + app.name + "**](" + app.link + ") - " + ("Free" if app.free else "Paid") + " " + (" with IAP -" if app.IAP else " - ") + " Rating: " + app.rating + "/100 - "
                        my_reply += "Search for '" + app_name + "' on the [**Play Store**](https://play.google.com/store/search?q=" + urllib.quote_plus(app_name.encode("utf-8")) + ")\n\n"

                        logger.info("'" + app_name + "' found. Name: " + app.name)
                    else:
                        my_reply +="I am sorry, I can't find any app named '" + app_name + "'.\n\n"
                        logger.info("Can't find any app named '" + app_name + "'")

    if nOfRequestedApps > Config.maxAppsPerComment:
        my_reply = "You requested more than " + str(Config.maxAppsPerComment) + " apps. I will only link to the first " + str(Config.maxAppsPerComment) + " apps.\n\n" + my_reply

    my_reply += Config.closingFormula

    if nOfFoundApps == 0:   #return None because we don't want to answer
        my_reply = None

    return my_reply

def findApp(app_name):
    logger.debug("Searching for '" + app_name + "'")
    app_name = app_name.lower()
    app = None

    if len(app_name)>0:
        #app = searchInDatabase(app_name)
        #if app:
        #     return app
        # else:
        try:
            app = PlayStore.search(app_name)
            return app
        except PlayStore.AppNotFoundException as e:
            return None
    else:
        return None

def doReply(comment,myReply):
    logger.debug("Replying to '" + comment.id + "'")

    tryAgain = True
    while tryAgain:
        tryAgain = False
        try:
            # "#	\n\n###	\n\n#####	\n"
            comment.reply(myReply)
            logger.info("Successfully replied to comment '" + comment.id + "'\n")
            break
        except praw.errors.RateLimitExceeded as timeError:
            logger.warning("Doing too much, sleeping for " + str(timeError.sleep_time))
            time.sleep(timeError.sleep_time)
            tryAgain = True
        except Exception as e:
            logger.error("Exception '" + str(e) + "' occured while replying to '" + comment.id + "'!")
            stopBot()

#building the logger
import logging
logger = logging.getLogger('LinkMeBot')
logger.setLevel(Config.loggingLevel)
fh = logging.FileHandler(Config.logFile)
fh.setLevel(Config.loggingLevel)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

#main method
if __name__ == "__main__":

    logger.info("Starting up")

    if os.path.isfile("/tmp/botRunning"):
        logger.warning("Bot is already running!")
        stopBot(False)
    else:
        with open('/tmp/botRunning', 'a'):
            pass

    logger.debug("Loading already_done")
    try:
        already_done = pickle.load( open( "done.p", "rb" ) )
    except IOError:
        already_done = []

    logger.debug("Logging in")

    try:
        r = praw.Reddit("/u/PlayStoreLinks__Bot by /u/cris9696 V2.0")
        r.login(Config.username, Config.password)
        logger.info("Successfully logged in")

    except praw.errors.RateLimitExceeded as error:
        logger.error("The Bot is doing too much! Sleeping for " + str(error.sleep_time) + " and then shutting down!")
        time.sleep(error.sleep_time)
        stopBot()

    except Exception as e:
        logger.error("Exception '" + str(e) + "' occured on login!")
        stopBot()

    subreddits = r.get_subreddit("+".join(Config.subreddits))

    link_me_regex = re.compile("\\blink[\s]*medebug[\s]*:[\s]*(.*?)(?:\.|;|$)", re.M | re.I)

    try:
        logger.debug("Getting the comments")
        comments = subreddits.get_comments()
        logger.info("Comments successfully downloaded")
    except Exception as e:
        logger.error("Exception '" + str(e) + "' occured while getting comments!")
        stopBot()

    for comment in comments:

        #to avoid injection of stuff
        clean_comment = removeRedditFormatting(comment.body)

        #match the request
        link_me_requests = link_me_regex.findall(clean_comment)
        #if it matches
        if len(link_me_requests) > 0:
            if not isDone(comment): #we check if we have not already answered to the comment
                logger.debug("Generating reply to '" + comment.id + "'")
                reply = generateReply(link_me_requests)
                if reply is not None:
                    doReply(comment,reply)
                else:
                    logger.info("No Apps found for comment '" + comment.id + "'. Ignoring reply.")
    stopBot()
<|endoftext|>