<commit_before>#!/usr/bin/env python

# standard library
from __future__ import division
import os
import json
import shutil
import distutils
import distutils.dir_util
import webbrowser
import datetime

# external
from sympy.physics.mechanics import ReferenceFrame, Point

# local
from .camera import PerspectiveCamera
from .server import run_server
from .light import PointLight
from ..system import System

__all__ = ['Scene']

try:
    import IPython
    from IPython.lib import backgroundjobs as bg
    from IPython.html import widgets
    from IPython.display import clear_output, display, Javascript

except ImportError:
    IPython = None

class Scene(object):
    """
    Scene class holds all the data required for the visualizations/
    animation of a system.

    It has methods for inputting the numerical data from the numerical
    integrations of Equations of Motions and convert them to JSON
    values, which can be then parsed by Javascripts(webgls).

    A scene object takes a ReferenceFrame, and a Point as required
    arguments. The reference_frame and point act as the inertial
    frame and origin with respect to which all objects are oriented
    and rendered in the visualizations

    A scene needs to be supplied with visualization_frames, Cameras,
    and Light objects, as optional arguments.
    A scene can also be supplied with the height and width of the
    browser window where visualization would be displayed.
    Default is 800 * 800.

    """
    def __init__(self, reference_frame, origin, *visualization_frames,
                 **kwargs):
        """Initializes a Scene instance.

        Parameters
        ==========
        reference_frame : ReferenceFrame
            All the transformations would be carried out with respect to
            this reference frame.
        origin : Point
            All the transformations would be carried out with respect to
            this point.
        visualization_frames : VisualizationFrame
            One or more visualization frames which are to be visualized in
            the scene.
        name : str, optional
            Name of Scene object.
        width : int or float, optional
            The width of the canvas used for visualizations. Default is
            800px.
        height : int or float, optional
            Height of the canvas used for visualizations. Default is 800px.
        camera : Camera, optional
            The camera with which to display the object. Default is
            PerspectiveCamera, with reference_frame and origin same as
            defined for this scene.
        """

        self._reference_frame = reference_frame
        self._origin = origin

        self.visualization_frames = list(visualization_frames)

        try:
            self._name = kwargs['name']
        except KeyError:
            self._name = 'unnamed'

        try:
            self._width = kwargs['width']
        except KeyError:
            self._width = 800

        try:
            self._height = kwargs['height']
        except KeyError:
            self._height = 800

        try:
            self.cameras = kwargs['cameras']
        except KeyError:
            self.cameras = [PerspectiveCamera('DefaultCamera',
                            self._reference_frame,
                            self._origin.locatenew(
                                'p_camera',
                                10*self._reference_frame.z))]

        try:
            self.lights = kwargs['lights']
        except KeyError:
            self.lights = [PointLight('DefaultLight',
                           self._reference_frame,
                           self._origin.locatenew(
                               'p_light',
                               10*self._reference_frame.z))]

    @property
    def name(self):
        """
        Returns Name of Scene.
        """
        return self._name

    @name.setter
    def name(self, new_name):
        """
        sets name of scene.
        """
        if not isinstance(new_name, str):
            raise TypeError('Name should be a valid str.')
        else:
            self._name = new_name

    @property
    def origin(self):
        """
        returns Origin of the Scene.
        """
        return self._origin

    @origin.setter
    def origin(self, new_origin):
        """
        sets origin of the scene
        """
        if not isinstance(new_origin, Point):
            raise TypeError('''origin should be a valid Point Object''')
        else:
            self._origin = new_origin

    @property
    def reference_frame(self):
        """
        returns reference_frame of the Scene.
        """
        return self._reference_frame

    @reference_frame.setter
    def reference_frame(self, new_reference_frame):
        """
        Sets reference frame for the scene.
        """
        if not isinstance(new_reference_frame, ReferenceFrame):
            raise TypeError('''reference_frame should be a valid
                                ReferenceFrame object.''')
        else:
            self._reference_frame = new_reference_frame

    def generate_visualization_json(self, dynamic_variables,
                                    constant_variables, dynamic_values,
                                    constant_values, fps=30,
                                    outfile_prefix=None):
        """
        generate_visualization_json() method generates a json str, which is
        saved to file.

        Parameters
        ==========
        dynamic_variables : Sympifyable list or tuple
            This contains all the dynamic symbols or state variables
            which are required for solving the transformation matrices
            of all the frames of the scene.

        constant_variables : Sympifyable list or tuple
            This contains all the symbols for the parameters which are
            used for defining various objects in the system.

        dynamic_values : list or tuple
            initial states of the system. The list or tuple
            should be respective to the state_sym.

        constant_values : list or tuple
            values of the parameters. The list or tuple
            should be respective to the par_sym.

        fps : int
            fps at which animation should be displayed.
            Please not that this fps should not exceed
            the hardware limit of the display device to
            be used. Default is 30fps.

        outfile_prefix : str
            A prefix to be put while saving the scene_desc
            and simulation_data files. Files will be named
            as `outfile_prefix_scene_desc.json` and
            `outfile_prefix_simulation_data.json`. If not specified
            a timestamp shall be used as the prefix.

        Returns
        =======

        The dictionary contains following keys:
        1) Width of the scene.
        2) Height of the scene.
        3) name of the scene.
        4) frames in the scene, which contains sub-dictionaries
           of all the visualization frames information.

        """

        if outfile_prefix is None:
            outfile_prefix = "_".join(str(datetime.datetime.now()).\
                                  split(".")[0].split(" "))

        #Saving the arguments for re-running simulations
        self.constant_variables = constant_variables
        self.dynamic_variables = dynamic_variables
        self.constant_values = constant_values
        self.dynamic_values = dynamic_values
        self.outfile_prefix = outfile_prefix
        self.fps = fps
        constant_map = dict(zip(constant_variables, constant_values))
        constant_variables_str = [str(i) for i in constant_variables]
        constant_map_for_json = dict(zip(constant_variables_str, constant_values))
        self.scene_json_file = outfile_prefix + "_scene_desc.json"
        self.simulation_json_file = outfile_prefix + "_simulation_data.json"

        self._simulation_data_dict = self.generate_simulation_dict(dynamic_variables,
                                                           constant_variables,
                                                           dynamic_values,
                                                           constant_values)
        self._scene_data_dict = self.generate_scene_dict(constant_map=constant_map)
        self._scene_data_dict["simulationData"] = self.simulation_json_file

        self._scene_data_dict["timeDelta"] = 1/fps
        self._scene_data_dict["timeSteps"] = len(dynamic_values)

        self._scene_data_dict["constant_map"] = constant_map_for_json
        scene_data_outfile = open(self.scene_json_file, 'w')
        scene_data_outfile.write(json.dumps(self._scene_data_dict, indent=4,
                                 separators=(',', ': ')))
        scene_data_outfile.close()

        simulation_data_outfile = open(self.simulation_json_file, 'w')
        simulation_data_outfile.write(json.dumps(self._simulation_data_dict, indent=4,
                                 separators=(',', ': ')))
        simulation_data_outfile.close()

    def generate_scene_dict(self, constant_map={}):
        """
        This method is used to create the dictionary compatible with
        PyDy visualizer. This JSON file contains all the relevant information
        required by PyDy visualizer to draw the scene on the canvas.

        """

        self._scene_info = {}
        self._scene_info["source"] = "PyDy"
        self._scene_info["name"] = self._name
        self._scene_info["newtonian_frame"] = str(self._reference_frame)
        self._scene_info["workspaceSize"] = 0.2#This should be accomodated in scene
                                                #instead of width/height of scene

        self._scene_info["objects"] = {}
        self._scene_info["cameras"] = {}
        self._scene_info["lights"] = {}

        for frame in self.visualization_frames:
            _object_info = frame.generate_scene_dict(constant_map=constant_map)
            self._scene_info["objects"].update(_object_info)

        for camera in self.cameras:
            _object_info = camera.generate_scene_dict()
            self._scene_info["cameras"].update(_object_info)

        for light in self.lights:
            _object_info = light.generate_scene_dict()
            self._scene_info["lights"].update(_object_info)

        return self._scene_info

    def generate_simulation_dict(self, dynamic_variables,
                                    constant_variables, dynamic_values,
                                    constant_values):
        """
        This method is used to create the JSON file compatible with
        PyDy visualizer. This JSON file consists of all the simulation data
        along with references to the objects, for allowing motion to the
        objects in the PyDy visualizer.

        """
        self._simulation_info = {}

        for frame in self.visualization_frames:
            frame.generate_transformation_matrix(self._reference_frame,
                                                 self._origin)
            frame.generate_numeric_transform_function(dynamic_variables,
                                                      constant_variables)
            frame.evaluate_transformation_matrix(dynamic_values,
                                                         constant_values)

            self._simulation_info.update(frame.generate_simulation_dict())

        for camera in self.cameras:
            camera.generate_transformation_matrix(self._reference_frame,
                                                 self._origin)
            camera.generate_numeric_transform_function(dynamic_variables,
                                                      constant_variables)
            camera.evaluate_transformation_matrix(dynamic_values,
                                                         constant_values)

            self._simulation_info.update(camera.generate_simulation_dict())

        for light in self.lights:
            light.generate_transformation_matrix(self._reference_frame,
                                                 self._origin)
            light.generate_numeric_transform_function(dynamic_variables,
                                                      constant_variables)
            light.evaluate_transformation_matrix(dynamic_values,
                                                         constant_values)

            self._simulation_info.update(light.generate_simulation_dict())

        return self._simulation_info

    def generate_visualization_json_system(self, system, **kwargs):
        """ Alternative method compatible with System class
        for generating visualization JSON.

        Parameters
        ----------
        system: object, pydy.System

        Keyword arguments are same as generate_visualization_json.

        """
        if not isinstance(system,System):
            raise TypeError("{} should be a valid pydy.System object".format(system))

        self.generate_visualization_json(system.states,
                                          system.constants_symbols,
                                          system.integrate(),
                                          system.constants.values(), **kwargs)

    def create_static_html(self, overwrite=False, silent=False):

        """Creates a directory named ``static`` in the current working
        directory which contains all of the HTML, CSS, and Javascript files
        required to run the visualization. Simply open ``static/index.html``
        in a WebGL compliant browser to view and interact with the
        visualization.

        This method can also be used to output files for embedding the
        visualizations in the static webpages. Simply copy the contents of
        static directory in the relevant directory for embedding in a static
        website.

        Parameters
        ----------
        overwrite : boolean, optional, default=False
            If True, the directory named ``static`` in the current working
            directory will be overwritten.
        Silent : boolean, optional, default=False
            If True, no messages will be displayed to
            STDOUT
        """

        dst = os.path.join(os.getcwd(), 'static')

        if os.path.exists(dst) and overwrite is False:
            ans = raw_input("The 'static' directory already exists. Would "
                            + "you like to overwrite the contents? [y|n]\n")
            if ans == 'y':
                distutils.dir_util.remove_tree(dst)
            else:
                if not silent: print "Aborted!"
                return

        src = os.path.join(os.path.dirname(__file__), 'static')
        if not silent: print("Copying static data.")
        distutils.dir_util.copy_tree(src, dst)
        if not silent: print("Copying Simulation data.")
        _scene_outfile_loc = os.path.join(os.getcwd(), 'static', self.scene_json_file)
        _simulation_outfile_loc = os.path.join(os.getcwd(), 'static', self.simulation_json_file)
        scene_outfile = open(_scene_outfile_loc, "w")
        simulation_outfile = open(_simulation_outfile_loc, "w")

        scene_outfile.write(json.dumps(self._scene_data_dict, indent=4,
                                separators=(',', ': ')))
        scene_outfile.close()
        simulation_outfile.write(json.dumps(self._simulation_data_dict, indent=4,
                                separators=(',', ': ')))
        simulation_outfile.close()
        if not silent: print("To view the visualization, open {}".format(
                            os.path.join(dst, 'index.html')) +
                            " in a WebGL compliant browser.")

    def remove_static_html(self, force=False):
        """Removes the ``static`` directory from the current working
        directory.

        Parameters
        ----------
        force : boolean, optional, default=False
            If true, no warning is issued before the removal of the
            directory.

        """
        if not os.path.exists('static'):
            print "All Done!"
            return

        if not force:
            ans = raw_input("Are you sure you would like to delete the " +
                                "'static' directory? [y|n]\n")
            if ans == 'y':
                force = True

        if force:
            distutils.dir_util.remove_tree(os.path.join(os.getcwd(),
                                                            'static'))
            print "All Done!"
        else:
            print "aborted!"

    def display(self):
        """
        display method can be used in two ways.
        When called from IPython notebook, it shows the visualization
        in the form of output cell in the IPython notebook.
        If it is called from python interpreter or
        IPython interpreter(not notebook), It generates an html file,
        in the current directory, which can be opened in the webgl
        compliant browser for viewing the visualizations.

        The simulation data is used from this scene, hence
        all simulation data generation methods should be called before
        calling this method

        """
        self.create_static_html()
        run_server(scene_file=self.scene_json_file)

    def display_ipython(self):
        """
        Method to display the visualization inside the
        Ipython notebook. It is only supported by IPython
        versions>=2.0.0

        """

        self.create_static_html(silent=True)
        self._widget_dict = {}
        self.container = widgets.ContainerWidget()
        components = []
        for var, init_val in \
            zip(self.constant_variables, self.constant_values):
            self._widget_dict[str(var)] = widgets.FloatTextWidget(value=init_val,
                                                              description=str(var))
            components.append(self._widget_dict[str(var)])

        self.button = widgets.ButtonWidget(description="Rerun Simulations")
        def button_click(clicked):
            self.button.add_class('disabled')
            self.button.description = 'Rerunning Simulation ...'
            self.constant_values = []
            for i in self._widget_dict.values():
                self.constant_values.append(i.value)
            self.generate_visualization_json(self.dynamic_variables,
                                    self.constant_variables, self.dynamic_values,
                                    self.constant_values,fps=self.fps,
                                    outfile_prefix=self.outfile_prefix)
            self.create_static_html(overwrite=True, silent=True)
            display(Javascript("DynamicsVisualizer.Parser.loadScene()"));
            self.button.remove_class('disabled')

            self.button.description = 'Rerun Simulation'

        self.button.on_click(button_click)
        #components.append(button)
        html_file = open("static/index_ipython.html")
        self.html_widget = widgets.HTMLWidget(value=html_file.read().format(load_url='static/' + self.scene_json_file))
        self.container.children = components
        self.container.set_css({"max-height": "10em",
                                "overflow-y": "scroll",
                                "display":"block"
                                })
        self.html_widget.set_css({"display":"block",
                                  "float":"left"
                                  })
        display(self.container)
        display(self.button)
        display(self.html_widget)
        self.button.add_class('btn-info')

<commit_msg>Save system as a var in Class<commit_after>#!/usr/bin/env python

# standard library
from __future__ import division
import os
import json
import shutil
import distutils
import distutils.dir_util
import webbrowser
import datetime

# external
from sympy.physics.mechanics import ReferenceFrame, Point

# local
from .camera import PerspectiveCamera
from .server import run_server
from .light import PointLight
from ..system import System

__all__ = ['Scene']

try:
    import IPython
    from IPython.lib import backgroundjobs as bg
    from IPython.html import widgets
    from IPython.display import clear_output, display, Javascript

except ImportError:
    IPython = None

class Scene(object):
    """
    Scene class holds all the data required for the visualizations/
    animation of a system.

    It has methods for inputting the numerical data from the numerical
    integrations of Equations of Motions and convert them to JSON
    values, which can be then parsed by Javascripts(webgls).

    A scene object takes a ReferenceFrame, and a Point as required
    arguments. The reference_frame and point act as the inertial
    frame and origin with respect to which all objects are oriented
    and rendered in the visualizations

    A scene needs to be supplied with visualization_frames, Cameras,
    and Light objects, as optional arguments.
    A scene can also be supplied with the height and width of the
    browser window where visualization would be displayed.
    Default is 800 * 800.

    """
    def __init__(self, reference_frame, origin, *visualization_frames,
                 **kwargs):
        """Initializes a Scene instance.

        Parameters
        ==========
        reference_frame : ReferenceFrame
            All the transformations would be carried out with respect to
            this reference frame.
        origin : Point
            All the transformations would be carried out with respect to
            this point.
        visualization_frames : VisualizationFrame
            One or more visualization frames which are to be visualized in
            the scene.
        name : str, optional
            Name of Scene object.
        width : int or float, optional
            The width of the canvas used for visualizations. Default is
            800px.
        height : int or float, optional
            Height of the canvas used for visualizations. Default is 800px.
        camera : Camera, optional
            The camera with which to display the object. Default is
            PerspectiveCamera, with reference_frame and origin same as
            defined for this scene.
        """

        self._reference_frame = reference_frame
        self._origin = origin

        self.visualization_frames = list(visualization_frames)

        try:
            self._name = kwargs['name']
        except KeyError:
            self._name = 'unnamed'

        try:
            self._width = kwargs['width']
        except KeyError:
            self._width = 800

        try:
            self._height = kwargs['height']
        except KeyError:
            self._height = 800

        try:
            self.cameras = kwargs['cameras']
        except KeyError:
            self.cameras = [PerspectiveCamera('DefaultCamera',
                            self._reference_frame,
                            self._origin.locatenew(
                                'p_camera',
                                10*self._reference_frame.z))]

        try:
            self.lights = kwargs['lights']
        except KeyError:
            self.lights = [PointLight('DefaultLight',
                           self._reference_frame,
                           self._origin.locatenew(
                               'p_light',
                               10*self._reference_frame.z))]

    @property
    def name(self):
        """
        Returns Name of Scene.
        """
        return self._name

    @name.setter
    def name(self, new_name):
        """
        sets name of scene.
        """
        if not isinstance(new_name, str):
            raise TypeError('Name should be a valid str.')
        else:
            self._name = new_name

    @property
    def origin(self):
        """
        returns Origin of the Scene.
        """
        return self._origin

    @origin.setter
    def origin(self, new_origin):
        """
        sets origin of the scene
        """
        if not isinstance(new_origin, Point):
            raise TypeError('''origin should be a valid Point Object''')
        else:
            self._origin = new_origin

    @property
    def reference_frame(self):
        """
        returns reference_frame of the Scene.
        """
        return self._reference_frame

    @reference_frame.setter
    def reference_frame(self, new_reference_frame):
        """
        Sets reference frame for the scene.
        """
        if not isinstance(new_reference_frame, ReferenceFrame):
            raise TypeError('''reference_frame should be a valid
                                ReferenceFrame object.''')
        else:
            self._reference_frame = new_reference_frame

    def generate_visualization_json(self, dynamic_variables,
                                    constant_variables, dynamic_values,
                                    constant_values, fps=30,
                                    outfile_prefix=None):
        """
        generate_visualization_json() method generates a json str, which is
        saved to file.

        Parameters
        ==========
        dynamic_variables : Sympifyable list or tuple
            This contains all the dynamic symbols or state variables
            which are required for solving the transformation matrices
            of all the frames of the scene.

        constant_variables : Sympifyable list or tuple
            This contains all the symbols for the parameters which are
            used for defining various objects in the system.

        dynamic_values : list or tuple
            initial states of the system. The list or tuple
            should be respective to the state_sym.

        constant_values : list or tuple
            values of the parameters. The list or tuple
            should be respective to the par_sym.

        fps : int
            fps at which animation should be displayed.
            Please not that this fps should not exceed
            the hardware limit of the display device to
            be used. Default is 30fps.

        outfile_prefix : str
            A prefix to be put while saving the scene_desc
            and simulation_data files. Files will be named
            as `outfile_prefix_scene_desc.json` and
            `outfile_prefix_simulation_data.json`. If not specified
            a timestamp shall be used as the prefix.

        Returns
        =======

        The dictionary contains following keys:
        1) Width of the scene.
        2) Height of the scene.
        3) name of the scene.
        4) frames in the scene, which contains sub-dictionaries
           of all the visualization frames information.

        """

        if outfile_prefix is None:
            outfile_prefix = "_".join(str(datetime.datetime.now()).\
                                  split(".")[0].split(" "))

        #Saving the arguments for re-running simulations
        self.constant_variables = constant_variables
        self.dynamic_variables = dynamic_variables
        self.constant_values = constant_values
        self.dynamic_values = dynamic_values
        self.outfile_prefix = outfile_prefix
        self.fps = fps
        constant_map = dict(zip(constant_variables, constant_values))
        constant_variables_str = [str(i) for i in constant_variables]
        constant_map_for_json = dict(zip(constant_variables_str, constant_values))
        self.scene_json_file = outfile_prefix + "_scene_desc.json"
        self.simulation_json_file = outfile_prefix + "_simulation_data.json"

        self._simulation_data_dict = self.generate_simulation_dict(dynamic_variables,
                                                           constant_variables,
                                                           dynamic_values,
                                                           constant_values)
        self._scene_data_dict = self.generate_scene_dict(constant_map=constant_map)
        self._scene_data_dict["simulationData"] = self.simulation_json_file

        self._scene_data_dict["timeDelta"] = 1/fps
        self._scene_data_dict["timeSteps"] = len(dynamic_values)

        self._scene_data_dict["constant_map"] = constant_map_for_json
        scene_data_outfile = open(self.scene_json_file, 'w')
        scene_data_outfile.write(json.dumps(self._scene_data_dict, indent=4,
                                 separators=(',', ': ')))
        scene_data_outfile.close()

        simulation_data_outfile = open(self.simulation_json_file, 'w')
        simulation_data_outfile.write(json.dumps(self._simulation_data_dict, indent=4,
                                 separators=(',', ': ')))
        simulation_data_outfile.close()

    def generate_scene_dict(self, constant_map={}):
        """
        This method is used to create the dictionary compatible with
        PyDy visualizer. This JSON file contains all the relevant information
        required by PyDy visualizer to draw the scene on the canvas.

        """

        self._scene_info = {}
        self._scene_info["source"] = "PyDy"
        self._scene_info["name"] = self._name
        self._scene_info["newtonian_frame"] = str(self._reference_frame)
        self._scene_info["workspaceSize"] = 0.2#This should be accomodated in scene
                                                #instead of width/height of scene

        self._scene_info["objects"] = {}
        self._scene_info["cameras"] = {}
        self._scene_info["lights"] = {}

        for frame in self.visualization_frames:
            _object_info = frame.generate_scene_dict(constant_map=constant_map)
            self._scene_info["objects"].update(_object_info)

        for camera in self.cameras:
            _object_info = camera.generate_scene_dict()
            self._scene_info["cameras"].update(_object_info)

        for light in self.lights:
            _object_info = light.generate_scene_dict()
            self._scene_info["lights"].update(_object_info)

        return self._scene_info

    def generate_simulation_dict(self, dynamic_variables,
                                    constant_variables, dynamic_values,
                                    constant_values):
        """
        This method is used to create the JSON file compatible with
        PyDy visualizer. This JSON file consists of all the simulation data
        along with references to the objects, for allowing motion to the
        objects in the PyDy visualizer.

        """
        self._simulation_info = {}

        for frame in self.visualization_frames:
            frame.generate_transformation_matrix(self._reference_frame,
                                                 self._origin)
            frame.generate_numeric_transform_function(dynamic_variables,
                                                      constant_variables)
            frame.evaluate_transformation_matrix(dynamic_values,
                                                         constant_values)

            self._simulation_info.update(frame.generate_simulation_dict())

        for camera in self.cameras:
            camera.generate_transformation_matrix(self._reference_frame,
                                                 self._origin)
            camera.generate_numeric_transform_function(dynamic_variables,
                                                      constant_variables)
            camera.evaluate_transformation_matrix(dynamic_values,
                                                         constant_values)

            self._simulation_info.update(camera.generate_simulation_dict())

        for light in self.lights:
            light.generate_transformation_matrix(self._reference_frame,
                                                 self._origin)
            light.generate_numeric_transform_function(dynamic_variables,
                                                      constant_variables)
            light.evaluate_transformation_matrix(dynamic_values,
                                                         constant_values)

            self._simulation_info.update(light.generate_simulation_dict())

        return self._simulation_info

    def generate_visualization_json_system(self, system, **kwargs):
        """ Alternative method compatible with System class
        for generating visualization JSON.

        Parameters
        ----------
        system: object, pydy.System

        Keyword arguments are same as generate_visualization_json.

        """
        if not isinstance(system,System):
            self.system = None
            raise TypeError("{} should be a valid pydy.System object".format(system))
        else:
            #save system
            self.system = system

        self.generate_visualization_json(system.states,
                                          system.constants_symbols,
                                          system.integrate(),
                                          system.constants.values(), **kwargs)

    def create_static_html(self, overwrite=False, silent=False):

        """Creates a directory named ``static`` in the current working
        directory which contains all of the HTML, CSS, and Javascript files
        required to run the visualization. Simply open ``static/index.html``
        in a WebGL compliant browser to view and interact with the
        visualization.

        This method can also be used to output files for embedding the
        visualizations in the static webpages. Simply copy the contents of
        static directory in the relevant directory for embedding in a static
        website.

        Parameters
        ----------
        overwrite : boolean, optional, default=False
            If True, the directory named ``static`` in the current working
            directory will be overwritten.
        Silent : boolean, optional, default=False
            If True, no messages will be displayed to
            STDOUT
        """

        dst = os.path.join(os.getcwd(), 'static')

        if os.path.exists(dst) and overwrite is False:
            ans = raw_input("The 'static' directory already exists. Would "
                            + "you like to overwrite the contents? [y|n]\n")
            if ans == 'y':
                distutils.dir_util.remove_tree(dst)
            else:
                if not silent: print "Aborted!"
                return

        src = os.path.join(os.path.dirname(__file__), 'static')
        if not silent: print("Copying static data.")
        distutils.dir_util.copy_tree(src, dst)
        if not silent: print("Copying Simulation data.")
        _scene_outfile_loc = os.path.join(os.getcwd(), 'static', self.scene_json_file)
        _simulation_outfile_loc = os.path.join(os.getcwd(), 'static', self.simulation_json_file)
        scene_outfile = open(_scene_outfile_loc, "w")
        simulation_outfile = open(_simulation_outfile_loc, "w")

        scene_outfile.write(json.dumps(self._scene_data_dict, indent=4,
                                separators=(',', ': ')))
        scene_outfile.close()
        simulation_outfile.write(json.dumps(self._simulation_data_dict, indent=4,
                                separators=(',', ': ')))
        simulation_outfile.close()
        if not silent: print("To view the visualization, open {}".format(
                            os.path.join(dst, 'index.html')) +
                            " in a WebGL compliant browser.")

    def remove_static_html(self, force=False):
        """Removes the ``static`` directory from the current working
        directory.

        Parameters
        ----------
        force : boolean, optional, default=False
            If true, no warning is issued before the removal of the
            directory.

        """
        if not os.path.exists('static'):
            print "All Done!"
            return

        if not force:
            ans = raw_input("Are you sure you would like to delete the " +
                                "'static' directory? [y|n]\n")
            if ans == 'y':
                force = True

        if force:
            distutils.dir_util.remove_tree(os.path.join(os.getcwd(),
                                                            'static'))
            print "All Done!"
        else:
            print "aborted!"

    def display(self):
        """
        display method can be used in two ways.
        When called from IPython notebook, it shows the visualization
        in the form of output cell in the IPython notebook.
        If it is called from python interpreter or
        IPython interpreter(not notebook), It generates an html file,
        in the current directory, which can be opened in the webgl
        compliant browser for viewing the visualizations.

        The simulation data is used from this scene, hence
        all simulation data generation methods should be called before
        calling this method

        """
        self.create_static_html()
        run_server(scene_file=self.scene_json_file)

    def display_ipython(self):
        """
        Method to display the visualization inside the
        Ipython notebook. It is only supported by IPython
        versions>=2.0.0

        """

        self.create_static_html(silent=True)
        self._widget_dict = {}
        self.container = widgets.ContainerWidget()
        components = []
        for var, init_val in \
            zip(self.constant_variables, self.constant_values):
            self._widget_dict[str(var)] = widgets.FloatTextWidget(value=init_val,
                                                              description=str(var))
            components.append(self._widget_dict[str(var)])

        self.button = widgets.ButtonWidget(description="Rerun Simulations")
        def button_click(clicked):
            self.button.add_class('disabled')
            self.button.description = 'Rerunning Simulation ...'
            self.constant_values = []
            for i in self._widget_dict.values():
                self.constant_values.append(i.value)
            self.generate_visualization_json(self.dynamic_variables,
                                    self.constant_variables, self.dynamic_values,
                                    self.constant_values,fps=self.fps,
                                    outfile_prefix=self.outfile_prefix)
            self.create_static_html(overwrite=True, silent=True)
            display(Javascript("DynamicsVisualizer.Parser.loadScene()"));
            self.button.remove_class('disabled')

            self.button.description = 'Rerun Simulation'

        self.button.on_click(button_click)
        #components.append(button)
        html_file = open("static/index_ipython.html")
        self.html_widget = widgets.HTMLWidget(value=html_file.read().format(load_url='static/' + self.scene_json_file))
        self.container.children = components
        self.container.set_css({"max-height": "10em",
                                "overflow-y": "scroll",
                                "display":"block"
                                })
        self.html_widget.set_css({"display":"block",
                                  "float":"left"
                                  })
        display(self.container)
        display(self.button)
        display(self.html_widget)
        self.button.add_class('btn-info')

<|endoftext|>


<commit_before>import json, ast, urllib, requests, sys

class user:
	# Example:
	# client = kippt_wrapper.user('myUsername','kjsdfklj2lhg323423klj42')
	def __init__(self, username, apitoken):
		self.username = username
		self.apitoken = apitoken
		self.header = {'X-Kippt-Username': username, 'X-Kippt-API-Token': apitoken, 'X-Kippt-Client': 'Kippt-Python-Wrapper,me@ThomasBiddle.com,https://github.com/thomasbiddle/Kippt-Projects', 'content-type': 'application/vnd.kippt.20120609+json'}

	# Check if our credentials are valid.
	# Example:
	# user.checkAuth()
	#
	# Return True on success and False on failure.
	def checkAuth(self):
		r = requests.get('https://kippt.com/api/account/', headers=self.header)
		if r.status_code is 200: return True
		else: return False

	# Get our lists.
	# Example:
	# meta, lists = user.getLists(offset = 5)
	# meta, lists = user.getLists(50, 5) # (limit, offset)
	# meta, lists = user.getLists()
	# x = meta['total_count']
	# for i in lists: print i['title'] ( Returns Python list of Kippt Lists )
	#
	# Available values in meta:
	# total_count, limit, offset
	# Available values in each list:
	# rss_url, updated, title, created, slug, id, resource_uri
	#
	# Returns data on success, and false on failure.
	def getLists(self, limit = 0, offset = 0):
		url = 'https://kippt.com/api/lists?limit=' + str(limit) + '&offset=' + str(offset)
		r = requests.get(url, headers=self.header)
		if r.status_code is 200: return r.json['meta'], r.json['objects']
		else: return False, False

	# Get a list.
	# Example:
	# myList = user.getList(54433)
	# x = myList['title']
	#
	# Available values in list:
	# rss_url, updated, title, created, slug, id, resource_uri
	#
	# Returns data on success, and false on failure.
	def getList(self, id):
		r = requests.get('https://kippt.com/api/lists/' + str(id), headers=self.header)
		if r.status_code is 200: return r.json
		else: return False

	# Get list collaborators
	# Example:
	# myCollabs = user.getListCollab(54433)
	# for i in myCollabs: print i['username']
	#
	# Available values in list:
	# username, avatar_url, id, resource_uri
	#
	# Returns data on success, and false on failure.
	def getListCollab(self, id):
		r = requests.get('https://kippt.com/api/lists/' + str(id) + '/collaborators', headers=self.header)
		if r.status_code is 200: return r.json
		else: return False

	# Get our clips.
	# Example:
	# myClips = user.getClips()
	# myClips = user.getClips(54332, 20, 5) # (listID, limit, offset)
	# myClips = user.getClips(limit = 20)
	#
	# Available values in meta:
	# total_count, limit, offset
	# Available values in clip:
	# id, url, title, list, notes, is_starred, url_domain, created, updated, resource_uri
	#
	# Returns data on success, and false on failure.
	def getClips(self, listID = None, limit = 0, offset = 0):
		url = 'https://kippt.com/api/clips?limit=' + str(limit) + '&offset=' + str(offset)
		if not listID is None: url = url + '&list=' + str(listID)
		r = requests.get(url, headers=self.header)
		if r.status_code is 200: return r.json['meta'], r.json['objects']
		else: return False, False

	# Get a clip.
	# Example:
	# myClip = user.getClip(2027593)
	# x = myClip['title']
	#
	# Available values in clip:
	# id, url, title, list, notes, is_starred, url_domain, created, updated, resource_uri
	#
	# Returns data on success, and false on failure.
	def getClip(self, id):
		r = requests.get('https://kippt.com/api/clips/' + str(id), headers=self.header)
		if r.status_code is 200: return r.json
		else: return False

	# Search for a query.
	# Example:
	# mySearch = user.search("Programming")
	#
	# Available values in meta:
	# total_count, limit, offset
	# Available values in clip:
	# id, url, title, list, notes, is_starred, url_domain, created, updated, resource_uri
	#
	# Returns data on success, and false on failure.
	def search(self, query, limit = 0, offset = 0):
		query = urllib.quote_plus(query)
		r = requests.get('https://kippt.com/api/search/clips/?q=' + query + '&limit=' + str(limit) + '&offset=' + str(offset), headers=self.header)
		if r.status_code is 200: return r.json['meta'], r.json['objects']
		else: return False, False

	# Create a list.
	# Examples:
	# user.createList('My New List!')
	#
	# Will return data on success and False on failure.
	def createList(self, name):
		clipdata = {'title': name}
		r = requests.post('https://kippt.com/api/lists/', data=json.dumps(clipdata), headers=self.header)
		if r.status_code is 201: return r.json
		else: return False

	# Add a clip.
	# Examples:
	# user.addClip('www.kippt.com')
	# user.addClip('www.kippt.com',title="My Title!")
	# user.addClip('www.kippt.com',1234,starred="true",notes='My Notes!')
	#
	# Will return data on success and False on failure.
	def addClip(self, url, listID=0, title = None, starred = None, notes = None, ):
		clipdata = {'url': url, 'list': '/api/lists/' + str(listID)}
		if not title is None: clipdata['title'] = title
		if not starred is None: clipdata['is_starred'] = starred
		if not notes is None: clipdata['notes'] = notes
		r = requests.post('https://kippt.com/api/clips/', data=json.dumps(clipdata), headers=self.header)
		if r.status_code is 201: return r.json
		else: return False

	# Create a list.
	# Examples:
	# newList = user.createList('Programming')
	# print newList
	def createList(self, name):
		clipdata = {'title': name}
		r = requests.post('https://kippt.com/api/lists/', data=json.dumps(clipdata), headers=self.header)
		if r.status_code is 201: return r.json
		else: return False

	# Delete a clip ( Only clip owners can modify or delete clips, not collaborators! )
	# Examples:
	# user.deleteClip(2028643)
	#
	# Will return True on success, and False on failure
	def deleteClip(self, id):
		r = requests.delete('https://kippt.com/api/clips/' + str(id), headers=self.header)
		if r.status_code is 204: return True
		else: return False

	# Delete a List
	# Examples:
	# user.deleteList(54433)
	#
	# Will return True on success, and False on failure
	def deleteList(self, id):
		r = requests.delete('https://kippt.com/api/lists/' + str(id), headers=self.header)
		if r.status_code is 204: return True
		else: return False

	# Update a Clip ( Only clip owners can modify or delete clips, not collaborators! )
	# Example:
	# pyRespUpdateClip(id=2027593, list_uri='/api/lists/55284/')
	#
	# Returns True on success, False on failure.
	def updateClip(self, id, title = None, notes = None, listID = None, starred = None):
		clipdata = {}
		if not title is None: clipdata['title'] = title
		if not notes is None: clipdata['notes'] = notes
		if not starred is None: clipdata['is_starred'] = starred
		if not listID is None: clipdata['list'] = '/api/lists/55284/' + listID
		r = requests.put('https://kippt.com/api/clips/' + str(id), data=json.dumps(clipdata), headers=self.header)
		if r.status_code is 200: return True
		else: return False

	# Updating List ( With Python Requests )
	# Example:
	# client.updateList(55284, title="New Title!")
	#
	# Returns True on success, False on failure.
	def updateList(self, id, title = None):
		clipdata = {}
		if not title is None: clipdata['title'] = title
		r = requests.put('https://kippt.com/api/lists/' + str(id), data=json.dumps(clipdata), headers=self.header)
		if r.status_code is 200: return True
		else: return False

# Testing purposes ( Faster than using the interpreter every time ;) )
if __name__ == '__main__':
	if len(sys.argv) != 3:
		print("Looks like your forgot something, or added too much!")
		print("Please start the script like so: python kippt_wrapper.py <username> <API_Token>")
	else:
		client = user(sys.argv[1],sys.argv[2])
		print(client.getLists())

<commit_msg>pep8-ified, fixed python3 bug<commit_after>import json, ast, requests, sys

# python 2/3 mismatch, no workaround
try:
    from urllib import quote_plus
except ImportError:
    from urllib.parse import quote_plus

class user:
    # Example:
    # client = kippt_wrapper.user('myUsername','kjsdfklj2lhg323423klj42')
    def __init__(self, username, apitoken):
        self.username = username
        self.apitoken = apitoken
        self.header = {'X-Kippt-Username': username,
                       'X-Kippt-API-Token': apitoken,
                       'X-Kippt-Client': 'Kippt-Python-Wrapper,me@ThomasBiddle.com,https://github.com/thomasbiddle/Kippt-Projects',
                       'content-type': 'application/vnd.kippt.20120609+json'}

    # Check if our credentials are valid.
    # Example:
    # user.checkAuth()
    #
    # Return True on success and False on failure.
    def checkAuth(self):
        r = requests.get('https://kippt.com/api/account/', headers=self.header)
        if r.status_code is 200: return True
        else: return False

    # Get our lists.
    # Example:
    # meta, lists = user.getLists(offset = 5)
    # meta, lists = user.getLists(50, 5) # (limit, offset)
    # meta, lists = user.getLists()
    # x = meta['total_count']
    # for i in lists: print i['title'] ( Returns Python list of Kippt Lists )
    #
    # Available values in meta:
    # total_count, limit, offset
    # Available values in each list:
    # rss_url, updated, title, created, slug, id, resource_uri
    #
    # Returns data on success, and false on failure.
    def getLists(self, limit = 0, offset = 0):
        url = 'https://kippt.com/api/lists?limit=' + str(limit) + '&offset=' + str(offset)
        r = requests.get(url, headers=self.header)
        if r.status_code is 200: return r.json['meta'], r.json['objects']
        else: return False, False

    # Get a list.
    # Example:
    # myList = user.getList(54433)
    # x = myList['title']
    #
    # Available values in list:
    # rss_url, updated, title, created, slug, id, resource_uri
    #
    # Returns data on success, and false on failure.
    def getList(self, id):
        r = requests.get('https://kippt.com/api/lists/' + str(id), headers=self.header)
        if r.status_code is 200: return r.json
        else: return False

    # Get list collaborators
    # Example:
    # myCollabs = user.getListCollab(54433)
    # for i in myCollabs: print i['username']
    #
    # Available values in list:
    # username, avatar_url, id, resource_uri
    #
    # Returns data on success, and false on failure.
    def getListCollab(self, id):
        r = requests.get('https://kippt.com/api/lists/' + str(id) + '/collaborators', headers=self.header)
        if r.status_code is 200: return r.json
        else: return False

    # Get our clips.
    # Example:
    # myClips = user.getClips()
    # myClips = user.getClips(54332, 20, 5) # (listID, limit, offset)
    # myClips = user.getClips(limit = 20)
    #
    # Available values in meta:
    # total_count, limit, offset
    # Available values in clip:
    # id, url, title, list, notes, is_starred, url_domain, created, updated, resource_uri
    #
    # Returns data on success, and false on failure.
    def getClips(self, listID = None, limit = 0, offset = 0):
        params = { "limit": str(limit),
                   "offset": str(offset) }
        url = 'https://kippt.com/api/clips?limit={limit}&offset={offset}'.format(**params)
        if not listID is None: url = url + '&list=' + str(listID)
        r = requests.get(url, headers=self.header)
        if r.status_code is 200: return r.json['meta'], r.json['objects']
        else: return False, False

    # Get a clip.
    # Example:
    # myClip = user.getClip(2027593)
    # x = myClip['title']
    #
    # Available values in clip:
    # id, url, title, list, notes, is_starred, url_domain, created, updated, resource_uri
    #
    # Returns data on success, and false on failure.
    def getClip(self, id):
        r = requests.get('https://kippt.com/api/clips/' + str(id), headers=self.header)
        if r.status_code is 200: return r.json
        else: return False

    # Search for a query.
    # Example:
    # mySearch = user.search("Programming")
    #
    # Available values in meta:
    # total_count, limit, offset
    # Available values in clip:
    # id, url, title, list, notes, is_starred, url_domain, created, updated, resource_uri
    #
    # Returns data on success, and false on failure.
    def search(self, query, limit = 0, offset = 0):
        query = quote_plus(query)
        params = { "query": query,
                    "limit": str(limit),
                    "offset": str(offset)
        }
        r = requests.get('https://kippt.com/api/search/clips/?q={query}&limit={limit}&offset={offset}'.format(**params),
                        headers=self.header)
        if r.status_code is 200: return r.json['meta'], r.json['objects']
        else: return False, False

    # Create a list.
    # Examples:
    # user.createList('My New List!')
    #
    # Will return data on success and False on failure.
    def createList(self, name):
        clipdata = {'title': name}
        r = requests.post('https://kippt.com/api/lists/', data=json.dumps(clipdata), headers=self.header)
        if r.status_code is 201: return r.json
        else: return False

    # Add a clip.
    # Examples:
    # user.addClip('www.kippt.com')
    # user.addClip('www.kippt.com',title="My Title!")
    # user.addClip('www.kippt.com',1234,starred="true",notes='My Notes!')
    #
    # Will return data on success and False on failure.
    def addClip(self, url, listID=0, title = None, starred = None, notes = None, ):
        clipdata = {'url': url, 'list': '/api/lists/' + str(listID)}
        if not title is None: clipdata['title'] = title
        if not starred is None: clipdata['is_starred'] = starred
        if not notes is None: clipdata['notes'] = notes
        r = requests.post('https://kippt.com/api/clips/', data=json.dumps(clipdata), headers=self.header)
        if r.status_code is 201: return r.json
        else: return False

    # Create a list.
    # Examples:
    # newList = user.createList('Programming')
    # print newList
    def createList(self, name):
        clipdata = {'title': name}
        r = requests.post('https://kippt.com/api/lists/', data=json.dumps(clipdata), headers=self.header)
        if r.status_code is 201: return r.json
        else: return False

    # Delete a clip ( Only clip owners can modify or delete clips, not collaborators! )
    # Examples:
    # user.deleteClip(2028643)
    #
    # Will return True on success, and False on failure
    def deleteClip(self, id):
        r = requests.delete('https://kippt.com/api/clips/' + str(id), headers=self.header)
        if r.status_code is 204: return True
        else: return False

    # Delete a List
    # Examples:
    # user.deleteList(54433)
    #
    # Will return True on success, and False on failure
    def deleteList(self, id):
        r = requests.delete('https://kippt.com/api/lists/' + str(id), headers=self.header)
        if r.status_code is 204: return True
        else: return False

    # Update a Clip ( Only clip owners can modify or delete clips, not collaborators! )
    # Example:
    # pyRespUpdateClip(id=2027593, list_uri='/api/lists/55284/')
    #
    # Returns True on success, False on failure.
    def updateClip(self, id, title = None, notes = None, listID = None, starred = None):
        clipdata = {}
        if not title is None: clipdata['title'] = title
        if not notes is None: clipdata['notes'] = notes
        if not starred is None: clipdata['is_starred'] = starred
        if not listID is None: clipdata['list'] = '/api/lists/55284/' + listID
        r = requests.put('https://kippt.com/api/clips/' + str(id), data=json.dumps(clipdata), headers=self.header)
        if r.status_code is 200: return True
        else: return False

    # Updating List ( With Python Requests )
    # Example:
    # client.updateList(55284, title="New Title!")
    #
    # Returns True on success, False on failure.
    def updateList(self, id, title = None):
        clipdata = {}
        if not title is None: clipdata['title'] = title
        r = requests.put('https://kippt.com/api/lists/' + str(id), data=json.dumps(clipdata), headers=self.header)
        if r.status_code is 200: return True
        else: return False

# Testing purposes ( Faster than using the interpreter every time ;) )
if __name__ == '__main__':
    if len(sys.argv) != 3:
        print("Looks like your forgot something, or added too much!")
        print("Please start the script like so: python kippt_wrapper.py <username> <API_Token>")
    else:
        client = user(sys.argv[1],sys.argv[2])
        print(client.getLists())

<|endoftext|>


<commit_before>
from entity import Entity
from component import Component
from .decorators import *
import components
from application import Application<commit_msg>add profile function to pyecs exports<commit_after>
from entity import Entity
from component import Component
from .decorators import *
import components
from application import Application, profile<|endoftext|>


<commit_before>"""
Python EDA Package

boolalg.py
"""

__copyright__ = "Copyright (c) 2012, Chris Drake"
__license__ = "All rights reserved"

__version__ = "0.0.1"
<commit_msg>Update to version 0.1.0<commit_after>"""
Python EDA Package

boolalg.py
"""

__copyright__ = "Copyright (c) 2012, Chris Drake"
__license__ = "All rights reserved"

__version__ = "0.1.0"
<|endoftext|>


<commit_before>import re
import os
import sys

from nltk.corpus import stopwords
stop = stopwords.words('english')

import gensim
import string
import redis
import json
import gzip
from unidecode import unidecode

import logging
import argparse

from phrases_extractor import get_phrases

price_obj = redis.Redis("localhost", port=6379, db=1)

logger = logging.getLogger(__name__)

def parse(filename):
    g = gzip.open(filename,"r")
    for l in g:
      yield json.dumps(eval(l))

def getRange(price):

    price  = float(price)
    if price <= 10:
      return 10
    elif 10 < price <= 100:
      return 100
    elif 100 < price <= 250:
      return 250
    elif 250 < price <= 500:
      return 500
    elif 500 < price <= 1000:
      return 1000
    elif 1000 < price <= 2000:
      return 2000
    elif 2000 < price <= 5000:
      return 5000
    elif price > 5000:
      return 9999

def process_phrases(text):
    text = unidecode(text)
    phrases = get_phrases(text)
    for phrase in phrases:
      text = text.replace(phrase, phrase.replace(' ', '_'))
    return text

def pre_process(m):
    m = unidecode(m)
    m = m.lower().strip() #normalize
    #print m
    #m=re.sub("[^a-zA-Z]", " ",m) #keep text only
    #m="".join(l for l in m if l not in string.punctuation) #remove punctuation
    m = m.strip("./@!#")
    m=[i for i in m.split() if i not in stop] #remove stopwords
    #m=m.split()
    print m
    return m

#iterator on Amazon Reviews
class MySentences(object):
    def __init__(self, dirname):
        self.dirname = dirname

    def __iter__(self):
        print os.listdir(self.dirname)
        for fname in os.listdir(self.dirname):
            logger.info('Processing file %s', fname)
            count=0
            l=0
            for e in parse((os.path.join(self.dirname, fname))):
                review= json.loads(e).get("reviewText")
                title = json.loads(e).get("summary")
                category = fname.strip('reviews json gz . _')
                all_text = title + ". " + review
                all_text = process_phrases(all_text)

                if len(review.split())>10: # don't consider reviews less than 10 words
                  productId = json.loads(e).get("asin")
                  try:
                    price = getRange(price_obj.get(productId))
                  except:
                    price = 0

                  final_data = "<" + str(price) + "> <" + category + "> <" + productId + "> " + all_text + " <" + str(price) + "> <" + category + "> <" + productId + ">"

                  sents = pre_process(final_data)
                  count +=1
                  l+=len(sents)
                  print 'Count: '," ",count," ",l," ",sents
                  yield sents

def train(data_folder, model_path):
    sentences = MySentences(data_folder)
    model = gensim.models.Word2Vec(sentences, min_count=25, workers=8, negative=10, sg=1, size = 300, sample=1e-3, hs=1, window = 10) #a1
    #model = gensim.models.Word2Vec(sentences, min_count=10, workers=6, negative=3, sg=0, size = 300, sample=1e-5, hs=0, window = 5) #a2
    # ./word2vec -train train100B.txt -read-vocab voc -output vectors.bin -cbow 1 -size 300 -window 5 -negative 3 -hs 0 -sample 1e-5 -threads 12 -binary 1 -min-count 10
    model.init_sims(replace=True)
    model.save(model_path)

if __name__ == "__main__":
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s", ' '.join(sys.argv))

    # check and process cmdline input
    program = os.path.basename(sys.argv[0])
    if len(sys.argv) < 2:
        print(globals()['__doc__'] % locals())
        sys.exit(1)

    parser = argparse.ArgumentParser()

    parser.add_argument(
        "-d", "--data", required=True,
        help="Input data folder.")
    parser.add_argument(
        "-m", "--model_output", required=True,
        help="Output model file, in word2vec format")
    args = parser.parse_args()

    data_folder, model_path = (args.data, args.model_output)
    logger.info('Picking data from %s...', data_folder)

    train(data_folder, model_path)

    logger.info("Saving model file %s in %s", model_file, os.path.abspath(model_file))
    logger.info("Finished running %s", program)

<commit_msg>Model 3<commit_after>import re
import os
import sys

from nltk.corpus import stopwords
stop = stopwords.words('english')

import gensim
import string
import redis
import json
import gzip
from unidecode import unidecode

import logging
import argparse

from phrases_extractor import get_phrases

price_obj = redis.Redis("localhost", port=6379, db=1)

logger = logging.getLogger(__name__)

def parse(filename):
    g = gzip.open(filename,"r")
    for l in g:
      yield json.dumps(eval(l))

def getRange(price):

    price  = float(price)
    if price <= 10:
      return 10
    elif 10 < price <= 100:
      return 100
    elif 100 < price <= 250:
      return 250
    elif 250 < price <= 500:
      return 500
    elif 500 < price <= 1000:
      return 1000
    elif 1000 < price <= 2000:
      return 2000
    elif 2000 < price <= 5000:
      return 5000
    elif price > 5000:
      return 9999

def process_phrases(text):
    text = unidecode(text)
    phrases = get_phrases(text)
    for phrase in phrases:
      text = text.replace(phrase, phrase.replace(' ', '_'))
    return text

def pre_process(m):
    m = unidecode(m)
    m = m.lower().strip() #normalize
    #print m
    #m=re.sub("[^a-zA-Z]", " ",m) #keep text only
    #m="".join(l for l in m if l not in string.punctuation) #remove punctuation
    m = m.strip("./@!#")
    m=[i for i in m.split() if i not in stop] #remove stopwords
    #m=m.split()
    print m
    return m

#iterator on Amazon Reviews
class MySentences(object):
    def __init__(self, dirname):
        self.dirname = dirname

    def __iter__(self):
        print os.listdir(self.dirname)
        for fname in os.listdir(self.dirname):
            logger.info('Processing file %s', fname)
            count=0
            l=0
            for e in parse((os.path.join(self.dirname, fname))):
                review= json.loads(e).get("reviewText")
                title = json.loads(e).get("summary")
                category = fname.strip('reviews json gz . _')
                all_text = title + ". " + review
                all_text = process_phrases(all_text)

                if len(review.split())>10: # don't consider reviews less than 10 words
                  productId = json.loads(e).get("asin")
                  try:
                    price = getRange(price_obj.get(productId))
                  except:
                    price = 0

                  final_data = "<" + str(price) + "> <" + category + "> <" + productId + "> " + all_text + " <" + str(price) + "> <" + category + "> <" + productId + ">"

                  sents = pre_process(final_data)
                  count +=1
                  l+=len(sents)
                  print 'Count: '," ",count," ",l," ",sents
                  yield sents

def train(data_folder, model_path):
    sentences = MySentences(data_folder)
    #model = gensim.models.Word2Vec(sentences, min_count=25, workers=8, negative=10, sg=1, size = 300, sample=1e-3, hs=1, window = 10) #a1
    #model = gensim.models.Word2Vec(sentences, min_count=10, workers=6, negative=3, sg=0, size = 300, sample=1e-5, hs=0, window = 5) #a2
    model = gensim.models.Word2Vec(sentences, min_count=10, workers=4, negative=5, sg=0, size = 300, sample=1e-3, hs=1, window = 5) #a3
    # ./word2vec -train train100B.txt -read-vocab voc -output vectors.bin -cbow 1 -size 300 -window 5 -negative 3 -hs 0 -sample 1e-5 -threads 12 -binary 1 -min-count 10
    model.init_sims(replace=True)
    model.save(model_path)

if __name__ == "__main__":
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s", ' '.join(sys.argv))

    # check and process cmdline input
    program = os.path.basename(sys.argv[0])
    if len(sys.argv) < 2:
        print(globals()['__doc__'] % locals())
        sys.exit(1)

    parser = argparse.ArgumentParser()

    parser.add_argument(
        "-d", "--data", required=True,
        help="Input data folder.")
    parser.add_argument(
        "-m", "--model_output", required=True,
        help="Output model file, in word2vec format")
    args = parser.parse_args()

    data_folder, model_path = (args.data, args.model_output)
    logger.info('Picking data from %s...', data_folder)

    train(data_folder, model_path)

    logger.info("Saving model file %s in %s", model_file, os.path.abspath(model_file))
    logger.info("Finished running %s", program)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import time
import signal
import ConfigParser
from optparse import OptionParser
import logging
from p2pnet import TcpServer
from p2pnet import PeerManager
from manager import ChainManager

logger = logging.getLogger(__name__)

def create_config():

    config = ConfigParser.ConfigParser()
    # set some defaults, which may be overwritten
    config.add_section('network')
    config.set('network', 'listen_host', 'localhost')
    config.set('network', 'listen_port', '30303')
    config.set('network', 'num_peers', '5')
    config.set('network', 'remote_port', '30303')
    config.set('network', 'remote_host', '')
    config.set('network', 'client_id', 'Ethereum(py)/0.0.1')

    config.add_section('misc')
    config.set('misc', 'verbosity', '1')
    config.set('misc', 'config_file', None)

    config.add_section('wallet')

    # NODE_ID == pubkey, needed in order to work with Ethereum(++)
    config.set('wallet', 'pub_key', 'J\x02U\xfaFs\xfa\xa3\x0f\xc5\xab\xfd<U\x0b\xfd\xbc\r<\x97=5\xf7&F:\xf8\x1cT\xa02\x81\xcf\xff"\xc5\xf5\x96[8\xacc\x01R\x98wW\xa3\x17\x82G\x85I\xc3o|\x84\xcbD6\xbay\xd6\xd9')

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage,  version="%prog 0.1a")
    parser.add_option("-l", "--listen",
                      dest="listen_port",
                      default=config.get('network', 'listen_port'),
                      help="<port>  Listen on the given port for incoming connected (default: 30303)."
                      )
    parser.add_option("-r", "--remote",
                      dest="remote_host",
                      help="<host> Connect to remote host (try: 54.201.28.117 or 54.204.10.41)"
                      )
    parser.add_option("-p", "--port",
                      dest="remote_port",
                      default=config.get('network', 'remote_port'),
                      help="<port> Connect to remote port (default: 30303)"
                      )
    parser.add_option("-v", "--verbose",
                      dest="verbosity",
                      default=config.get('misc', 'verbosity'),
                      help="<0 - 9>  Set the log verbosity from 0 to 9 (default: 1)")
    parser.add_option("-x", "--peers",
                      dest="num_peers",
                      default=config.get('network', 'num_peers'),
                      help="<number>  Attempt to connect to given number of peers (default: 5)")
    parser.add_option("-C", "--config",
                      dest="config_file",
                      help="read coniguration")

    (options, args) = parser.parse_args()
    # set network options
    for attr in ('listen_port', 'remote_host', 'remote_port', 'num_peers'):
        config.set('network', attr, getattr(
            options, attr) or config.get('network', attr))
    # set misc options
    for attr in ('verbosity', 'config_file'):
        config.set(
            'misc', attr, getattr(options, attr) or config.get('misc', attr))

    if len(args) != 0:
        parser.error("wrong number of arguments")
        sys.exit(1)

    if config.get('misc', 'config_file'):
        config.read(config.get('misc', 'config_file'))

    return config

def main():
    config = create_config()

    # Setup logging according to config:
    log_level = config.getint('misc', 'verbosity')
    if log_level == 2:
        logging.basicConfig(format='%(message)s', level=logging.WARN)
    elif log_level > 2:
        logging.basicConfig(
            format='[%(asctime)s] %(name)s %(levelname)s %(threadName)s: %(message)s', level=logging.DEBUG)
    else:
        logging.basicConfig(format='%(message)s', level=logging.INFO)

    # peer manager
    peer_manager = PeerManager(config=config)

    # chain manager
    chain_manager = ChainManager(config=config)

    # start tcp server
    try:
        tcp_server = TcpServer(peer_manager,
                               config.get('network', 'listen_host'),
                               config.getint('network', 'listen_port'))
    except IOError as e:
        logger.error("Could not start TCP server: \"{0}\"".format(str(e)))
        sys.exit(1)

    peer_manager.local_address = (tcp_server.ip, tcp_server.port)
    tcp_server.start()
    peer_manager.start()
    chain_manager.start()

    # handle termination signals
    def signal_handler(signum=None, frame=None):
        logger.info('Signal handler called with signal {0}'.format(signum))
        peer_manager.stop()
        chain_manager.stop()
        # tcp_server checks for peer_manager.stopped()
    for sig in [signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT, signal.SIGINT]:
        signal.signal(sig, signal_handler)

    # connect peer
    if config.get('network', 'remote_host'):
        peer_manager.connect_peer(
            config.get('network', 'remote_host'),
            config.getint('network', 'remote_port'))

    # loop
    while not peer_manager.stopped():
        time.sleep(0.1)

    logger.info('extiting')
    # tcp_server.join() # does not work!
    peer_manager.join()

if __name__ == '__main__':
    main()<commit_msg>test code to create the dummy blockchain after three peers are conencted<commit_after>#!/usr/bin/env python
import sys
import time
import signal
import ConfigParser
from optparse import OptionParser
import logging
from p2pnet import TcpServer
from p2pnet import PeerManager
from manager import ChainManager

logger = logging.getLogger(__name__)

def create_config():

    config = ConfigParser.ConfigParser()
    # set some defaults, which may be overwritten
    config.add_section('network')
    config.set('network', 'listen_host', 'localhost')
    config.set('network', 'listen_port', '30303')
    config.set('network', 'num_peers', '5')
    config.set('network', 'remote_port', '30303')
    config.set('network', 'remote_host', '')
    config.set('network', 'client_id', 'Ethereum(py)/0.0.1')

    config.add_section('misc')
    config.set('misc', 'verbosity', '1')
    config.set('misc', 'config_file', None)

    config.add_section('wallet')

    # NODE_ID == pubkey, needed in order to work with Ethereum(++)
    config.set('wallet', 'pub_key', 'J\x02U\xfaFs\xfa\xa3\x0f\xc5\xab\xfd<U\x0b\xfd\xbc\r<\x97=5\xf7&F:\xf8\x1cT\xa02\x81\xcf\xff"\xc5\xf5\x96[8\xacc\x01R\x98wW\xa3\x17\x82G\x85I\xc3o|\x84\xcbD6\xbay\xd6\xd9')

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage,  version="%prog 0.1a")
    parser.add_option("-l", "--listen",
                      dest="listen_port",
                      default=config.get('network', 'listen_port'),
                      help="<port>  Listen on the given port for incoming connected (default: 30303)."
                      )
    parser.add_option("-r", "--remote",
                      dest="remote_host",
                      help="<host> Connect to remote host (try: 54.201.28.117 or 54.204.10.41)"
                      )
    parser.add_option("-p", "--port",
                      dest="remote_port",
                      default=config.get('network', 'remote_port'),
                      help="<port> Connect to remote port (default: 30303)"
                      )
    parser.add_option("-v", "--verbose",
                      dest="verbosity",
                      default=config.get('misc', 'verbosity'),
                      help="<0 - 9>  Set the log verbosity from 0 to 9 (default: 1)")
    parser.add_option("-x", "--peers",
                      dest="num_peers",
                      default=config.get('network', 'num_peers'),
                      help="<number>  Attempt to connect to given number of peers (default: 5)")
    parser.add_option("-C", "--config",
                      dest="config_file",
                      help="read coniguration")

    (options, args) = parser.parse_args()
    # set network options
    for attr in ('listen_port', 'remote_host', 'remote_port', 'num_peers'):
        config.set('network', attr, getattr(
            options, attr) or config.get('network', attr))
    # set misc options
    for attr in ('verbosity', 'config_file'):
        config.set(
            'misc', attr, getattr(options, attr) or config.get('misc', attr))

    if len(args) != 0:
        parser.error("wrong number of arguments")
        sys.exit(1)

    if config.get('misc', 'config_file'):
        config.read(config.get('misc', 'config_file'))

    return config

def main():
    config = create_config()

    # Setup logging according to config:
    log_level = config.getint('misc', 'verbosity')
    if log_level == 2:
        logging.basicConfig(format='%(message)s', level=logging.WARN)
    elif log_level > 2:
        logging.basicConfig(
            format='[%(asctime)s] %(name)s %(levelname)s %(threadName)s: %(message)s', level=logging.DEBUG)
    else:
        logging.basicConfig(format='%(message)s', level=logging.INFO)

    # peer manager
    peer_manager = PeerManager(config=config)

    # chain manager
    chain_manager = ChainManager(config=config)

    # start tcp server
    try:
        tcp_server = TcpServer(peer_manager,
                               config.get('network', 'listen_host'),
                               config.getint('network', 'listen_port'))
    except IOError as e:
        logger.error("Could not start TCP server: \"{0}\"".format(str(e)))
        sys.exit(1)

    peer_manager.local_address = (tcp_server.ip, tcp_server.port)
    tcp_server.start()
    peer_manager.start()
    chain_manager.start()

    # handle termination signals
    def signal_handler(signum=None, frame=None):
        logger.info('Signal handler called with signal {0}'.format(signum))
        peer_manager.stop()
        chain_manager.stop()
        # tcp_server checks for peer_manager.stopped()
    for sig in [signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT, signal.SIGINT]:
        signal.signal(sig, signal_handler)

    # connect peer
    if config.get('network', 'remote_host'):
        peer_manager.connect_peer(
            config.get('network', 'remote_host'),
            config.getint('network', 'remote_port'))

    # loop
    while not peer_manager.stopped():
        time.sleep(0.1)
        if len(peer_manager.get_connected_peer_addresses()) > 2:
            chain_manager.bootstrap_blockchain()

    logger.info('extiting')
    # tcp_server.join() # does not work!
    peer_manager.join()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import time
import signal
import ConfigParser
from optparse import OptionParser
import logging
import logging.config
from p2pnet import TcpServer
from p2pnet import PeerManager
from manager import ChainManager

logger = logging.getLogger(__name__)

def create_config():

    config = ConfigParser.ConfigParser()
    # set some defaults, which may be overwritten
    config.add_section('network')
    config.set('network', 'listen_host', 'localhost')
    config.set('network', 'listen_port', '30303')
    config.set('network', 'num_peers', '5')
    config.set('network', 'remote_port', '30303')
    config.set('network', 'remote_host', '')
    config.set('network', 'client_id', 'Ethereum(py)/0.0.1')

    config.add_section('misc')
    config.set('misc', 'verbosity', '1')
    config.set('misc', 'config_file', None)
    config.set('misc', 'logging', None)

    config.add_section('wallet')

    # NODE_ID == pubkey, needed in order to work with Ethereum(++)
    config.set('wallet', 'pub_key', 'J\x02U\xfaFs\xfa\xa3\x0f\xc5\xab\xfd<U\x0b\xfd\xbc\r<\x97=5\xf7&F:\xf8\x1cT\xa02\x81\xcf\xff"\xc5\xf5\x96[8\xacc\x01R\x98wW\xa3\x17\x82G\x85I\xc3o|\x84\xcbD6\xbay\xd6\xd9')

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage,  version="%prog 0.1a")
    parser.add_option("-l", "--listen",
                      dest="listen_port",
                      default=config.get('network', 'listen_port'),
                      help="<port>  Listen on the given port for incoming connected (default: 30303)."
                      )
    parser.add_option("-r", "--remote",
                      dest="remote_host",
                      help="<host> Connect to remote host"
                      )
    parser.add_option("-p", "--port",
                      dest="remote_port",
                      default=config.get('network', 'remote_port'),
                      help="<port> Connect to remote port (default: 30303)"
                      )
    parser.add_option("-v", "--verbose",
                      dest="verbosity",
                      default=config.get('misc', 'verbosity'),
                      help="<0 - 3>  Set the log verbosity from 0 to 3 (default: 1)")
    parser.add_option("-L", "--logging",
                      dest="logging",
                      default=config.get('misc', 'logging'),
                      help="<logger1:LEVEL,logger2:LEVEL> set the console log level for logger1, logger2, etc.\
                            Empty loggername means root-logger, e.g. 'pyethereum.wire:DEBUG,:INFO'. Overrides '-v'")
    parser.add_option("-x", "--peers",
                      dest="num_peers",
                      default=config.get('network', 'num_peers'),
                      help="<number>  Attempt to connect to given number of peers (default: 5)")
    parser.add_option("-C", "--config",
                      dest="config_file",
                      help="read coniguration")

    (options, args) = parser.parse_args()

    # set network options
    for attr in ('listen_port', 'remote_host', 'remote_port', 'num_peers'):
        config.set('network', attr, getattr(
            options, attr) or config.get('network', attr))
    # set misc options
    for attr in ('verbosity', 'config_file'):
        config.set(
            'misc', attr, getattr(options, attr) or config.get('misc', attr))

    if len(args) != 0:
        parser.error("wrong number of arguments")
        sys.exit(1)

    if config.get('misc', 'config_file'):
        config.read(config.get('misc', 'config_file'))

    #configure logging
    loggerlevels = getattr(options, 'logging') or ''
    configure_logging(loggerlevels, verbosity=config.getint('misc', 'verbosity'))

    return config

def configure_logging(loggerlevels, verbosity=1):
    logconfig = dict(version=1,
                    disable_existing_loggers=False,
                    formatters=dict(
                        debug=dict(
                            format='[%(asctime)s] %(name)s %(levelname)s %(threadName)s: %(message)s'
                        ),
                        minimal=dict(
                            format='%(message)s'
                        ),
                    ),
                    handlers=dict(
                        default={
                            'level': 'INFO',
                            'class': 'logging.StreamHandler',
                            'formatter': 'minimal'
                        },
                        verbose={
                            'level': 'DEBUG',
                            'class': 'logging.StreamHandler',
                            'formatter': 'debug'
                        },
                    ),
                    loggers=dict()
                    )

    for loggerlevel in filter(lambda _: ':' in _, loggerlevels.split(',')):
        name, level = loggerlevel.split(':')
        logconfig['loggers'][name] = dict(handlers=['verbose'], level=level, propagate=False)

    if len(logconfig['loggers']) == 0:
        logconfig['loggers'][''] = dict(handlers=['default'], level={0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG'}.get(verbosity), propagate=True)

    logging.config.dictConfig(logconfig)
    logging.debug("logging set up like that: {0}".format(logconfig))

def main():
    config = create_config()

    # peer manager
    peer_manager = PeerManager(config=config)

    # chain manager
    chain_manager = ChainManager(config=config)

    # start tcp server
    try:
        tcp_server = TcpServer(peer_manager,
                               config.get('network', 'listen_host'),
                               config.getint('network', 'listen_port'))
    except IOError as e:
        logger.error("Could not start TCP server: \"{0}\"".format(str(e)))
        sys.exit(1)

    peer_manager.local_address = (tcp_server.ip, tcp_server.port)
    tcp_server.start()
    peer_manager.start()
    chain_manager.start()

    # handle termination signals
    def signal_handler(signum=None, frame=None):
        logger.info('Signal handler called with signal {0}'.format(signum))
        peer_manager.stop()
        chain_manager.stop()
        # tcp_server checks for peer_manager.stopped()
    for sig in [signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT, signal.SIGINT]:
        signal.signal(sig, signal_handler)

    # connect peer
    if config.get('network', 'remote_host'):
        peer_manager.connect_peer(
            config.get('network', 'remote_host'),
            config.getint('network', 'remote_port'))

    # loop
    while not peer_manager.stopped():
        time.sleep(0.1)

    logger.info('extiting')
    # tcp_server.join() # does not work!
    peer_manager.join()

if __name__ == '__main__':
    main()
<commit_msg>Update eth.py<commit_after>#!/usr/bin/env python
import sys
import time
import signal
import ConfigParser
from optparse import OptionParser
import logging
import logging.config
from p2pnet import TcpServer
from p2pnet import PeerManager
from manager import ChainManager

logger = logging.getLogger(__name__)

def create_config():

    config = ConfigParser.ConfigParser()
    # set some defaults, which may be overwritten
    config.add_section('network')
    config.set('network', 'listen_host', 'localhost')
    config.set('network', 'listen_port', '30303')
    config.set('network', 'num_peers', '5')
    config.set('network', 'remote_port', '30303')
    config.set('network', 'remote_host', '')
    config.set('network', 'client_id', 'Ethereum(py)/0.0.1')

    config.add_section('misc')
    config.set('misc', 'verbosity', '1')
    config.set('misc', 'config_file', None)
    config.set('misc', 'logging', None)

    config.add_section('wallet')

    # NODE_ID == pubkey, needed in order to work with Ethereum(++)
    config.set('wallet', 'pub_key', 'J\x02U\xfaFs\xfa\xa3\x0f\xc5\xab\xfd<U\x0b\xfd\xbc\r<\x97=5\xf7&F:\xf8\x1cT\xa02\x81\xcf\xff"\xc5\xf5\x96[8\xacc\x01R\x98wW\xa3\x17\x82G\x85I\xc3o|\x84\xcbD6\xbay\xd6\xd9')

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage,  version="%prog 0.1a")
    parser.add_option("-l", "--listen",
                      dest="listen_port",
                      default=config.get('network', 'listen_port'),
                      help="<port>  Listen on the given port for incoming connected (default: 30303)."
                      )
    parser.add_option("-r", "--remote",
                      dest="remote_host",
                      help="<host> Connect to remote host"
                      )
    parser.add_option("-p", "--port",
                      dest="remote_port",
                      default=config.get('network', 'remote_port'),
                      help="<port> Connect to remote port (default: 30303)"
                      )
    parser.add_option("-v", "--verbose",
                      dest="verbosity",
                      default=config.get('misc', 'verbosity'),
                      help="<0 - 3>  Set the log verbosity from 0 to 3 (default: 1)")
    parser.add_option("-L", "--logging",
                      dest="logging",
                      default=config.get('misc', 'logging'),
                      help="<logger1:LEVEL,logger2:LEVEL> set the console log level for logger1, logger2, etc.\
                            Empty loggername means root-logger, e.g. 'pyethereum.wire:DEBUG,:INFO'. Overrides '-v'")
    parser.add_option("-x", "--peers",
                      dest="num_peers",
                      default=config.get('network', 'num_peers'),
                      help="<number>  Attempt to connect to given number of peers (default: 5)")
    parser.add_option("-C", "--config",
                      dest="config_file",
                      help="read coniguration")

    (options, args) = parser.parse_args()

    # set network options
    for attr in ('listen_port', 'remote_host', 'remote_port', 'num_peers'):
        config.set('network', attr, getattr(
            options, attr) or config.get('network', attr))
    # set misc options
    for attr in ('verbosity', 'config_file'):
        config.set(
            'misc', attr, getattr(options, attr) or config.get('misc', attr))

    if len(args) != 0:
        parser.error("wrong number of arguments")
        sys.exit(1)

    if config.get('misc', 'config_file'):
        config.read(config.get('misc', 'config_file'))

    #configure logging
    loggerlevels = getattr(options, 'logging') or ''
    configure_logging(loggerlevels, verbosity=config.getint('misc', 'verbosity'))

    return config

def configure_logging(loggerlevels, verbosity=1):
    logconfig = dict(version=1,
                    disable_existing_loggers=False,
                    formatters=dict(
                        debug=dict(
                            format='[%(asctime)s] %(name)s %(levelname)s %(threadName)s: %(message)s'
                        ),
                        minimal=dict(
                            format='%(message)s'
                        ),
                    ),
                    handlers=dict(
                        default={
                            'level': 'INFO',
                            'class': 'logging.StreamHandler',
                            'formatter': 'minimal'
                        },
                        verbose={
                            'level': 'DEBUG',
                            'class': 'logging.StreamHandler',
                            'formatter': 'debug'
                        },
                    ),
                    loggers=dict()
                    )

    for loggerlevel in filter(lambda _: ':' in _, loggerlevels.split(',')):
        name, level = loggerlevel.split(':')
        logconfig['loggers'][name] = dict(handlers=['verbose'], level=level, propagate=False)

    if len(logconfig['loggers']) == 0:
        logconfig['loggers'][''] = dict(handlers=['default'], level={0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG'}.get(verbosity), propagate=True)

    logging.config.dictConfig(logconfig)
    logging.debug("logging set up like that: {0}".format(logconfig))

def main():
    config = create_config()

    # peer manager
    peer_manager = PeerManager(config=config)

    # chain manager
    chain_manager = ChainManager(config=config)

    # start tcp server
    try:
        tcp_server = TcpServer(peer_manager,
                               config.get('network', 'listen_host'),
                               config.getint('network', 'listen_port'))
    except IOError as e:
        logger.error("Could not start TCP server: \"{0}\"".format(str(e)))
        sys.exit(1)

    peer_manager.local_address = (tcp_server.ip, tcp_server.port)
    tcp_server.start()
    peer_manager.start()
    chain_manager.start()

    # handle termination signals
    def signal_handler(signum=None, frame=None):
        logger.info('Signal handler called with signal {0}'.format(signum))
        peer_manager.stop()
        chain_manager.stop()
        # tcp_server checks for peer_manager.stopped()
    for sig in [signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT, signal.SIGINT]:
        signal.signal(sig, signal_handler)

    # connect peer
    if config.get('network', 'remote_host'):
        peer_manager.connect_peer(
            config.get('network', 'remote_host'),
            config.getint('network', 'remote_port'))

    # loop
    while not peer_manager.stopped():
        time.sleep(0.1)

    logger.info('exiting')
    # tcp_server.join() # does not work!
    peer_manager.join()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import asyncio
import json
import logging
import traceback

logger = logging.getLogger(__name__)

class ProcedureAlreadyRegisteredError(Exception):
    pass

class JsonRPCProtocol(asyncio.Protocol):
    DEFAULT_CHARSET = "utf-8"
    DEFAULT_CONTENT_TYPE = "application/jsonrpc"
    VERSION = "2.0"

    def __init__(self, **kwargs):
        self.charset = kwargs.get("content_type",
                                  self.DEFAULT_CHARSET)
        self.content_type = kwargs.get("content_type",
                                       self.DEFAULT_CONTENT_TYPE)
        self.deserializer = kwargs.get("deserializer",
                                       self.DEFAULT_CONTENT_TYPE)
        self.serializer = kwargs.get("serializer",
                                     self.DEFAULT_CONTENT_TYPE)

        self.procedure_options = {}
        self.procedures = {}
        self.transport = None

    def __call__(self):
        return self

    @staticmethod
    def extract_body(body):
        id = body.get("id", None)
        json_rpc_version = body.get("jsonrpc", None)
        method = body.get("method", None)
        params = body.get("params", None)

        return id, json_rpc_version, method, params

    def _procedure_handler(self, body):
        id, version, method, params = JsonRPCProtocol.extract_body(body)

        if version is not JsonRPCProtocol.VERSION:
            logger.warn("Unknown message {}".format(body))
            return

        if not id:
            logger.debug("Notification message received.")
        elif not method:
            logger.debug("Response message received.")
        else:
            logger.debug("Request message received.")

    def connection_made(self, transport):
        self.transport = transport

    def data_received(self, data):
        logger.debug("Received {}".format(data))

        if not data:
            return

        _, body = data.decode(self.charset).split("\r\n\r\n")

        try:
            self._procedure_handler(json.loads(body))
        except:
            pass

    def register_procedure(self, procedure_name, **options):
        def decorator(f):
            if procedure_name in self.procedures:
                msg = "Procedure {} is already registered." \
                    .format(procedure_name)

                logger.error(msg)
                raise ProcedureAlreadyRegisteredError(msg)

            self.procedures[procedure_name] = f

            if options:
                self.procedure_options[procedure_name] = options

            logger.info("Registered procedure {} with options {}"
                        .format(procedure_name, options))

            return f
        return decorator

    def send_data(self, data):
        try:
            content_length = len(data.encode("utf-8")) if data else 0
            body = json.dumps(data)  # add serializer

            response = (
                "Content-Length: {}\r\n"
                "Content-Type: {}; charset={}\r\n\r\n"
                "{}".format(content_length,
                            self.content_type,
                            self.charset,
                            body)
            )

            logger.info("Sending data: {}".format(body))

            self.transport.write(response.encode(self.charset))
        except:
            logger.error(traceback.format_exc())
<commit_msg>Add json serializers<commit_after>import asyncio
import json
import logging
import traceback

logger = logging.getLogger(__name__)

class ProcedureAlreadyRegisteredError(Exception):
    pass

class JsonRPCRequestMessage:
    def __init__(self, id=None, jsonrpc=None, method=None, params=None):
        self.id = id
        self.version = jsonrpc
        self.method = method
        self.params = params

    @classmethod
    def from_dict(cls, data):
        if 'jsonrpc' not in data:
            return data

        return cls(**data)

class JsonRPCProtocol(asyncio.Protocol):
    DEFAULT_CHARSET = "utf-8"
    DEFAULT_CONTENT_TYPE = "application/jsonrpc"
    VERSION = "2.0"

    def __init__(self, **kwargs):
        self.charset = kwargs.get("content_type",
                                  self.DEFAULT_CHARSET)
        self.content_type = kwargs.get("content_type",
                                       self.DEFAULT_CONTENT_TYPE)

        self.procedure_options = {}
        self.procedures = {}
        self.transport = None

    def __call__(self):
        return self

    def _procedure_handler(self, message: JsonRPCRequestMessage):
        if message.version != JsonRPCProtocol.VERSION:
            logger.warn("Unknown message {}".format(message))
            return

        if message.id is None:
            logger.debug("Notification message received.")
        elif message.method is None:
            logger.debug("Response message received.")
        else:
            logger.debug("Request message received.")

    def connection_made(self, transport):
        self.transport = transport

    def data_received(self, data):
        logger.debug("Received {}".format(data))

        if not data:
            return

        _, body = data.decode(self.charset).split("\r\n\r\n")

        try:
            self._procedure_handler(json.loads(body, object_hook=JsonRPCRequestMessage.from_dict))
        except:
            pass

    def register_procedure(self, procedure_name, **options):
        def decorator(f):
            if procedure_name in self.procedures:
                msg = "Procedure {} is already registered." \
                    .format(procedure_name)

                logger.error(msg)
                raise ProcedureAlreadyRegisteredError(msg)

            self.procedures[procedure_name] = f

            if options:
                self.procedure_options[procedure_name] = options

            logger.info("Registered procedure {} with options {}"
                        .format(procedure_name, options))

            return f
        return decorator

    def send_data(self, data):
        try:
            content_length = len(data.encode("utf-8")) if data else 0
            body = json.dumps(data, default=lambda o: o.__dict__)

            response = (
                "Content-Length: {}\r\n"
                "Content-Type: {}; charset={}\r\n\r\n"
                "{}".format(content_length,
                            self.content_type,
                            self.charset,
                            body)
            )

            logger.info("Sending data: {}".format(body))

            self.transport.write(response.encode(self.charset))
        except:
            logger.error(traceback.format_exc())
<|endoftext|>


<commit_before>__version__ = '0.2.0'
<commit_msg>bump version<commit_after>__version__ = '0.2.1'
<|endoftext|>


<commit_before># Copyright 2016 ELIFE. All rights reserved.
# Use of this source code is governed by a MIT
# license that can be found in the LICENSE file.
from pyinform import _inform
from ctypes import byref, c_bool, c_char_p, c_int, POINTER

_strerror = _inform.inform_strerror
_strerror.argtypes = [POINTER(c_int)]
_strerror.restype = c_char_p

_is_success = _inform.inform_succeeded
_is_success.argtypes = [POINTER(c_int)]
_is_success.restype = c_bool

_is_failure = _inform.inform_failed
_is_failure.argtypes = [POINTER(c_int)]
_is_failure.restype = c_bool

def error_string(e):
    """
    Generate an error message from an integral error code `e`.
    """
    if not isinstance(e, c_int):
        return error_string(c_int(e))
    else:
        return _strerror(byref(e)).decode("utf-8")

def is_success(e):
    """
    Determine if an error code represents a success.
    """
    if not isinstance(e, c_int):
        return is_success(c_int(e))
    else:
        return _is_success(byref(e))

def is_failure(e):
    """
    Determine if an error code represents a failure.
    """
    if not isinstance(e, c_int):
        return is_failure(c_int(e))
    else:
        return _is_failure(byref(e))

class InformError(Exception):
    """
    InformError signifies an error occurred in a call to inform.
    """
    def __init__(self, e=-1, func=None):
        msg = error_string(e)

        if func is None:
            msg = "an inform error occurred - \"{}\"".format(msg)
        else:
            msg = "an inform error occurred in `{}` - \"{}\"".format(func, msg)

        super(InformError, self).__init__(msg)

        self.error_code = e if isinstance(e, c_int) else c_int(e)

def error_guard(e, func=None):
    """
    Raise an appropriately formated error if `e` is a failure
    """
    if is_failure(e):
        raise InformError(e,func)
<commit_msg>Create the `ErrorCode` synonym for `c_int`<commit_after># Copyright 2016 ELIFE. All rights reserved.
# Use of this source code is governed by a MIT
# license that can be found in the LICENSE file.
from ctypes import byref, c_bool, c_char_p, c_int, POINTER
from pyinform import _inform

_strerror = _inform.inform_strerror
_strerror.argtypes = [POINTER(c_int)]
_strerror.restype = c_char_p

_is_success = _inform.inform_succeeded
_is_success.argtypes = [POINTER(c_int)]
_is_success.restype = c_bool

_is_failure = _inform.inform_failed
_is_failure.argtypes = [POINTER(c_int)]
_is_failure.restype = c_bool

ErrorCode = c_int

def error_string(e):
    """
    Generate an error message from an integral error code `e`.
    """
    if not isinstance(e, ErrorCode):
        return error_string(ErrorCode(e))
    else:
        return _strerror(byref(e)).decode("utf-8")

def is_success(e):
    """
    Determine if an error code represents a success.
    """
    if not isinstance(e, ErrorCode):
        return is_success(ErrorCode(e))
    else:
        return _is_success(byref(e))

def is_failure(e):
    """
    Determine if an error code represents a failure.
    """
    if not isinstance(e, ErrorCode):
        return is_failure(ErrorCode(e))
    else:
        return _is_failure(byref(e))

class InformError(Exception):
    """
    InformError signifies an error occurred in a call to inform.
    """
    def __init__(self, e=-1, func=None):
        msg = error_string(e)

        if func is None:
            msg = "an inform error occurred - \"{}\"".format(msg)
        else:
            msg = "an inform error occurred in `{}` - \"{}\"".format(func, msg)

        super(InformError, self).__init__(msg)

        self.error_code = e if isinstance(e, ErrorCode) else ErrorCode(e)

def error_guard(e, func=None):
    """
    Raise an appropriately formated error if `e` is a failure
    """
    if is_failure(e):
        raise InformError(e,func)
<|endoftext|>