<commit_before>import abc
from cms.constants import RIGHT, LEFT, REFRESH
from django.conf import settings
from django.template.loader import render_to_string
from django.utils import simplejson

class BaseItem(object):
    __metaclass__ = abc.ABCMeta
    template = abc.abstractproperty()

    def __init__(self, position=LEFT):
        self.position = position

    @property
    def right(self):
        return self.position is RIGHT

    def render(self):
        return render_to_string(self.template, self.get_context())

    def get_context(self):
        return {}

class ToolbarAPIMixin(object):
    __metaclass__ = abc.ABCMeta

    def add_item(self, item):
        if not isinstance(item, BaseItem):
            raise ValueError("Items must be subclasses of cms.toolbar.items.BaseItem, %r isn't" % item)
        self.items.append(item)
        return item

    def add_sideframe_item(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=False,
                 on_close=REFRESH, position=LEFT):
        item = SideframeItem(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            close_on_url_change=close_on_url_change,
            on_close=on_close,
            position=position,
        )
        self.add_item(item)
        return item

    def add_modal_item(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=False,
                 on_close=REFRESH, position=LEFT):
        item = ModalItem(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            close_on_url_change=close_on_url_change,
            on_close=on_close,
            position=position,
        )
        self.add_item(item)
        return item

    def add_link_item(self, name, url, active=False, disabled=False, extra_classes=None, position=LEFT):
        item = LinkItem(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            position=position
        )
        self.add_item(item)
        return item

    def add_ajax_item(self, name, action, active=False, disabled=False, extra_classes=None, data=None, question=None,
                      position=LEFT):
        item = AjaxItem(name, action, self.csrf_token,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            data=data,
            question=question,
            position=position,
        )
        self.add_item(item)
        return item

    def add_break(self):
        item = Break()
        self.add_item(item)
        return item

class Menu(ToolbarAPIMixin, BaseItem):
    template = "cms/toolbar/menu/list.html"

    def __init__(self, name, csrf_token, url='#', sub_level=False, position=LEFT):
        super(Menu, self).__init__(position)
        self.items = []
        self.menus = {}
        self.url = url
        self.name = name
        self.sub_level = sub_level
        self.csrf_token = csrf_token

    def __repr__(self):
        return '<Menu:%s>' % unicode(self.name)

    def get_menu(self, key, verbose_name, position=LEFT):
        if key in self.menus:
            return self.menus[key]
        menu = Menu(verbose_name, self.csrf_token, sub_level=True, position=position)
        self.menus[key] = menu
        self.items.append(menu)
        return menu

    def get_items(self):
        return self.items

    def get_context(self):
        return {
            'items': self.get_items(),
            'url': self.url,
            'title': self.name,
            'sub_level': self.sub_level}

class LinkItem(BaseItem):
    template = "cms/toolbar/menu/link_item.html"

    def __init__(self, name, url, active=False, disabled=False, extra_classes=None, position=LEFT):
        self.position = position
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []

    def __repr__(self):
        return '<LinkItem:%s>' % unicode(self.name)

    def get_context(self):
        return {
            'url': self.url,
            'name': self.name,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
        }

class SideframeItem(BaseItem):
    template = "cms/toolbar/menu/sideframe_item.html"

    def __init__(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=True,
                 on_close=REFRESH, position=LEFT):
        self.position = position
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []
        self.on_close = on_close
        self.close_on_url_change = close_on_url_change

    def __repr__(self):
        return '<SideframeItem:%s>' % unicode(self.name)

    def get_context(self):
        return {
            'url': self.url,
            'name': self.name,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
            'on_close': self.on_close,
            'close_on_url_change': self.close_on_url_change,
        }

class AjaxItem(BaseItem):
    template = "cms/toolbar/menu/ajax_item.html"

    def __init__(self, name, action, csrf_token, data=None, active=False, disabled=False, extra_classes=None,
                 question=None, position=LEFT):
        self.position = position
        self.name = name
        self.action = action
        self.active = active
        self.disabled = disabled
        self.csrf_token = csrf_token
        self.data = data or {}
        self.extra_classes = extra_classes or []
        self.question = question

    def __repr__(self):
        return '<AjaxItem:%s>' % unicode(self.name)

    def get_context(self):
        data = {}
        data.update(self.data)
        data[settings.CSRF_COOKIE_NAME] = self.csrf_token
        data = simplejson.dumps(data)
        return {
            'action': self.action,
            'name': self.name,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
            'data': data,
            'question': self.question
        }

class ModalItem(BaseItem):
    template = "cms/toolbar/menu/modal_item.html"

    def __init__(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=True,
                 on_close=REFRESH, position=LEFT):
        self.position = position
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []
        self.close_on_url_change = close_on_url_change
        self.on_close = on_close

    def __repr__(self):
        return '<ModalItem:%s>' % unicode(self.name)

    def get_context(self):
        return {
            'name': self.name,
            'url': self.url,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
            'close_on_url_change': self.close_on_url_change,
            'on_close': self.on_close,
        }

class DialogItem(BaseItem):
    template = "cms/toolbar/menu/dialog_item.html"

class Break(BaseItem):
    template = "cms/toolbar/menu/break.html"

class Button(object):
    def __init__(self, name, url, active=False, disabled=False, extra_classes=None):
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []

    def __repr__(self):
        return '<Button:%s>' % unicode(self.name)

class ButtonList(BaseItem):
    template = "cms/toolbar/menu/button_list.html"

    def __init__(self, extra_classes=None, position=LEFT):
        self.position = position
        self.extra_classes = extra_classes or []
        self.buttons = []

    def add_item(self, item):
        if not isinstance(item, Button):
            raise ValueError("Expected instance of cms.toolbar.items.Button, got %r instead" % item)
        self.buttons.append(item)

    def add_button(self, name, url, active=False, disabled=False, extra_classes=None):
        item = Button(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes
        )
        self.buttons.append(item)
        return item

    def get_context(self):
        return {
            'buttons': self.buttons,
            'extra_classes': self.extra_classes
        }

class Dialog(BaseItem):
    template = "cms/toolbar/menu/dialog.html"

    def __init__(self, url, title, question, right=False):
        super(Dialog, self).__init__(right)
        self.url = url
        self.title = title
        self.question = question

    def __repr__(self):
        return '<Dialog:%s>' % self.title

    def get_context(self):
        return {'url': self.url, 'title': self.title, 'question': self.question}
<commit_msg>removed unused class<commit_after>import abc
from cms.constants import RIGHT, LEFT, REFRESH
from django.conf import settings
from django.template.loader import render_to_string
from django.utils import simplejson

class BaseItem(object):
    __metaclass__ = abc.ABCMeta
    template = abc.abstractproperty()

    def __init__(self, position=LEFT):
        self.position = position

    @property
    def right(self):
        return self.position is RIGHT

    def render(self):
        return render_to_string(self.template, self.get_context())

    def get_context(self):
        return {}

class ToolbarAPIMixin(object):
    __metaclass__ = abc.ABCMeta

    def add_item(self, item):
        if not isinstance(item, BaseItem):
            raise ValueError("Items must be subclasses of cms.toolbar.items.BaseItem, %r isn't" % item)
        self.items.append(item)
        return item

    def add_sideframe_item(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=False,
                 on_close=REFRESH, position=LEFT):
        item = SideframeItem(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            close_on_url_change=close_on_url_change,
            on_close=on_close,
            position=position,
        )
        self.add_item(item)
        return item

    def add_modal_item(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=False,
                 on_close=REFRESH, position=LEFT):
        item = ModalItem(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            close_on_url_change=close_on_url_change,
            on_close=on_close,
            position=position,
        )
        self.add_item(item)
        return item

    def add_link_item(self, name, url, active=False, disabled=False, extra_classes=None, position=LEFT):
        item = LinkItem(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            position=position
        )
        self.add_item(item)
        return item

    def add_ajax_item(self, name, action, active=False, disabled=False, extra_classes=None, data=None, question=None,
                      position=LEFT):
        item = AjaxItem(name, action, self.csrf_token,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes,
            data=data,
            question=question,
            position=position,
        )
        self.add_item(item)
        return item

    def add_break(self):
        item = Break()
        self.add_item(item)
        return item

class Menu(ToolbarAPIMixin, BaseItem):
    template = "cms/toolbar/menu/list.html"

    def __init__(self, name, csrf_token, url='#', sub_level=False, position=LEFT):
        super(Menu, self).__init__(position)
        self.items = []
        self.menus = {}
        self.url = url
        self.name = name
        self.sub_level = sub_level
        self.csrf_token = csrf_token

    def __repr__(self):
        return '<Menu:%s>' % unicode(self.name)

    def get_menu(self, key, verbose_name, position=LEFT):
        if key in self.menus:
            return self.menus[key]
        menu = Menu(verbose_name, self.csrf_token, sub_level=True, position=position)
        self.menus[key] = menu
        self.items.append(menu)
        return menu

    def get_items(self):
        return self.items

    def get_context(self):
        return {
            'items': self.get_items(),
            'url': self.url,
            'title': self.name,
            'sub_level': self.sub_level}

class LinkItem(BaseItem):
    template = "cms/toolbar/menu/link_item.html"

    def __init__(self, name, url, active=False, disabled=False, extra_classes=None, position=LEFT):
        self.position = position
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []

    def __repr__(self):
        return '<LinkItem:%s>' % unicode(self.name)

    def get_context(self):
        return {
            'url': self.url,
            'name': self.name,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
        }

class SideframeItem(BaseItem):
    template = "cms/toolbar/menu/sideframe_item.html"

    def __init__(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=True,
                 on_close=REFRESH, position=LEFT):
        self.position = position
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []
        self.on_close = on_close
        self.close_on_url_change = close_on_url_change

    def __repr__(self):
        return '<SideframeItem:%s>' % unicode(self.name)

    def get_context(self):
        return {
            'url': self.url,
            'name': self.name,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
            'on_close': self.on_close,
            'close_on_url_change': self.close_on_url_change,
        }

class AjaxItem(BaseItem):
    template = "cms/toolbar/menu/ajax_item.html"

    def __init__(self, name, action, csrf_token, data=None, active=False, disabled=False, extra_classes=None,
                 question=None, position=LEFT):
        self.position = position
        self.name = name
        self.action = action
        self.active = active
        self.disabled = disabled
        self.csrf_token = csrf_token
        self.data = data or {}
        self.extra_classes = extra_classes or []
        self.question = question

    def __repr__(self):
        return '<AjaxItem:%s>' % unicode(self.name)

    def get_context(self):
        data = {}
        data.update(self.data)
        data[settings.CSRF_COOKIE_NAME] = self.csrf_token
        data = simplejson.dumps(data)
        return {
            'action': self.action,
            'name': self.name,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
            'data': data,
            'question': self.question
        }

class ModalItem(BaseItem):
    template = "cms/toolbar/menu/modal_item.html"

    def __init__(self, name, url, active=False, disabled=False, extra_classes=None, close_on_url_change=True,
                 on_close=REFRESH, position=LEFT):
        self.position = position
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []
        self.close_on_url_change = close_on_url_change
        self.on_close = on_close

    def __repr__(self):
        return '<ModalItem:%s>' % unicode(self.name)

    def get_context(self):
        return {
            'name': self.name,
            'url': self.url,
            'active': self.active,
            'disabled': self.disabled,
            'extra_classes': self.extra_classes,
            'close_on_url_change': self.close_on_url_change,
            'on_close': self.on_close,
        }

class DialogItem(BaseItem):
    template = "cms/toolbar/menu/dialog_item.html"

class Break(BaseItem):
    template = "cms/toolbar/menu/break.html"

class Button(object):
    def __init__(self, name, url, active=False, disabled=False, extra_classes=None):
        self.name = name
        self.url = url
        self.active = active
        self.disabled = disabled
        self.extra_classes = extra_classes or []

    def __repr__(self):
        return '<Button:%s>' % unicode(self.name)

class ButtonList(BaseItem):
    template = "cms/toolbar/menu/button_list.html"

    def __init__(self, extra_classes=None, position=LEFT):
        self.position = position
        self.extra_classes = extra_classes or []
        self.buttons = []

    def add_item(self, item):
        if not isinstance(item, Button):
            raise ValueError("Expected instance of cms.toolbar.items.Button, got %r instead" % item)
        self.buttons.append(item)

    def add_button(self, name, url, active=False, disabled=False, extra_classes=None):
        item = Button(name, url,
            active=active,
            disabled=disabled,
            extra_classes=extra_classes
        )
        self.buttons.append(item)
        return item

    def get_context(self):
        return {
            'buttons': self.buttons,
            'extra_classes': self.extra_classes
        }
<|endoftext|>


<commit_before>import traceback
from pyprint.Printer import Printer

from coalib.misc.Decorators import enforce_signature
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.settings.FunctionMetadata import FunctionMetadata
from coalib.settings.Section import Section

class Bear(Printer, LogPrinter):
    """
    A bear contains the actual subroutine that is responsible for checking
    source code for certain specifications. However it can actually do
    whatever it wants with the files it gets. If you are missing some Result
    type, feel free to contact us and/or help us extending the coalib.

    This is the base class for every bear. If you want to write an bear, you
    will probably want to look at the GlobalBear and LocalBear classes that
    inherit from this class. In any case you'll want to overwrite at least the
    run method. You can send debug/warning/error messages through the
    debug(), warn(), err() functions. These will send the
    appropriate messages so that they are outputted. Be aware that if you use
    err(), you are expected to also terminate the bear run-through
    immediately.

    If you need some setup or teardown for your bear, feel free to overwrite
    the set_up() and tear_down() functions. They will be invoked
    before/after every run invocation.

    Settings are available at all times through self.section.
    """

    @enforce_signature
    def __init__(self,
                 section: Section,
                 message_queue,
                 timeout=0):
        Printer.__init__(self)
        LogPrinter.__init__(self, self)

        if not hasattr(message_queue, "put") and message_queue is not None:
            raise TypeError("message_queue has to be a Queue or None.")

        self.section = section
        self.message_queue = message_queue
        self.timeout = timeout

        cp = self.check_prerequisites()
        if cp is not True:
            error_string = ("The bear " + type(self).__name__ +
                            " does not fulfill all requirements.")
            if cp is not False:
                error_string += " " + cp

            self.warn(error_string)
            raise RuntimeError(error_string)

    def _print(self, output, **kwargs):
        self.debug(output)

    def log_message(self, log_message, timestamp=None, **kwargs):
        if self.message_queue is not None:
            self.message_queue.put(log_message)

    def run(self, *args, dependency_results=None, **kwargs):
        raise NotImplementedError

    def run_bear_from_section(self, args, kwargs):
        try:
            kwargs.update(
                self.get_metadata().create_params_from_section(self.section))
        except ValueError as err:
            self.warn("The bear {} cannot be executed.".format(
                type(self).__name__), str(err))

        return self.run(*args, **kwargs)

    def execute(self, *args, **kwargs):
        name = type(self).__name__
        try:
            self.debug("Running bear {}...".format(name))
            # If it's already a list it won't change it
            return list(self.run_bear_from_section(args, kwargs) or [])
        except:
            self.warn(
                "Bear {} failed to run. Take a look at debug messages for "
                "further information.".format(name))
            self.debug(
                "The bear {bear} raised an exception. If you are the writer "
                "of this bear, please make sure to catch all exceptions. If "
                "not and this error annoys you, you might want to get in "
                "contact with the writer of this bear.\n\nTraceback "
                "information is provided below:\n\n{traceback}"
                "\n".format(bear=name, traceback=traceback.format_exc()))

    @staticmethod
    def kind():
        """
        :return: The kind of the bear
        """
        raise NotImplementedError

    @classmethod
    def get_metadata(cls):
        """
        :return: Metadata for the run function. However parameters like
        self or parameters implicitly used by coala (e.g. filename for local
        bears) are already removed.
        """
        return FunctionMetadata.from_function(
            cls.run,
            omit={"self", "dependency_results"})

    @classmethod
    def missing_dependencies(cls, lst):
        """
        Checks if the given list contains all dependencies.

        :param lst: A list of all already resolved bear classes (not
        instances).
        :return A list of missing dependencies.
        """
        dep_classes = cls.get_dependencies()

        for item in lst:
            if item in dep_classes:
                dep_classes.remove(item)

        return dep_classes

    @staticmethod
    def get_dependencies():
        """
        Retrieves bear classes that are to be executed before this bear gets
        executed. The results of these bears will then be passed to the
        run method as a dict via the dependency_results argument. The dict
        will have the name of the Bear as key and the list of its results as
        results.

        :return A list of bear classes.
        """
        return []

    @classmethod
    def get_non_optional_settings(cls):
        """
        This method has to determine which settings are needed by this bear.
        The user will be prompted for needed settings that are not available
        in the settings file so don't include settings where a default value
        would do.

        :return: a dictionary of needed settings as keys and a tuple of help
        text and annotation as values
        """
        return cls.get_metadata().non_optional_params

    @staticmethod
    def check_prerequisites():
        """
        Checks whether needed runtime prerequisites of the bear are satisfied.

        This function gets executed at construction and returns True by
        default.

        :return: True if prerequisites are satisfied, else False or a string
                 that serves a more detailed description of what's missing.
        """
        return True
<commit_msg>Bear: Align documentation correctly<commit_after>import traceback
from pyprint.Printer import Printer

from coalib.misc.Decorators import enforce_signature
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.settings.FunctionMetadata import FunctionMetadata
from coalib.settings.Section import Section

class Bear(Printer, LogPrinter):
    """
    A bear contains the actual subroutine that is responsible for checking
    source code for certain specifications. However it can actually do
    whatever it wants with the files it gets. If you are missing some Result
    type, feel free to contact us and/or help us extending the coalib.

    This is the base class for every bear. If you want to write an bear, you
    will probably want to look at the GlobalBear and LocalBear classes that
    inherit from this class. In any case you'll want to overwrite at least the
    run method. You can send debug/warning/error messages through the
    debug(), warn(), err() functions. These will send the
    appropriate messages so that they are outputted. Be aware that if you use
    err(), you are expected to also terminate the bear run-through
    immediately.

    If you need some setup or teardown for your bear, feel free to overwrite
    the set_up() and tear_down() functions. They will be invoked
    before/after every run invocation.

    Settings are available at all times through self.section.
    """

    @enforce_signature
    def __init__(self,
                 section: Section,
                 message_queue,
                 timeout=0):
        Printer.__init__(self)
        LogPrinter.__init__(self, self)

        if not hasattr(message_queue, "put") and message_queue is not None:
            raise TypeError("message_queue has to be a Queue or None.")

        self.section = section
        self.message_queue = message_queue
        self.timeout = timeout

        cp = self.check_prerequisites()
        if cp is not True:
            error_string = ("The bear " + type(self).__name__ +
                            " does not fulfill all requirements.")
            if cp is not False:
                error_string += " " + cp

            self.warn(error_string)
            raise RuntimeError(error_string)

    def _print(self, output, **kwargs):
        self.debug(output)

    def log_message(self, log_message, timestamp=None, **kwargs):
        if self.message_queue is not None:
            self.message_queue.put(log_message)

    def run(self, *args, dependency_results=None, **kwargs):
        raise NotImplementedError

    def run_bear_from_section(self, args, kwargs):
        try:
            kwargs.update(
                self.get_metadata().create_params_from_section(self.section))
        except ValueError as err:
            self.warn("The bear {} cannot be executed.".format(
                type(self).__name__), str(err))

        return self.run(*args, **kwargs)

    def execute(self, *args, **kwargs):
        name = type(self).__name__
        try:
            self.debug("Running bear {}...".format(name))
            # If it's already a list it won't change it
            return list(self.run_bear_from_section(args, kwargs) or [])
        except:
            self.warn(
                "Bear {} failed to run. Take a look at debug messages for "
                "further information.".format(name))
            self.debug(
                "The bear {bear} raised an exception. If you are the writer "
                "of this bear, please make sure to catch all exceptions. If "
                "not and this error annoys you, you might want to get in "
                "contact with the writer of this bear.\n\nTraceback "
                "information is provided below:\n\n{traceback}"
                "\n".format(bear=name, traceback=traceback.format_exc()))

    @staticmethod
    def kind():
        """
        :return: The kind of the bear
        """
        raise NotImplementedError

    @classmethod
    def get_metadata(cls):
        """
        :return: Metadata for the run function. However parameters like `self`
                 or parameters implicitly used by coala (e.g. filename for
                 local bears) are already removed.
        """
        return FunctionMetadata.from_function(
            cls.run,
            omit={"self", "dependency_results"})

    @classmethod
    def missing_dependencies(cls, lst):
        """
        Checks if the given list contains all dependencies.

        :param lst: A list of all already resolved bear classes (not
                    instances).
        :return:    A list of missing dependencies.
        """
        dep_classes = cls.get_dependencies()

        for item in lst:
            if item in dep_classes:
                dep_classes.remove(item)

        return dep_classes

    @staticmethod
    def get_dependencies():
        """
        Retrieves bear classes that are to be executed before this bear gets
        executed. The results of these bears will then be passed to the
        run method as a dict via the dependency_results argument. The dict
        will have the name of the Bear as key and the list of its results as
        results.

        :return: A list of bear classes.
        """
        return []

    @classmethod
    def get_non_optional_settings(cls):
        """
        This method has to determine which settings are needed by this bear.
        The user will be prompted for needed settings that are not available
        in the settings file so don't include settings where a default value
        would do.

        :return: A dictionary of needed settings as keys and a tuple of help
                 text and annotation as values
        """
        return cls.get_metadata().non_optional_params

    @staticmethod
    def check_prerequisites():
        """
        Checks whether needed runtime prerequisites of the bear are satisfied.

        This function gets executed at construction and returns True by
        default.

        :return: True if prerequisites are satisfied, else False or a string
                 that serves a more detailed description of what's missing.
        """
        return True
<|endoftext|>


<commit_before>from math import sin, cos, atan, pi, sqrt, exp

def fwd_euler(f, x0, ti, te, deltaT):
    """
    Forward Euler Method for Ordinary Differential Equations

    Assume slope is constant between t[k] and t[k+1]

    Parameters
    ----------
    f: dx/dt = f(x, t)
    x0: initial state
    ti: initial time
    te: final time
    deltaT: time step

    Returns
    -------
    listT: 1-dimensional list of time at each time step
    listX: 2-dimensional list of state x at each time step
    """
    # number of time steps
    mTimeStep = int((te-ti) * 1.0 / deltaT)

    # number of states == length of initial state vector
    nStates = len(x0)

    # tuple of time step index
    #   0, 1, ...., mTimeStep-1
    listK = tuple(range(mTimeStep))
    # tuple of time step
    #   because time step will be constant,
    #   define as a tuple instead of a list
    listT = tuple(([ti + deltaT*i for i in listK]))
    # if ti, te, deltaT are given as 0.0, 1.0, 0.1
    #   then mTimeStep wil be 10
    #   and listT will be [0:0.1:0.9];
    #       len(listT) == 10

    # pre-allocate memory space
    #   to store state vector of each time step
    listX = [tuple(x0)]     # init x buffer

    # allocation loop
    #   at k = 0, x is x0
    #   at k = [1, 2, ..., n-1] x is not known
    #       so use [0.0] * nStates
    for k in listT[1:]:
        listX.append([0.0] * nStates)
    # end allocation loop
    # now 2d array of mTimeStep x nStates prepared

    xk = x0

    # time step loop
    for k in listK[:-1]:
        # derivatives are currently time step
        sk = f(xk, listT[k])

        # next step x
        xk1 = listX[k+1]

        # state loop
        for i in xrange(nStates):
            # apply forward Euler method
            xk1[i] = xk[i] + sk[i]*deltaT
        # end state loop at time step k

        # update xk to next step
        xk = xk1
    # end time step loop

    return listT, listX
# end function fwd_euler()

def mod_euler(f, x_init, t_start, t_end, delta_t):
    """
    Modified Euler Method Solver
    Usage: listT, listX = mod_euler(f, x_init, t_start, t_end, delta_t)

    Assume slope changes linearly between t[k] and t[k+1]

    Parameters
    ----------
    f : callable(x, t)
        Computes the derivatives of x at (x, t)
        dx/dt = f(x, t)
        Returns a list of [x0, x1, ... ]
    x_init : list or tuple
         initial state on x
    t_start : float
        Initial time
    t_end : float
        Ending time
    deltaT : float
        Sampling time
    delta_t: time step
    Returns
    -------
    t_list : list, shape(int(te-ti)/deltaT + 1,1)
    x_list : list, shape(int(te-ti)/deltaT + 1,len(x_init))

    Examples
    --------
    >>> lT, lX = mod_euler(lambda x, t:[(math.sin(math.pi*t) - x[0])*0.5],[0],0.0, 0.5, 0.1)
    >>> print lT
    [0.0, 0.10000000000000001, 0.20000000000000001, 0.30000000000000004, 0.40000000000000002, 0.5]
    >>> print lX
    [[0], [0.0077254248593736849], [0.029382595321196046], [0.062135518400607666], [0.10209697840236187], [0.14470734296725662]]
    """
    x_list = [tuple(x_init)]    # init x buffer
    t_list = [t_start]

    tk = t_start
    tk1 = tk + delta_t

    # to make t_end the last element of t_list
    t_end += ((-0.5) * delta_t)

    # time step loop
    while tk < t_end:
        xk = x_list[-1]

        # step 1 : same as forward Euler
        sk = f(xk, tk)
        xk1_p = [(x + sk[i]*delta_t) for i, x in enumerate(xk)]

        # step 2 : calculate slope at (xk + f(xk, tk) * delta_t, tk + delta_t)
        sk1_p = f(xk1_p, tk1)

        # step 3 : average of f(xk, tk) and f(xk + f(xk, tk) * delta_t, tk + delta_t)
        sk_c = [(0.5*(s+sk1_p[i])) for (i, s) in enumerate(sk)]

        # step 4 : go forward using averaged slope
        xk1_c = [(x + sk_c[i] * delta_t) for i, x in enumerate(xk)]

        x_list.append(xk1_c)

        # time advance
        tk += delta_t
        tk1 += delta_t

        # append time
        t_list.append(tk)

    return t_list, x_list

def runge_while (func, x0, ti, te, deltaT):
    """
    Runge Method Solver
    Usage: listT, listX = runge_while(func, x0, ti, te, deltaT)

    Parameters
    ----------
    func : callable(x, t)
        Computes the derivatives of x at 0.
        Returns a list of [x0, x1, ... ]
    x0 : list or tuple
        Initial condition on x
    ti : float
        Initial time
    te : float
        Ending time
    deltaT : float
        Sampling time

    Returns
    -------
    listT : list, shape(int(te-ti)/deltaT + 1,1)

    Examples
    --------
    >>> lT, lX = runge_while(lambda x, t:[(math.sin(math.pi*t) - x[0])*0.5],[0],0.0, 0.5, 0.1)
    >>> print lT
    [0.0, 0.10000000000000001, 0.20000000000000001, 0.30000000000000004, 0.40000000000000002, 0.5]
    >>> print lX
    [[0], [0.0076608912592762328], [0.029394418941171337], [0.062350250267466614], [0.10261479161461737], [0.14559255884915154]]
    """

    listX = [x0]    # init x buffer
    listT = [ti]

    deltaThalf = 0.5 * deltaT
    deltaTsixth = deltaT/6.0

    tk = ti
    tk_half = tk + 0.5 * deltaT
    tk1 = tk + deltaT

    te += (-deltaThalf)

    while tk < te:
        xk = listX[-1]

        # step 1
        k1 = func(xk, tk)

        # step 2
        xk1_p = [(xk[i] + k*deltaThalf) for (i, k) in enumerate(k1)]
        k2 = func(xk1_p, tk_half)

        # step 3
        xk2_p = [(xk[i] + k*deltaThalf) for (i, k) in enumerate(k2)]
        k3 = func(xk2_p, tk_half)

        # step 4
        xk3_p = [(xk[i] + k * deltaT) for (i, k) in enumerate(k3)]
        k4 = func(xk3_p, tk1)

        # step 5
        xk1_c = [x + deltaTsixth*(k1[i] + 2*(k2[i] + k3[i]) + k4[i]) for (i, x) in enumerate(xk)]

        listX.append(xk1_c)
        tk += deltaT
        tk_half += deltaT
        tk1 += deltaT
        listT.append(tk)

    return listT, listX

tau = 0.5
m = 10.0
c = 100.0
k = 1000.0

def func(xk, tk):
    """
    Differential equation

    m x2dot(t) + c xdot(t) + k x(t) = u(t)
    u(t) = 1

    Use m, c, k defined outside of this function

    Parameters
    ----------
    xk: state vector at time step k
        xk[0] = x
        xk[1] = xdot

    Returns
    -------
    ydot : list of derivatives
    """
    # step input
    u = 1

    y1, y2 = xk[0], xk[1]

    y1dot = y2
    y2dot = (u - (k*y1 + c*y2))/m

    return (y1dot, y2dot)
# end of function func()

def exact(t):
    """
    Exact solution of a  1-DOF mechanical vibration
    Ref : Rao, Mechanical Vibration, 2nd ed,
        ISBN 0-201-55693-6, Example 4.3
    """
    # step input
    u = 1
    # natural frequency (rad/sec)
    wn = sqrt(k/m)
    # damping ratio
    zeta = c/(2.0 * m * wn)

    s = sqrt(1.0 - zeta * zeta)
    s1 = 1.0 / s

    # damped frequency (rad/sec)
    wd = wn * s
    # phase (rad)
    phi = atan(zeta * s)

    y1 = (u/k) * (1.0 - s1 * exp(-zeta * wn * t) * cos(wd*t-phi))

    return (y1)
# end of function exact()

if "__main__" == __name__:
    help(fwd_euler)

    ti = 0.0
    te = 2.0
    delta_T = 0.01
    x0 = (0.0, 0.0)
    vT, vX = fwd_euler(func, x0, ti, te, delta_T)
    t_list_mod_euler, x_list_mod_euler = mod_euler(func, x0, ti, te, delta_T)
    t_list_runge, x_list_runge = runge_while(func, x0, ti, te, delta_T)

    delta_T = 0.001
    vT01, vX01 = fwd_euler(func, x0, ti, te, delta_T)

    # exact solution
    vXexact = tuple([exact(tk) for tk in vT])

    import pylab
    pylab.plot(vT, vX, label='fwd Euler(0.01)')
    pylab.plot(vT01, vX01, label='fwd Euler(0.001)')
    pylab.plot(t_list_mod_euler, x_list_mod_euler, label='Modified Euler(0.01)')
    pylab.plot(t_list_runge, x_list_runge, label='Runge(0.01)')
    pylab.plot(vT, vXexact, 'k', label='exact')
    pylab.legend(loc=0)
    pylab.grid(True)
    pylab.ylabel('x')
    pylab.xlabel('t')
    pylab.show()

    vP, vV = zip(*vX)
    vP01, vV01 = zip(*vX)
    p_list_mod_euler, v_list_mod_euler = zip(*x_list_mod_euler)
    p_list_runge, v_list_runge = zip(*x_list_runge)

    pylab.plot(vP,vV, label='fwd Euler (0.01)')
    pylab.plot(vP01, vV01, label='fwd Euler(0.001)')
    pylab.plot(p_list_mod_euler, v_list_mod_euler, label='Modified Euler(0.01)')
    pylab.plot(p_list_runge, v_list_runge, label='Runge(0.01)')
    pylab.legend(loc=0)
    pylab.grid(True)
    pylab.ylabel('xdot')
    pylab.xlabel('x')
    pylab.show()
<commit_msg>coding style<commit_after>from math import sin, cos, atan, pi, sqrt, exp

def fwd_euler(f, x0, ti, te, deltaT):
    """
    Forward Euler Method for Ordinary Differential Equations

    Assume slope is constant between t[k] and t[k+1]

    Parameters
    ----------
    f: dx/dt = f(x, t)
    x0: initial state
    ti: initial time
    te: final time
    deltaT: time step

    Returns
    -------
    listT: 1-dimensional list of time at each time step
    listX: 2-dimensional list of state x at each time step
    """
    # number of time steps
    mTimeStep = int((te-ti) * 1.0 / deltaT)

    # number of states == length of initial state vector
    nStates = len(x0)

    # tuple of time step index
    #   0, 1, ...., mTimeStep-1
    listK = tuple(range(mTimeStep))
    # tuple of time step
    #   because time step will be constant,
    #   define as a tuple instead of a list
    listT = tuple(([ti + deltaT*i for i in listK]))
    # if ti, te, deltaT are given as 0.0, 1.0, 0.1
    #   then mTimeStep wil be 10
    #   and listT will be [0:0.1:0.9];
    #       len(listT) == 10

    # pre-allocate memory space
    #   to store state vector of each time step
    listX = [tuple(x0)]     # init x buffer

    # allocation loop
    #   at k = 0, x is x0
    #   at k = [1, 2, ..., n-1] x is not known
    #       so use [0.0] * nStates
    for k in listT[1:]:
        listX.append([0.0] * nStates)
    # end allocation loop
    # now 2d array of mTimeStep x nStates prepared

    xk = x0

    # time step loop
    for k in listK[:-1]:
        # derivatives are currently time step
        sk = f(xk, listT[k])

        # next step x
        xk1 = listX[k+1]

        # state loop
        for i in xrange(nStates):
            # apply forward Euler method
            xk1[i] = xk[i] + sk[i]*deltaT
        # end state loop at time step k

        # update xk to next step
        xk = xk1
    # end time step loop

    return listT, listX
# end function fwd_euler()

def mod_euler(f, x_init, t_start, t_end, delta_t):
    """
    Modified Euler Method Solver
    Usage: listT, listX = mod_euler(f, x_init, t_start, t_end, delta_t)

    Assume slope changes linearly between t[k] and t[k+1]

    Parameters
    ----------
    f : callable(x, t)
        Computes the derivatives of x at (x, t)
        dx/dt = f(x, t)
        Returns a list of [x0, x1, ... ]
    x_init : list or tuple
         initial state of x
    t_start : float
        Initial time
    t_end : float
        Ending time
    deltaT : float
        Sampling time
    delta_t: time step
    Returns
    -------
    t_list : list, shape(int(te-ti)/deltaT + 1,1)
    x_list : list, shape(int(te-ti)/deltaT + 1,len(x_init))

    Examples
    --------
    >>> lT, lX = mod_euler(lambda x, t:[(math.sin(math.pi*t) - x[0])*0.5],[0],0.0, 0.5, 0.1)
    >>> print lT
    [0.0, 0.10000000000000001, 0.20000000000000001, 0.30000000000000004, 0.40000000000000002, 0.5]
    >>> print lX
    [[0], [0.0077254248593736849], [0.029382595321196046], [0.062135518400607666], [0.10209697840236187], [0.14470734296725662]]
    """
    x_list = [tuple(x_init)]    # init x buffer
    t_list = [t_start]

    tk = t_start
    tk1 = tk + delta_t

    # to make t_end the last element of t_list
    t_end += ((-0.5) * delta_t)

    # time step loop
    while tk < t_end:
        xk = x_list[-1]

        # step 1 : same as forward Euler
        sk = f(xk, tk)
        xk1_p = [(x + sk[i]*delta_t) for i, x in enumerate(xk)]

        # step 2 : calculate slope at (xk + f(xk, tk) * delta_t, tk + delta_t)
        sk1_p = f(xk1_p, tk1)

        # step 3 : average of f(xk, tk) and f(xk + f(xk, tk) * delta_t, tk + delta_t)
        sk_c = [(0.5*(s+sk1_p[i])) for (i, s) in enumerate(sk)]

        # step 4 : go forward using averaged slope
        xk1_c = [(x + sk_c[i] * delta_t) for i, x in enumerate(xk)]

        x_list.append(xk1_c)

        # time advance
        tk += delta_t
        tk1 += delta_t

        # append time
        t_list.append(tk)

    return t_list, x_list

def runge_while (f, x_init, t_init, t_end, delta_t):
    """
    Runge Method Solver
    Usage: listT, listX = runge_while(f, x_init, t_init, t_end, delta_t)

    Parameters
    ----------
    f : callable(x, t)
        Computes the derivatives of x at 0.
        Returns a list of [x_init, x1, ... ]
    x_init : list or tuple
        Initial state of x
    t_init : float
        Initial time
    t_end : float
        Ending time
    delta_t : float
        Sampling time

    Returns
    -------
    t_list : list, shape(int(te-ti)/deltaT + 1,1)
    x_list : list, shape(int(te-ti)/deltaT + 1,len(x_init))

    Examples
    --------
    >>> lT, lX = runge_while(lambda x, t:[(math.sin(math.pi*t) - x[0])*0.5],[0],0.0, 0.5, 0.1)
    >>> print lT
    [0.0, 0.10000000000000001, 0.20000000000000001, 0.30000000000000004, 0.40000000000000002, 0.5]
    >>> print lX
    [[0], [0.0076608912592762328], [0.029394418941171337], [0.062350250267466614], [0.10261479161461737], [0.14559255884915154]]
    """

    listX = [x_init]    # init x buffer
    listT = [t_init]

    deltaThalf = 0.5 * delta_t
    deltaTsixth = delta_t/6.0

    tk = t_init
    tk_half = tk + 0.5 * delta_t
    tk1 = tk + delta_t

    t_end += (-deltaThalf)

    while tk < t_end:
        xk = listX[-1]

        # step 1
        k1 = f(xk, tk)

        # step 2
        xk1_p = [(xk[i] + k*deltaThalf) for (i, k) in enumerate(k1)]
        k2 = f(xk1_p, tk_half)

        # step 3
        xk2_p = [(xk[i] + k*deltaThalf) for (i, k) in enumerate(k2)]
        k3 = f(xk2_p, tk_half)

        # step 4
        xk3_p = [(xk[i] + k * delta_t) for (i, k) in enumerate(k3)]
        k4 = f(xk3_p, tk1)

        # step 5
        xk1_c = [x + deltaTsixth*(k1[i] + 2*(k2[i] + k3[i]) + k4[i]) for (i, x) in enumerate(xk)]

        listX.append(xk1_c)
        tk += delta_t
        tk_half += delta_t
        tk1 += delta_t
        listT.append(tk)

    return listT, listX

tau = 0.5
m = 10.0
c = 100.0
k = 1000.0

def func(xk, tk):
    """
    Differential equation

    m x2dot(t) + c xdot(t) + k x(t) = u(t)
    u(t) = 1

    Use m, c, k defined outside of this function

    Parameters
    ----------
    xk: state vector at time step k
        xk[0] = x
        xk[1] = xdot

    Returns
    -------
    ydot : list of derivatives
    """
    # step input
    u = 1

    y1, y2 = xk[0], xk[1]

    y1dot = y2
    y2dot = (u - (k*y1 + c*y2))/m

    return (y1dot, y2dot)
# end of function func()

def exact(t):
    """
    Exact solution of a  1-DOF mechanical vibration
    Ref : Rao, Mechanical Vibration, 2nd ed,
        ISBN 0-201-55693-6, Example 4.3
    """
    # step input
    u = 1
    # natural frequency (rad/sec)
    wn = sqrt(k/m)
    # damping ratio
    zeta = c/(2.0 * m * wn)

    s = sqrt(1.0 - zeta * zeta)
    s1 = 1.0 / s

    # damped frequency (rad/sec)
    wd = wn * s
    # phase (rad)
    phi = atan(zeta * s)

    y1 = (u/k) * (1.0 - s1 * exp(-zeta * wn * t) * cos(wd*t-phi))

    return (y1)
# end of function exact()

if "__main__" == __name__:
    help(fwd_euler)

    ti = 0.0
    te = 2.0
    delta_T = 0.01
    x0 = (0.0, 0.0)
    vT, vX = fwd_euler(func, x0, ti, te, delta_T)
    t_list_mod_euler, x_list_mod_euler = mod_euler(func, x0, ti, te, delta_T)
    t_list_runge, x_list_runge = runge_while(func, x0, ti, te, delta_T)

    delta_T = 0.001
    vT01, vX01 = fwd_euler(func, x0, ti, te, delta_T)

    # exact solution
    vXexact = tuple([exact(tk) for tk in vT])

    import pylab
    pylab.plot(vT, vX, label='fwd Euler(0.01)')
    pylab.plot(vT01, vX01, label='fwd Euler(0.001)')
    pylab.plot(t_list_mod_euler, x_list_mod_euler, label='Modified Euler(0.01)')
    pylab.plot(t_list_runge, x_list_runge, label='Runge(0.01)')
    pylab.plot(vT, vXexact, 'k', label='exact')
    pylab.legend(loc=0)
    pylab.grid(True)
    pylab.ylabel('x')
    pylab.xlabel('t')
    pylab.show()

    vP, vV = zip(*vX)
    vP01, vV01 = zip(*vX)
    p_list_mod_euler, v_list_mod_euler = zip(*x_list_mod_euler)
    p_list_runge, v_list_runge = zip(*x_list_runge)

    pylab.plot(vP,vV, label='fwd Euler (0.01)')
    pylab.plot(vP01, vV01, label='fwd Euler(0.001)')
    pylab.plot(p_list_mod_euler, v_list_mod_euler, label='Modified Euler(0.01)')
    pylab.plot(p_list_runge, v_list_runge, label='Runge(0.01)')
    pylab.legend(loc=0)
    pylab.grid(True)
    pylab.ylabel('xdot')
    pylab.xlabel('x')
    pylab.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
from __future__ import absolute_import

import argparse
import logging
import os
import shutil

from formula import Ag
from wok.conf import Config

# TODO: Path modification during operation by os.environ
def main():
    # TODO: Add choices kwarg for install/remove based on avail formula
    mesg = """wok is a homebrew like installer.

    Available Programs:
    """
    parser = argparse.ArgumentParser(description=mesg,
                                     formatter_class=argparse.
                                     RawDescriptionHelpFormatter)
    parser.add_argument('-w', '--wokinit', nargs='?',
                        default=os.path.expanduser('~/.wok.yaml'),
                        help='yaml config file')
    ex_main = parser.add_mutually_exclusive_group()
    ex_main.add_argument('-i', '--install', dest='action', action='store_const',
                        const='i', help='install specified programs')
    ex_main.add_argument('-r', '--remove', dest='action', action='store_const',
                        const='r', help='remove specified programs')
    ex_main.add_argument('-u', '--update', dest='action', action='store_const',
                        const='u', help='update installed programs')
    parser.add_argument(dest='progs', metavar='PROG', nargs='*',
                        help='programs to install')
    args = parser.parse_args()

    config = Config(args.wokinit)
    build_dir = config.install_to

    if os.path.exists(build_dir):
        shutil.rmtree(build_dir)
    os.makedirs(build_dir)

    logging.debug('CLI: %s', args)
    logging.debug('Wok Config: %s', config)

    if args.progs is not None and 'ag' in args.progs:
        install_dir = os.path.join(config.install_to, Ag.__name__.lower())
        build = Ag(install_dir)
        build.download()
        build.build()
        assert build.verify()

if __name__ == '__main__':
    main()
<commit_msg>Clean up parsing.<commit_after>#!/usr/bin/env python
# encoding: utf-8
from __future__ import absolute_import

import argparse
import logging
import os
import shutil

from formula import Ag
from wok.conf import Config

# TODO: Path modification during operation by os.environ
def main():
    # TODO: Add choices kwarg for install/remove based on avail formula
    mesg = """wok is a homebrew like installer.

    Available Programs:
    """
    parser = argparse.ArgumentParser(description=mesg,
                                     formatter_class=argparse.
                                     RawDescriptionHelpFormatter)
    parser.add_argument('-w', '--wokinit',
                        default=os.path.expanduser('~/.wok.yaml'),
                        help='yaml config file')
    mut1 = parser.add_mutually_exclusive_group()
    mut1.add_argument('-i', '--install', nargs='+',
                        metavar='PROG', help='install specified programs')
    mut1.add_argument('-r', '--remove', nargs='+',
                        metavar='PROG', help='remove specified programs')
    mut1.add_argument('-u', '--update', default=False, action='store_true',
                        help='update installed programs')
    args = parser.parse_args()

    config = Config(args.wokinit)
    build_dir = config.install_to

    if os.path.exists(build_dir):
        shutil.rmtree(build_dir)
    os.makedirs(build_dir)

    logging.debug('CLI: %s', args)
    logging.debug('Wok Config: %s', config)

    if args.install is not None and 'ag' in args.install:
        install_dir = os.path.join(config.install_to, Ag.__name__.lower())
        build = Ag(install_dir)
        build.download()
        build.build()
        assert build.verify()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""

Tensorflow implementation of WV-DR algorithm wrapper class(sklearn style)

:authro: Fido Wang (wanghailin317@gmail.com)
:refer: http://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf

"""
import tensorflow as tf
import os
import numpy as np
from option import Option
import math
import time
import collections
import random
import pickle

def generate_batch_para(doc_ids, word_ids, batch_size, num_skips, window_size):
    """
    batch generator for Skip-Gram Model(Distributed Representation of Word Vecotors)
    :param doc_ids: list of document indices
    :param word_ids: list of word indices
    :param batch_size: number of words in each mini-batch
    :param num_skips: number of sample for each target word window
    :param window_size: number of words between the target word
    """
    data_index = 0
    assert batch_size % num_skips == 0
    assert num_skips <= 2 * window_size
    labels = np.ndarray(shape=(batch_size), dtype=np.int32)
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    span = 2 * window_size + 1
    buffer = collections.deque(maxlen=span)
    buffer_para = collections.deque(maxlen=span)

    i = 0
    while data_index < len(word_ids):
        if len(buffer) == span and len(set(buffer_para)) == 1:
            target = window_size
            targets_to_avoid = [window_size]
            for j in range(num_skips):
                while target in targets_to_avoid:
                    target = random.randint(0, span - 1)
                labels[i + j] = buffer[target]
                batch[i + j] = buffer[window_size]
            i += num_skips
        buffer.append(word_ids[data_index])
        buffer_para.append(doc_ids[data_index])
        # data_index = (data_index + 1) % len(word_ids)
        data_index = (data_index + 1)
        if i == batch_size:
            yield batch, labels[:, None], doc_ids[data_index]
            i = 0
            labels = np.ndarray(shape=(batch_size), dtype=np.int32)
            batch = np.ndarray(shape=(batch_size), dtype=np.int32)

class Skipgram(object):
    """
    Skip-Gram word embedder

    Support Functional Style setting thus all function begin with 'set' and 'build'
    will return an object,the Embedding Estimator itself.
    """

    def __init__(self, options, vocab=None):
        assert isinstance(options, Option)
        self._options = options
        self.__inputs, self.__labels, self.__lr = None, None, None
        self.word_embeddings = None
        self.__normalized_word_embeddings = None
        self.__cost, self.__optimizer, self. __summary = None, None ,None
        self.vocab = vocab
        self.reversed_vocab = None if vocab is None else {v: k for k,v in vocab.items()}
        self.use_subsampling = False
        self._session = None
        self.saver = None

    def set_vocab(self, vocab):
        self.vocab = vocab
        self.reversed_vocab = None if vocab is None else {v: k for k, v in vocab.items()}
        return self

    def use_subsampling(self, switch=True):
        self.use_subsampling = switch
        return self

    def _get_inputs(self):
        """
        Create TF Placeholders for input, targets, and learning rate.
        :return: Tuple of Placeholders (input, targets, learning rate)
        """
        opts = self._options
        inputs_ = tf.placeholder(tf.int32, [opts.batch_size], name='input')
        labels_ = tf.placeholder(tf.int32, [opts.batch_size, 1], name='target')
        lr_ = tf.placeholder(tf.float32, name='learning_rate')
        return inputs_, labels_, lr_

    def _get_embedding_layer(self, input_data):
        """
        Create embedding for <input_data>.
        :param input_data: TF placeholder for text input.
        :return: Embedded input.
        """
        opts = self._options
        embedding = tf.Variable(tf.random_uniform((opts.vocab_size, opts.embed_dim), -1, 1))
        return embedding, tf.nn.embedding_lookup(embedding, input_data)

    def build_graph(self):
        """
        Create Graph and Initialize tf Session for training
        """
        train_graph = tf.Graph()
        opts = self._options
        with train_graph.as_default():
            self.__inputs, self.__labels, self.__lr = self._get_inputs()
            embeddings, embed = self._get_embedding_layer(self.__inputs)

            norm_w = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
            self.__normalized_word_embeddings = embeddings / norm_w

            weights = tf.Variable(
                tf.truncated_normal((opts.vocab_size, opts.embed_dim),
                                    stddev=1.0 / math.sqrt(opts.embed_dim))
            )
            biases = tf.Variable(tf.zeros(opts.vocab_size))
            if opts.loss == 'softmax':
                loss = tf.nn.sampled_softmax_loss(weights=weights,
                                                  biases=biases,
                                                  labels=self.__labels,
                                                  inputs=embed,
                                                  num_sampled=opts.negative_sample_size,
                                                  num_classes=opts.vocab_size)
            elif opts.loss == 'nce':
                loss = tf.nn.nce_loss(weights=weights,
                                      biases=biases,
                                      labels=self.__labels,
                                      inputs=embed,
                                      num_sampled=opts.negative_sample_size,
                                      num_classes=opts.vocab_size)
            self.__cost = tf.reduce_mean(loss)
            tf.summary.scalar("w2v_loss", self.__cost)

            if opts.train_method == 'Adam':
                self.__optimizer = tf.train.AdamOptimizer(self.__lr).minimize(self.__cost)
            else:
                self.__optimizer = tf.train.GradientDescentOptimizer(self.__lr).minimize(self.__cost)
            self.__summary = tf.summary.merge_all()
            self._session = tf.Session(graph=train_graph)
            self.saver = tf.train.Saver()

        return self

    @staticmethod
    def _get_batches(self, doc_ids, word_ids, batch_size, skip_size, window_size):
        n_batches = len(word_ids) // batch_size
        words = word_ids[:n_batches * batch_size]
        for idx in range(0, len(words), batch_size):
            x, y = [], []
            batch = words[idx:idx + batch_size]
            for ii in range(len(batch)):
                batch_x = batch[ii]
                batch_y = self._get_target(batch, ii, window_size)
                y.extend(batch_y)
                x.extend([batch_x] * len(batch_y))
            yield np.array(x), np.array(y)[:, None]

    @staticmethod
    def _get_target(words, idx, window_size=5):
        R = np.random.randint(1, window_size + 1)
        start = idx - R if (idx - R) > 0 else 0
        stop = idx + R
        target_words = set(words[start:idx] + words[idx + 1:stop + 1])

        return list(target_words)

    def _choose_batch_generator(self, chosen='words'):
        if chosen == 'words':
            self._generate_batches = self._get_batches
        if chosen == 'docs':
            self._generate_batches = generate_batch_para

    def fit(self, train_data):
        opts = self._options
        iteration = 1
        loss = 0
        if type(train_data[0]) is int:
            word_ids = train_data
            doc_ids = [0] * len(train_data)
            self._choose_batch_generator(chosen='words')
        elif type(train_data[0]) is list:
            doc_ids = [[i] * len(j) for i, j in enumerate(train_data)]
            doc_ids = [item for sublist in doc_ids for item in sublist]
            word_ids = [item for sublist in train_data for item in sublist]
            self._choose_batch_generator(chosen='docs')

        with self._session as session:
            session.run(tf.global_variables_initializer())
            for e in range(1, opts.epochs_to_train):
                batches = self._generate_batches(doc_ids, word_ids, opts.batch_size, opts.window_size, opts.window_size)
                start = time.time()
                learning_rate = opts.learning_rate
                for x, y, train_idx in batches:
                    period_loss = 1000000000
                    learning_rate = opts.learning_rate if learning_rate < period_loss else learning_rate * 0.1
                    feed = {self.__inputs: x,
                            self.__labels: y,
                            self.__lr: learning_rate}
                    train_loss, _ = session.run([self.__cost, self.__optimizer], feed_dict=feed)

                    loss += train_loss
                    if iteration % opts.statistics_interval == 0:
                        period_loss = loss / opts.statistics_interval
                        end = time.time()
                        print("Epoch {}/{}".format(e, opts.epochs_to_train - 1),
                              "Iteration: {}".format(iteration),
                              "Current Doc: {}".format(train_idx),
                              "Avg. Training loss: {:.4f}".format(period_loss),
                              "{:.4f} sec/batch".format((end - start) * 1.0 / opts.statistics_interval))
                        loss = 0
                        start = time.time()
                    if iteration % opts.checkpoint_interval == 0:
                        self.saver.save(self._session,
                                        "word2vec",
                                        global_step=iteration)
                    iteration += 1
            self.word_embeddings = self.__normalized_word_embeddings.eval()
            self.saver.save(self._session, "final_word2vec")

    def transform_w(self, word_index):
        return self.word_embeddings[word_index, :]

    def transform_doc(self, word_indexs):
        doc_embeddings = [self.word_embeddings[i, :] for i in word_indexs]
        return doc_embeddings

    def save(self, path):
        """
        To save trained model and its params.
        """
        save_path = self.saver.save(self._session,'model.data')
        # save parameters of the model
        params = self._options
        pickle.dump(params,
                  open(os.path.join(path, 'model_params.json'), 'wb'), pickle.HIGHEST_PROTOCOL)

        # save dictionary, reverse_dictionary
        pickle.dump(self.vocab,
                  open(os.path.join(path, 'model_dict.json'), 'wb'), pickle.HIGHEST_PROTOCOL)

        print("Model saved in file: %s" % save_path)
        return save_path

    def _restore(self, path):
        with self.graph.as_default():
            self.saver.restore(self._session, path)
        with self._session as session:
            session.run(tf.global_variables_initializer())
            self.word_embeddings = self.__normalized_word_embeddings.eval()

    @classmethod
    def restore(cls, path):
        """
        To restore a saved model.
        """
        # load params of the model
        path_dir = os.path.dirname(path)
        params = pickle.load(open(os.path.join(path_dir, 'model_params.json'), 'rb'))
        # init an instance of this class
        estimator = Skipgram(params)
        estimator.build_graph();
        estimator._restore(os.path.join(path_dir, 'model.data'))
        # bind dictionaries
        estimator.set_vocab(pickle.load(open(os.path.join(path_dir, 'model_dict.json'), 'rb')))
        estimator._restore(os.path.join(path_dir, 'model.data'))

        return estimator
<commit_msg>- fix typo issues<commit_after>"""

Tensorflow implementation of WV-DR algorithm wrapper class(sklearn style)

:authro: Fido Wang (wanghailin317@gmail.com)
:refer: http://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf

"""
import tensorflow as tf
import os
import numpy as np
from option import Option
import math
import time
import collections
import random
import pickle

def generate_batch_para(doc_ids, word_ids, batch_size, num_skips, window_size):
    """
    batch generator for Skip-Gram Model(Distributed Representation of Word Vecotors)
    :param doc_ids: list of document indices
    :param word_ids: list of word indices
    :param batch_size: number of words in each mini-batch
    :param num_skips: number of sample for each target word window
    :param window_size: number of words between the target word
    """
    data_index = 0
    assert batch_size % num_skips == 0
    assert num_skips <= 2 * window_size
    labels = np.ndarray(shape=(batch_size), dtype=np.int32)
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    span = 2 * window_size + 1
    buffer = collections.deque(maxlen=span)
    buffer_para = collections.deque(maxlen=span)

    i = 0
    while data_index < len(word_ids):
        if len(buffer) == span and len(set(buffer_para)) == 1:
            target = window_size
            targets_to_avoid = [window_size]
            for j in range(num_skips):
                while target in targets_to_avoid:
                    target = random.randint(0, span - 1)
                labels[i + j] = buffer[target]
                batch[i + j] = buffer[window_size]
            i += num_skips
        buffer.append(word_ids[data_index])
        buffer_para.append(doc_ids[data_index])
        # data_index = (data_index + 1) % len(word_ids)
        data_index = (data_index + 1)
        if i == batch_size:
            yield batch, labels[:, None], doc_ids[data_index]
            i = 0
            labels = np.ndarray(shape=(batch_size), dtype=np.int32)
            batch = np.ndarray(shape=(batch_size), dtype=np.int32)

class Skipgram(object):
    """
    Skip-Gram word embedder

    Support Functional Style setting thus all function begin with 'set' and 'build'
    will return an object,the Embedding Estimator itself.
    """

    def __init__(self, options, vocab=None):
        assert isinstance(options, Option)
        self._options = options
        self.__inputs, self.__labels, self.__lr = None, None, None
        self.word_embeddings = None
        self.__normalized_word_embeddings = None
        self.__cost, self.__optimizer, self. __summary = None, None ,None
        self.vocab = vocab
        self.reversed_vocab = None if vocab is None else {v: k for k,v in vocab.items()}
        self.use_subsampling = False
        self._session = None
        self.saver = None

    def set_vocab(self, vocab):
        self.vocab = vocab
        self.reversed_vocab = None if vocab is None else {v: k for k, v in vocab.items()}
        return self

    def use_subsampling(self, switch=True):
        self.use_subsampling = switch
        return self

    def _get_inputs(self):
        """
        Create TF Placeholders for input, targets, and learning rate.
        :return: Tuple of Placeholders (input, targets, learning rate)
        """
        opts = self._options
        inputs_ = tf.placeholder(tf.int32, [opts.batch_size], name='input')
        labels_ = tf.placeholder(tf.int32, [opts.batch_size, 1], name='target')
        lr_ = tf.placeholder(tf.float32, name='learning_rate')
        return inputs_, labels_, lr_

    def _get_embedding_layer(self, input_data):
        """
        Create embedding for <input_data>.
        :param input_data: TF placeholder for text input.
        :return: Embedded input.
        """
        opts = self._options
        embedding = tf.Variable(tf.random_uniform((opts.vocab_size, opts.embed_dim), -1, 1))
        return embedding, tf.nn.embedding_lookup(embedding, input_data)

    def build_graph(self):
        """
        Create Graph and Initialize tf Session for training
        """
        train_graph = tf.Graph()
        self.graph = train_graph
        opts = self._options
        with train_graph.as_default():
            self.__inputs, self.__labels, self.__lr = self._get_inputs()
            embeddings, embed = self._get_embedding_layer(self.__inputs)

            norm_w = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
            self.__normalized_word_embeddings = embeddings / norm_w

            weights = tf.Variable(
                tf.truncated_normal((opts.vocab_size, opts.embed_dim),
                                    stddev=1.0 / math.sqrt(opts.embed_dim))
            )
            biases = tf.Variable(tf.zeros(opts.vocab_size))
            if opts.loss == 'softmax':
                loss = tf.nn.sampled_softmax_loss(weights=weights,
                                                  biases=biases,
                                                  labels=self.__labels,
                                                  inputs=embed,
                                                  num_sampled=opts.negative_sample_size,
                                                  num_classes=opts.vocab_size)
            elif opts.loss == 'nce':
                loss = tf.nn.nce_loss(weights=weights,
                                      biases=biases,
                                      labels=self.__labels,
                                      inputs=embed,
                                      num_sampled=opts.negative_sample_size,
                                      num_classes=opts.vocab_size)
            self.__cost = tf.reduce_mean(loss)
            tf.summary.scalar("w2v_loss", self.__cost)

            if opts.train_method == 'Adam':
                self.__optimizer = tf.train.AdamOptimizer(self.__lr).minimize(self.__cost)
            else:
                self.__optimizer = tf.train.GradientDescentOptimizer(self.__lr).minimize(self.__cost)
            self.__summary = tf.summary.merge_all()
            self._session = tf.Session(graph=train_graph)
            self.saver = tf.train.Saver()

        return self

    @staticmethod
    def _get_batches(self, doc_ids, word_ids, batch_size, skip_size, window_size):
        n_batches = len(word_ids) // batch_size
        words = word_ids[:n_batches * batch_size]
        for idx in range(0, len(words), batch_size):
            x, y = [], []
            batch = words[idx:idx + batch_size]
            for ii in range(len(batch)):
                batch_x = batch[ii]
                batch_y = self._get_target(batch, ii, window_size)
                y.extend(batch_y)
                x.extend([batch_x] * len(batch_y))
            yield np.array(x), np.array(y)[:, None]

    @staticmethod
    def _get_target(words, idx, window_size=5):
        R = np.random.randint(1, window_size + 1)
        start = idx - R if (idx - R) > 0 else 0
        stop = idx + R
        target_words = set(words[start:idx] + words[idx + 1:stop + 1])

        return list(target_words)

    def _choose_batch_generator(self, chosen='words'):
        if chosen == 'words':
            self._generate_batches = self._get_batches
        if chosen == 'docs':
            self._generate_batches = generate_batch_para

    def fit(self, train_data):
        opts = self._options
        iteration = 1
        loss = 0
        if type(train_data[0]) is int:
            word_ids = train_data
            doc_ids = [0] * len(train_data)
            self._choose_batch_generator(chosen='words')
        elif type(train_data[0]) is list:
            doc_ids = [[i] * len(j) for i, j in enumerate(train_data)]
            doc_ids = [item for sublist in doc_ids for item in sublist]
            word_ids = [item for sublist in train_data for item in sublist]
            self._choose_batch_generator(chosen='docs')

        with self._session as session:
            session.run(tf.global_variables_initializer())
            for e in range(1, opts.epochs_to_train):
                batches = self._generate_batches(doc_ids, word_ids, opts.batch_size, opts.window_size, opts.window_size)
                start = time.time()
                learning_rate = opts.learning_rate
                for x, y, train_idx in batches:
                    period_loss = 1000000000
                    learning_rate = opts.learning_rate if learning_rate < period_loss else learning_rate * 0.1
                    feed = {self.__inputs: x,
                            self.__labels: y,
                            self.__lr: learning_rate}
                    train_loss, _ = session.run([self.__cost, self.__optimizer], feed_dict=feed)

                    loss += train_loss
                    if iteration % opts.statistics_interval == 0:
                        period_loss = loss / opts.statistics_interval
                        end = time.time()
                        print("Epoch {}/{}".format(e, opts.epochs_to_train - 1),
                              "Iteration: {}".format(iteration),
                              "Current Doc: {}".format(train_idx),
                              "Avg. Training loss: {:.4f}".format(period_loss),
                              "{:.4f} sec/batch".format((end - start) * 1.0 / opts.statistics_interval))
                        loss = 0
                        start = time.time()
                    if iteration % opts.checkpoint_interval == 0:
                        self.saver.save(self._session,
                                        "word2vec",
                                        global_step=iteration)
                    iteration += 1
            self.word_embeddings = self.__normalized_word_embeddings.eval()
            self.saver.save(self._session, "final_word2vec")

    def transform_w(self, word_index):
        return self.word_embeddings[word_index, :]

    def transform_doc(self, word_indexs):
        doc_embeddings = [self.word_embeddings[i, :] for i in word_indexs]
        return doc_embeddings

    def save(self, path):
        """
        To save trained model and its params.
        """
        save_path = self.saver.save(self._session,'model.data')
        # save parameters of the model
        params = self._options
        pickle.dump(params,
                  open(os.path.join(path, 'model_params.json'), 'wb'), pickle.HIGHEST_PROTOCOL)

        # save dictionary, reverse_dictionary
        pickle.dump(self.vocab,
                  open(os.path.jooin(path, 'model_dict.json'), 'wb'), pickle.HIGHEST_PROTOCOL)

        print("Model saved in file: %s" % save_path)
        return save_path

    def _restore(self, path):
        with self.graph.as_default():
            self.saver.restore(self._session, path)
        with self._session as session:
            session.run(tf.global_variables_initializer())
            self.word_embeddings = self.__normalized_word_embeddings.eval()

    @classmethod
    def restore(cls, path):
        """
        To restore a saved model.
        """
        # load params of the model
        path_dir = os.path.dirname(path)
        params = pickle.load(open(os.path.join(path_dir, 'model_params.json'), 'rb'))
        # init an instance of this class
        estimator = Skipgram(params)
        estimator.build_graph();
        estimator._restore(os.path.join(path_dir, 'model.data'))
        # bind dictionaries
        estimator.set_vocab(pickle.load(open(os.path.join(path_dir, 'model_dict.json'), 'rb')))

        return estimator
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import sys

outfile = sys.argv[1]

bio_df = pd.read_csv('~/biographies/metadata/hathi_ic_biog.tsv', sep='\t')

bioindex = pd.read_csv('/media/secure_volume/index/bioindex.tsv',sep='\t')

merge_bioindex = pd.merge(
    bio_df, bioindex,
    how='inner',
    left_on='docid',
    right_on='htid').loc[
    :,['imprintdate','authgender','mainid','htid','docid','filesuffix']
    ]

def make_balanced(df):
    balanced_df = pd.DataFrame()
    for yr in np.arange(1923,2001):
        for gender in ['F','M','U']:
            subset_df = df.loc[(df.imprintdate == str(yr)) & (df.authgender == gender),:]
            df_shape = subset_df.shape
            if gender == 'U':
                subset_df = subset_df.head(25)
            if df_shape[0] > 50:
                subset_df = subset_df.head(50)
            else:
                print('Only', df_shape[0], 'rows written for:')
                print(yr, '-->',gender)
            balanced_df = pd.concat([balanced_df, subset_df], axis=0)
    return balanced_df

balanced_df = make_balanced(merge_bioindex)

balanced_df.to_csv(outfile, index=False, sep='\t')

print(balanced_df.shape[0], 'rows written to', outfile)
<commit_msg>updated years in df using Ted's date functions<commit_after>import pandas as pd
import numpy as np
import sys

outfile = sys.argv[1]

bio_df = pd.read_csv('~/biographies/metadata/hathi_ic_biog.tsv', sep='\t')

bioindex = pd.read_csv('/media/secure_volume/index/bioindex.tsv',sep='\t')

merge_bioindex = pd.merge(
    bio_df, bioindex,
    how='inner',
    left_on='docid',
    right_on='htid').loc[
    :,['infer_date','authgender','mainid','htid','filesuffix']
    ]

# date functions taken from https://github.com/tedunderwood/library/blob/master/SonicScrewdriver.py
def infer_date(datetype, firstdate, seconddate, textdate):
    '''Receives a date type and three dates, as strings, with no guarantee that any
    of the dates will be numeric. The logic of the data here is defined by
    MARC standards for controlfield 008:
    http://www.loc.gov/marc/bibliographic/concise/bd008a.html
    Returns a date that represents either a shaky consensus
    about the earliest attested date for this item, or 0, indicating no
    consensus.
    '''
    try:
        intdate = int(firstdate)
    except:
        # No readable date
        if firstdate.endswith('uu'):
            # Two missing places is too many.
            intdate = 0
        elif firstdate.endswith('u'):
            # but one is okay
            try:
                decade = int(firstdate[0:3])
                intdate = decade * 10
            except:
                # something's weird. fail.
                intdate = 0
        else:
            intdate = 0

    if intdate == 0:
        try:
            intdate = int(textdate)
        except:
            intdate = 0

    try:
        intsecond = int(seconddate)
    except:
        intsecond = 0

    if intsecond - intdate > 80 and intsecond < 2100:
        # A gap of more than eighty years is too much.
        # This is usually an estimated date that could be anywhere within
        # the nineteenth century.
        # note that we specify intsecond < 2100 because otherwise things
        # dated 9999 throw an error
        intdate = 0

    if datetype == 't' and intsecond > 0 and intsecond < intdate:
        intdate = intsecond
        # This is a case where we have both a publication date and
        # a copyright date. Accept the copyright date. We're going
        # for 'the earliest attested date for the item.'

    if intdate < 1000 and intsecond > 1700 and intsecond < 2100:
        intdate = intsecond

    return intdate

def date_row(row):
    datetype = row["datetype"]
    firstdate = row["startdate"]
    secondate = row["enddate"]
    if "imprintdate" in row:
        textdate = row["imprintdate"]
    else:
        textdate = row["textdate"]
    intdate = infer_date(datetype, firstdate, secondate, textdate)
    return intdate

def make_balanced(df):
    balanced_df = pd.DataFrame()
    for yr in np.arange(1923,2001):
        for gender in ['F','M','U']:
            subset_df = df.loc[(df.infer_date == str(yr)) & (df.authgender == gender),:]
            df_shape = subset_df.shape
            if gender == 'U':
                subset_df = subset_df.head(25)
            if df_shape[0] > 50:
                subset_df = subset_df.head(50)
            else:
                print('Only', df_shape[0], 'rows written for:')
                print(yr, '-->',gender)
            balanced_df = pd.concat([balanced_df, subset_df], axis=0)
    return balanced_df

# add infer_date to the df
merge_dict = merge_bioindex.to_dict(orient='records')
for row in merge_dict:
    row['infer_date'] = date_row(row)

new_bioindex = pd.DataFrame()
new_bioindex.from_records(df_dict)

# create gender-balanced subset df
balanced_df = make_balanced(new_bioindex)

# write to given outfile
balanced_df.to_csv(outfile, index=False, sep='\t')

print(balanced_df.shape[0], 'rows written to', outfile)
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
from prepare_data import data_manip, generate_train, generate_test, data_merge, fs_boruta, greedy_elim, additional_feature, rmsle
from sklearn.ensemble import RandomForestRegressor
from sklearn.cross_validation import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.decomposition import PCA
import sys

# generate test and training data and merge them together for manipulation (dummies and hierachical)
df_train = generate_train()
df_test = generate_test()
df_merge = pd.concat([df_train, df_test])
df_merge = additional_feature(df_merge)

# create dummies and hierachical changes and split data into train and test
df_merge_transform = data_manip(df_merge)
df_train = df_merge_transform[df_merge_transform['source'] == 'train'].copy(deep=True)
df_train.drop('source', axis=1, inplace=True)
df_train= df_train.apply(pd.to_numeric)
df_test = df_merge_transform[df_merge_transform['source'] == 'test'].copy(deep=True)
df_test.drop('source', axis=1, inplace=True)
df_test= df_test.apply(pd.to_numeric)

# perform boruta feature selection
bool_fs_selection = True
if bool_fs_selection:
    key_features = greedy_elim(df_train)
    key_features = [i for i in key_features]
    print(key_features)
else:
    key_features = [i for i in df_test.columns]
    #key_features = ['1stFlrSF', '2ndFlrSF', 'BsmtFinSF1', 'BsmtQual', 'GarageArea', 'GarageCars',
 #'GrLivArea', 'LotArea', 'OverallQual', 'TotRmsAbvGrd', 'TotalBsmtSF',
 #'YearBuilt', 'YearRemodAdd']

# train and output test data
bool_split_data = False
df_train = df_train[key_features + ['SalePrice']]
df_test = df_test[key_features]
X = df_train.drop('SalePrice', axis=1)
y = df_train['SalePrice']
forest = RandomForestRegressor()
if bool_split_data:
    X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.3, random_state=42)
    forest.fit(X_train,y_train)
    predictions = forest.predict(X_test)
    predictions = np.array(predictions)
    y_test = np.array(y_test)
    print(rmsle(predictions, y_test))

else:
    forest.fit(X,y)
    df_test.fillna(0, inplace=True)
    predictions = pd.DataFrame(forest.predict(df_test), index=[i for i in range(1461,2920)])
    predictions.columns = ['SalePrice']
    predictions.index.name = 'Id'
    predictions.to_csv('output.csv')

<commit_msg>commit<commit_after>import pandas as pd
import numpy as np
from prepare_data import data_manip, generate_train, generate_test, data_merge, fs_boruta, greedy_elim, additional_feature, rmsle
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.cross_validation import train_test_split
from sklearn.linear_model import Lasso
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVC
from sklearn.metrics import mean_squared_error
from sklearn.decomposition import PCA
import sys

# generate test and training data and merge them together for manipulation (dummies and hierachical)
df_train = generate_train()
df_test = generate_test()
df_merge = pd.concat([df_train, df_test])
df_merge = additional_feature(df_merge)

# create dummies and hierachical changes and split data into train and test
df_merge_transform = data_manip(df_merge)
df_train = df_merge_transform[df_merge_transform['source'] == 'train'].copy(deep=True)
df_train.drop('source', axis=1, inplace=True)
df_train= df_train.apply(pd.to_numeric)
df_test = df_merge_transform[df_merge_transform['source'] == 'test'].copy(deep=True)
df_test.drop('source', axis=1, inplace=True)
df_test= df_test.apply(pd.to_numeric)

# perform boruta feature selection
bool_fs_selection = False
if bool_fs_selection:
    key_features = greedy_elim(df_train)
    key_features = [i for i in key_features]
    print(key_features)
else:
    key_features = ['1stFlrSF', '2ndFlrSF', '3SsnPorch', 'BedroomAbvGr', 'BsmtCond', 'BsmtExposure', 'BsmtFinSF1',
                    'BsmtFinSF2', 'BsmtFinType1', 'BsmtFinType2', 'BsmtFullBath', 'BsmtHalfBath', 'BsmtQual',
                    'BsmtUnfSF', 'EnclosedPorch', 'ExterCond', 'ExterQual', 'FireplaceQu', 'Fireplaces', 'FullBath',
                    'Functional', 'GarageArea', 'GarageCars', 'GarageCond', 'GarageFinish', 'GarageQual', 'GrLivArea',
                    'HalfBath', 'HeatingQC', 'KitchenAbvGr', 'KitchenQual', 'LandSlope', 'LotArea', 'LotShape', 'LowQualFinSF',
                    'MiscVal', 'MoSold', 'OpenPorchSF', 'OverallCond', 'OverallQual', 'PavedDrive', 'ScreenPorch',
                    'TotRmsAbvGrd', 'TotalBsmtSF', 'WoodDeckSF', 'YearBuilt', 'YearRemodAdd', 'YrSold', 'Bungalow',
                    'MSSubClass_20', 'MSSubClass_30', 'MSSubClass_50', 'MSSubClass_60', 'MSSubClass_70', 'MSSubClass_80',
                    'MSSubClass_120', 'MSZoning_C (all)', 'MSZoning_FV', 'MSZoning_RL', 'MSZoning_RM', 'Alley_Grvl', 'Alley_NA',
                    'Alley_Pave', 'LandContour_Bnk', 'LandContour_HLS', 'LandContour_Low', 'LandContour_Lvl',
                    'LotConfig_Corner', 'LotConfig_CulDSac', 'LotConfig_FR2', 'LotConfig_FR3', 'LotConfig_Inside',
                    'Neighborhood_BrkSide', 'Neighborhood_ClearCr', 'Neighborhood_CollgCr', 'Neighborhood_Crawfor',
                    'Neighborhood_Edwards', 'Neighborhood_Gilbert', 'Neighborhood_IDOTRR', 'Neighborhood_Mitchel',
                    'Neighborhood_NAmes', 'Neighborhood_NWAmes', 'Neighborhood_NoRidge', 'Neighborhood_NridgHt',
                    'Neighborhood_OldTown', 'Neighborhood_Sawyer', 'Neighborhood_SawyerW', 'Neighborhood_Somerst',
                    'Neighborhood_StoneBr','Neighborhood_Timber', 'Condition1_Artery', 'Condition1_Feedr',
                    'Condition1_Norm', 'Condition1_RRAe', 'BldgType_1Fam', 'BldgType_Duplex', 'BldgType_TwnhsE',
                    'HouseStyle_1.5Fin', 'HouseStyle_1Story', 'HouseStyle_2Story', 'HouseStyle_SLvl', 'RoofStyle_Flat',
                    'RoofStyle_Gable', 'RoofStyle_Gambrel', 'RoofStyle_Hip', 'RoofMatl_CompShg', 'Exterior1st_BrkFace',
                    'Exterior1st_CemntBd', 'Exterior1st_HdBoard', 'Exterior1st_MetalSd', 'Exterior1st_Plywood',
                    'Exterior1st_VinylSd', 'Exterior1st_Wd Sdng', 'Exterior1st_WdShing', 'Exterior2nd_Brk Cmn',
                    'Exterior2nd_CmentBd', 'Exterior2nd_HdBoard', 'Exterior2nd_MetalSd', 'Exterior2nd_Plywood',
                    'Exterior2nd_Stucco', 'Exterior2nd_VinylSd', 'Exterior2nd_Wd Sdng', 'Exterior2nd_Wd Shng',
                    'MasVnrType_BrkCmn', 'MasVnrType_BrkFace', 'MasVnrType_None', 'MasVnrType_Stone', 'Foundation_BrkTil',
                    'Foundation_CBlock', 'Foundation_PConc', 'CentralAir_N', 'CentralAir_Y', 'Electrical_FuseA',
                    'Electrical_SBrkr', 'GarageType_Attchd', 'GarageType_BuiltIn', 'GarageType_Detchd', 'GarageType_NA',
                    'MiscFeature_NA', 'Fence_GdPrv', 'Fence_GdWo', 'Fence_MnPrv', 'Fence_NA', 'SaleType_COD','SaleType_New',
                    'SaleType_WD', 'SaleCondition_Abnorml', 'SaleCondition_Family', 'SaleCondition_Normal',
                    'SaleCondition_Partial']

# train and output test data
bool_split_data = True
df_train = df_train[key_features + ['SalePrice']]
df_test = df_test[key_features]
X = df_train.drop('SalePrice', axis=1)
y = df_train['SalePrice']

# fit model 1
model_1 = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.013)
#model_1 = RandomForestRegressor(n_estimators=3000, max_features=0.25)
#model_1 = Lasso(normalize=True, max_iter=100000)
#model_1 = KNeighborsRegressor()
#model_1 = SVC()

#if bool_split_data:
print(key_features)
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.3, random_state=42)

# train model and output accuracy
df_test.fillna(0, inplace=True)
model_1.fit(X_train, y_train)
predictions = model_1.predict(X_test)
print(predictions)
df_compare = pd.DataFrame({'test': y_test, 'predictions': predictions}, index = y_test.index)
print(df_compare.head())
df_compare.to_csv('compare.csv')
sys.exit()

predictions = np.array(predictions)
y_test = np.array(y_test)
print(rmsle(predictions, y_test))
sys.exit()

# output data
predictions = pd.DataFrame(model_1.predict(df_test), index=[i for i in range(1461,2920)])
predictions.columns = ['SalePrice']
predictions.index.name = 'Id'
predictions.to_csv('boosted.csv')

<|endoftext|>


<commit_before>import argparse
import bz2
import io
import sys
import urllib.request

import networkx as nx
import arpy

def read_wot(keysfile, namesfile, sigsfile):
    G = nx.MultiDiGraph()

    keys = list()
    for name in namesfile:
        keyid = int.from_bytes(keysfile.read(4), byteorder="big")
        keys.append(keyid)
        G.add_node(keyid, name=name.decode("utf-8", errors="replace"))

    for owner in keys:
        numsigs = int.from_bytes(sigsfile.read(4), byteorder="big")
        for i in range(numsigs):
            signer_index = int.from_bytes(sigsfile.read(4), byteorder="big")
            signer_index = signer_index & 0x0FFFFFFF
            signer = keys[signer_index]
            G.add_edge(signer, owner)

    return G

def latest_wot():
    url = "http://wot.christoph-egger.org/download/latest.wot"
    return urllib.request.urlopen(url)

def extract_wot(wot_file):
    wot_files = {}

    with bz2.BZ2File(wot_file) as wot_ar:
        wot_archive = arpy.Archive(fileobj=wot_ar)
        for f in wot_archive:
            filename = f.header.name.decode()
            contents = io.BytesIO(f.read())
            wot_files[filename] = contents

    return wot_files

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Generate web of trust graph")
    parser.add_argument(
        "-w", "--wot",
        dest="wot_filename",
        help="use local .wot file",
    )
    parser.add_argument(
        "-f", "--format",
        choices=["gexf", "graphml", "yaml"],
        dest="file_format",
        default="gexf",
        help="specify file format of outputted graph",
    )
    parser.add_argument(
        "-o", "--output",
        help="output filename",
    )
    args = parser.parse_args()

    if args.wot_filename:
        wot_file = open(args.wot_filename, "rb")
    else:
        wot_file = latest_wot()

    files = extract_wot(wot_file)

    G = read_wot(files["keys"], files["names"], files["signatures"])

    if args.output:
        outfile = open(args.output, "wb")
    else:
        outfile = sys.stdout.buffer

    if args.file_format == "gexf":
        nx.write_gexf(G, outfile)
    elif args.file_format == "graphml":
        nx.write_graphml(G, outfile)
    elif args.file_format == "yaml":
        nx.write_yaml(G, outfile)
<commit_msg>Add hashbang<commit_after>#!/usr/bin/env python3

import argparse
import bz2
import io
import sys
import urllib.request

import networkx as nx
import arpy

def read_wot(keysfile, namesfile, sigsfile):
    G = nx.MultiDiGraph()

    keys = list()
    for name in namesfile:
        keyid = int.from_bytes(keysfile.read(4), byteorder="big")
        keys.append(keyid)
        G.add_node(keyid, name=name.decode("utf-8", errors="replace"))

    for owner in keys:
        numsigs = int.from_bytes(sigsfile.read(4), byteorder="big")
        for i in range(numsigs):
            signer_index = int.from_bytes(sigsfile.read(4), byteorder="big")
            signer_index = signer_index & 0x0FFFFFFF
            signer = keys[signer_index]
            G.add_edge(signer, owner)

    return G

def latest_wot():
    url = "http://wot.christoph-egger.org/download/latest.wot"
    return urllib.request.urlopen(url)

def extract_wot(wot_file):
    wot_files = {}

    with bz2.BZ2File(wot_file) as wot_ar:
        wot_archive = arpy.Archive(fileobj=wot_ar)
        for f in wot_archive:
            filename = f.header.name.decode()
            contents = io.BytesIO(f.read())
            wot_files[filename] = contents

    return wot_files

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Generate web of trust graph")
    parser.add_argument(
        "-w", "--wot",
        dest="wot_filename",
        help="use local .wot file",
    )
    parser.add_argument(
        "-f", "--format",
        choices=["gexf", "graphml", "yaml"],
        dest="file_format",
        default="gexf",
        help="specify file format of outputted graph",
    )
    parser.add_argument(
        "-o", "--output",
        help="output filename",
    )
    args = parser.parse_args()

    if args.wot_filename:
        wot_file = open(args.wot_filename, "rb")
    else:
        wot_file = latest_wot()

    files = extract_wot(wot_file)

    G = read_wot(files["keys"], files["names"], files["signatures"])

    if args.output:
        outfile = open(args.output, "wb")
    else:
        outfile = sys.stdout.buffer

    if args.file_format == "gexf":
        nx.write_gexf(G, outfile)
    elif args.file_format == "graphml":
        nx.write_graphml(G, outfile)
    elif args.file_format == "yaml":
        nx.write_yaml(G, outfile)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Get article image file from title or file
"""

from __future__ import print_function

import os
import argparse
import json
import mimetypes
import re
import wp_query

__author__ = "siznax"
__version__ = "25 Sep 2015"

def _extract(images):
    output = []
    for item in images:
        match = re.search(r"\[\[(.*)\]\]", item)
        if match:
            item = match.group(1)
        match = re.search(r"([^\|]*)|", item)
        if match:
            item = match.group(1)
        output.append(item)
    return output

def _walk(data):
    images = []
    for page in data["query"]["pages"]:
        for rev in page['revisions']:
            for line in rev['content'].split("\n"):
                if re.search(r'image {0,}=', line):
                    image = line.split("=")[1].strip()
                    images.append(image)
    return images

def _main(titles):
    if os.path.exists(titles):
        fname = titles
        ftype = mimetypes.guess_type(fname)[0]
        if ftype != 'application/json':
            raise ValueError("invalid file type: %s" % ftype)
        with open(fname) as fh:
            data = json.loads(fh.read())
    else:
        data = json.loads(wp_query.data(titles))
    images = _walk(data)
    images = _extract(images)
    for img in images:
        print(img)

if __name__ == "__main__":
    desc = "Get image file from article if extant"
    argp = argparse.ArgumentParser(description=desc)
    argp.add_argument("titles", help="article titles or filename")
    args = argp.parse_args()
    _main(args.titles)
<commit_msg>enable multiple title args in wp_image.py<commit_after>#!/usr/bin/env python
"""
Get article image file from title or file
"""

from __future__ import print_function

import os
import argparse
import json
import mimetypes
import re
import wp_query

__author__ = "siznax"
__version__ = "25 Sep 2015"

def _extract(images):
    output = []
    for item in images:
        match = re.search(r"\[\[(.*)\]\]", item)
        if match:
            item = match.group(1)
        match = re.search(r"([^\|]*)|", item)
        if match:
            item = match.group(1)
        output.append(item)
    return output

def _walk(data):
    images = []
    for page in data["query"]["pages"]:
        for rev in page['revisions']:
            for line in rev['content'].split("\n"):
                if re.search(r'image {0,}=', line):
                    image = line.split("=")[1].strip()
                    images.append(image)
    return images

def _main(titles):
    if os.path.exists(titles[0]):
        fname = titles[0]
        ftype = mimetypes.guess_type(fname)[0]
        if ftype != 'application/json':
            raise ValueError("invalid file type: %s" % ftype)
        with open(fname) as fh:
            data = json.loads(fh.read())
    else:
        data = json.loads(wp_query.data(titles))
    images = _walk(data)
    images = _extract(images)
    for img in images:
        print(img)

if __name__ == "__main__":
    desc = "Get image file from article if extant"
    argp = argparse.ArgumentParser(description=desc)
    argp.add_argument("titles", nargs='+',
                      help="article titles or filename")
    args = argp.parse_args()
    _main(args.titles)
<|endoftext|>