<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import heapq
import itertools

class Event(object):
    """
    Class that represents single event to be handled

    Provides information about:

    subject - subject which sends event
    name - name of an event
    parameters - event parameters
    return_value - event return value
    processed - whether event has been processed or not

    Example:

    subject = self
    e = Event(subject, 'test', {'a' :1})
    e.subject           # subject
    e.name              # 'test'
    e.parameters        # {'a': 1}
    e['a']              # 1
    e.parameters['a']   # 1
    e.return_value      # None
    e.return_value = 'a'
    e.return_value      # 'a'
    e.processed         # False
    e.mark_procesed()
    e.processed         # True
    """

    def __init__(self, subject, name, parameters={}):
        """
        Initializes class instance
        """
        self.subject = subject
        self.name = name
        self.parameters = parameters
        self.return_value = None
        self._processed = False

    @property
    def processed(self):
        """
        Returns information whether event has been processed
        """
        return self._processed

    def mark_processed(self):
        """
        Marks event as processed
        """
        self._processed = True
        return self

    def __getitem__(self, key):
        """
        Allows to treat class instances as dicts
        """
        return self.parameters[key]

    def __setitem__(self, key, value):
        """
        Allows to treat class instances as dicts
        """
        self.parameters[key] = value

    def __delitem__(self, key):
        """
        Allows to treat class instances as dicts
        """
        del self.parameters[key]

    def __contains__(self, item):
        """
        Allows to treat class instances as dicts
        """
        return item in self.parameters

class Dispatcher(object):
    """
    Dispatches given events according to previously set listeners
    """

    def __init__(self):
        """
        Initializes class instance
        """
        self._listeners = {}
        self.counter = itertools.count()

    def attach(self, name, listener, priority=0):
        """
        Attaches new listener to dispatcher
        """
        if name not in self._listeners:
            self._listeners[name] = []
        heapq.heappush(self._listeners[name], self._prepare(listener, priority))

    def _prepare(self, listener, priority=0):
        """
        Prepares internal listener structure
        """
        return (priority, next(self.counter), listener)

    def has_listeners(self, name):
        """
        Returns information whether there are any listeners
        attached to given event name
        """
        return name in self._listeners and len(self._listeners[name])>0

    def get_listeners(self, name):
        """
        Fetches list of listeners attached to given event.
        Returns iterator
        """
        if name not in self._listeners:
            return []
        return (l[2] for l in self._listeners[name])

    def notify(self, event):
        """
        Notifies each listener about new event
        """
        for listener in self.get_listeners(event.name):
            listener(event)
        return event;

    def notify_until(self, event):
        """
        Notifies listeners about new event until any of then returns True
        """
        for listener in self.get_listeners(event.name):
            if listener(event):
                event.mark_processed()
                break
        return event

    def filter(self, event, value):
        """
        Filters given value using given event
        """
        for listener in self.get_listeners(event.name):
            value = listener(event, value)
        event.return_value = value
        return event

class Listener(object):
    """
    Class that simplifies attaching listeners
    """

    def register(self, dispatcher):
        """
        Registers event listeners to given dispatcher
        """
        for t in self.mapping.iteritems():
            try:
                priority = t[2]
            except IndexError:
                priority = 100
            dispatcher.connect(t[0], t[1], priority)

    @property
    def mapping(self):
        """
        Returns list of listeners to be attached to dispatcher.
        [(event name, listener, priority), (event name, listener, priority)]
        """
        raise NotImplementedError('Return list of tuples with ' +\
            '(event, callback priority) mappings')
<commit_msg>Made Event parameters read-only<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import heapq
import itertools

class Event(object):
    """
    Class that represents single event to be handled

    Provides information about:

    subject - subject which sends event
    name - name of an event
    parameters - event parameters
    return_value - event return value
    processed - whether event has been processed or not

    Example:

    subject = self
    e = Event(subject, 'test', {'a' :1})
    e.subject           # subject
    e.name              # 'test'
    e.parameters        # {'a': 1}
    e['a']              # 1
    e.parameters['a']   # 1
    e.return_value      # None
    e.return_value = 'a'
    e.return_value      # 'a'
    e.processed         # False
    e.mark_procesed()
    e.processed         # True
    """

    def __init__(self, subject, name, parameters={}):
        """
        Initializes class instance
        """
        self.subject = subject
        self.name = name
        self.parameters = parameters
        self.return_value = None
        self._processed = False

    @property
    def processed(self):
        """
        Returns information whether event has been processed
        """
        return self._processed

    def mark_processed(self):
        """
        Marks event as processed
        """
        self._processed = True
        return self

    def __getitem__(self, key):
        """
        Allows to treat class instances as dicts
        """
        return self.parameters[key]

    def __setitem__(self, key, value):
        """
        Allows to treat class instances as dicts
        """
        raise RuntimeError('Event is read-only')

    def __delitem__(self, key):
        """
        Allows to treat class instances as dicts
        """
        raise RuntimeError('Event is read-only')

    def __contains__(self, item):
        """
        Allows to treat class instances as dicts
        """
        return item in self.parameters

class Dispatcher(object):
    """
    Dispatches given events according to previously set listeners
    """

    def __init__(self):
        """
        Initializes class instance
        """
        self._listeners = {}
        self.counter = itertools.count()

    def attach(self, name, listener, priority=0):
        """
        Attaches new listener to dispatcher
        """
        if name not in self._listeners:
            self._listeners[name] = []
        heapq.heappush(self._listeners[name], self._prepare(listener, priority))

    def _prepare(self, listener, priority=0):
        """
        Prepares internal listener structure
        """
        return (priority, next(self.counter), listener)

    def has_listeners(self, name):
        """
        Returns information whether there are any listeners
        attached to given event name
        """
        return name in self._listeners and len(self._listeners[name])>0

    def get_listeners(self, name):
        """
        Fetches list of listeners attached to given event.
        Returns iterator
        """
        if name not in self._listeners:
            return []
        return (l[2] for l in self._listeners[name])

    def notify(self, event):
        """
        Notifies each listener about new event
        """
        for listener in self.get_listeners(event.name):
            listener(event)
        return event;

    def notify_until(self, event):
        """
        Notifies listeners about new event until any of then returns True
        """
        for listener in self.get_listeners(event.name):
            if listener(event):
                event.mark_processed()
                break
        return event

    def filter(self, event, value):
        """
        Filters given value using given event
        """
        for listener in self.get_listeners(event.name):
            value = listener(event, value)
        event.return_value = value
        return event

class Listener(object):
    """
    Class that simplifies attaching listeners
    """

    def register(self, dispatcher):
        """
        Registers event listeners to given dispatcher
        """
        for t in self.mapping.iteritems():
            try:
                priority = t[2]
            except IndexError:
                priority = 100
            dispatcher.connect(t[0], t[1], priority)

    @property
    def mapping(self):
        """
        Returns list of listeners to be attached to dispatcher.
        [(event name, listener, priority), (event name, listener, priority)]
        """
        raise NotImplementedError('Return list of tuples with ' +\
            '(event, callback priority) mappings')
<|endoftext|>


<commit_before><commit_msg>added plotting for orbits<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding:utf-8 -*-

import BaseHTTPServer
import sys
import time
import urlparse
import json
import hashlib
import hmac
from email.parser import Parser
import smtplib

HOST_NAME = sys.argv[1]
PORT_NUMBER = int(sys.argv[2])
SECRET_KEY = sys.argv[3]
DEBUG = len(sys.argv) >= 5;

# from https://stackoverflow.com/questions/1265665/python-check-if-a-string-represents-an-int-without-using-try-except
def is_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False

def send_email(to, subject, body):

    msg = Parser().parsestr('From: <github_notifications@lists.prplfoundation.org>\n' +
        'To: <'+ to + '>\n' +
        'Subject: '+ subject + '\n' +
        '\n' +
        '' + body + '\n')
    if not DEBUG:
        s = smtplib.SMTP('localhost')
        s.sendmail('github_notifications@lists.prplfoundation.org', [to], msg.as_string())
        s.quit()
    else:
        print msg

def is_pull_request_url(potential_url):
    try:
        parse_result = urlparse.urlparse(potential_url)
        path_array = parse_result[2].split('/')
        return (path_array[len(path_array) - 2] == 'pull') and is_int(path_array[len(path_array) - 1])
    except ValueError:
        return False

def handle_pull_request_opened(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'There\'s a new pull request by %s on the prpl %s repository\n\n' % (payload['sender']['login'], email_and_name['name'])

    body = body + '%s\n%s\nDescription: %s\n' % (payload['pull_request']['title'], payload['pull_request']['html_url'], payload['pull_request']['body'])
    #add link
    send_email(email_and_name['email'], "New pull request - %s" % (payload['pull_request']['title']), body)

def handle_pull_request_closed(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])

    was_merged = payload['pull_request']['merged']
    merged_or_closed = "merged" if was_merged else "closed"
    body = 'There\'s a %s pull request on the prpl %s repository\n\n' % (merged_or_closed, email_and_name['name'])

    body = body + '%s\n%s\nDescription: %s\n' % (payload['pull_request']['title'], payload['pull_request']['html_url'], payload['pull_request']['body'])
    merged_or_closed = "Merged" if was_merged else "Closed"
    send_email(email_and_name['email'], "%s pull request - %s" % (merged_or_closed, payload['pull_request']['title']), body)

def handle_pull_request_review(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'New comment by %s on prpl %s repository\n\n%s\nDescription:%s\n' % (payload['comment']['user']['login'], email_and_name['name'], payload['comment']['html_url'], payload['comment']['body'])
    send_email(email_and_name['email'], 'New comment on pull request', body)

def handle_pull_request_comment(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'New comment by %s on prpl %s repository\n\n%s\nDescription:%s\n' % (payload['comment']['user']['login'], email_and_name['name'], payload['comment']['html_url'], payload['comment']['body'])
    send_email(email_and_name['email'], 'New comment on pull request', body)

def handle_issue_comment(payload, repos):
    if is_pull_request_url(payload['issue']['html_url']):
        handle_pull_request_comment(payload, repos)

def handle_issue_opened(payload,repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'New issue by %s on prpl %s repository\n\n%s\nDescription: %s\n' % (payload['issue']['user']['login'], email_and_name['name'], payload['issue']['html_url'], payload['issue']['body'])
    send_email(email_and_name['email'], 'New issue posted - %s' % (payload['issue']['title']), body)

def handle_issue_closed(payload,repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'Closed issue by %s on prpl %s repository\n\n%s\nDescription: %s\n' % (payload['issue']['user']['login'], email_and_name['name'], payload['issue']['html_url'], payload['issue']['body'])
    send_email(email_and_name['email'], 'Issue Closed - %s' % (payload['issue']['title']), body)

def handle_hook(event, payload, repos):
    if event == 'pull_request':
        if payload['action'] == 'opened':
            return handle_pull_request_opened(payload, repos)
        elif payload['action'] == 'closed':
            return handle_pull_request_closed(payload, repos)
    elif event == 'pull_request_review_comment':
        return handle_pull_request_review(payload, repos)

    elif event == 'issue_comment':
        return handle_issue_comment(payload, repos)
    elif event == 'issues':
        if payload['action'] == 'opened':
            return handle_issue_opened(payload,repos)
        elif payload['action'] == 'closed':
            return handle_issue_closed(payload, repos)
    else:
        pass

def verify_signature(payload, hub_signature):
    signature = 'sha1=' + hmac.new(SECRET_KEY, payload, hashlib.sha1).hexdigest()
    #should use compare_digest but isn't in our current python implementation.
    #also, we're not protecting nuclear secrets so we should be fine
    return signature == hub_signature

class JsonRepos:
    def __init__(self,json_file):
        self.json_file = json_file
        self.json = json.load(open(json_file))

    def get_email_and_name(self, repo_name):
        return self.json[repo_name]

class HookHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    server_version = "HookHandler/0.1"
    def do_GET(s):
        s.send_response(200)

    def do_POST(s):
        repos = JsonRepos('repos.json')
        length = int(s.headers['Content-Length'])
        full_payload = s.rfile.read(length).decode('utf-8')
        post_data = urlparse.parse_qs(full_payload)
        payload = json.loads(post_data['payload'][0])
        if not DEBUG and not verify_signature(full_payload, s.headers['X-Hub-Signature']):
            s.send_error(403)
            return

        event = s.headers['X-GitHub-Event']
        handle_hook(event, payload, repos)

        s.send_response(200)

if __name__ == '__main__':
    server_class = BaseHTTPServer.HTTPServer
    httpd = server_class((HOST_NAME, PORT_NUMBER), HookHandler)
    print time.asctime(), "Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER)
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        pass
    httpd.server_close()
    print time.asctime(), "Server Stops - %s:%s" % (HOST_NAME, PORT_NUMBER)
<commit_msg>Encode to UTF8 and protect against SMTP header insertion in the subject<commit_after>#!/usr/bin/env python
#-*- coding:utf-8 -*-

import BaseHTTPServer
import sys
import time
import urlparse
import json
import hashlib
import hmac
import email
from email.parser import Parser
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart

HOST_NAME = sys.argv[1]
PORT_NUMBER = int(sys.argv[2])
SECRET_KEY = sys.argv[3]
DEBUG = len(sys.argv) >= 5;

# from https://stackoverflow.com/questions/1265665/python-check-if-a-string-represents-an-int-without-using-try-except
def is_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False

def send_email(to, subject, body):

    subject.replace('\n', '  ')
    subject.replace('\r', '  ')
    to.replace('\n', '  ')
    to.replace('\r', '  ')
    outer = MIMEMultipart()
    outer['Subject'] = subject
    outer['To'] = to
    outer['From'] = 'github_notifications@lists.prplfoundation.org'
    outer['Date'] = email.utils.formatdate(localtime = True)
    outer.preamble = 'GitHub notification mails are now in MIME to allow UTF8.\n'
    outer.attach(MIMEText(body, 'plain', 'utf8'))
    msg = outer.as_string()

    if not DEBUG:
        s = smtplib.SMTP('localhost')
        s.sendmail('github_notifications@lists.prplfoundation.org', [to], msg)
        s.quit()
    else:
        print msg

def is_pull_request_url(potential_url):
    try:
        parse_result = urlparse.urlparse(potential_url)
        path_array = parse_result[2].split('/')
        return (path_array[len(path_array) - 2] == 'pull') and is_int(path_array[len(path_array) - 1])
    except ValueError:
        return False

def handle_pull_request_opened(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'There\'s a new pull request by %s on the prpl %s repository\n\n' % (payload['sender']['login'], email_and_name['name'])

    body = body + '%s\n%s\nDescription: %s\n' % (payload['pull_request']['title'], payload['pull_request']['html_url'], payload['pull_request']['body'])
    #add link
    send_email(email_and_name['email'], "New pull request - %s" % (payload['pull_request']['title']), body)

def handle_pull_request_closed(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])

    was_merged = payload['pull_request']['merged']
    merged_or_closed = "merged" if was_merged else "closed"
    body = 'There\'s a %s pull request on the prpl %s repository\n\n' % (merged_or_closed, email_and_name['name'])

    body = body + '%s\n%s\nDescription: %s\n' % (payload['pull_request']['title'], payload['pull_request']['html_url'], payload['pull_request']['body'])
    merged_or_closed = "Merged" if was_merged else "Closed"
    send_email(email_and_name['email'], "%s pull request - %s" % (merged_or_closed, payload['pull_request']['title']), body)

def handle_pull_request_review(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'New comment by %s on prpl %s repository\n\n%s\nDescription:%s\n' % (payload['comment']['user']['login'], email_and_name['name'], payload['comment']['html_url'], payload['comment']['body'])
    send_email(email_and_name['email'], 'New comment on pull request', body)

def handle_pull_request_comment(payload, repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'New comment by %s on prpl %s repository\n\n%s\nDescription:%s\n' % (payload['comment']['user']['login'], email_and_name['name'], payload['comment']['html_url'], payload['comment']['body'])
    send_email(email_and_name['email'], 'New comment on pull request', body)

def handle_issue_comment(payload, repos):
    if is_pull_request_url(payload['issue']['html_url']):
        handle_pull_request_comment(payload, repos)

def handle_issue_opened(payload,repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'New issue by %s on prpl %s repository\n\n%s\nDescription: %s\n' % (payload['issue']['user']['login'], email_and_name['name'], payload['issue']['html_url'], payload['issue']['body'])
    send_email(email_and_name['email'], 'New issue posted - %s' % (payload['issue']['title']), body)

def handle_issue_closed(payload,repos):
    email_and_name = repos.get_email_and_name(payload['repository']['name'])
    body = 'Closed issue by %s on prpl %s repository\n\n%s\nDescription: %s\n' % (payload['issue']['user']['login'], email_and_name['name'], payload['issue']['html_url'], payload['issue']['body'])
    send_email(email_and_name['email'], 'Issue Closed - %s' % (payload['issue']['title']), body)

def handle_hook(event, payload, repos):
    if event == 'pull_request':
        if payload['action'] == 'opened':
            return handle_pull_request_opened(payload, repos)
        elif payload['action'] == 'closed':
            return handle_pull_request_closed(payload, repos)
    elif event == 'pull_request_review_comment':
        return handle_pull_request_review(payload, repos)

    elif event == 'issue_comment':
        return handle_issue_comment(payload, repos)
    elif event == 'issues':
        if payload['action'] == 'opened':
            return handle_issue_opened(payload,repos)
        elif payload['action'] == 'closed':
            return handle_issue_closed(payload, repos)
    else:
        pass

def verify_signature(payload, hub_signature):
    signature = 'sha1=' + hmac.new(SECRET_KEY, payload, hashlib.sha1).hexdigest()
    #should use compare_digest but isn't in our current python implementation.
    #also, we're not protecting nuclear secrets so we should be fine
    return signature == hub_signature

class JsonRepos:
    def __init__(self,json_file):
        self.json_file = json_file
        self.json = json.load(open(json_file))

    def get_email_and_name(self, repo_name):
        return self.json[repo_name]

class HookHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    server_version = "HookHandler/0.1"
    def do_GET(s):
        s.send_response(200)

    def do_POST(s):
        repos = JsonRepos('repos.json')
        length = int(s.headers['Content-Length'])
        full_payload = s.rfile.read(length).decode('utf-8')
        post_data = urlparse.parse_qs(full_payload)
        payload = json.loads(post_data['payload'][0])
        if not DEBUG and not verify_signature(full_payload, s.headers['X-Hub-Signature']):
            s.send_error(403)
            return

        event = s.headers['X-GitHub-Event']
        handle_hook(event, payload, repos)

        s.send_response(200)

if __name__ == '__main__':
    server_class = BaseHTTPServer.HTTPServer
    httpd = server_class((HOST_NAME, PORT_NUMBER), HookHandler)
    print time.asctime(), "Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER)
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        pass
    httpd.server_close()
    print time.asctime(), "Server Stops - %s:%s" % (HOST_NAME, PORT_NUMBER)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Electrum - lightweight Bitcoin client
# Copyright (C) 2012 thomasv@gitorious
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Kivy GUI

import sys
import os
from typing import TYPE_CHECKING

KIVY_GUI_PATH = os.path.abspath(os.path.dirname(__file__))
os.environ['KIVY_DATA_DIR'] = os.path.join(KIVY_GUI_PATH, 'data')

try:
    sys.argv = ['']
    import kivy
except ImportError:
    # This error ideally shouldn't be raised with pre-built packages
    sys.exit("Error: Could not import kivy. Please install it using the "
             "instructions mentioned here `https://kivy.org/#download` .")

# minimum required version for kivy
kivy.require('1.8.0')

from electrum.logging import Logger
from electrum.gui import BaseElectrumGui

if TYPE_CHECKING:
    from electrum.simple_config import SimpleConfig
    from electrum.daemon import Daemon
    from electrum.plugin import Plugins

class ElectrumGui(BaseElectrumGui, Logger):

    def __init__(self, *, config: 'SimpleConfig', daemon: 'Daemon', plugins: 'Plugins'):
        BaseElectrumGui.__init__(self, config=config, daemon=daemon, plugins=plugins)
        Logger.__init__(self)
        self.logger.debug('ElectrumGUI: initialising')
        self.network = daemon.network

    def main(self):
        from .main_window import ElectrumWindow
        w = ElectrumWindow(
            config=self.config,
            network=self.network,
            plugins=self.plugins,
            gui_object=self,
        )
        w.run()
<commit_msg>kivy: implement ElectrumGui.stop<commit_after>#!/usr/bin/env python
#
# Electrum - lightweight Bitcoin client
# Copyright (C) 2012 thomasv@gitorious
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Kivy GUI

import sys
import os
from typing import TYPE_CHECKING

KIVY_GUI_PATH = os.path.abspath(os.path.dirname(__file__))
os.environ['KIVY_DATA_DIR'] = os.path.join(KIVY_GUI_PATH, 'data')

try:
    sys.argv = ['']
    import kivy
except ImportError:
    # This error ideally shouldn't be raised with pre-built packages
    sys.exit("Error: Could not import kivy. Please install it using the "
             "instructions mentioned here `https://kivy.org/#download` .")

# minimum required version for kivy
kivy.require('1.8.0')

from electrum.logging import Logger
from electrum.gui import BaseElectrumGui

if TYPE_CHECKING:
    from electrum.simple_config import SimpleConfig
    from electrum.daemon import Daemon
    from electrum.plugin import Plugins

class ElectrumGui(BaseElectrumGui, Logger):

    def __init__(self, *, config: 'SimpleConfig', daemon: 'Daemon', plugins: 'Plugins'):
        BaseElectrumGui.__init__(self, config=config, daemon=daemon, plugins=plugins)
        Logger.__init__(self)
        self.logger.debug('ElectrumGUI: initialising')
        self.network = daemon.network

    def main(self):
        from .main_window import ElectrumWindow
        w = ElectrumWindow(
            config=self.config,
            network=self.network,
            plugins=self.plugins,
            gui_object=self,
        )
        w.run()

    def stop(self) -> None:
        from kivy.app import App
        from kivy.clock import Clock
        app = App.get_running_app()
        if not app:
            return
        Clock.schedule_once(lambda dt: app.stop())
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#  Copyright 2017-2018 University Of Southern California
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
from __future__ import print_function

import os

__author__ = 'Rafael Ferreira da Silva'

class Type:
    INSTALLED = 'INSTALLED'
    STAGEABLE = 'STAGEABLE'

class TransformationCatalog:
    def __init__(self, workflow_dir, filename='tc.txt'):
        """
        Create a Pegasus transformation catalog.
        :param workflow_dir: Path to the workflow directory
        :param filename: catalog filename (default: rc.txt)
        """
        self.workflow_dir = workflow_dir
        self.filename = filename
        self._executables = []
        self._containers = []

    def add(self, executable):
        """
        Add an executable to the transformation catalog.
        :param executable: A DAX3 Executable object
        """
        if not executable:
            raise Exception('An executable should be provided.')

        if executable not in self._executables:
            self._executables.append(executable)

    def add_container(self, container):
        """
        Add a container to the transformation catalog.
        :param container: A DAX3 Container object
        """
        if not container:
            raise Exception('A container should be provided.')

        if container not in self._containers:
            self._containers.append(container)

    def write(self, force=False):
        """
        Write the catalog to a file.
        :param force: whether to overwrite the catalog file
        """
        catalog_file = self.workflow_dir + '/' + self.filename

        if not os.path.isfile(catalog_file) or force:
            with open(catalog_file, 'w') as ppf:
                # executables
                for e in self._executables:
                    # executable name
                    name = e.name
                    if e.namespace:
                        name = e.namespace + '::' + name
                    if e.version:
                        name = name + ':' + e.version
                    ppf.write('tr %s {\n' % name)

                    # profiles
                    for p in e.profiles:
                        ppf.write(
                            '\tprofile %s "%s" "%s"\n' %
                            (p.namespace, p.key, p.value)
                        )

                    # pfns
                    installed = 'INSTALLED'
                    if not e.installed:
                        installed = 'STAGEABLE'

                    for pfn in e.pfns:
                        ppf.write('\tsite %s {\n' % pfn.site)
                        # profiles
                        for p in pfn.profiles:
                            ppf.write(
                                '\t\tprofile %s "%s" "%s"\n' %
                                (p.namespace, p.key, p.value)
                            )

                        ppf.write('\t\tpfn "%s"\n' % pfn.url)
                        if e.arch:
                            ppf.write('\t\tarch "%s"\n' % e.arch)
                        if e.os:
                            ppf.write('\t\tos "%s"\n' % e.os)
                        if e.osrelease:
                            ppf.write('\t\tosrelease "%s"\n' % e.osrelease)
                        if e.osversion:
                            ppf.write('\t\tosversion "%s"\n' % e.osversion)
                        ppf.write('\t\ttype "%s"\n' % installed)

                        # reference to container
                        if e.container:
                            ppf.write('\t\tcontainer "%s"\n' % e.container)

                        ppf.write('\t}\n')

                    ppf.write('}\n\n')

                # containers
                for c in self._containers:
                    ppf.write('cont %s {\n' % c.name)
                    ppf.write('\ttype "%s"\n' % c.type)
                    ppf.write('\timage "%s"\n' % c.image)

                    if c.imagesite:
                        ppf.write('\timage_site "%s"\n' % c.imagesite)

                    if c.dockerfile:
                        ppf.write('\tdockerfile "%s"\n' % c.dockerfile)

                    # mount
                    for m in c.mount:
                        ppf.write('\tmount "%s"\n' % m)

                    # profiles
                    for p in c.profiles:
                        ppf.write(
                            '\tprofile %s "%s" "%s"\n' %
                            (p.namespace, p.key, p.value)
                        )

                    ppf.write('}\n\n')

        else:
            print(
                '\x1b[0;35mWARNING: Transformation Catalog (%s) already exists. Use "force=True" '
                'to overwrite it.\n\x1b[0m' % catalog_file
            )
<commit_msg>write the container name into the tc<commit_after>#!/usr/bin/env python
#
#  Copyright 2017-2018 University Of Southern California
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
from __future__ import print_function

import os

__author__ = 'Rafael Ferreira da Silva'

class Type:
    INSTALLED = 'INSTALLED'
    STAGEABLE = 'STAGEABLE'

class TransformationCatalog:
    def __init__(self, workflow_dir, filename='tc.txt'):
        """
        Create a Pegasus transformation catalog.
        :param workflow_dir: Path to the workflow directory
        :param filename: catalog filename (default: rc.txt)
        """
        self.workflow_dir = workflow_dir
        self.filename = filename
        self._executables = []
        self._containers = []

    def add(self, executable):
        """
        Add an executable to the transformation catalog.
        :param executable: A DAX3 Executable object
        """
        if not executable:
            raise Exception('An executable should be provided.')

        if executable not in self._executables:
            self._executables.append(executable)

    def add_container(self, container):
        """
        Add a container to the transformation catalog.
        :param container: A DAX3 Container object
        """
        if not container:
            raise Exception('A container should be provided.')

        if container not in self._containers:
            self._containers.append(container)

    def write(self, force=False):
        """
        Write the catalog to a file.
        :param force: whether to overwrite the catalog file
        """
        catalog_file = self.workflow_dir + '/' + self.filename

        if not os.path.isfile(catalog_file) or force:
            with open(catalog_file, 'w') as ppf:
                # executables
                for e in self._executables:
                    # executable name
                    name = e.name
                    if e.namespace:
                        name = e.namespace + '::' + name
                    if e.version:
                        name = name + ':' + e.version
                    ppf.write('tr %s {\n' % name)

                    # profiles
                    for p in e.profiles:
                        ppf.write(
                            '\tprofile %s "%s" "%s"\n' %
                            (p.namespace, p.key, p.value)
                        )

                    # pfns
                    installed = 'INSTALLED'
                    if not e.installed:
                        installed = 'STAGEABLE'

                    for pfn in e.pfns:
                        ppf.write('\tsite %s {\n' % pfn.site)
                        # profiles
                        for p in pfn.profiles:
                            ppf.write(
                                '\t\tprofile %s "%s" "%s"\n' %
                                (p.namespace, p.key, p.value)
                            )

                        ppf.write('\t\tpfn "%s"\n' % pfn.url)
                        if e.arch:
                            ppf.write('\t\tarch "%s"\n' % e.arch)
                        if e.os:
                            ppf.write('\t\tos "%s"\n' % e.os)
                        if e.osrelease:
                            ppf.write('\t\tosrelease "%s"\n' % e.osrelease)
                        if e.osversion:
                            ppf.write('\t\tosversion "%s"\n' % e.osversion)
                        ppf.write('\t\ttype "%s"\n' % installed)

                        # reference to container
                        if e.container:
                            ppf.write('\t\tcontainer "%s"\n' %
                                      e.container.name
                            )

                        ppf.write('\t}\n')

                    ppf.write('}\n\n')

                # containers
                for c in self._containers:
                    ppf.write('cont %s {\n' % c.name)
                    ppf.write('\ttype "%s"\n' % c.type)
                    ppf.write('\timage "%s"\n' % c.image)

                    if c.imagesite:
                        ppf.write('\timage_site "%s"\n' % c.imagesite)

                    if c.dockerfile:
                        ppf.write('\tdockerfile "%s"\n' % c.dockerfile)

                    # mount
                    for m in c.mount:
                        ppf.write('\tmount "%s"\n' % m)

                    # profiles
                    for p in c.profiles:
                        ppf.write(
                            '\tprofile %s "%s" "%s"\n' %
                            (p.namespace, p.key, p.value)
                        )

                    ppf.write('}\n\n')

        else:
            print(
                '\x1b[0;35mWARNING: Transformation Catalog (%s) already exists. Use "force=True" '
                'to overwrite it.\n\x1b[0m' % catalog_file
            )
<|endoftext|>


<commit_before>#
# Copyright 2014 Thomas Rabaix <thomas.rabaix@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from tornado.web import RequestHandler
from tornado.httpserver import HTTPRequest
import jinja2

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

def get_dummy_connection():
    return SubConnection(SubContext('127.0.0.1', 'http'))

class SubRequestHandler(RequestHandler):
    def get_buffer(self):
        return b"".join(self._write_buffer)

class SubConnection(object):
    def __init__(self, context):
        self.context = context

    def set_close_callback(self, callback):
        self.on_connection_close = callback

class SubContext(object):
    def __init__(self, remote_ip, protocol):
        self.remote_ip = remote_ip
        self.protocol = protocol

class Core(object):
    def __init__(self, node_manager, context_creator, dispatcher, application, render_type):
        self.node_manager = node_manager
        self.context_creator = context_creator
        self.dispatcher = dispatcher
        self.application = application
        self.render_type = render_type

    def render_node(self, node, defaults=None):
        defaults = defaults or {}

        # load the node
        node = self.node_manager.get_node(node)

        if not node:
            return "<!-- unable to found the node -->"

        # load the related node's handler
        handler = self.node_manager.get_handler(node)

        if not handler:
            return "<!-- unable to found the node handler -->"

        # build the execution context
        context = self.context_creator.build(node, handler, defaults)

        request_handler = SubRequestHandler(self.application, HTTPRequest('GET', '/_internal', connection=get_dummy_connection()))

        # render the response
        handler.execute(request_handler, context)

        return jinja2.Markup(self.unicode(request_handler.get_buffer()))

    def render_node_event(self, event_name, options=None):
        event = self.dispatcher.dispatch(event_name, options or {})

        if not event.has('node'):
            return "<!-- no listener registered for event: %s -->" % event_name

        return jinja2.Markup(self.unicode(self.render_node(event.get('node'))))

    def render(self, path, type=None):
        if not type:
            type = self.render_type

        if type == 'esi':
            return jinja2.Markup('<esi:include src="%s" />' % path)

        elif type == 'ssi':
            o = urlparse(path)
            return jinja2.Markup('<!--# include virtual="%s?%s&_element=no-debug" -->' % (o.path, o.query))

    def unicode(self, content):
        if isinstance(content, unicode):
            return content

        return content.decode("utf-8")

class ResponseListener(object):
    def __init__(self, templating):
        self.templating = templating

    def handle(self, event):
        result = event.get('result')

        if not isinstance(result, tuple):
            return

        request_handler = event.get('request_handler')

        status_code = 500
        template = None
        params = {}
        headers = {
            'Content-Type': 'text/html; charset=utf-8'
        }

        if len(result) == 3:
            status_code, template, params = result

        if len(result) == 4:
            status_code, template, params, headers = result

        for name, value in headers.iteritems():
            if not request_handler.has_header(name):
                request_handler.set_header(name, value)

        request_handler.set_status(status_code)
        request_handler.write(self.templating.get_template(template).render(params))

        if event.has('node_handler'):
            event.get('node_handler').finalize(request_handler)

        event.set('result', None)<commit_msg>[jinja] add missing SafeMarkup call<commit_after>#
# Copyright 2014 Thomas Rabaix <thomas.rabaix@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from tornado.web import RequestHandler
from tornado.httpserver import HTTPRequest
import jinja2

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

def get_dummy_connection():
    return SubConnection(SubContext('127.0.0.1', 'http'))

class SubRequestHandler(RequestHandler):
    def get_buffer(self):
        return b"".join(self._write_buffer)

class SubConnection(object):
    def __init__(self, context):
        self.context = context

    def set_close_callback(self, callback):
        self.on_connection_close = callback

class SubContext(object):
    def __init__(self, remote_ip, protocol):
        self.remote_ip = remote_ip
        self.protocol = protocol

class Core(object):
    def __init__(self, node_manager, context_creator, dispatcher, application, render_type):
        self.node_manager = node_manager
        self.context_creator = context_creator
        self.dispatcher = dispatcher
        self.application = application
        self.render_type = render_type

    def render_node(self, node, defaults=None):
        defaults = defaults or {}

        # load the node
        node = self.node_manager.get_node(node)

        if not node:
            return jinja2.Markup("<!-- unable to found the node -->")

        # load the related node's handler
        handler = self.node_manager.get_handler(node)

        if not handler:
            return jinja2.Markup("<!-- unable to found the node handler -->")

        # build the execution context
        context = self.context_creator.build(node, handler, defaults)

        request_handler = SubRequestHandler(self.application, HTTPRequest('GET', '/_internal', connection=get_dummy_connection()))

        # render the response
        handler.execute(request_handler, context)

        return jinja2.Markup(self.unicode(request_handler.get_buffer()))

    def render_node_event(self, event_name, options=None):
        event = self.dispatcher.dispatch(event_name, options or {})

        if not event.has('node'):
            return jinja2.Markup("<!-- no listener registered for event: %s -->" % event_name)

        return jinja2.Markup(self.unicode(self.render_node(event.get('node'))))

    def render(self, path, type=None):
        if not type:
            type = self.render_type

        if type == 'esi':
            return jinja2.Markup('<esi:include src="%s" />' % path)

        elif type == 'ssi':
            o = urlparse(path)
            return jinja2.Markup('<!--# include virtual="%s?%s&_element=no-debug" -->' % (o.path, o.query))

    def unicode(self, content):
        if isinstance(content, unicode):
            return content

        return content.decode("utf-8")

class ResponseListener(object):
    def __init__(self, templating):
        self.templating = templating

    def handle(self, event):
        result = event.get('result')

        if not isinstance(result, tuple):
            return

        request_handler = event.get('request_handler')

        status_code = 500
        template = None
        params = {}
        headers = {
            'Content-Type': 'text/html; charset=utf-8'
        }

        if len(result) == 3:
            status_code, template, params = result

        if len(result) == 4:
            status_code, template, params, headers = result

        for name, value in headers.iteritems():
            if not request_handler.has_header(name):
                request_handler.set_header(name, value)

        request_handler.set_status(status_code)
        request_handler.write(self.templating.get_template(template).render(params))

        if event.has('node_handler'):
            event.get('node_handler').finalize(request_handler)

        event.set('result', None)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json

from braces.views import CsrfExemptMixin
from oauth2_provider.ext.rest_framework import OAuth2Authentication
from oauth2_provider.models import Application, AccessToken
from oauth2_provider.settings import oauth2_settings
from oauth2_provider.views.mixins import OAuthLibMixin
from rest_framework import permissions
from rest_framework import status
from rest_framework.decorators import api_view, authentication_classes, permission_classes
from rest_framework.response import Response
from rest_framework.views import APIView

from .oauth2_backends import KeepRequestCore
from .oauth2_endpoints import SocialTokenServer

class ConvertTokenView(CsrfExemptMixin, OAuthLibMixin, APIView):
    """
    Implements an endpoint to provide access tokens

    The endpoint is used in the following flows:

    * Authorization code
    * Password
    * Client credentials
    """
    server_class = SocialTokenServer
    validator_class = oauth2_settings.OAUTH2_VALIDATOR_CLASS
    oauthlib_backend_class = KeepRequestCore
    permission_classes = (permissions.AllowAny,)

    def post(self, request, *args, **kwargs):

        # Use the rest framework `.data` to fake the post body of the django request.
        request._request.POST = request._request.POST.copy()
        for key, value in request.data.iteritems():
            request._request.POST[key] = value

        url, headers, body, status = self.create_token_response(request._request)
        response = Response(data=json.loads(body), status=status)

        for k, v in headers.items():
            response[k] = v
        return response

@api_view(['POST'])
@authentication_classes([OAuth2Authentication])
@permission_classes([permissions.IsAuthenticated])
def invalidate_sessions(request):
    client_id = request.POST.get("client_id", None)
    if client_id is None:
        return Response({
            "client_id": ["This field is required."]
        }, status=status.HTTP_400_BAD_REQUEST)

    try:
        app = Application.objects.get(client_id=client_id)
    except Application.DoesNotExist:
        return Response({
            "detail": "The application linked to the provided client_id could not be found."
        }, status=status.HTTP_400_BAD_REQUEST)

    tokens = AccessToken.objects.filter(user=request.user, application=app)
    tokens.delete()
    return Response({}, status=status.HTTP_204_NO_CONTENT)
<commit_msg>changing iteritems to items (python 3)<commit_after># -*- coding: utf-8 -*-
import json

from braces.views import CsrfExemptMixin
from oauth2_provider.ext.rest_framework import OAuth2Authentication
from oauth2_provider.models import Application, AccessToken
from oauth2_provider.settings import oauth2_settings
from oauth2_provider.views.mixins import OAuthLibMixin
from rest_framework import permissions
from rest_framework import status
from rest_framework.decorators import api_view, authentication_classes, permission_classes
from rest_framework.response import Response
from rest_framework.views import APIView

from .oauth2_backends import KeepRequestCore
from .oauth2_endpoints import SocialTokenServer

class ConvertTokenView(CsrfExemptMixin, OAuthLibMixin, APIView):
    """
    Implements an endpoint to provide access tokens

    The endpoint is used in the following flows:

    * Authorization code
    * Password
    * Client credentials
    """
    server_class = SocialTokenServer
    validator_class = oauth2_settings.OAUTH2_VALIDATOR_CLASS
    oauthlib_backend_class = KeepRequestCore
    permission_classes = (permissions.AllowAny,)

    def post(self, request, *args, **kwargs):

        # Use the rest framework `.data` to fake the post body of the django request.
        request._request.POST = request._request.POST.copy()
        for key, value in request.data.items():
            request._request.POST[key] = value

        url, headers, body, status = self.create_token_response(request._request)
        response = Response(data=json.loads(body), status=status)

        for k, v in headers.items():
            response[k] = v
        return response

@api_view(['POST'])
@authentication_classes([OAuth2Authentication])
@permission_classes([permissions.IsAuthenticated])
def invalidate_sessions(request):
    client_id = request.POST.get("client_id", None)
    if client_id is None:
        return Response({
            "client_id": ["This field is required."]
        }, status=status.HTTP_400_BAD_REQUEST)

    try:
        app = Application.objects.get(client_id=client_id)
    except Application.DoesNotExist:
        return Response({
            "detail": "The application linked to the provided client_id could not be found."
        }, status=status.HTTP_400_BAD_REQUEST)

    tokens = AccessToken.objects.filter(user=request.user, application=app)
    tokens.delete()
    return Response({}, status=status.HTTP_204_NO_CONTENT)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, unicode_literals

import numpy as np

from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.regularizers import l1, l2, l1l2
from keras.utils.np_utils import to_categorical
from scipy.sparse import issparse

class KerasMultilayerPerceptron(object):
    _MAX_BATCH_SIZE = 3000

    def __init__(self, layers=list(), layers_activation='tanh', classification_layer_activation='softmax',
                 layers_initialization='uniform', dropout_layers=None, optimizer='adam',
                 loss='categorical_crossentropy', epochs=100, batch_size=50, metrics=None, l1=0.01, l2=0.01,
                 verbosity=0):
        self._layers = layers
        self._layers_activation = layers_activation
        self._classification_layer_activation = classification_layer_activation
        self._layers_initialization = layers_initialization
        self._dropout_layers = dropout_layers
        self._optimizer = optimizer
        self._loss = loss
        self._epochs = epochs
        self._batch_size = batch_size
        self._metrics = metrics
        self._l1 = l1
        self._l2 = l2
        self._verbosity = 0
        self._model = None
        self._classes = None

    def _build_network(self, input_size, output_size):
        self._model = Sequential()

        for lidx, layer_dim in enumerate(self._layers + [output_size]):
            layer_config = {
                'output_dim': layer_dim,
                'input_dim': None if self._model.layers else input_size,
                'init': self._layers_initialization,
            }

            if lidx < len(self._layers):
                layer_config['activation'] = self._layers_activation
            else:
                layer_config['activation'] = self._classification_layer_activation

            if self._l1 > 0 and self._l2 > 0:
                layer_config['W_regularizer'] = l1l2(self._l1, self._l2)
                layer_config['b_regularizer'] = l1l2(self._l1, self._l2)
            elif self._l1 > 0:
                layer_config['W_regularizer'] = l1(self._l1)
                layer_config['b_regularizer'] = l1(self._l1)
            elif self._l2 > 0:
                layer_config['W_regularizer'] = l2(self._l2)
                layer_config['b_regularizer'] = l2(self._l2)

            self._model.add(Dense(**layer_config))

            if self._dropout_layers:
                self._model.add(Dropout(self._dropout_layers.pop(0)))

        self._model.compile(loss=self._loss, optimizer=self._optimizer, metrics=self._metrics)

    def fit(self, x, y):
        self._classes = np.unique(y)
        self._build_network(x.shape[1], len(self._classes))

        if self._epochs < 0 and self._batch_size < 0:
            self._batch_size = min(x.shape[0], self._MAX_BATCH_SIZE)
            self._epochs = np.int(np.ceil(x.shape[0] / self._batch_size))
        elif self._epochs < 0:
            self._epochs = np.int(np.ceil(x.shape[0] / self._batch_size))
        elif self._batch_size < 0:
            self._batch_size = np.int(np.ceil(x.shape[0] / self._epochs))

        x = x.toarray() if issparse(x) else x

        return self._model.fit(x, to_categorical(np.searchsorted(self._classes, y)),
                               nb_epoch=self._epochs, batch_size=self._batch_size, verbose=self._verbosity)

    def predict(self, x):
        """
        Predicts the model classes (defined in self._classes) for the input data
        """
        assert self._model is not None, "The model needs to be trained"

        x = x.toarray() if issparse(x) else x
        predicted_classes = self._model.predict_classes(x, batch_size=self._batch_size, verbose=self._verbosity)

        return self._classes[predicted_classes]

    def predict_proba(self, x):
        """
        Predicts the model class probability for the input data
        """
        assert self._model is not None, "The model needs to be trained"

        x = x.toarray() if issparse(x) else x
        return self._model.predict(x, verbose=self._verbosity)
<commit_msg>Verbosity set for keras<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, unicode_literals

import numpy as np

from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.regularizers import l1, l2, l1l2
from keras.utils.np_utils import to_categorical
from scipy.sparse import issparse

class KerasMultilayerPerceptron(object):
    _MAX_BATCH_SIZE = 3000

    def __init__(self, layers=list(), layers_activation='tanh', classification_layer_activation='softmax',
                 layers_initialization='uniform', dropout_layers=None, optimizer='adam',
                 loss='categorical_crossentropy', epochs=100, batch_size=50, metrics=None, l1=0.01, l2=0.01,
                 verbosity=0):
        self._layers = layers
        self._layers_activation = layers_activation
        self._classification_layer_activation = classification_layer_activation
        self._layers_initialization = layers_initialization
        self._dropout_layers = dropout_layers
        self._optimizer = optimizer
        self._loss = loss
        self._epochs = epochs
        self._batch_size = batch_size
        self._metrics = metrics
        self._l1 = l1
        self._l2 = l2
        self._verbosity = verbosity
        self._model = None
        self._classes = None

    def _build_network(self, input_size, output_size):
        self._model = Sequential()

        for lidx, layer_dim in enumerate(self._layers + [output_size]):
            layer_config = {
                'output_dim': layer_dim,
                'input_dim': None if self._model.layers else input_size,
                'init': self._layers_initialization,
            }

            if lidx < len(self._layers):
                layer_config['activation'] = self._layers_activation
            else:
                layer_config['activation'] = self._classification_layer_activation

            if self._l1 > 0 and self._l2 > 0:
                layer_config['W_regularizer'] = l1l2(self._l1, self._l2)
                layer_config['b_regularizer'] = l1l2(self._l1, self._l2)
            elif self._l1 > 0:
                layer_config['W_regularizer'] = l1(self._l1)
                layer_config['b_regularizer'] = l1(self._l1)
            elif self._l2 > 0:
                layer_config['W_regularizer'] = l2(self._l2)
                layer_config['b_regularizer'] = l2(self._l2)

            self._model.add(Dense(**layer_config))

            if self._dropout_layers:
                self._model.add(Dropout(self._dropout_layers.pop(0)))

        self._model.compile(loss=self._loss, optimizer=self._optimizer, metrics=self._metrics)

    def fit(self, x, y):
        self._classes = np.unique(y)
        self._build_network(x.shape[1], len(self._classes))

        if self._epochs < 0 and self._batch_size < 0:
            self._batch_size = min(x.shape[0], self._MAX_BATCH_SIZE)
            self._epochs = np.int(np.ceil(x.shape[0] / self._batch_size))
        elif self._epochs < 0:
            self._epochs = np.int(np.ceil(x.shape[0] / self._batch_size))
        elif self._batch_size < 0:
            self._batch_size = np.int(np.ceil(x.shape[0] / self._epochs))

        x = x.toarray() if issparse(x) else x

        return self._model.fit(x, to_categorical(np.searchsorted(self._classes, y)),
                               nb_epoch=self._epochs, batch_size=self._batch_size, verbose=self._verbosity)

    def predict(self, x):
        """
        Predicts the model classes (defined in self._classes) for the input data
        """
        assert self._model is not None, "The model needs to be trained"

        x = x.toarray() if issparse(x) else x
        predicted_classes = self._model.predict_classes(x, batch_size=self._batch_size, verbose=self._verbosity)

        return self._classes[predicted_classes]

    def predict_proba(self, x):
        """
        Predicts the model class probability for the input data
        """
        assert self._model is not None, "The model needs to be trained"

        x = x.toarray() if issparse(x) else x
        return self._model.predict(x, verbose=self._verbosity)
<|endoftext|>


<commit_before>from __future__ import division
from numpy import prod
import contextlib

from .. import util
from ..train import Trainer
from ..ops import NumpyOps
from ..mem import Memory
from ..util import get_ops
from ... import check
from ... import describe
from ...check import equal_length, has_shape, is_sequence, is_float, is_array

class Model(object):
    '''Model base class.'''
    name = 'model'
    id = 0
    lsuv = False
    ops = NumpyOps()
    Trainer = Trainer
    descriptions = []
    on_data_hooks = []
    on_init_hooks = [] # Use this to add layers
    _operators = {}

    @classmethod
    @contextlib.contextmanager
    def define_operators(cls, operators):
        '''Bind operators to specified functions for the scope of the context:

        Example
        -------

            model = Model()
            other = Model()
            with Model.define_operators({"+": lambda self, other: "plus"}):
                print(model + other)
                # "plus"
            print(model + other)
            # Raises TypeError --- binding limited to scope of with block.
        '''
        old_ops = dict(cls._operators)
        for op, func in operators.items():
            cls._operators[op] = func
        yield
        cls._operators = old_ops

    @classmethod
    @contextlib.contextmanager
    def use_device(cls, device):
        '''Change the device to execute on for the scope of the block.'''
        if device == cls.ops.device:
            yield
        else:
            curr_ops = cls.ops
            cls.ops = get_ops(device)
            yield
            cls.ops = curr_ops

    @property
    def input_shape(self):
        raise NotImplementedError

    @property
    def output_shape(self):
        raise NotImplementedError

    def __init__(self, *args, **kwargs):
        self.name = self.__class__.name
        kwargs = self._update_defaults(args, kwargs)
        self._mem = Memory(self.ops)
        self._dims = {}
        if not hasattr(self, '_layers'):
            self._layers = []
        self.descriptions = dict(self.descriptions)
        self.on_init_hooks = list(self.on_init_hooks)
        self.on_data_hooks = list(self.on_data_hooks)

        for attr, install in self.descriptions.items():
            install(attr, self)
        for hook in self.on_init_hooks:
            hook(self, *args, **kwargs)
        self.set_id()

    def _update_defaults(self, args, kwargs):
        new_kwargs = {}
        for key, value in kwargs.items():
            if hasattr(self, key):
                setattr(self, key, value)
            else:
                new_kwargs[key] = value
        return new_kwargs

    def set_id(self):
        Model.id += 1
        self.id = Model.id
        for layer in self._layers:
            layer.set_id()

    #@check.args(equal_length)
    @check.arg(1, is_sequence)
    def begin_training(self, train_X, train_y=None, **trainer_cfg):
        for hook in self.on_data_hooks:
            hook(self, train_X, train_y)
        return self.Trainer(self, **trainer_cfg)

    @check.arg(2, is_float)
    @check.arg(1, has_shape(('nB', 'nI')))
    def begin_update(self, X, drop=0.0):
        raise NotImplementedError

    def predict(self, X):
        y, _ = self.begin_update(X)
        return y

    def predict_one(self, x):
        X = self.ops.expand_dims(x, axis=0)
        return self.predict(X)[0]

    @contextlib.contextmanager
    def use_params(self, params): # pragma: no cover
        backup = None
        weights = self._mem.weights
        if self.id in params:
            param = params[self.id]
            backup = weights.copy()
            weights[:] = param
        if hasattr(self, '_layers'):
            contexts = [layer.use_params(params) for layer in self._layers]
            for context in contexts:
                next(context.gen)
        yield
        if backup is not None:
            self._mem.weights[:] = backup
        for i, context in enumerate(contexts):
            # This is ridiculous, but apparently it's what you
            # have to do to make this work across Python 2/3?
            try:
                next(context.gen)
            except StopIteration:
                pass

    def __call__(self, x):
        '''
        x
            Must match expected type
            Must match expected shape
        '''
        return self.predict(x)

    def pipe(self, stream, batch_size=1000):
        for batch in util.minibatch(stream, batch_size):
            ys = self.predict(batch)
            for y in ys:
                yield y

    def update(self, stream, batch_size=1000):
        for X, y in util.minibatch(stream, batch_size=batch_size):
            output, finish_update = self.begin_update(X)
            gradient = finish_update(y)
            yield gradient

    def evaluate(self, X, y):
        '''
        x
            Must match expected type
            Must match expected shape
        y
            Must match expected type
        '''
        scores = self.ops.xp.vstack(self.pipe(X))
        scores = scores.reshape(y.shape)
        if len(scores.shape) == 1:
            correct = ((scores >= 0.5) == (y >= 0.5)).sum()
        else:
            correct = (scores.argmax(axis=1) == y.argmax(axis=1)).sum()
        return correct / y.shape[0]

    @check.operator_is_defined('+')
    def __add__(self, other):
        '''Apply the function bound to the '+' operator.'''
        return self._operators['+'](self, other)

    @check.operator_is_defined('-')
    def __sub__(self, other):
        '''Apply the function bound to the '-' operator.'''
        return self._operators['-'](self, other)

    @check.operator_is_defined('*')
    def __mul__(self, other):
        '''Apply the function bound to the '*' operator.'''
        return self._operators['*'](self, other)

    @check.operator_is_defined('@')
    def __matmul__(self, other):
        '''Apply the function bound to the '@' operator.'''
        return self._operators['@'](self, other)

    @check.operator_is_defined('/')
    def __div__(self, other):
        '''Apply the function bound to the '/' operator.'''
        return self._operators['/'](self, other)

    @check.operator_is_defined('/')
    def __truediv__(self, other): # pragma: no cover
        '''Apply the function bound to the '/' operator.'''
        return self._operators['/'](self, other)

    @check.operator_is_defined('//')
    def __floordiv__(self, other):
        '''Apply the function bound to the '//' operator.'''
        return self._operators['//'](self, other)

    @check.operator_is_defined('%')
    def __mod__(self, other):
        '''Apply the function bound to the '%' operator.'''
        return self._operators['%'](self, other)

    @check.operator_is_defined('**')
    def __pow__(self, other, modulo=None):
        '''Apply the function bound to the '**' operator.'''
        return self._operators['**'](self, other)

    @check.operator_is_defined('<<')
    def __lshift__(self, other):
        '''Apply the function bound to the '<<' operator.'''
        return self._operators['<<'](self, other)

    @check.operator_is_defined('>>')
    def __rshift__(self, other):
        '''Apply the function bound to the '>>' operator.'''
        return self._operators['>>'](self, other)

    @check.operator_is_defined('&')
    def __and__(self, other):
        '''Apply the function bound to the '&' operator.'''
        return self._operators['&'](self, other)

    @check.operator_is_defined('^')
    def __xor__(self, other):
        '''Apply the function bound to the '^' operator.'''
        return self._operators['^'](self, other)

    @check.operator_is_defined('|')
    def __or__(self, other):
        '''Apply the function bound to the '|' operator.'''
        return self._operators['|'](self, other)

#def list_gradients(self):
#    pass
#def check_input(self, x, expect_batch=False):
#    if self.layers:
#        return self.layers[0].check_input(x, expect_batch=expect_batch)
#    if is_batch(x):
#        shape = x.shape[1:]
#    else:
#        raise ShapeError.expected_batch(locals(), globals())
#    if shape != self.input_shape:
#        raise ShapeError.dim_mismatch(self.input_shape, shape)
#    else:
#        return True
#def allocate_params(model):
#    '''Allocate all trainable parameters of a model, including for its sublayers,
#    so that parameters can be contiguous in memory.'''
#    # Get total size
#    size = sum(prod(shape) for shape in model.describe_all_params)
#    params = Params(total=size)
#    for i, layer in enumerate(all_layers(model)):
#        layer.name = '%s-%d' % (layer.name, i)
#        for name, shape, init in describe_params(model):
#            params.add(name, shape)
#            if init is not None:
#                init(params.get(name), inplace=True)
#    return params
#
#
#def set_dimensions_given_data(model, X, y):
#    pass
#
#
#def initialize_weights_given_data(model, X, y):
#    for name, weights in model.weights.items():
#        init = model.get_initialzer(name)
#
#    for key, (name, shape, init_func) in model.description.weights:
#        if init_func is not None:
#            weights = model.w.get(key)
#            init_func(weights, X, y)
#
#
#
#def all_layers(model):
#    '''Iterate over all layers in the model.'''
#    queue = [model]
#    seen = set()
#    for layer in queue:
#        yield layer
#        queue.extend(layer.layers)
#        assert id(layer) not in seen # Catch loops!
#        seen.add(id(layer))
#
#
#def describe_all_params(model):
#    for layer in model.all_layers:
#        for name, shape, init in layer.describe_params:
#            yield name, shape, init
#
#
<commit_msg>Add logloss option, fix GPU use_params in Model<commit_after>from __future__ import division
from numpy import prod
import contextlib

from .. import util
from ..train import Trainer
from ..ops import NumpyOps
from ..mem import Memory
from ..util import get_ops, copy_array
from ... import check
from ... import describe
from ...check import equal_length, has_shape, is_sequence, is_float, is_array

class Model(object):
    '''Model base class.'''
    name = 'model'
    id = 0
    lsuv = False
    ops = NumpyOps()
    Trainer = Trainer
    descriptions = []
    on_data_hooks = []
    on_init_hooks = [] # Use this to add layers
    _operators = {}

    @classmethod
    @contextlib.contextmanager
    def define_operators(cls, operators):
        '''Bind operators to specified functions for the scope of the context:

        Example
        -------

            model = Model()
            other = Model()
            with Model.define_operators({"+": lambda self, other: "plus"}):
                print(model + other)
                # "plus"
            print(model + other)
            # Raises TypeError --- binding limited to scope of with block.
        '''
        old_ops = dict(cls._operators)
        for op, func in operators.items():
            cls._operators[op] = func
        yield
        cls._operators = old_ops

    @classmethod
    @contextlib.contextmanager
    def use_device(cls, device):
        '''Change the device to execute on for the scope of the block.'''
        if device == cls.ops.device:
            yield
        else:
            curr_ops = cls.ops
            cls.ops = get_ops(device)
            yield
            cls.ops = curr_ops

    @property
    def input_shape(self):
        raise NotImplementedError

    @property
    def output_shape(self):
        raise NotImplementedError

    def __init__(self, *args, **kwargs):
        self.name = self.__class__.name
        kwargs = self._update_defaults(args, kwargs)
        self._mem = Memory(self.ops)
        self._dims = {}
        if not hasattr(self, '_layers'):
            self._layers = []
        self.descriptions = dict(self.descriptions)
        self.on_init_hooks = list(self.on_init_hooks)
        self.on_data_hooks = list(self.on_data_hooks)

        for attr, install in self.descriptions.items():
            install(attr, self)
        for hook in self.on_init_hooks:
            hook(self, *args, **kwargs)
        self.set_id()

    def _update_defaults(self, args, kwargs):
        new_kwargs = {}
        for key, value in kwargs.items():
            if hasattr(self, key):
                setattr(self, key, value)
            else:
                new_kwargs[key] = value
        return new_kwargs

    def set_id(self):
        Model.id += 1
        self.id = Model.id
        for layer in self._layers:
            layer.set_id()

    #@check.args(equal_length)
    @check.arg(1, is_sequence)
    def begin_training(self, train_X, train_y=None, **trainer_cfg):
        for hook in self.on_data_hooks:
            hook(self, train_X, train_y)
        return self.Trainer(self, **trainer_cfg)

    @check.arg(2, is_float)
    @check.arg(1, has_shape(('nB', 'nI')))
    def begin_update(self, X, drop=0.0):
        raise NotImplementedError

    def predict(self, X):
        y, _ = self.begin_update(X)
        return y

    def predict_one(self, x):
        X = self.ops.expand_dims(x, axis=0)
        return self.predict(X)[0]

    @contextlib.contextmanager
    def use_params(self, params): # pragma: no cover
        backup = None
        weights = self._mem.weights
        if self.id in params:
            param = params[self.id]
            backup = weights.copy()
            copy_array(weights, param)
        if hasattr(self, '_layers'):
            contexts = [layer.use_params(params) for layer in self._layers]
            for context in contexts:
                next(context.gen)
        yield
        if backup is not None:
            copy_array(self._mem.weights, backup)
        for i, context in enumerate(contexts):
            # This is ridiculous, but apparently it's what you
            # have to do to make this work across Python 2/3?
            try:
                next(context.gen)
            except StopIteration:
                pass

    def __call__(self, x):
        '''
        x
            Must match expected type
            Must match expected shape
        '''
        return self.predict(x)

    def pipe(self, stream, batch_size=1000):
        for batch in util.minibatch(stream, batch_size):
            ys = self.predict(batch)
            for y in ys:
                yield y

    def update(self, stream, batch_size=1000):
        for X, y in util.minibatch(stream, batch_size=batch_size):
            output, finish_update = self.begin_update(X)
            gradient = finish_update(y)
            yield gradient

    def evaluate(self, X, y):
        '''
        x
            Must match expected type
            Must match expected shape
        y
            Must match expected type
        '''
        scores = self.ops.xp.vstack(self.pipe(X))
        scores = scores.reshape(y.shape)
        if len(scores.shape) == 1:
            correct = ((scores >= 0.5) == (y >= 0.5)).sum()
        else:
            correct = (scores.argmax(axis=1) == y.argmax(axis=1)).sum()
        return correct / y.shape[0]

    def evaluate_logloss(self, X, y):
        yh = self.ops.xp.vstack(self.pipe(X))
        yh = yh.reshape(y.shape)
        assert len(yh.shape) == 1
        losses = -y * self.ops.xp.log(yh + 1e-8) - (1-y) * self.ops.xp.log((1-yh)+1e-8)
        return losses.mean()

    @check.operator_is_defined('+')
    def __add__(self, other):
        '''Apply the function bound to the '+' operator.'''
        return self._operators['+'](self, other)

    @check.operator_is_defined('-')
    def __sub__(self, other):
        '''Apply the function bound to the '-' operator.'''
        return self._operators['-'](self, other)

    @check.operator_is_defined('*')
    def __mul__(self, other):
        '''Apply the function bound to the '*' operator.'''
        return self._operators['*'](self, other)

    @check.operator_is_defined('@')
    def __matmul__(self, other):
        '''Apply the function bound to the '@' operator.'''
        return self._operators['@'](self, other)

    @check.operator_is_defined('/')
    def __div__(self, other):
        '''Apply the function bound to the '/' operator.'''
        return self._operators['/'](self, other)

    @check.operator_is_defined('/')
    def __truediv__(self, other): # pragma: no cover
        '''Apply the function bound to the '/' operator.'''
        return self._operators['/'](self, other)

    @check.operator_is_defined('//')
    def __floordiv__(self, other):
        '''Apply the function bound to the '//' operator.'''
        return self._operators['//'](self, other)

    @check.operator_is_defined('%')
    def __mod__(self, other):
        '''Apply the function bound to the '%' operator.'''
        return self._operators['%'](self, other)

    @check.operator_is_defined('**')
    def __pow__(self, other, modulo=None):
        '''Apply the function bound to the '**' operator.'''
        return self._operators['**'](self, other)

    @check.operator_is_defined('<<')
    def __lshift__(self, other):
        '''Apply the function bound to the '<<' operator.'''
        return self._operators['<<'](self, other)

    @check.operator_is_defined('>>')
    def __rshift__(self, other):
        '''Apply the function bound to the '>>' operator.'''
        return self._operators['>>'](self, other)

    @check.operator_is_defined('&')
    def __and__(self, other):
        '''Apply the function bound to the '&' operator.'''
        return self._operators['&'](self, other)

    @check.operator_is_defined('^')
    def __xor__(self, other):
        '''Apply the function bound to the '^' operator.'''
        return self._operators['^'](self, other)

    @check.operator_is_defined('|')
    def __or__(self, other):
        '''Apply the function bound to the '|' operator.'''
        return self._operators['|'](self, other)

#def list_gradients(self):
#    pass
#def check_input(self, x, expect_batch=False):
#    if self.layers:
#        return self.layers[0].check_input(x, expect_batch=expect_batch)
#    if is_batch(x):
#        shape = x.shape[1:]
#    else:
#        raise ShapeError.expected_batch(locals(), globals())
#    if shape != self.input_shape:
#        raise ShapeError.dim_mismatch(self.input_shape, shape)
#    else:
#        return True
#def allocate_params(model):
#    '''Allocate all trainable parameters of a model, including for its sublayers,
#    so that parameters can be contiguous in memory.'''
#    # Get total size
#    size = sum(prod(shape) for shape in model.describe_all_params)
#    params = Params(total=size)
#    for i, layer in enumerate(all_layers(model)):
#        layer.name = '%s-%d' % (layer.name, i)
#        for name, shape, init in describe_params(model):
#            params.add(name, shape)
#            if init is not None:
#                init(params.get(name), inplace=True)
#    return params
#
#
#def set_dimensions_given_data(model, X, y):
#    pass
#
#
#def initialize_weights_given_data(model, X, y):
#    for name, weights in model.weights.items():
#        init = model.get_initialzer(name)
#
#    for key, (name, shape, init_func) in model.description.weights:
#        if init_func is not None:
#            weights = model.w.get(key)
#            init_func(weights, X, y)
#
#
#
#def all_layers(model):
#    '''Iterate over all layers in the model.'''
#    queue = [model]
#    seen = set()
#    for layer in queue:
#        yield layer
#        queue.extend(layer.layers)
#        assert id(layer) not in seen # Catch loops!
#        seen.add(id(layer))
#
#
#def describe_all_params(model):
#    for layer in model.all_layers:
#        for name, shape, init in layer.describe_params:
#            yield name, shape, init
#
#
<|endoftext|>