<commit_before>__author__ = 'chris'
import sys, os
import gzip
import pickle
import stun
import json
import random

import nacl.signing, nacl.hash, nacl.encoding

from cStringIO import StringIO

from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import task
from twisted.web import resource, server

from binascii import unhexlify, hexlify

from random import shuffle

from seed import peers

from guid import guid

from dht import log
from dht.node import Node
from dht.network import Server
from dht.crawling import NodeSpiderCrawl
from dht.utils import digest, deferredDict
from dht import kprotocol

sys.path.append(os.path.dirname(__file__))
application = service.Application("OpenBazaar_seed_server")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)

# Load the keys
if os.path.isfile('keys.pickle'):
    keys = pickle.load(open("keys.pickle", "r"))
    privkey = keys["kademlia_key"]
    signing_key_hex = keys["signing_privkey"]
    signing_key = nacl.signing.SigningKey(signing_key_hex, encoder=nacl.encoding.HexEncoder)
else:
    print "Generating GUID, stand by..."
    privkey = hexlify(guid.generate())
    signing_key = nacl.signing.SigningKey.generate()
    keys = {
            'kademlia_key': privkey,
            'signing_privkey': signing_key.encode(encoder=nacl.encoding.HexEncoder),
            'signing_pubkey': signing_key.verify_key.encode(encoder=nacl.encoding.HexEncoder)
            }
    pickle.dump(keys, open("keys.pickle", "wb"))

# Create the guid
privkey = nacl.signing.SigningKey(privkey, encoder=nacl.encoding.HexEncoder)
pubkey = privkey.verify_key
signed_pubkey = privkey.sign(str(pubkey))
h = nacl.hash.sha512(signed_pubkey)
guid = unhexlify(h[:40])

# Stun
response = stun.get_ip_info(stun_host="stun.l.google.com", source_port=0, stun_port=19302)
ip_address = response[1]
port = 18467

# Start the kademlia server
this_node = Node(guid, ip_address, port, signed_pubkey)

if os.path.isfile('cache.pickle'):
    kserver = Server.loadState('cache.pickle', ip_address, port)
else:
    kserver = Server(this_node)

kserver.saveStateRegularly('cache.pickle', 10)
udpserver = internet.UDPServer(18467, kserver.protocol)
udpserver.setServiceParent(application)

class WebResource(resource.Resource):
    def __init__(self, kserver):
        resource.Resource.__init__(self)
        self.kserver = kserver
        self.nodes = []
        for bucket in self.kserver.protocol.router.buckets:
            for node in bucket.getNodes():
                self.nodes.append(node)
        self.nodes.append(this_node)
        loopingCall = task.LoopingCall(self.crawl)
        loopingCall.start(60, True)

    def crawl(self):
        def gather_results(result):
            for proto in result:
                n = kprotocol.Node()
                n.ParseFromString(proto)
                if n.merchant:
                    node = Node(n.guid, n.ip, n.port, n.signedPublicKey, n.merchant, n.server_port, n.transport)
                else:
                    node = Node(n.guid, n.ip, n.port, n.signedPublicKey)
                self.nodes.append(node)
            shuffle(self.nodes)

        def start_crawl(results):
            for node, result in results.items():
                if not result[0]:
                    self.nodes.remove(node)
            node = Node(digest(random.getrandbits(255)))
            nearest = self.kserver.protocol.router.findNeighbors(node)
            spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest, 100, 4)
            d = spider.find().addCallback(gather_results)

        ds = {}
        for bucket in self.kserver.protocol.router.buckets:
            for node in bucket.getNodes():
                if node not in self.nodes:
                    self.nodes.append(node)
        for node in self.nodes:
            if node is not this_node:
                ds[node] = self.kserver.protocol.callPing(node)
        deferredDict(ds).addCallback(start_crawl)

    def getChild(self, child, request):
        return self

    def render_GET(self, request):
        print request.args
        log.msg("Received a request for nodes, responding...")
        if "format" in request.args:
            if request.args["format"][0] == "json":
                json_list = []
                if "type" in request.args and request.args["type"][0] == "vendors":
                    print "getting list of vendors"
                    for node in self.nodes:
                        if node.merchant is True:
                            print "found vendor"
                            node_dic = {}
                            node_dic["ip"] = node.ip
                            node_dic["port"] = node.port
                            json_list.append(node_dic)
                    sig = signing_key.sign(str(json_list))
                    resp = {"peers" : json_list, "signature" : hexlify(sig[:64])}
                    request.write(json.dumps(resp, indent=4))
                else:
                    for node in self.nodes[:50]:
                        node_dic = {}
                        node_dic["ip"] = node.ip
                        node_dic["port"] = node.port
                        json_list.append(node_dic)
                    sig = signing_key.sign(str(json_list))
                    resp = {"peers" : json_list, "signature" : hexlify(sig[:64])}
                    request.write(json.dumps(resp, indent=4))
            elif request.args["format"][0] == "protobuf":
                proto = peers.PeerSeeds()
                for node in self.nodes[:50]:
                    peer = peers.PeerData()
                    peer.ip_address = node.ip
                    peer.port = node.port
                    peer.vendor = node.merchant
                    proto.peer_data.append(peer.SerializeToString())

                sig = signing_key.sign("".join(proto.peer_data))
                proto.signature = sig
                uncompressed_data = proto.SerializeToString()
                buf = StringIO()
                f = gzip.GzipFile(mode='wb', fileobj=buf)
                try:
                    f.write(uncompressed_data)
                finally:
                    f.close()
                resp = buf.getvalue()
                request.write(resp)
        else:
            proto = peers.PeerSeeds()
            if "type" in request.args and request.args["type"][0] == "vendors":
                for node in self.nodes:
                    if node.merchant is True:
                        peer = peers.PeerData()
                        peer.ip_address = node.ip
                        peer.port = node.port
                        peer.vendor = node.merchant
                        proto.peer_data.append(peer.SerializeToString())

                sig = signing_key.sign("".join(proto.peer_data))
                proto.signature = sig
                uncompressed_data = proto.SerializeToString()
                buf = StringIO()
                f = gzip.GzipFile(mode='wb', fileobj=buf)
                try:
                    f.write(uncompressed_data)
                finally:
                    f.close()
                resp = buf.getvalue()
                request.write(resp)
            else:
                for node in self.nodes[:50]:
                    peer = peers.PeerData()
                    peer.ip_address = node.ip
                    peer.port = node.port
                    peer.vendor = node.merchant
                    proto.peer_data.append(peer.SerializeToString())

                sig = signing_key.sign("".join(proto.peer_data))
                proto.signature = sig
                uncompressed_data = proto.SerializeToString()
                buf = StringIO()
                f = gzip.GzipFile(mode='wb', fileobj=buf)
                try:
                    f.write(uncompressed_data)
                finally:
                    f.close()
                resp = buf.getvalue()
                request.write(resp)
        request.finish()
        return server.NOT_DONE_YET

server_protocol = server.Site(WebResource(kserver))
seed_server = internet.TCPServer(8080, server_protocol)
seed_server.setServiceParent(application)<commit_msg>switched seed node list to dictionary<commit_after>__author__ = 'chris'
import sys, os
import gzip
import pickle
import stun
import json
import random

import nacl.signing, nacl.hash, nacl.encoding

from cStringIO import StringIO

from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import task
from twisted.web import resource, server

from binascii import unhexlify, hexlify

from random import shuffle

from seed import peers

from guid import guid

from dht import log
from dht.node import Node
from dht.network import Server
from dht.crawling import NodeSpiderCrawl
from dht.utils import digest, deferredDict
from dht import kprotocol

sys.path.append(os.path.dirname(__file__))
application = service.Application("OpenBazaar_seed_server")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)

# Load the keys
if os.path.isfile('keys.pickle'):
    keys = pickle.load(open("keys.pickle", "r"))
    privkey = keys["kademlia_key"]
    signing_key_hex = keys["signing_privkey"]
    signing_key = nacl.signing.SigningKey(signing_key_hex, encoder=nacl.encoding.HexEncoder)
else:
    print "Generating GUID, stand by..."
    privkey = hexlify(guid.generate())
    signing_key = nacl.signing.SigningKey.generate()
    keys = {
            'kademlia_key': privkey,
            'signing_privkey': signing_key.encode(encoder=nacl.encoding.HexEncoder),
            'signing_pubkey': signing_key.verify_key.encode(encoder=nacl.encoding.HexEncoder)
            }
    pickle.dump(keys, open("keys.pickle", "wb"))

# Create the guid
privkey = nacl.signing.SigningKey(privkey, encoder=nacl.encoding.HexEncoder)
pubkey = privkey.verify_key
signed_pubkey = privkey.sign(str(pubkey))
h = nacl.hash.sha512(signed_pubkey)
guid = unhexlify(h[:40])

# Stun
response = stun.get_ip_info(stun_host="stun.l.google.com", source_port=0, stun_port=19302)
ip_address = response[1]
port = 18467

# Start the kademlia server
this_node = Node(guid, ip_address, port, signed_pubkey)

if os.path.isfile('cache.pickle'):
    kserver = Server.loadState('cache.pickle', ip_address, port)
else:
    kserver = Server(this_node)

kserver.saveStateRegularly('cache.pickle', 10)
udpserver = internet.UDPServer(18467, kserver.protocol)
udpserver.setServiceParent(application)

class WebResource(resource.Resource):
    def __init__(self, kserver):
        resource.Resource.__init__(self)
        self.kserver = kserver
        self.nodes = {}
        for bucket in self.kserver.protocol.router.buckets:
            for node in bucket.getNodes():
                self.nodes[node.id] = node
        self.nodes[this_node.id] = this_node
        loopingCall = task.LoopingCall(self.crawl)
        loopingCall.start(60, True)

    def crawl(self):
        def gather_results(result):
            for proto in result:
                n = kprotocol.Node()
                n.ParseFromString(proto)
                if n.merchant:
                    node = Node(n.guid, n.ip, n.port, n.signedPublicKey, n.merchant, n.server_port, n.transport)
                else:
                    node = Node(n.guid, n.ip, n.port, n.signedPublicKey)
                if node.id not in self.nodes:
                    self.nodes[node.id] = node
            shuffle(self.nodes)

        def start_crawl(results):
            for node, result in results.items():
                if not result[0]:
                    del self.nodes[node.id]
            node = Node(digest(random.getrandbits(255)))
            nearest = self.kserver.protocol.router.findNeighbors(node)
            spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest, 100, 4)
            d = spider.find().addCallback(gather_results)

        ds = {}
        for bucket in self.kserver.protocol.router.buckets:
            for node in bucket.getNodes():
                if node not in self.nodes:
                    self.nodes[node.id] = node
        for node in self.nodes.values():
            if node.id != this_node.id:
                ds[node] = self.kserver.protocol.callPing(node)
        deferredDict(ds).addCallback(start_crawl)

    def getChild(self, child, request):
        return self

    def render_GET(self, request):
        print request.args
        log.msg("Received a request for nodes, responding...")
        if "format" in request.args:
            if request.args["format"][0] == "json":
                json_list = []
                if "type" in request.args and request.args["type"][0] == "vendors":
                    print "getting list of vendors"
                    for node in self.nodes.values():
                        if node.merchant is True:
                            print "found vendor"
                            node_dic = {}
                            node_dic["ip"] = node.ip
                            node_dic["port"] = node.port
                            json_list.append(node_dic)
                    sig = signing_key.sign(str(json_list))
                    resp = {"peers" : json_list, "signature" : hexlify(sig[:64])}
                    request.write(json.dumps(resp, indent=4))
                else:
                    for node in self.nodes.values()[:50]:
                        node_dic = {}
                        node_dic["ip"] = node.ip
                        node_dic["port"] = node.port
                        json_list.append(node_dic)
                    sig = signing_key.sign(str(json_list))
                    resp = {"peers" : json_list, "signature" : hexlify(sig[:64])}
                    request.write(json.dumps(resp, indent=4))
            elif request.args["format"][0] == "protobuf":
                proto = peers.PeerSeeds()
                for node in self.nodes.values()[:50]:
                    peer = peers.PeerData()
                    peer.ip_address = node.ip
                    peer.port = node.port
                    peer.vendor = node.merchant
                    proto.peer_data.append(peer.SerializeToString())

                sig = signing_key.sign("".join(proto.peer_data))
                proto.signature = sig
                uncompressed_data = proto.SerializeToString()
                buf = StringIO()
                f = gzip.GzipFile(mode='wb', fileobj=buf)
                try:
                    f.write(uncompressed_data)
                finally:
                    f.close()
                resp = buf.getvalue()
                request.write(resp)
        else:
            proto = peers.PeerSeeds()
            if "type" in request.args and request.args["type"][0] == "vendors":
                for node in self.nodes.values():
                    if node.merchant is True:
                        peer = peers.PeerData()
                        peer.ip_address = node.ip
                        peer.port = node.port
                        peer.vendor = node.merchant
                        proto.peer_data.append(peer.SerializeToString())

                sig = signing_key.sign("".join(proto.peer_data))
                proto.signature = sig
                uncompressed_data = proto.SerializeToString()
                buf = StringIO()
                f = gzip.GzipFile(mode='wb', fileobj=buf)
                try:
                    f.write(uncompressed_data)
                finally:
                    f.close()
                resp = buf.getvalue()
                request.write(resp)
            else:
                for node in self.nodes.values()[:50]:
                    peer = peers.PeerData()
                    peer.ip_address = node.ip
                    peer.port = node.port
                    peer.vendor = node.merchant
                    proto.peer_data.append(peer.SerializeToString())

                sig = signing_key.sign("".join(proto.peer_data))
                proto.signature = sig
                uncompressed_data = proto.SerializeToString()
                buf = StringIO()
                f = gzip.GzipFile(mode='wb', fileobj=buf)
                try:
                    f.write(uncompressed_data)
                finally:
                    f.close()
                resp = buf.getvalue()
                request.write(resp)
        request.finish()
        return server.NOT_DONE_YET

server_protocol = server.Site(WebResource(kserver))
seed_server = internet.TCPServer(8080, server_protocol)
seed_server.setServiceParent(application)<|endoftext|>


<commit_before>import json
import logging
import os
import shutil
import sys
import atexit
import webbrowser

from functools import wraps
from uuid import uuid1
from threading import Event
from cefpython3 import cefpython as cef
from copy import copy

from .js.css import disable_text_select
from webview import _js_bridge_call
from webview.util import parse_api_js, default_html

sys.excepthook = cef.ExceptHook
instances = {}

logger = logging.getLogger(__name__)

class JSBridge:
    def __init__(self, eval_events, api, uid):
        self.results = {}
        self.eval_events = eval_events
        self.api = api
        self.uid = uid

    def return_result(self, result, uid):
        self.results[uid] = json.loads(result) if result else None
        self.eval_events[uid].set()

    def call(self, func_name, param):
        _js_bridge_call(self.uid, self.api, func_name, param)

class Browser:
    def __init__(self, handle, browser, api, text_select, uid):
        self.handle = handle
        self.browser = browser
        self.api = api
        self.text_select = text_select
        self.uid = uid

        self.eval_events = {}
        self.js_bridge = JSBridge(self.eval_events, api, uid)
        self.initialized = False
        self.loaded = Event()

    def initialize(self):
        if self.initialized:
            return

        if self.api:
            self.browser.ExecuteJavascript(parse_api_js(self.api))

        if not self.text_select:
            self.browser.ExecuteJavascript(disable_text_select)

        self.initialized = True
        self.loaded.set()

    def close(self):
        self.browser.CloseBrowser(True)

    def evaluate_js(self, code):
        self.loaded.wait()
        eval_script = """
            try {{
                window.external.return_result({0}, '{1}');
            }} catch(e) {{
                console.error(e.stack);
                window.external.return_result(null, '{1}');
            }}
        """

        id_ = uuid1().hex[:8]
        self.eval_events[id_] = Event()
        self.browser.ExecuteJavascript(eval_script.format(code, id_))
        self.eval_events[id_].wait()  # result is obtained via JSBridge.return_result

        result = copy(self.js_bridge.results[id_])

        del self.eval_events[id_]
        del self.js_bridge.results[id_]

        return result

    def get_current_url(self):
        self.loaded.wait()
        return self.browser.GetUrl()

    def load_url(self, url):
        self.browser.LoadUrl(url)

    def load_html(self, html):
        self.browser.LoadUrl('data:text/html,{0}'.format(html))

def find_instance(browser):
    for instance in instances.values():
        if instance.browser is browser:
            return instance

    return None

class LoadHandler(object):
    def OnBeforePopup(self, **args):
        url = args['target_url']
        user_gesture = args['user_gesture']

        if user_gesture:
            webbrowser.open(url)

        return True

    def OnLoadingStateChange(self, browser, is_loading, **_):
        instance = find_instance(browser)

        if instance is not None:
            if is_loading:
                instance.initialized = False
            else:
                instance.initialize()
        else:
            logger.debug('CEF instance is not found %s ' % browser)

def _cef_call(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        uid = args[-1]

        if uid not in instances:
            raise Exception('CEF window with uid {0} does not exist'.format(uid))

        _webview_ready.wait()
        return func(*args, **kwargs)

    return wrapper

_webview_ready = None

def init(webview_ready, debug):
    global _initialized, _webview_ready
    _webview_ready = webview_ready

    if not _initialized:
        settings = {
            'multi_threaded_message_loop': True,
            'context_menu': {
                'enabled': debug
            }
        }
        cef.Initialize(settings=settings)
        cef.DpiAware.EnableHighDpiSupport()
        _initialized = True

def create_browser(uid, handle, alert_func, url=None, js_api=None, text_select=False):
    def _create():
        real_url = url or 'data:text/html,{0}'.format(default_html)
        cef_browser = cef.CreateBrowserSync(window_info=window_info, url=real_url)
        browser = Browser(handle, cef_browser, js_api, text_select, uid)

        bindings = cef.JavascriptBindings()
        bindings.SetObject('external', browser.js_bridge)
        bindings.SetFunction('alert', alert_func)

        cef_browser.SetJavascriptBindings(bindings)
        cef_browser.SetClientHandler(LoadHandler())

        instances[uid] = browser
        _webview_ready.set()

    window_info = cef.WindowInfo()
    window_info.SetAsChild(handle)
    cef.PostTask(cef.TID_UI, _create)

@_cef_call
def load_html(html, uid):
    instance = instances[uid]
    instance.load_html(html)

@_cef_call
def load_url(url, uid):
    instance = instances[uid]
    instance.load_url(url)

@_cef_call
def evaluate_js(code, uid):
    instance = instances[uid]
    return instance.evaluate_js(code)

@_cef_call
def get_current_url(uid):
    instance = instances[uid]
    url = instance.get_current_url()

    if url.startswith('data:text/html,'):
        return None
    else:
        return url

@_cef_call
def resize(width, height, uid):
    hwnd = instances[uid].handle
    lparam = width << 16 & height
    cef.WindowUtils.OnSize(hwnd, 5, 0, lparam)

@_cef_call
def close_window(uid):
    instance = instances[uid]
    instance.close()
    del instances[uid]

def shutdown():
    try:
        if os.path.exists('blob_storage'):
            shutil.rmtree('blob_storage')

        if os.path.exists('webrtc_event_logs'):
            shutil.rmtree('webrtc_event_logs')

        if os.path.exists('error.log'):
            shutil.rmtree('error.log')

    except Exception as e:
        logger.debug(e, exc_info=True)
    cef.Shutdown()

_initialized = False
<commit_msg>[CEF] Fix JS logic not persisting after navigating to another page<commit_after>import json
import logging
import os
import shutil
import sys
import webbrowser

from functools import wraps
from uuid import uuid1
from threading import Event
from cefpython3 import cefpython as cef
from copy import copy

from .js.css import disable_text_select
from webview import _js_bridge_call
from webview.util import parse_api_js, default_html

sys.excepthook = cef.ExceptHook
instances = {}

logger = logging.getLogger(__name__)

class JSBridge:
    def __init__(self, eval_events, api, uid):
        self.results = {}
        self.eval_events = eval_events
        self.api = api
        self.uid = uid

    def return_result(self, result, uid):
        self.results[uid] = json.loads(result) if result else None
        self.eval_events[uid].set()

    def call(self, func_name, param):
        _js_bridge_call(self.uid, self.api, func_name, param)

class Browser:
    def __init__(self, handle, browser, api, text_select, uid):
        self.handle = handle
        self.browser = browser
        self.api = api
        self.text_select = text_select
        self.uid = uid

        self.eval_events = {}
        self.js_bridge = JSBridge(self.eval_events, api, uid)
        self.initialized = False
        self.loaded = Event()

    def initialize(self):
        if self.initialized:
            return

        self.browser.GetJavascriptBindings().Rebind()

        if self.api:
            self.browser.ExecuteJavascript(parse_api_js(self.api))

        if not self.text_select:
            self.browser.ExecuteJavascript(disable_text_select)

        self.initialized = True
        self.loaded.set()

    def close(self):
        self.browser.CloseBrowser(True)

    def evaluate_js(self, code):
        self.loaded.wait()
        eval_script = """
            try {{
                window.external.return_result({0}, '{1}');
            }} catch(e) {{
                console.error(e.stack);
                window.external.return_result(null, '{1}');
            }}
        """

        id_ = uuid1().hex[:8]
        self.eval_events[id_] = Event()
        self.browser.ExecuteJavascript(eval_script.format(code, id_))
        self.eval_events[id_].wait()  # result is obtained via JSBridge.return_result

        result = copy(self.js_bridge.results[id_])

        del self.eval_events[id_]
        del self.js_bridge.results[id_]

        return result

    def get_current_url(self):
        self.loaded.wait()
        return self.browser.GetUrl()

    def load_url(self, url):
        self.initialized = False
        self.loaded.clear()
        self.browser.LoadUrl(url)

    def load_html(self, html):
        self.initialized = False
        self.loaded.clear()
        self.browser.LoadUrl('data:text/html,{0}'.format(html))

def find_instance(browser):
    for instance in instances.values():
        if instance.browser is browser:
            return instance

    return None

class LoadHandler(object):
    def OnBeforePopup(self, **args):
        url = args['target_url']
        user_gesture = args['user_gesture']

        if user_gesture:
            webbrowser.open(url)

        return True

    def OnLoadingStateChange(self, browser, is_loading, **_):
        instance = find_instance(browser)

        if instance is not None:
            if is_loading:
                instance.initialized = False
            else:
                instance.initialize()
        else:
            logger.debug('CEF instance is not found %s ' % browser)

def _cef_call(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        uid = args[-1]

        if uid not in instances:
            raise Exception('CEF window with uid {0} does not exist'.format(uid))

        _webview_ready.wait()
        return func(*args, **kwargs)

    return wrapper

_webview_ready = None

def init(webview_ready, debug):
    global _initialized, _webview_ready
    _webview_ready = webview_ready

    if not _initialized:
        settings = {
            'multi_threaded_message_loop': True,
            'context_menu': {
                'enabled': debug
            }
        }
        cef.Initialize(settings=settings)
        cef.DpiAware.EnableHighDpiSupport()
        _initialized = True

def create_browser(uid, handle, alert_func, url=None, js_api=None, text_select=False):
    def _create():
        real_url = url or 'data:text/html,{0}'.format(default_html)
        cef_browser = cef.CreateBrowserSync(window_info=window_info, url=real_url)
        browser = Browser(handle, cef_browser, js_api, text_select, uid)

        bindings = cef.JavascriptBindings()
        bindings.SetObject('external', browser.js_bridge)
        bindings.SetFunction('alert', alert_func)

        cef_browser.SetJavascriptBindings(bindings)
        cef_browser.SetClientHandler(LoadHandler())

        instances[uid] = browser
        _webview_ready.set()

    window_info = cef.WindowInfo()
    window_info.SetAsChild(handle)
    cef.PostTask(cef.TID_UI, _create)

@_cef_call
def load_html(html, uid):
    instance = instances[uid]
    instance.load_html(html)

@_cef_call
def load_url(url, uid):
    instance = instances[uid]
    instance.load_url(url)

@_cef_call
def evaluate_js(code, uid):
    instance = instances[uid]
    return instance.evaluate_js(code)

@_cef_call
def get_current_url(uid):
    instance = instances[uid]
    url = instance.get_current_url()

    if url.startswith('data:text/html,'):
        return None
    else:
        return url

@_cef_call
def resize(width, height, uid):
    hwnd = instances[uid].handle
    lparam = width << 16 & height
    cef.WindowUtils.OnSize(hwnd, 5, 0, lparam)

@_cef_call
def close_window(uid):
    instance = instances[uid]
    instance.close()
    del instances[uid]

def shutdown():
    try:
        if os.path.exists('blob_storage'):
            shutil.rmtree('blob_storage')

        if os.path.exists('webrtc_event_logs'):
            shutil.rmtree('webrtc_event_logs')

        if os.path.exists('error.log'):
            shutil.rmtree('error.log')

    except Exception as e:
        logger.debug(e, exc_info=True)

    cef.Shutdown()

_initialized = False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Weckan -- Web application using CKAN model
# By: Axel Haustant <axel@haustant.fr>
#
# Copyright (C) 2013 Axel Haustant
# http://github.com/etalab/weckan
#
# This file is part of Weckan.
#
# Weckan is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Weckan is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from ckan.model import User

def get_user_from_request(request):
    '''Simple user fetching from request'''
    oauth2 = request.cookies.get('oauth2', None)

    if not oauth2:
        return None

    end = oauth2.index('!') if '!' in oauth2 else len(oauth2)
    username = oauth2[40:end].decode('utf8')
    return User.by_name(username)
<commit_msg>Fetch User from repoze.who<commit_after># -*- coding: utf-8 -*-

# Weckan -- Web application using CKAN model
# By: Axel Haustant <axel@haustant.fr>
#
# Copyright (C) 2013 Axel Haustant
# http://github.com/etalab/weckan
#
# This file is part of Weckan.
#
# Weckan is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Weckan is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from ckan.model import User

def get_user_from_request(request):
    '''Simple user fetching from request'''
    username = request.environ.get('repoze.who.identity', {}).get('repoze.who.userid')
    return User.get(username) if username else None
<|endoftext|>


<commit_before>"""
Functions and classes that implement different learning rules.

$Id$
"""
__version__ = "$Revision$"

import weave

def hebbian_div_norm_c(input_activity, self_activity, rows, cols, len, cfs, alpha):
    """
    An inline C implementation of Hebbian learning with divisive normalization
    for the whole sheet.
    """

    hebbian_div_norm_code = """
        float *wi, *wj;
        double *x, *inpi, *inpj;
        int *slice;
        int rr1, rr2, cc1, cc2, rc;
        int i, j, r, l;
        PyObject *cf, *cfsr;
        PyObject *sarray = PyString_FromString("slice_array");
        PyObject *weights = PyString_FromString("weights");
        double load, delta;
        double totald;

        x = self_activity;
        for (r=0; r<rows; ++r) {
            cfsr = PyList_GetItem(cfs,r);
            for (l=0; l<cols; ++l) {
                load = *x++;
                if (load != 0) {
                    load *= alpha;

                    cf = PyList_GetItem(cfsr,l);
                    wi = (float *)(((PyArrayObject*)PyObject_GetAttr(cf,weights))->data);
                    wj = wi;
                    slice = (int *)(((PyArrayObject*)PyObject_GetAttr(cf,sarray))->data);
                    rr1 = *slice++;
                    rr2 = *slice++;
                    cc1 = *slice++;
                    cc2 = *slice;

                    totald = 0.0;

                    inpj = input_activity+len*rr1+cc1;
                    for (i=rr1; i<rr2; ++i) {
                        inpi = inpj;
                        for (j=cc1; j<cc2; ++j) {
                            delta = load * *inpi;
                            *wi += delta;
                            totald += delta;
                            ++wi;
                            ++inpi;
                        }
                        inpj += len;
                    }

                    // normalize the weights
                    totald += 1.0;
                    totald = 1.0/totald;
                    rc = (rr2-rr1)*(cc2-cc1);

                    for (i=0; i<rc; ++i) {
                        *wj *= totald;
                        ++wj;
                    }
                }
            }
        }
    """

    weave.inline(hebbian_div_norm_code, ['input_activity', 'self_activity', 'rows', 'cols', 'len', 'cfs', 'alpha'])

def hebbian_c(input_activity, self_activity, rows, cols, len, cfs, alpha):
    """
    An inline C implementation of Hebbian learning for the whole sheet.
    """

    hebbian_code = """
        float *wi, *wj;
        double *x, *inpi, *inpj;
        int *slice;
        int rr1, rr2, cc1, cc2, rc;
        int i, j, r, l;
        PyObject *cf, *cfsr;
        PyObject *sarray = PyString_FromString("slice_array");
        PyObject *weights = PyString_FromString("weights");
        double load, delta;
        double totald;

        x = self_activity;
        for (r=0; r<rows; ++r) {
            cfsr = PyList_GetItem(cfs,r);
            for (l=0; l<cols; ++l) {
                load = *x++;
                if (load != 0) {
                    load *= alpha;

                    cf = PyList_GetItem(cfsr,l);
                    wi = (float *)(((PyArrayObject*)PyObject_GetAttr(cf,weights))->data);
                    wj = wi;
                    slice = (int *)(((PyArrayObject*)PyObject_GetAttr(cf,sarray))->data);
                    rr1 = *slice++;
                    rr2 = *slice++;
                    cc1 = *slice++;
                    cc2 = *slice;

                    totald = 0.0;

                    inpj = input_activity+len*rr1+cc1;
                    for (i=rr1; i<rr2; ++i) {
                        inpi = inpj;
                        for (j=cc1; j<cc2; ++j) {
                            delta = load * *inpi;
                            *wi += delta;
                            totald += delta;
                            ++wi;
                            ++inpi;
                        }
                        inpj += len;
                    }
                }
            }
        }
    """

    weave.inline(hebbian_code, ['input_activity', 'self_activity', 'rows', 'cols', 'len', 'cfs', 'alpha'])

def hebbian(input_activity, unit_activity, weights, alpha):
    """
    Hebbian learning for one single unit.
    """
    weights += alpha * unit_activity * input_activity

################################################################
# Normalization methods

def divisive_normalization(weights):
    """
    Normalize the numpy array weights to normalize_total by divisive
    normalization.
    """

    s = sum(weights.flat)
    if s != 0:
        factor = 1.0/s
        weights *= factor

################################################################
# Wrapper functions of the learning and normalization methods

def apply_learn_norm_fn(proj, inp, act, learn_fn, norm_fn):
    """
    Apply the specified learn_fn and then norm_fn to the connections fields
    in proj using input inp and output activity act.
    """

    rows,cols = act.shape
    alpha = proj.learning_rate

    if learn_fn.func_name == "hebbian_c":
        # hebbian_c is an function for the whole sheet
        cfs = proj.cfs
        len, len2 = inp.shape
        if proj.normalize:
            # Hebbian learning with divisive normalization per projection.
            # It is much faster to do both of them in one function.
            if norm_fn.func_name == "divisive_normalization":
                hebbian_div_norm_c(inp, act, rows, cols, len, cfs, alpha)
            else:
                hebbian_c(inp, act, rows, cols, len, cfs, alpha)
                norm = proj.normalize
                for r in range(rows):
                    for c in range(cols):
                        norm_fn(cfs[r][c].weights)
        else:
            hebbian_c(inp, act, rows, cols, len, cfs, alpha)
    else:
        # apply learning rule and normalization to each unit
        norm = proj.normalize
        for r in range(rows):
            for c in range(cols):
                cf = proj.cf(r,c)
                learn_fn(cf.get_input_matrix(inp), act[r,c], cf.weights, alpha)
                if norm:
                    norm_fn(cf.weights)

<commit_msg>Minor fix to comment<commit_after>"""
Functions and classes that implement different learning rules.

$Id$
"""
__version__ = "$Revision$"

import weave

def hebbian_div_norm_c(input_activity, self_activity, rows, cols, len, cfs, alpha):
    """
    An inline C implementation of Hebbian learning with divisive normalization
    for the whole sheet.
    """

    hebbian_div_norm_code = """
        float *wi, *wj;
        double *x, *inpi, *inpj;
        int *slice;
        int rr1, rr2, cc1, cc2, rc;
        int i, j, r, l;
        PyObject *cf, *cfsr;
        PyObject *sarray = PyString_FromString("slice_array");
        PyObject *weights = PyString_FromString("weights");
        double load, delta;
        double totald;

        x = self_activity;
        for (r=0; r<rows; ++r) {
            cfsr = PyList_GetItem(cfs,r);
            for (l=0; l<cols; ++l) {
                load = *x++;
                if (load != 0) {
                    load *= alpha;

                    cf = PyList_GetItem(cfsr,l);
                    wi = (float *)(((PyArrayObject*)PyObject_GetAttr(cf,weights))->data);
                    wj = wi;
                    slice = (int *)(((PyArrayObject*)PyObject_GetAttr(cf,sarray))->data);
                    rr1 = *slice++;
                    rr2 = *slice++;
                    cc1 = *slice++;
                    cc2 = *slice;

                    totald = 0.0;

                    inpj = input_activity+len*rr1+cc1;
                    for (i=rr1; i<rr2; ++i) {
                        inpi = inpj;
                        for (j=cc1; j<cc2; ++j) {
                            delta = load * *inpi;
                            *wi += delta;
                            totald += delta;
                            ++wi;
                            ++inpi;
                        }
                        inpj += len;
                    }

                    // normalize the weights
                    totald += 1.0;
                    totald = 1.0/totald;
                    rc = (rr2-rr1)*(cc2-cc1);

                    for (i=0; i<rc; ++i) {
                        *wj *= totald;
                        ++wj;
                    }
                }
            }
        }
    """

    weave.inline(hebbian_div_norm_code, ['input_activity', 'self_activity', 'rows', 'cols', 'len', 'cfs', 'alpha'])

def hebbian_c(input_activity, self_activity, rows, cols, len, cfs, alpha):
    """
    An inline C implementation of Hebbian learning for the whole sheet.
    """

    hebbian_code = """
        float *wi, *wj;
        double *x, *inpi, *inpj;
        int *slice;
        int rr1, rr2, cc1, cc2, rc;
        int i, j, r, l;
        PyObject *cf, *cfsr;
        PyObject *sarray = PyString_FromString("slice_array");
        PyObject *weights = PyString_FromString("weights");
        double load, delta;
        double totald;

        x = self_activity;
        for (r=0; r<rows; ++r) {
            cfsr = PyList_GetItem(cfs,r);
            for (l=0; l<cols; ++l) {
                load = *x++;
                if (load != 0) {
                    load *= alpha;

                    cf = PyList_GetItem(cfsr,l);
                    wi = (float *)(((PyArrayObject*)PyObject_GetAttr(cf,weights))->data);
                    wj = wi;
                    slice = (int *)(((PyArrayObject*)PyObject_GetAttr(cf,sarray))->data);
                    rr1 = *slice++;
                    rr2 = *slice++;
                    cc1 = *slice++;
                    cc2 = *slice;

                    totald = 0.0;

                    inpj = input_activity+len*rr1+cc1;
                    for (i=rr1; i<rr2; ++i) {
                        inpi = inpj;
                        for (j=cc1; j<cc2; ++j) {
                            delta = load * *inpi;
                            *wi += delta;
                            totald += delta;
                            ++wi;
                            ++inpi;
                        }
                        inpj += len;
                    }
                }
            }
        }
    """

    weave.inline(hebbian_code, ['input_activity', 'self_activity', 'rows', 'cols', 'len', 'cfs', 'alpha'])

def hebbian(input_activity, unit_activity, weights, alpha):
    """
    Hebbian learning for one single unit.
    """
    weights += alpha * unit_activity * input_activity

################################################################
# Normalization methods

def divisive_normalization(weights):
    """
    Normalize the numpy array weights to sum to 1.0 by divisive
    normalization.
    """

    s = sum(weights.flat)
    if s != 0:
        factor = 1.0/s
        weights *= factor

################################################################
# Wrapper functions of the learning and normalization methods

def apply_learn_norm_fn(proj, inp, act, learn_fn, norm_fn):
    """
    Apply the specified learn_fn and then norm_fn to the connections fields
    in proj using input inp and output activity act.
    """

    rows,cols = act.shape
    alpha = proj.learning_rate

    if learn_fn.func_name == "hebbian_c":
        # hebbian_c is an function for the whole sheet
        cfs = proj.cfs
        len, len2 = inp.shape
        if proj.normalize:
            # Hebbian learning with divisive normalization per projection.
            # It is much faster to do both of them in one function.
            if norm_fn.func_name == "divisive_normalization":
                hebbian_div_norm_c(inp, act, rows, cols, len, cfs, alpha)
            else:
                hebbian_c(inp, act, rows, cols, len, cfs, alpha)
                norm = proj.normalize
                for r in range(rows):
                    for c in range(cols):
                        norm_fn(cfs[r][c].weights)
        else:
            hebbian_c(inp, act, rows, cols, len, cfs, alpha)
    else:
        # apply learning rule and normalization to each unit
        norm = proj.normalize
        for r in range(rows):
            for c in range(cols):
                cf = proj.cf(r,c)
                learn_fn(cf.get_input_matrix(inp), act[r,c], cf.weights, alpha)
                if norm:
                    norm_fn(cf.weights)

<|endoftext|>


<commit_before>"""
PlotGroupTemplate class, and global repository of such objects.

These classes allow users to specify different types of plots in a way
that is independent of particular models or Sheets.

$Id$
"""
__version__='$Revision$'

from topo.base.parameterizedobject import ParameterizedObject
from topo.misc.keyedlist import KeyedList
from topo.base.parameterclasses import Parameter, BooleanParameter, Filename

class PlotGroupTemplate(ParameterizedObject):
    """
    Class specifying how to construct a PlotGroup from the objects in a Simulation.

    A PlotGroupTemplate is a data structure that specifies how to
    construct a set of related plots, when later given a set of Sheets
    in a Simulation.  The template can be modified however the user
    wishes, allowing the user to control what information is shown in
    plots, how the data is displayed, and so on.  A single template is
    sufficient for any model, because the template does not include
    any information about specific Sheets.  Thus the templates can be
    modified whenever the user desires, but do not *usually* need to
    be modified.
    """

    command = Parameter("pass",
      doc="Command string to run before plotting, if any.")
    template_plot_type=Parameter('bitmap',
      doc="Whether the plots are bitmap images or curves, to determine which GUI components are needed")
    normalize = BooleanParameter(False,
      doc="Default value for the normalize option for the plot")
    image_location = Filename(doc='Paths to search for images to be loaded.')

    def __init__(self, plot_templates=[], static_images = [],**params):
        """
        A PlotGroupTemplate is constructed from a name, an (optional)
        command that the user specifies should be called before using the
        PlotGroupTemplate, and a plot_templates list.  The plot_templates
        list should contain tuples (plot_name, plot_template).  Each
        plot_template is a dictionary of (name, value) pairs, where each
        name specifies a plotting channel (such as Hue or Confidence), and
        the value is the name of a SheetView (such as Activity or
        OrientationPreference).  Current channels include Strength,
        Hue, and Confidence, but others will be added in the future
        such as Red, Green, Blue, Outline, etc.

        For instance, one could define an Orientation-colored Activity
        plot as::

          pgt.plot_templates['Activity'] =
            PlotGroupTemplate(name='Activity', command='measure_activity()',
                              plot_templates=[('Activity',
                                               {'Strength'   : 'Activity',
                                                'Hue'        : 'OrientationPreference',
                                                'Confidence' : None})])

        This specifies that there will be up to one Plot named Activity in
        the final PlotGroup per Sheet, although there could be no plots at
        all if no Sheet has a SheetView named Activity.  The Plot will be
        colored by OrientationPreference if such a SheetView exists, and
        the value channel will be determined by the SheetView Activity.

        Here's a more complicated example specifying two different plots
        in the same PlotGroup::

          PlotGroupTemplate(name='Orientation Preference', command = 'measure_or_pref()',
                            plot_templates=[('Orientation Preference',
                                             {'Strength'   : None,
                                              'Hue'        : 'OrientationPreference',
                                              'Confidence' : None}),
                                            ('Orientation Selectivity',
                                             {'Strength'   : 'OrientationSelectivity',
                                              'Hue'        : None,
                                              'Confidence' : None})])

        Here the PlotGroup will contain up to two Plots per Sheet,
        depending on which Sheets have OrientationPreference and
        OrientationSelectivity.
        """

        super(PlotGroupTemplate,self).__init__(**params)

	self.plot_templates = KeyedList(plot_templates)
	self.static_images = KeyedList(static_images)

    ### JCALERT! We might eventually write these two functions 'Python-like'
    ### by using keyword argument to specify each channel and then get the dictionnary
    ### of all remaining argument....
    def add_plot(self,name,specification_tuple_list):
	dict={}
	for key,value in specification_tuple_list:
	    dict[key]=value
	self.plot_templates.append((name,dict))

    def add_static_image(self,name,file_path):
        self.image_location = file_path
	self.static_images.append((name,self.image_location))

###############################################################################
# Specific PlotGroupTemplates are stored in this repository,
# to which users can add their own as needed
plotgroup_templates = KeyedList()

###############################################################################
# Sample plots; users can override any of these as necessary
#
# JABALERT: Should eventually remove anything mentioning OrientationPreference
# or OrientationSelectivity from here, and instead set those up in
# measure_or_pref or somewhere like that.  That way, there will be no special
# treatment of any particular input feature.

# JABALERT:
# We should also be able to store a documentation string describing each plot
# (for hovering help text) within each template.
# JC: we should also maybe add the situate option (and auto-refresh?)
### we might want to pass a plotgroup_type to the template
### (see corresponding alert in PlotGroupPanel)

def new_plotgroup_template(name,command,template_plot_type='bitmap',normalize=False):
    pgt = PlotGroupTemplate(name=name,command=command,template_plot_type=template_plot_type,normalize=normalize)
    plotgroup_templates[pgt.name]=pgt
    return pgt

pgt = new_plotgroup_template(name='Activity',command='update_activity()')
pgt.add_plot('Activity',[('Strength','Activity'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Connection Fields',command='update_connectionfields()',normalize=True)
pgt.add_plot('Connection Fields',[('Strength','Weights'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Projection Activity',command='update_projectionactivity()',normalize=True)
pgt.add_plot('ProjectionActivity',[('Strength','ProjectionActivity'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Projection',command='update_projections()',normalize=True)
pgt.add_plot('Projection',[('Strength','Weights'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Orientation Preference',command='measure_or_pref()')
pgt.add_plot('Orientation Preference',[('Hue','OrientationPreference')])
pgt.add_plot('Orientation Preference&Selectivity',[('Hue','OrientationPreference'),
						   ('Confidence','OrientationSelectivity')])
pgt.add_plot('Orientation Selectivity',[('Strength','OrientationSelectivity')])
pgt.add_static_image('Color Key','topo/commands/or_key_white_vert_small.png')

pgt = new_plotgroup_template(name='Position Preference',command='measure_position_pref() ; topographic_grid()',normalize=True)
pgt.add_plot('X Preference',[('Strength','XPreference')])
pgt.add_plot('Y Preference',[('Strength','YPreference')])
pgt.add_plot('Position Preference',[('Red','XPreference'),('Green','YPreference')])

pgt = new_plotgroup_template(name='Center of Gravity',command='measure_cog() ; topographic_grid(xsheet_view_name="XCoG",ysheet_view_name="YCoG")',normalize=True)
pgt.add_plot('X CoG',[('Strength','XCoG')])
pgt.add_plot('Y CoG',[('Strength','YCoG')])
pgt.add_plot('CoG',[('Red','XCoG'),('Green','YCoG')])

pgt = new_plotgroup_template(name='Ocular Preference',command='measure_od_pref()')
pgt.add_plot('Ocular Preference',[('Strength','OcularPreference')])
pgt.add_plot('Ocular Selectivity',[('Strength','OcularSelectivity')])

pgt = new_plotgroup_template(name='PhaseDisparity Preference',command='measure_phasedisparity()')
pgt.add_plot('PhaseDisparity Preference',[('Hue','PhasedisparityPreference')])
pgt.add_plot('PhaseDisparity Selectivity',[('Strength','PhasedisparitySelectivity')])
pgt.add_static_image('Color Key','topo/commands/disp_key_white_vert_small.png')

pgt = new_plotgroup_template(name='Orientation Tuning Fullfield',command='measure_or_tuning_fullfield(); tuning_curve(x_axis="orientation", plot_type=pylab.plot, x_ticks=(0,pi/4,pi/2,3*pi/4,pi),x_labels=("0","$\pi/4$","$\pi/2$","$3\pi/4$","$\pi$"),x_lim=(0,pi), unit="radians")',template_plot_type="curve")

pgt = new_plotgroup_template(name='Contrast Response Fullfield',command='measure_contrast_response_fullfield(); tuning_curve(x_axis="contrast", plot_type=pylab.semilogx,x_ticks=None,x_labels=None,x_lim=None, unit="%")',template_plot_type="curve")

pgt = new_plotgroup_template(name='Orientation Tuning',command='measure_or_tuning(); tuning_curve(x_axis="orientation",plot_type=pylab.plot,x_ticks=(0,pi/4,pi/2,3*pi/4,pi),x_labels=("0","$\pi/4$","$\pi/2$","$3\pi/4$","$\pi$"),x_lim=(0,pi), unit="radians")',template_plot_type="curve")

pgt = new_plotgroup_template(name='Contrast Response',command='measure_contrast_response(); tuning_curve(x_axis="contrast",plot_type=pylab.semilogx,x_ticks=None,x_labels=None,x_lim=None,unit="%")',template_plot_type="curve")

pgt = new_plotgroup_template(name='Size Tuning',command='measure_size_response(); tuning_curve(x_axis="size",plot_type=pylab.plot, x_ticks=None, x_labels=None,x_lim=None, unit="Diameter of stimulus")',template_plot_type="curve")

<commit_msg>added flag for determining wether or not to plot before opening gui window<commit_after>"""
PlotGroupTemplate class, and global repository of such objects.

These classes allow users to specify different types of plots in a way
that is independent of particular models or Sheets.

$Id$
"""
__version__='$Revision$'

from topo.base.parameterizedobject import ParameterizedObject
from topo.misc.keyedlist import KeyedList
from topo.base.parameterclasses import Parameter, BooleanParameter, Filename

class PlotGroupTemplate(ParameterizedObject):
    """
    Class specifying how to construct a PlotGroup from the objects in a Simulation.

    A PlotGroupTemplate is a data structure that specifies how to
    construct a set of related plots, when later given a set of Sheets
    in a Simulation.  The template can be modified however the user
    wishes, allowing the user to control what information is shown in
    plots, how the data is displayed, and so on.  A single template is
    sufficient for any model, because the template does not include
    any information about specific Sheets.  Thus the templates can be
    modified whenever the user desires, but do not *usually* need to
    be modified.
    """

    command = Parameter("pass",
      doc="Command string to run before plotting, if any.")
    template_plot_type=Parameter('bitmap',
      doc="Whether the plots are bitmap images or curves, to determine which GUI components are needed")
    initial_plot=BooleanParameter(True,
      doc="Whether to call the plot command initially, to avoid waiting for long processes before changing the update command")
    normalize = BooleanParameter(False,
      doc="Default value for the normalize option for the plot")
    image_location = Filename(doc='Paths to search for images to be loaded.')

    def __init__(self, plot_templates=[], static_images = [],**params):
        """
        A PlotGroupTemplate is constructed from a name, an (optional)
        command that the user specifies should be called before using the
        PlotGroupTemplate, and a plot_templates list.  The plot_templates
        list should contain tuples (plot_name, plot_template).  Each
        plot_template is a dictionary of (name, value) pairs, where each
        name specifies a plotting channel (such as Hue or Confidence), and
        the value is the name of a SheetView (such as Activity or
        OrientationPreference).  Current channels include Strength,
        Hue, and Confidence, but others will be added in the future
        such as Red, Green, Blue, Outline, etc.

        For instance, one could define an Orientation-colored Activity
        plot as::

          pgt.plot_templates['Activity'] =
            PlotGroupTemplate(name='Activity', command='measure_activity()',
                              plot_templates=[('Activity',
                                               {'Strength'   : 'Activity',
                                                'Hue'        : 'OrientationPreference',
                                                'Confidence' : None})])

        This specifies that there will be up to one Plot named Activity in
        the final PlotGroup per Sheet, although there could be no plots at
        all if no Sheet has a SheetView named Activity.  The Plot will be
        colored by OrientationPreference if such a SheetView exists, and
        the value channel will be determined by the SheetView Activity.

        Here's a more complicated example specifying two different plots
        in the same PlotGroup::

          PlotGroupTemplate(name='Orientation Preference', command = 'measure_or_pref()',
                            plot_templates=[('Orientation Preference',
                                             {'Strength'   : None,
                                              'Hue'        : 'OrientationPreference',
                                              'Confidence' : None}),
                                            ('Orientation Selectivity',
                                             {'Strength'   : 'OrientationSelectivity',
                                              'Hue'        : None,
                                              'Confidence' : None})])

        Here the PlotGroup will contain up to two Plots per Sheet,
        depending on which Sheets have OrientationPreference and
        OrientationSelectivity.
        """

        super(PlotGroupTemplate,self).__init__(**params)

	self.plot_templates = KeyedList(plot_templates)
	self.static_images = KeyedList(static_images)

    ### JCALERT! We might eventually write these two functions 'Python-like'
    ### by using keyword argument to specify each channel and then get the dictionnary
    ### of all remaining argument....
    def add_plot(self,name,specification_tuple_list):
	dict={}
	for key,value in specification_tuple_list:
	    dict[key]=value
	self.plot_templates.append((name,dict))

    def add_static_image(self,name,file_path):
        self.image_location = file_path
	self.static_images.append((name,self.image_location))

###############################################################################
# Specific PlotGroupTemplates are stored in this repository,
# to which users can add their own as needed
plotgroup_templates = KeyedList()

###############################################################################
# Sample plots; users can override any of these as necessary
#
# JABALERT: Should eventually remove anything mentioning OrientationPreference
# or OrientationSelectivity from here, and instead set those up in
# measure_or_pref or somewhere like that.  That way, there will be no special
# treatment of any particular input feature.

# JABALERT:
# We should also be able to store a documentation string describing each plot
# (for hovering help text) within each template.
# JC: we should also maybe add the situate option (and auto-refresh?)
### we might want to pass a plotgroup_type to the template
### (see corresponding alert in PlotGroupPanel)

def new_plotgroup_template(name,command,initial_plot,template_plot_type='bitmap',normalize=False):
    pgt = PlotGroupTemplate(name=name,command=command,initial_plot=initial_plot,template_plot_type=template_plot_type,normalize=normalize)
    plotgroup_templates[pgt.name]=pgt
    return pgt

pgt = new_plotgroup_template(name='Activity',command='update_activity()',initial_plot=True)
pgt.add_plot('Activity',[('Strength','Activity'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Connection Fields',command='update_connectionfields()',initial_plot=True,normalize=True)
pgt.add_plot('Connection Fields',[('Strength','Weights'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Projection Activity',command='update_projectionactivity()',initial_plot=True,normalize=True)
pgt.add_plot('ProjectionActivity',[('Strength','ProjectionActivity'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Projection',command='update_projections()',initial_plot=True, normalize=True)
pgt.add_plot('Projection',[('Strength','Weights'),('Hue','OrientationPreference'),('Confidence','OrientationSelectivity')])

pgt = new_plotgroup_template(name='Orientation Preference',command='measure_or_pref()',initial_plot=False)
pgt.add_plot('Orientation Preference',[('Hue','OrientationPreference')])
pgt.add_plot('Orientation Preference&Selectivity',[('Hue','OrientationPreference'),
						   ('Confidence','OrientationSelectivity')])
pgt.add_plot('Orientation Selectivity',[('Strength','OrientationSelectivity')])
pgt.add_static_image('Color Key','topo/commands/or_key_white_vert_small.png')

pgt = new_plotgroup_template(name='Position Preference',command='measure_position_pref() ; topographic_grid()',initial_plot=False,normalize=True)
pgt.add_plot('X Preference',[('Strength','XPreference')])
pgt.add_plot('Y Preference',[('Strength','YPreference')])
pgt.add_plot('Position Preference',[('Red','XPreference'),('Green','YPreference')])

pgt = new_plotgroup_template(name='Center of Gravity',command='measure_cog() ; topographic_grid(xsheet_view_name="XCoG",ysheet_view_name="YCoG")',initial_plot=False,normalize=True)
pgt.add_plot('X CoG',[('Strength','XCoG')])
pgt.add_plot('Y CoG',[('Strength','YCoG')])
pgt.add_plot('CoG',[('Red','XCoG'),('Green','YCoG')])

pgt = new_plotgroup_template(name='Ocular Preference',command='measure_od_pref()',initial_plot=False)
pgt.add_plot('Ocular Preference',[('Strength','OcularPreference')])
pgt.add_plot('Ocular Selectivity',[('Strength','OcularSelectivity')])

pgt = new_plotgroup_template(name='PhaseDisparity Preference',command='measure_phasedisparity()',initial_plot=False)
pgt.add_plot('PhaseDisparity Preference',[('Hue','PhasedisparityPreference')])
pgt.add_plot('PhaseDisparity Selectivity',[('Strength','PhasedisparitySelectivity')])
pgt.add_static_image('Color Key','topo/commands/disp_key_white_vert_small.png')

pgt = new_plotgroup_template(name='Orientation Tuning Fullfield',command='measure_or_tuning_fullfield(); tuning_curve(x_axis="orientation", plot_type=pylab.plot, x_ticks=(0,pi/4,pi/2,3*pi/4,pi),x_labels=("0","$\pi/4$","$\pi/2$","$3\pi/4$","$\pi$"),x_lim=(0,pi), unit="radians")',initial_plot=False,template_plot_type="curve")

pgt = new_plotgroup_template(name='Contrast Response Fullfield',command='measure_contrast_response_fullfield(); tuning_curve(x_axis="contrast", plot_type=pylab.semilogx,x_ticks=None,x_labels=None,x_lim=None, unit="%")',initial_plot=False,template_plot_type="curve")

pgt = new_plotgroup_template(name='Orientation Tuning',command='measure_or_tuning(); tuning_curve(x_axis="orientation",plot_type=pylab.plot,x_ticks=(0,pi/4,pi/2,3*pi/4,pi),x_labels=("0","$\pi/4$","$\pi/2$","$3\pi/4$","$\pi$"),x_lim=(0,pi), unit="radians")',initial_plot=False,template_plot_type="curve")

pgt = new_plotgroup_template(name='Contrast Response',command='measure_contrast_response(); tuning_curve(x_axis="contrast",plot_type=pylab.semilogx,x_ticks=None,x_labels=None,x_lim=None,unit="%")',initial_plot=False,template_plot_type="curve")

pgt = new_plotgroup_template(name='Size Tuning',command='measure_size_response(); tuning_curve(x_axis="size",plot_type=pylab.plot, x_ticks=None, x_labels=None,x_lim=None, unit="Diameter of stimulus")',initial_plot=False,template_plot_type="curve")

<|endoftext|>


<commit_before>import logging

try:
    import pycurl
except ImportError:
    pass

__version__ = "0.1.0"

_logger = logging.getLogger("tor_async_util.%s" % __name__)

def is_libcurl_compiled_with_async_dns_resolver():
    """If you've configured Tornado to use the async curl_httpclient,
    per this (http://tornado.readthedocs.org/en/latest/httpclient.html)
    warning, you'll want to make sure that libcurl has been compiled
    with async DNS resolver. This function wraps up all the gory details
    for answering this question and returns True if libcurl has been
    compiled with async DNS resolver otherwise returns False.
    """
    try:
        version_info = pycurl.version_info()
        features = version_info[4]
        # to understand CURL_VERSION_ASYNCHDNS see
        # https://github.com/bagder/curl/blob/master/include/curl/curl.h#L2286
        CURL_VERSION_ASYNCHDNS = 1 << 7
        return (features & CURL_VERSION_ASYNCHDNS) == CURL_VERSION_ASYNCHDNS
    except Exception as ex:
        fmt = (
            "Error trying to figure out if libcurl is complied with "
            "async DNS resolver - %s"
        )
        _logger.debug(fmt, ex)
        return False
<commit_msg>bump version number from 0.1.0 to 1.0.0<commit_after>import logging

try:
    import pycurl
except ImportError:
    pass

__version__ = "1.0.0"

_logger = logging.getLogger("tor_async_util.%s" % __name__)

def is_libcurl_compiled_with_async_dns_resolver():
    """If you've configured Tornado to use the async curl_httpclient,
    per this (http://tornado.readthedocs.org/en/latest/httpclient.html)
    warning, you'll want to make sure that libcurl has been compiled
    with async DNS resolver. This function wraps up all the gory details
    for answering this question and returns True if libcurl has been
    compiled with async DNS resolver otherwise returns False.
    """
    try:
        version_info = pycurl.version_info()
        features = version_info[4]
        # to understand CURL_VERSION_ASYNCHDNS see
        # https://github.com/bagder/curl/blob/master/include/curl/curl.h#L2286
        CURL_VERSION_ASYNCHDNS = 1 << 7
        return (features & CURL_VERSION_ASYNCHDNS) == CURL_VERSION_ASYNCHDNS
    except Exception as ex:
        fmt = (
            "Error trying to figure out if libcurl is complied with "
            "async DNS resolver - %s"
        )
        _logger.debug(fmt, ex)
        return False
<|endoftext|>


<commit_before>from __future__ import division

import math
import random

import torch

from .utils import RandomShuffler
from .batch import Batch
from .dataset import Dataset

class Iterator(object):
    """Defines an iterator that loads batches of data from a Dataset.

    Attributes:
        dataset: The Dataset object to load Examples from.
        batch_size: Batch size.
        batch_size_fn: Function of three arguments (new example to add, current
            count of examples in the batch, and current effective batch size)
            that returns the new effective batch size resulting from adding
            that example to a batch. This is useful for dynamic batching, where
            this function would add to the current effective batch size the
            number of tokens in the new example.
        sort_key: A key to use for sorting examples in order to batch together
            examples with similar lengths and minimize padding. The sort_key
            provided to the Iterator constructor overrides the sort_key
            attribute of the Dataset, or defers to it if None.
        train: Whether the iterator represents a train set.
        repeat: Whether to repeat the iterator for multiple epochs.
        shuffle: Whether to shuffle examples between epochs.
        sort: Whether to sort examples according to self.sort_key.
            Note that repeat, shuffle, and sort default to train, train, and
            (not train).
        sort_within_batch: Whether to sort (in descending order according to
            self.sort_key) within each batch. If None, defaults to self.sort.
            If self.sort is True and this is False, the batch is left in the
            original (ascending) sorted order.
        device: Device to create batches on. Use -1 for CPU and None for the
            currently active GPU device.
    """

    def __init__(self, dataset, batch_size, sort_key=None, device=None,
                 batch_size_fn=None, train=True,
                 repeat=None, shuffle=None, sort=None,
                 sort_within_batch=None):
        self.batch_size, self.train, self.dataset = batch_size, train, dataset
        self.batch_size_fn = batch_size_fn
        self.iterations = 0
        self.repeat = train if repeat is None else repeat
        self.shuffle = train if shuffle is None else shuffle
        self.sort = not train if sort is None else sort
        if sort_within_batch is None:
            self.sort_within_batch = self.sort
        else:
            self.sort_within_batch = sort_within_batch
        if sort_key is None:
            self.sort_key = dataset.sort_key
        else:
            self.sort_key = sort_key
        self.device = device
        if not torch.cuda.is_available() and self.device is None:
            self.device = -1

        self.random_shuffler = RandomShuffler()

        # For state loading/saving only
        self._iterations_this_epoch = 0
        self._random_state_this_epoch = None
        self._restored_from_state = False

    @classmethod
    def splits(cls, datasets, batch_sizes=None, **kwargs):
        """Create Iterator objects for multiple splits of a dataset.

        Arguments:
            datasets: Tuple of Dataset objects corresponding to the splits. The
                first such object should be the train set.
            batch_sizes: Tuple of batch sizes to use for the different splits,
                or None to use the same batch_size for all splits.
            Remaining keyword arguments: Passed to the constructor of the
                iterator class being used.
        """
        if batch_sizes is None:
            batch_sizes = [kwargs.pop('batch_size')] * len(datasets)
        ret = []
        for i in range(len(datasets)):
            train = i == 0
            ret.append(cls(
                datasets[i], batch_size=batch_sizes[i], train=train, **kwargs))
        return tuple(ret)

    def data(self):
        """Return the examples in the dataset in order, sorted, or shuffled."""
        if self.sort:
            xs = sorted(self.dataset, key=self.sort_key)
        elif self.shuffle:
            xs = [self.dataset[i] for i in self.random_shuffler(range(len(self.dataset)))]
        else:
            xs = self.dataset
        return xs

    def init_epoch(self):
        """Set up the batch generator for a new epoch."""

        if self._restored_from_state:
            self.random_shuffler.random_state = self._random_state_this_epoch
        else:
            self._random_state_this_epoch = self.random_shuffler.random_state

        self.create_batches()

        if self._restored_from_state:
            self._restored_from_state = False
        else:
            self._iterations_this_epoch = 0

        if not self.repeat:
            self.iterations = 0

    def create_batches(self):
        self.batches = batch(self.data(), self.batch_size, self.batch_size_fn)

    @property
    def epoch(self):
        return self.iterations / len(self)

    def __len__(self):
        if self.batch_size_fn is not None:
            raise NotImplementedError
        return math.ceil(len(self.dataset) / self.batch_size)

    def __iter__(self):
        while True:
            self.init_epoch()
            for idx, minibatch in enumerate(self.batches):
                # fast-forward if loaded from state
                if self._iterations_this_epoch > idx:
                    continue
                self.iterations += 1
                self._iterations_this_epoch += 1
                if self.sort_within_batch:
                    # NOTE: `rnn.pack_padded_sequence` requires that a minibatch
                    # be sorted by decreasing order, which requires reversing
                    # relative to typical sort keys
                    if self.sort:
                        minibatch.reverse()
                    else:
                        minibatch.sort(key=self.sort_key, reverse=True)
                yield Batch(minibatch, self.dataset, self.device,
                            self.train)
            if not self.repeat:
                raise StopIteration

    def state_dict(self):
        return {
            "iterations": self.iterations,
            "iterations_this_epoch": self._iterations_this_epoch,
            "random_state_this_epoch": self._random_state_this_epoch}

    def load_state_dict(self, state_dict):
        self.iterations = state_dict["iterations"]
        self._iterations_this_epoch = state_dict["iterations_this_epoch"]
        self._random_state_this_epoch = state_dict["random_state_this_epoch"]
        self._restored_from_state = True

class BPTTIterator(Iterator):
    """Defines an iterator for language modeling tasks that use BPTT.

    Provides contiguous streams of examples together with targets that are
    one timestep further forward, for language modeling training with
    backpropagation through time (BPTT). Expects a Dataset with a single
    example and a single field called 'text' and produces Batches with text and
    target attributes.

    Attributes:
        dataset: The Dataset object to load Examples from.
        batch_size: Batch size.
        bptt_len: Length of sequences for backpropagation through time.
        sort_key: A key to use for sorting examples in order to batch together
            examples with similar lengths and minimize padding. The sort_key
            provided to the Iterator constructor overrides the sort_key
            attribute of the Dataset, or defers to it if None.
        train: Whether the iterator represents a train set.
        repeat: Whether to repeat the iterator for multiple epochs.
        shuffle: Whether to shuffle examples between epochs.
        sort: Whether to sort examples according to self.sort_key.
            Note that repeat, shuffle, and sort default to train, train, and
            (not train).
        device: Device to create batches on. Use -1 for CPU and None for the
            currently active GPU device.
    """

    def __init__(self, dataset, batch_size, bptt_len, **kwargs):
        self.bptt_len = bptt_len
        super(BPTTIterator, self).__init__(dataset, batch_size, **kwargs)

    def __len__(self):
        return math.ceil((len(self.dataset[0].text) / self.batch_size - 1) /
                         self.bptt_len)

    def __iter__(self):
        text = self.dataset[0].text
        TEXT = self.dataset.fields['text']
        TEXT.eos_token = None
        text = text + ([TEXT.pad_token] * int(math.ceil(len(text) / self.batch_size) *
                                              self.batch_size - len(text)))
        data = TEXT.numericalize(
            [text], device=self.device, train=self.train)
        data = data.view(self.batch_size, -1).t().contiguous()
        dataset = Dataset(examples=self.dataset.examples, fields=[
            ('text', TEXT), ('target', TEXT)])
        while True:
            for i in range(0, len(self) * self.bptt_len, self.bptt_len):
                seq_len = min(self.bptt_len, len(data) - i - 1)
                yield Batch.fromvars(
                    dataset, self.batch_size, train=self.train,
                    text=data[i:i + seq_len],
                    target=data[i + 1:i + 1 + seq_len])
            if not self.repeat:
                raise StopIteration

class BucketIterator(Iterator):
    """Defines an iterator that batches examples of similar lengths together.

    Minimizes amount of padding needed while producing freshly shuffled
    batches for each new epoch. See pool for the bucketing procedure used.
    """

    def create_batches(self):
        if self.sort:
            self.batches = batch(self.data(), self.batch_size,
                                 self.batch_size_fn)
        else:
            self.batches = pool(self.data(), self.batch_size,
                                self.sort_key, self.batch_size_fn,
                                random_shuffler=self.random_shuffler)

def batch(data, batch_size, batch_size_fn=None):
    """Yield elements from data in chunks of batch_size."""
    if batch_size_fn is None:
        def batch_size_fn(new, count, sofar):
            return count
    minibatch, size_so_far = [], 0
    for ex in data:
        minibatch.append(ex)
        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)
        if size_so_far == batch_size:
            yield minibatch
            minibatch, size_so_far = [], 0
        elif size_so_far > batch_size:
            yield minibatch[:-1]
            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)
    if minibatch:
        yield minibatch

def pool(data, batch_size, key, batch_size_fn=lambda new, count, sofar: count,
         random_shuffler=None):
    """Sort within buckets, then batch, then shuffle batches.

    Partitions data into chunks of size 100*batch_size, sorts examples within
    each chunk using sort_key, then batch these examples and shuffle the
    batches.
    """
    if random_shuffler is None:
        random_shuffler = random.shuffle
    for p in batch(data, batch_size * 100, batch_size_fn):
        p_batch = batch(sorted(p, key=key), batch_size, batch_size_fn)
        for b in random_shuffler(list(p_batch)):
            yield b
<commit_msg>avoid deprecated StopIteration usage (#244)<commit_after>from __future__ import division

import math
import random

import torch

from .utils import RandomShuffler
from .batch import Batch
from .dataset import Dataset

class Iterator(object):
    """Defines an iterator that loads batches of data from a Dataset.

    Attributes:
        dataset: The Dataset object to load Examples from.
        batch_size: Batch size.
        batch_size_fn: Function of three arguments (new example to add, current
            count of examples in the batch, and current effective batch size)
            that returns the new effective batch size resulting from adding
            that example to a batch. This is useful for dynamic batching, where
            this function would add to the current effective batch size the
            number of tokens in the new example.
        sort_key: A key to use for sorting examples in order to batch together
            examples with similar lengths and minimize padding. The sort_key
            provided to the Iterator constructor overrides the sort_key
            attribute of the Dataset, or defers to it if None.
        train: Whether the iterator represents a train set.
        repeat: Whether to repeat the iterator for multiple epochs.
        shuffle: Whether to shuffle examples between epochs.
        sort: Whether to sort examples according to self.sort_key.
            Note that repeat, shuffle, and sort default to train, train, and
            (not train).
        sort_within_batch: Whether to sort (in descending order according to
            self.sort_key) within each batch. If None, defaults to self.sort.
            If self.sort is True and this is False, the batch is left in the
            original (ascending) sorted order.
        device: Device to create batches on. Use -1 for CPU and None for the
            currently active GPU device.
    """

    def __init__(self, dataset, batch_size, sort_key=None, device=None,
                 batch_size_fn=None, train=True,
                 repeat=None, shuffle=None, sort=None,
                 sort_within_batch=None):
        self.batch_size, self.train, self.dataset = batch_size, train, dataset
        self.batch_size_fn = batch_size_fn
        self.iterations = 0
        self.repeat = train if repeat is None else repeat
        self.shuffle = train if shuffle is None else shuffle
        self.sort = not train if sort is None else sort
        if sort_within_batch is None:
            self.sort_within_batch = self.sort
        else:
            self.sort_within_batch = sort_within_batch
        if sort_key is None:
            self.sort_key = dataset.sort_key
        else:
            self.sort_key = sort_key
        self.device = device
        if not torch.cuda.is_available() and self.device is None:
            self.device = -1

        self.random_shuffler = RandomShuffler()

        # For state loading/saving only
        self._iterations_this_epoch = 0
        self._random_state_this_epoch = None
        self._restored_from_state = False

    @classmethod
    def splits(cls, datasets, batch_sizes=None, **kwargs):
        """Create Iterator objects for multiple splits of a dataset.

        Arguments:
            datasets: Tuple of Dataset objects corresponding to the splits. The
                first such object should be the train set.
            batch_sizes: Tuple of batch sizes to use for the different splits,
                or None to use the same batch_size for all splits.
            Remaining keyword arguments: Passed to the constructor of the
                iterator class being used.
        """
        if batch_sizes is None:
            batch_sizes = [kwargs.pop('batch_size')] * len(datasets)
        ret = []
        for i in range(len(datasets)):
            train = i == 0
            ret.append(cls(
                datasets[i], batch_size=batch_sizes[i], train=train, **kwargs))
        return tuple(ret)

    def data(self):
        """Return the examples in the dataset in order, sorted, or shuffled."""
        if self.sort:
            xs = sorted(self.dataset, key=self.sort_key)
        elif self.shuffle:
            xs = [self.dataset[i] for i in self.random_shuffler(range(len(self.dataset)))]
        else:
            xs = self.dataset
        return xs

    def init_epoch(self):
        """Set up the batch generator for a new epoch."""

        if self._restored_from_state:
            self.random_shuffler.random_state = self._random_state_this_epoch
        else:
            self._random_state_this_epoch = self.random_shuffler.random_state

        self.create_batches()

        if self._restored_from_state:
            self._restored_from_state = False
        else:
            self._iterations_this_epoch = 0

        if not self.repeat:
            self.iterations = 0

    def create_batches(self):
        self.batches = batch(self.data(), self.batch_size, self.batch_size_fn)

    @property
    def epoch(self):
        return self.iterations / len(self)

    def __len__(self):
        if self.batch_size_fn is not None:
            raise NotImplementedError
        return math.ceil(len(self.dataset) / self.batch_size)

    def __iter__(self):
        while True:
            self.init_epoch()
            for idx, minibatch in enumerate(self.batches):
                # fast-forward if loaded from state
                if self._iterations_this_epoch > idx:
                    continue
                self.iterations += 1
                self._iterations_this_epoch += 1
                if self.sort_within_batch:
                    # NOTE: `rnn.pack_padded_sequence` requires that a minibatch
                    # be sorted by decreasing order, which requires reversing
                    # relative to typical sort keys
                    if self.sort:
                        minibatch.reverse()
                    else:
                        minibatch.sort(key=self.sort_key, reverse=True)
                yield Batch(minibatch, self.dataset, self.device,
                            self.train)
            if not self.repeat:
                return

    def state_dict(self):
        return {
            "iterations": self.iterations,
            "iterations_this_epoch": self._iterations_this_epoch,
            "random_state_this_epoch": self._random_state_this_epoch}

    def load_state_dict(self, state_dict):
        self.iterations = state_dict["iterations"]
        self._iterations_this_epoch = state_dict["iterations_this_epoch"]
        self._random_state_this_epoch = state_dict["random_state_this_epoch"]
        self._restored_from_state = True

class BPTTIterator(Iterator):
    """Defines an iterator for language modeling tasks that use BPTT.

    Provides contiguous streams of examples together with targets that are
    one timestep further forward, for language modeling training with
    backpropagation through time (BPTT). Expects a Dataset with a single
    example and a single field called 'text' and produces Batches with text and
    target attributes.

    Attributes:
        dataset: The Dataset object to load Examples from.
        batch_size: Batch size.
        bptt_len: Length of sequences for backpropagation through time.
        sort_key: A key to use for sorting examples in order to batch together
            examples with similar lengths and minimize padding. The sort_key
            provided to the Iterator constructor overrides the sort_key
            attribute of the Dataset, or defers to it if None.
        train: Whether the iterator represents a train set.
        repeat: Whether to repeat the iterator for multiple epochs.
        shuffle: Whether to shuffle examples between epochs.
        sort: Whether to sort examples according to self.sort_key.
            Note that repeat, shuffle, and sort default to train, train, and
            (not train).
        device: Device to create batches on. Use -1 for CPU and None for the
            currently active GPU device.
    """

    def __init__(self, dataset, batch_size, bptt_len, **kwargs):
        self.bptt_len = bptt_len
        super(BPTTIterator, self).__init__(dataset, batch_size, **kwargs)

    def __len__(self):
        return math.ceil((len(self.dataset[0].text) / self.batch_size - 1) /
                         self.bptt_len)

    def __iter__(self):
        text = self.dataset[0].text
        TEXT = self.dataset.fields['text']
        TEXT.eos_token = None
        text = text + ([TEXT.pad_token] * int(math.ceil(len(text) / self.batch_size) *
                                              self.batch_size - len(text)))
        data = TEXT.numericalize(
            [text], device=self.device, train=self.train)
        data = data.view(self.batch_size, -1).t().contiguous()
        dataset = Dataset(examples=self.dataset.examples, fields=[
            ('text', TEXT), ('target', TEXT)])
        while True:
            for i in range(0, len(self) * self.bptt_len, self.bptt_len):
                seq_len = min(self.bptt_len, len(data) - i - 1)
                yield Batch.fromvars(
                    dataset, self.batch_size, train=self.train,
                    text=data[i:i + seq_len],
                    target=data[i + 1:i + 1 + seq_len])
            if not self.repeat:
                raise StopIteration

class BucketIterator(Iterator):
    """Defines an iterator that batches examples of similar lengths together.

    Minimizes amount of padding needed while producing freshly shuffled
    batches for each new epoch. See pool for the bucketing procedure used.
    """

    def create_batches(self):
        if self.sort:
            self.batches = batch(self.data(), self.batch_size,
                                 self.batch_size_fn)
        else:
            self.batches = pool(self.data(), self.batch_size,
                                self.sort_key, self.batch_size_fn,
                                random_shuffler=self.random_shuffler)

def batch(data, batch_size, batch_size_fn=None):
    """Yield elements from data in chunks of batch_size."""
    if batch_size_fn is None:
        def batch_size_fn(new, count, sofar):
            return count
    minibatch, size_so_far = [], 0
    for ex in data:
        minibatch.append(ex)
        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)
        if size_so_far == batch_size:
            yield minibatch
            minibatch, size_so_far = [], 0
        elif size_so_far > batch_size:
            yield minibatch[:-1]
            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)
    if minibatch:
        yield minibatch

def pool(data, batch_size, key, batch_size_fn=lambda new, count, sofar: count,
         random_shuffler=None):
    """Sort within buckets, then batch, then shuffle batches.

    Partitions data into chunks of size 100*batch_size, sorts examples within
    each chunk using sort_key, then batch these examples and shuffle the
    batches.
    """
    if random_shuffler is None:
        random_shuffler = random.shuffle
    for p in batch(data, batch_size * 100, batch_size_fn):
        p_batch = batch(sorted(p, key=key), batch_size, batch_size_fn)
        for b in random_shuffler(list(p_batch)):
            yield b
<|endoftext|>