<commit_before>#!/usr/bin/env python
import sys, os
import copy
import time
import subprocess
import json
import urllib2
import argparse
import importlib
import inspect
from panoptic.modules.base import PanopticStatPlugin

def submit_stats(stats, url):
    req = urllib2.Request(url)
    req.add_header('Content-Type', 'application/json')
    return urllib2.urlopen(req, serialize(stats))

def collect(plugins, kwargs):
    stats = {}
    for plugin in plugins:
        p = plugin(**kwargs)
        p.sample()
        stats[plugin] = p
    stats['time'] = time.time()
    stats['hostname'] = open('/etc/hostname').read().strip()
    return stats

def serialize(stats):
    json_stats = {}
    for s in stats:
        if inspect.isclass(s):
            json_stats[s.__name__] = stats[s].as_json()
    return json.dumps(json_stats)

def collect_and_submit(stats, kwargs):
    new_stats = collect(loaded_plugins, kwargs)
    sub_stats = copy.deepcopy(new_stats)
    for p in loaded_plugins:
        if p.diffable:
            sub_stats[p] = new_stats[p] - stats[p]
    submit_stats(sub_stats, args.url)
    return new_stats

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('url', metavar='collector_url', help="URL of the collector server.")
    parser.add_argument('plugin', nargs='+')
    parser.add_argument('--interface', metavar='interface', help="Network interface to collect data from.")
    parser.add_argument('--timeout', metavar='timeout', help="Number of seconds to collect data.", default=0)
    parser.add_argument('--sample-rate', metavar='rate', help="Number of seconds between data points.", default=1)
    args = parser.parse_args()

    kwargs = {}
    if args.interface:
        kwargs['interface'] = args.interface

    loaded_plugins = []
    for plugin in args.plugin:
        try:
            m = importlib.import_module('panoptic.modules.%s' % plugin)
            # dynamic module loading
            # also known as "WTF HAX"
            for i in m.__dict__:
                if i == 'PanopticStatPlugin': continue
                if inspect.isclass(m.__dict__[i]):
                    if issubclass(m.__dict__[i], PanopticStatPlugin):
                        loaded_plugins.append(m.__dict__[i])
        except ImportError:
            print "Plugin %s not found." % plugin
            sys.exit(1)

    for p in loaded_plugins:
        if p.needs_root and os.geteuid():
            print "Plugin %s needs root and you are not root." % p.__name__
            sys.exit(1)

    stats = collect(loaded_plugins, kwargs)
    start_time = time.time()
    timeout = int(args.timeout)
    print "Running..."
    while (timeout == 0 or time.time() < start_time + timeout):
        stats = collect_and_submit(stats, kwargs)
        time.sleep(args.sample_rate)
    print "Done."
<commit_msg>Add more important statistics to the client submission<commit_after>#!/usr/bin/env python
import sys, os
import copy
import time
import subprocess
import json
import urllib2
import argparse
import importlib
import inspect
from panoptic.modules.base import PanopticStatPlugin

def submit_stats(stats, url):
    req = urllib2.Request(url)
    req.add_header('Content-Type', 'application/json')
    print "submitting..."
    return urllib2.urlopen(req, serialize(stats))

def collect(plugins, kwargs):
    stats = {}
    for plugin in plugins:
        p = plugin(**kwargs)
        p.sample()
        stats[plugin] = p
    return stats

def serialize(stats):
    json_stats = {}
    json_stats['time'] = time.time()
    json_stats['hostname'] = open('/etc/hostname').read().strip()

    for s in stats:
        if inspect.isclass(s):
            json_stats[s.__name__] = stats[s].stats
    return json.dumps(json_stats)

def collect_and_submit(stats, kwargs):
    new_stats = collect(loaded_plugins, kwargs)
    sub_stats = copy.deepcopy(new_stats)
    for p in loaded_plugins:
        if p.diffable:
            sub_stats[p] = new_stats[p] - stats[p]
    submit_stats(sub_stats, args.url)
    return new_stats

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('url', metavar='collector_url', help="URL of the collector server.")
    parser.add_argument('plugin', nargs='+')
    parser.add_argument('--interface', metavar='interface', help="Network interface to collect data from.")
    parser.add_argument('--timeout', metavar='timeout', help="Number of seconds to collect data.", default=0)
    parser.add_argument('--sample-rate', metavar='rate', help="Number of seconds between data points.", default=1)
    args = parser.parse_args()

    kwargs = {}
    if args.interface:
        kwargs['interface'] = args.interface

    loaded_plugins = []
    for plugin in args.plugin:
        try:
            m = importlib.import_module('panoptic.modules.%s' % plugin)
            # dynamic module loading
            # also known as "WTF HAX"
            for i in m.__dict__:
                if i == 'PanopticStatPlugin': continue
                if inspect.isclass(m.__dict__[i]):
                    if issubclass(m.__dict__[i], PanopticStatPlugin):
                        loaded_plugins.append(m.__dict__[i])
        except ImportError:
            print "Plugin %s not found." % plugin
            sys.exit(1)

    for p in loaded_plugins:
        if p.needs_root and os.geteuid():
            print "Plugin %s needs root and you are not root." % p.__name__
            sys.exit(1)

    stats = collect(loaded_plugins, kwargs)
    start_time = time.time()
    timeout = int(args.timeout)
    print "Running..."
    while (timeout == 0 or time.time() < start_time + timeout):
        stats = collect_and_submit(stats, kwargs)
        time.sleep(args.sample_rate)
    print "Done."
<|endoftext|>


<commit_before># vim: set fileencoding=utf-8 :

import sys, socket, time
from helper import *

socket.setdefaulttimeout(5)

from random import choice
import logging

log = logging.getLogger('main')

try:
    import config
except:
    class config:
        keepalive = True
        dns_reset = 60 * 60 * 8 # we cache dns->ip and after this time we will refresh our cacheentry

GZIP = True
try:
    import StringIO, gzip
except:
    GZIP = False

if 'MSG_WAITALL' in socket.__dict__:
    EASY_RECV = True
else:
    EASY_RECV = False

class http(object):
    conns = {} # this will store all keep-alive connections in form (host, state)
    dns_cache = {} # will translate host to ip ... 'dns_name.org': (ip, timestamp)
    host_page_port_cache = {} # cache for get_host_page_port this just avoids recalculation
    encoding = '' # when the url might have umlauts the encoding will convert it

    def __init__(self, url, log = None):
        cleanUrl = url.replace("\r","").replace("\n","").replace("\t","")
        self.origUrl = cleanUrl
        self.host, self.page, self.port = http.extract_host_page_port(cleanUrl)
        self.request = {}
        self.request['http_version'] = '1.1'
        self.request['method']       = 'GET'
        self.request['header']       = [] # can be set from outside
        self.post = ''
        if GZIP:
            self.request['header'].append('Accept-Encoding: gzip')
        self.log = log
        self.redirection = ''
        self.cookies = [] # list should later be a dict it's just my lazyness :/

    @classmethod
    def extract_host_page_port(cls, url, force = False):
        ''' returns tuple (host, page, port) force will avoid cache '''
        if not force:
            if url in cls.host_page_port_cache:
                return cls.host_page_port_cache[url]
        if url.startswith('http://'):                       # we don't need this
            url = url[7:]
        p  = url.find(':')                                  # port
        br = url.find('/')                                  # example.org:123/abc
        if br == -1:                                        # example.org:123?abc=1
            br = url.find('?')
            if br == -1:                                    # example.org:123
                br = len(url)
        if p != -1 and br > p:                              # br > p cause: example.org/bla=http://muh.org
            port = int(url[p+1:br])
            host = url[:p]
        else:
            port = 80
            host = url[:br]
        page = url[br:]
        if page == '':
            page = '/'
        page = page
        cls.host_page_port_cache[url] = (host, page, port)
        return (host, page, port)

    @classmethod
    def get_ip(cls, host, force = False):
        if not force and host in cls.dns_cache:
            ipList, last_update = cls.dns_cache[host]
            if last_update < time.time() + config.dns_reset:
                return cls.get_ip(host, True)
        else:
            ip, aliasList, ipList = socket.gethostbyname_ex(host)
            #ipList.append(ip)
            cls.dns_cache[host] = (ipList, time.time())
        if len(ipList) > 1:
            return choice(ipList)
        return ipList[0]

    def connect(self, force = False):
        if self.request['http_version'] == '1.1' and config.keepalive:
            self.keepalive = True
        else:
            self.keepalive = False
        if self.keepalive and not force:
            if self.host in http.conns and http.conns[self.host][0] == 'CONN_OPEN':
                return http.conns[self.host][1]
        c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        c.settimeout(10)
        try:
            self.ip = http.get_ip(self.host)
            c.connect((self.ip, self.port))
        except socket.timeout, (txt):
            if self.log:
                self.log.error('error in connect to %s:%d timeout: %s' % (self.host, self.port, txt))
            if self.host in http.conns:
                del http.conns[self.host]
            return None
        except socket.error, (e, txt):
            # socket.gaierror: (-2, 'Name or service not known')
            if self.log:
                self.log.error('error in connect to %s:%d errorcode:%d and %s' % (self.host, self.port, e, txt.decode('utf-8')))
            if self.host in http.conns:
                del http.conns[self.host]
            return None

        if self.keepalive:
            http.conns[self.host] = (c, 'CONN_IN_USE')
        return c

    def open(self, post = ''):
        if post:
            self.post = post

        self.c = self.connect()
        if not self.c:
            return
        header = []
        if self.post:
            self.request['method'] = 'POST'
        header.append('%s %s HTTP/%s' % (self.request['method'], self.page, self.request['http_version']))
        header.append('HOST: %s' % self.host)
        for i in self.request['header']:
            header.append(i)
        if self.post:
            if 'content_type' in self.request:
                header.append('Content-Type: %s' % self.request['content_type'])
            else:
                header.append('Content-Type: application/x-www-form-urlencoded')
            header.append('Content-Length: %d' % len(self.post))
            header.append('\r\n%s' % self.post)
        send = '\r\n'.join(header)
        if self.encoding:
            send = unicode(send).encode(self.encoding)
        if not self.post:
            send += '\r\n\r\n'
        self.c.sendall(send)
        try:
            self.get_head()
        except:
            self.log.error("error when receiving head")

    def recv(self, size = 4096, precision = False):
        ''' a blocking recv function - which should also work on windows and solaris
            this is the lowest level of recv, which i can call from this class '''
        data = ''
        if self.buf:
            data = self.buf[:size]
            self.buf = self.buf[size:]
            size -= len(data)
            if size == 0:
                return data
        call = self.recv_blocking
        data += self.recv_with_reconnect_call(call, size)
        if precision:
            self.buf = data[size:]
            return data[:size]
        return data

    def recv_blocking(self, size = 4096):
        data = ''
        if EASY_RECV:
            data += self.c.recv(size, socket.MSG_WAITALL)
        else:
            while size > 0:
                chunk = self.c.recv(size)
                if chunk == '':
                    break
                data += chunk
                size -= len(chunk)
        return data

    def recv_with_reconnect_call(self, call, arg):
        ''' a wrapper around the socketrecv to allow reconnect on closed sockets '''
        try:
            return call(arg)
        except socket.timeout, (txt):
            if self.log:
                self.log.error('error in connect to %s:%d timeout: %s' % (self.host, self.port, txt))
            if self.host in http.conns:
                del http.conns[self.host]
            return None
        except socket.error, (e, err):
            # error: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgeset
            # gaierror: (-2,eerror: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgesetzt')
            if e == 104:
                self.c = self.connect(True)
                if not self.c:
                    return None
                return call(arg)
            else:
                if self.host in http.conns:
                    del http.conns[self.host] # we have a strange error here, so we just delete this host - cause it will surely produce more errors
                self.log.error('crecv has a problem with %d, %d, %s' % (e, err.eerror[0], err.eerror[1]))
        # return an empty sting in case of error
        return ''

    def get_chunks(self):
        ''' getting chunks - through some strange implementation, i will first recv everything and then just strip off the chunk-informations '''
        # TODO implement it better - currently it is quite slow
        body = self.buf
        # first we download the whole file
        while True:
            if body.endswith('\n0\r\n\r\n'):
                break
            body += self.c.recv(4096)
        body = body[:-5]

        # after that we create a new return string and eliminate all chunk-trash
        x = body.find('\r\n')
        body2 = ''
        while x > 0:
            length = int(body[:x], 16)
            body2 += body[(x + 2):(x + 2 + length)]
            body = body[x + 4 + length:]
            if not body:
                return body2
            x = body.find('\r\n')
        return ''

    def get_head(self):
        ''' just get the answering head - we need at least this, to receive the body (else we won't know if the body is chunked and so on)
        also returns all already gathered pieces of the body '''
        self.buf = None # reset it first (important)
        self.buf = self.recv_with_reconnect_call(self.c.recv, 4096)
        if self.buf is None:
            return None
        x = self.buf.find('\r\n\r\n')
        deadlockStop = 0
        lastData = ""
        while x == -1:
            deadlockStop+=1
            if deadlockStop == 23:
                self.log.error("stopping getHead.. Deadlock")
                return None
            data = self.recv()
            if data == lastData:
                self.log.error("stopping getHead.. receiving always the same")
                self.log.error((self.host, self.page))
                self.log.error(data)
                return None
            lastData = data
            self.buf += data
            x = self.buf.find('\r\n\r\n')
        self.head = header(self.buf[:x+2]) # keep the \r\n at the end, so we can search easier
        self.buf = self.buf[x+4:]
        if self.head.get('set-cookie'):
            self.cookies.append(self.head.get('set-cookie'))
        if self.keepalive:
            if self.head.get('connection') and self.head.get('connection') == 'close':
                if self.host in http.conns:
                    del http.conns[self.host]

        if self.head.status == 301 or self.head.status == 302 or self.head.status == 303: # 302 == found, 303 == see other
            if self.redirection == self.head.get('Location'):
                self.log.error("redirection loop")
            self.redirection = self.head.get('Location')
            if not self.redirection.startswith('http://'):
                self.redirection = 'http://'+self.host+self.redirection
            self.log.info("redirect "+self.origUrl+" -to-> "+self.redirection)
            self.host, self.page, self.port = http.extract_host_page_port(self.redirection)
            self.origUrl = self.redirection[:]
            self.open()

    def finnish(self):
        ''' when a download gets ended, this function will mark the connection as free for future requests '''
        if not self.c:
            return
        if self.keepalive:
            http.conns[self.host] = (self.c, 'CONN_OPEN')
        else:
            self.c.close()

    def get(self):
        if self.head.get('Transfer-Encoding') == 'chunked':
            body = self.get_chunks()
        else:
            length = self.head.get('Content-Length')
            if not length:
                length = 9999999 # very big - to make sure we download everything
                if self.log:
                    self.log.warning('there was no content length in the header')
                    self.log.warning(repr(self.head.plain))
            else:
                length = int(length)
            # if delta > 0: - i think this isn't needed
            body = self.recv(length)

        self.finnish() # close connection or free it for future requests

        if GZIP and self.head.get('Content-Encoding') == 'gzip':
            compressedstream = StringIO.StringIO(body)
            gzipper   = gzip.GzipFile(fileobj = compressedstream)
            body = gzipper.read()
        return body

    def get_redirection(self):
        if self.redirection:
            return self.redirection
        return ''

    def __del__(self):
        # when we delete this object, we can free the connection for future use
        if self.keepalive:
            if self.host in http.conns:
                if http.conns[self.host][1] != 'CONN_OPEN':
                    if self.log:
                        self.log.debug('creating a dirty connection')
                    self.finnish()

class header(object):
    def __init__(self, head):
        self.plain = head
        self.plain_lower = head.lower()
        # HTTP/1.0 200 OK
        # HTTP/1.0 300 Moved Permanently
        self.version = head[5:8]                            # 1.0
        self.status  = int(head[9:12])                      # 200
        x = head.find('\r')
        self.status_str  = head[13:x]                       # OK / permanently moved..
        self.cache = {}
        while True:
            y = head.find(':', x + 3) # + 3 is just a guess about the minlength of keywords
            if y == -1:
                break
            keyword = head[x+2:y].lower()
            x = head.find('\r', y + 3)
            value = head[y+2:x]
            self.cache[keyword] = value

    def get(self, what):
        what = what.lower()
        try:
            return self.cache[what]
        except:
            return None
    def __str__(self):
        return self.plain

if __name__ == '__main__':
    def tick_time(t):
        for i in xrange(0, t):
            time.sleep(3)
            print i

    a = http('http://dtwow.eu')
    a.request['header'].append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.request['header'].append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.request['header'].append('Accept-Language: en-us,en;q=0.5')
    a.request['header'].append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    a.get()
    #print a.head.plain()
    #print a.get()
    a = http('http://dtwow.eu')
    a.request['header'].append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.request['header'].append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.request['header'].append('Accept-Language: en-us,en;q=0.5')
    a.request['header'].append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    a.get()
    a.head.get('bla')
    a.head.get('bl1')
    a.head.get('bl2')
    a.head.get('bl3')
<commit_msg>error resistance in http<commit_after># vim: set fileencoding=utf-8 :

import sys, socket, time
from helper import *

socket.setdefaulttimeout(5)

from random import choice
import logging

log = logging.getLogger('main')

try:
    import config
except:
    class config:
        keepalive = True
        dns_reset = 60 * 60 * 8 # we cache dns->ip and after this time we will refresh our cacheentry

GZIP = True
try:
    import StringIO, gzip
except:
    GZIP = False

if 'MSG_WAITALL' in socket.__dict__:
    EASY_RECV = True
else:
    EASY_RECV = False

class http(object):
    conns = {} # this will store all keep-alive connections in form (host, state)
    dns_cache = {} # will translate host to ip ... 'dns_name.org': (ip, timestamp)
    host_page_port_cache = {} # cache for get_host_page_port this just avoids recalculation
    encoding = '' # when the url might have umlauts the encoding will convert it

    def __init__(self, url, log = None):
        cleanUrl = url.replace("\r","").replace("\n","").replace("\t","")
        self.origUrl = cleanUrl
        self.host, self.page, self.port = http.extract_host_page_port(cleanUrl)
        self.request = {}
        self.request['http_version'] = '1.1'
        self.request['method']       = 'GET'
        self.request['header']       = [] # can be set from outside
        self.post = ''
        if GZIP:
            self.request['header'].append('Accept-Encoding: gzip')
        self.log = log
        self.redirection = ''
        self.cookies = [] # list should later be a dict it's just my lazyness :/

    @classmethod
    def extract_host_page_port(cls, url, force = False):
        ''' returns tuple (host, page, port) force will avoid cache '''
        if not force:
            if url in cls.host_page_port_cache:
                return cls.host_page_port_cache[url]
        if url.startswith('http://'):                       # we don't need this
            url = url[7:]
        p  = url.find(':')                                  # port
        br = url.find('/')                                  # example.org:123/abc
        if br == -1:                                        # example.org:123?abc=1
            br = url.find('?')
            if br == -1:                                    # example.org:123
                br = len(url)
        if p != -1 and br > p:                              # br > p cause: example.org/bla=http://muh.org
            port = int(url[p+1:br])
            host = url[:p]
        else:
            port = 80
            host = url[:br]
        page = url[br:]
        if page == '':
            page = '/'
        page = page
        cls.host_page_port_cache[url] = (host, page, port)
        return (host, page, port)

    @classmethod
    def get_ip(cls, host, force = False):
        if not force and host in cls.dns_cache:
            ipList, last_update = cls.dns_cache[host]
            if last_update < time.time() + config.dns_reset:
                return cls.get_ip(host, True)
        else:
            ip, aliasList, ipList = socket.gethostbyname_ex(host)
            #ipList.append(ip)
            cls.dns_cache[host] = (ipList, time.time())
        if len(ipList) > 1:
            return choice(ipList)
        return ipList[0]

    def connect(self, force = False):
        if self.request['http_version'] == '1.1' and config.keepalive:
            self.keepalive = True
        else:
            self.keepalive = False
        if self.keepalive and not force:
            if self.host in http.conns and http.conns[self.host][0] == 'CONN_OPEN':
                return http.conns[self.host][1]
        c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        c.settimeout(10)
        try:
            self.ip = http.get_ip(self.host)
            c.connect((self.ip, self.port))
        except socket.timeout, (txt):
            if self.log:
                self.log.error('error in connect to %s:%d timeout: %s' % (self.host, self.port, txt))
            if self.host in http.conns:
                del http.conns[self.host]
            return None
        except socket.error, (e, txt):
            # socket.gaierror: (-2, 'Name or service not known')
            if self.log:
                self.log.error('error in connect to %s:%d errorcode:%d and %s' % (self.host, self.port, e, txt.decode('utf-8')))
            if self.host in http.conns:
                del http.conns[self.host]
            return None

        if self.keepalive:
            http.conns[self.host] = (c, 'CONN_IN_USE')
        return c

    def open(self, post = ''):
        if post:
            self.post = post

        self.c = self.connect()
        if not self.c:
            return
        header = []
        if self.post:
            self.request['method'] = 'POST'
        header.append('%s %s HTTP/%s' % (self.request['method'], self.page, self.request['http_version']))
        header.append('HOST: %s' % self.host)
        for i in self.request['header']:
            header.append(i)
        if self.post:
            if 'content_type' in self.request:
                header.append('Content-Type: %s' % self.request['content_type'])
            else:
                header.append('Content-Type: application/x-www-form-urlencoded')
            header.append('Content-Length: %d' % len(self.post))
            header.append('\r\n%s' % self.post)
        send = '\r\n'.join(header)
        if self.encoding:
            send = unicode(send).encode(self.encoding)
        if not self.post:
            send += '\r\n\r\n'
        self.c.sendall(send)
        try:
            self.get_head()
        except:
            self.log.error("error when receiving head")

    def recv(self, size = 4096, precision = False):
        ''' a blocking recv function - which should also work on windows and solaris
            this is the lowest level of recv, which i can call from this class '''
        data = ''
        if self.buf:
            data = self.buf[:size]
            self.buf = self.buf[size:]
            size -= len(data)
            if size == 0:
                return data
        call = self.recv_blocking
        data += self.recv_with_reconnect_call(call, size)
        if precision:
            self.buf = data[size:]
            return data[:size]
        return data

    def recv_blocking(self, size = 4096):
        data = ''
        if EASY_RECV:
            data += self.c.recv(size, socket.MSG_WAITALL)
        else:
            while size > 0:
                chunk = self.c.recv(size)
                if chunk == '':
                    break
                data += chunk
                size -= len(chunk)
        return data

    def recv_with_reconnect_call(self, call, arg):
        ''' a wrapper around the socketrecv to allow reconnect on closed sockets '''
        try:
            return call(arg)
        except socket.timeout, (txt):
            if self.log:
                self.log.error('error in connect to %s:%d timeout: %s' % (self.host, self.port, txt))
            if self.host in http.conns:
                del http.conns[self.host]
            return None
        except socket.error, (e, err):
            # error: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgeset
            # gaierror: (-2,eerror: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgesetzt')
            if e == 104:
                self.c = self.connect(True)
                if not self.c:
                    return None
                return call(arg)
            else:
                if self.host in http.conns:
                    del http.conns[self.host] # we have a strange error here, so we just delete this host - cause it will surely produce more errors
                self.log.error('crecv has a problem with %d, %d, %s' % (e, err.eerror[0], err.eerror[1]))
        # return an empty sting in case of error
        return ''

    def get_chunks(self):
        ''' getting chunks - through some strange implementation, i will first recv everything and then just strip off the chunk-informations '''
        # TODO implement it better - currently it is quite slow
        body = self.buf
        if not body:
            return None
        # first we download the whole file
        while True:
            if body.endswith('\n0\r\n\r\n'):
                break
            body += self.c.recv(4096)
        body = body[:-5]

        # after that we create a new return string and eliminate all chunk-trash
        x = body.find('\r\n')
        body2 = ''
        while x > 0:
            length = int(body[:x], 16)
            body2 += body[(x + 2):(x + 2 + length)]
            body = body[x + 4 + length:]
            if not body:
                return body2
            x = body.find('\r\n')
        return ''

    def get_head(self):
        ''' just get the answering head - we need at least this, to receive the body (else we won't know if the body is chunked and so on)
        also returns all already gathered pieces of the body '''
        self.buf = None # reset it first (important)
        self.buf = self.recv_with_reconnect_call(self.c.recv, 4096)
        if self.buf is None:
            return None
        x = self.buf.find('\r\n\r\n')
        deadlockStop = 0
        lastData = ""
        while x == -1:
            deadlockStop+=1
            if deadlockStop == 23:
                self.log.error("stopping getHead.. Deadlock")
                return None
            data = self.recv()
            if data == lastData:
                self.log.error("stopping getHead.. receiving always the same")
                self.log.error((self.host, self.page))
                self.log.error(data)
                return None
            lastData = data
            self.buf += data
            x = self.buf.find('\r\n\r\n')
        self.head = header(self.buf[:x+2]) # keep the \r\n at the end, so we can search easier
        self.buf = self.buf[x+4:]
        if self.head.get('set-cookie'):
            self.cookies.append(self.head.get('set-cookie'))
        if self.keepalive:
            if self.head.get('connection') and self.head.get('connection') == 'close':
                if self.host in http.conns:
                    del http.conns[self.host]

        if self.head.status == 301 or self.head.status == 302 or self.head.status == 303: # 302 == found, 303 == see other
            if self.redirection == self.head.get('Location'):
                self.log.error("redirection loop")
            self.redirection = self.head.get('Location')
            if not self.redirection.startswith('http://'):
                self.redirection = 'http://'+self.host+self.redirection
            self.log.info("redirect "+self.origUrl+" -to-> "+self.redirection)
            self.host, self.page, self.port = http.extract_host_page_port(self.redirection)
            self.origUrl = self.redirection[:]
            self.open()

    def finnish(self):
        ''' when a download gets ended, this function will mark the connection as free for future requests '''
        if not self.c:
            return
        if self.keepalive:
            http.conns[self.host] = (self.c, 'CONN_OPEN')
        else:
            self.c.close()

    def get(self):
        if self.head.get('Transfer-Encoding') == 'chunked':
            body = self.get_chunks()
        else:
            length = self.head.get('Content-Length')
            if not length:
                length = 9999999 # very big - to make sure we download everything
                if self.log:
                    self.log.warning('there was no content length in the header')
                    self.log.warning(repr(self.head.plain))
            else:
                length = int(length)
            # if delta > 0: - i think this isn't needed
            body = self.recv(length)

        self.finnish() # close connection or free it for future requests

        if GZIP and self.head.get('Content-Encoding') == 'gzip':
            compressedstream = StringIO.StringIO(body)
            gzipper   = gzip.GzipFile(fileobj = compressedstream)
            body = gzipper.read()
        return body

    def get_redirection(self):
        if self.redirection:
            return self.redirection
        return ''

    def __del__(self):
        # when we delete this object, we can free the connection for future use
        if self.keepalive:
            if self.host in http.conns:
                if http.conns[self.host][1] != 'CONN_OPEN':
                    if self.log:
                        self.log.debug('creating a dirty connection')
                    self.finnish()

class header(object):
    def __init__(self, head):
        self.plain = head
        self.plain_lower = head.lower()
        # HTTP/1.0 200 OK
        # HTTP/1.0 300 Moved Permanently
        self.version = head[5:8]                            # 1.0
        self.status  = int(head[9:12])                      # 200
        x = head.find('\r')
        self.status_str  = head[13:x]                       # OK / permanently moved..
        self.cache = {}
        while True:
            y = head.find(':', x + 3) # + 3 is just a guess about the minlength of keywords
            if y == -1:
                break
            keyword = head[x+2:y].lower()
            x = head.find('\r', y + 3)
            value = head[y+2:x]
            self.cache[keyword] = value

    def get(self, what):
        what = what.lower()
        try:
            return self.cache[what]
        except:
            return None
    def __str__(self):
        return self.plain

if __name__ == '__main__':
    def tick_time(t):
        for i in xrange(0, t):
            time.sleep(3)
            print i

    a = http('http://dtwow.eu')
    a.request['header'].append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.request['header'].append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.request['header'].append('Accept-Language: en-us,en;q=0.5')
    a.request['header'].append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    a.get()
    #print a.head.plain()
    #print a.get()
    a = http('http://dtwow.eu')
    a.request['header'].append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.request['header'].append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.request['header'].append('Accept-Language: en-us,en;q=0.5')
    a.request['header'].append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    a.get()
    a.head.get('bla')
    a.head.get('bl1')
    a.head.get('bl2')
    a.head.get('bl3')
<|endoftext|>


<commit_before>#!/usr/bin/python -OO

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.    If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage MCPrpcCLI
# @author Joseph Perry <joseph@artefactual.com>

import gearman
admin = gearman.admin_client.GearmanAdminClient(host_list=["127.0.0.1"])
for client in admin. get_workers():
    if client["client_id"] != "-": #exclude server task connections
        print client["client_id"], client["ip"]

for stat in admin.get_status():
    if stat["running"] != 0 or stat["queued"] != 0:
        print stat
<commit_msg>Related to testing for gearman admin connection.<commit_after>#!/usr/bin/python -OO

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.    If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage MCPrpcCLI
# @author Joseph Perry <joseph@artefactual.com>

import gearman
admin = gearman.admin_client.GearmanAdminClient(host_list=["127.0.0.1"])
#print admin.get_status()
#print admin.get_workers()

for client in admin.get_workers():
    if client["client_id"] != "-": #exclude server task connections
        print client["client_id"], client["ip"]

for stat in admin.get_status():
    if stat["running"] != 0 or stat["queued"] != 0:
        print stat
<|endoftext|>


<commit_before>import requests
import datetime

BASE_URL = "https://www.stanza.co/api/schedules/almanacacademiccalendar/"

class Calendar(object):
    def __init__(self):
        pass

    def pull_3year(self):
        """Returns a list (in JSON format) containing all the events from the Penn iCal Calendar.

        List contains events in chronological order.

        Each element of the list is a dictionary, containing:
            - Name of the event 'name'
            - Start date 'start'
            - End date 'end'
        """
        events = []
        for term in ["fall", "summer", "spring"]:
            url = "{}{}{}term.ics".format(BASE_URL, datetime.datetime.now().year, term)
            resp = requests.get(url)
            resp.raise_for_status()
            r = resp.text
            lines = r.split("\n")
            d = {}
            for line in lines:
                if line == "BEGIN:VEVENT":
                    d = {}
                elif line.startswith("DTSTART"):
                    raw_date = line.split(":")[1]
                    start_date = datetime.datetime.strptime(raw_date, '%Y%m%d').date()
                    d['start'] = start_date.strftime('%Y-%m-%d')
                elif line.startswith("DTEND"):
                    raw_date = line.split(":")[1]
                    end_date = datetime.datetime.strptime(raw_date, '%Y%m%d').date()
                    d['end'] = end_date.strftime('%Y-%m-%d')
                elif line.startswith("SUMMARY"):
                    name = line.split(":")[1]
                    d['name'] = str(name).strip()
                elif line == "END:VEVENT":
                    events.append(d)

        events.sort(key=lambda d: d['start'])
        return events
<commit_msg>Fixed calendar crash when rawdate includes timezone by stripping timezone<commit_after>import requests
import datetime

BASE_URL = "https://www.stanza.co/api/schedules/almanacacademiccalendar/"

class Calendar(object):
    def __init__(self):
        pass

    def pull_3year(self):
        """Returns a list (in JSON format) containing all the events from the Penn iCal Calendar.

        List contains events in chronological order.

        Each element of the list is a dictionary, containing:
            - Name of the event 'name'
            - Start date 'start'
            - End date 'end'
        """
        events = []
        for term in ["fall", "summer", "spring"]:
            url = "{}{}{}term.ics".format(BASE_URL, datetime.datetime.now().year, term)
            resp = requests.get(url)
            resp.raise_for_status()
            r = resp.text
            lines = r.split("\n")
            d = {}
            for line in lines:
                if line == "BEGIN:VEVENT":
                    d = {}
                elif line.startswith("DTSTART"):
                    raw_date = line.split(":")[1][0:8]
                    start_date = datetime.datetime.strptime(raw_date, '%Y%m%d').date()
                    d['start'] = start_date.strftime('%Y-%m-%d')
                elif line.startswith("DTEND"):
                    raw_date = line.split(":")[1][0:8]
                    end_date = datetime.datetime.strptime(raw_date, '%Y%m%d').date()
                    d['end'] = end_date.strftime('%Y-%m-%d')
                elif line.startswith("SUMMARY"):
                    name = line.split(":")[1]
                    d['name'] = str(name).strip()
                elif line == "END:VEVENT":
                    events.append(d)

        events.sort(key=lambda d: d['start'])
        return events
<|endoftext|>


<commit_before>import Queue
import json

class Result:
    def __init__(self):
        self.results = {}

    def add_result(self, client, url, result):
        if client not in self.results:
            self.results[client] = {}
        if url not in self.results[client]:
            self.results[client][url] = []
        self.results[client][url].append(result)

    def write(self, result_file):
        stream = open(result_file, 'w')
        stream.write(json.dumps(self.results))
        stream.close()
<commit_msg>Remove unused import<commit_after>import json

class Result:
    def __init__(self):
        self.results = {}

    def add_result(self, client, url, result):
        if client not in self.results:
            self.results[client] = {}
        if url not in self.results[client]:
            self.results[client][url] = []
        self.results[client][url].append(result)

    def write(self, result_file):
        stream = open(result_file, 'w')
        stream.write(json.dumps(self.results))
        stream.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2010, 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
# Copyright 2012 Google, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
:module: nosy
:synopsis: Rewrite of Jeff Winkler's nosy script tailored to testing watchdog
:platform: OS-independent
"""

import os.path
import sys
import stat
import time
import subprocess
from fnmatch import fnmatch

def match_patterns(pathname, patterns):
  """Returns ``True`` if the pathname matches any of the given patterns."""
  for pattern in patterns:
    if fnmatch(pathname, pattern):
      return True
  return False

def filter_paths(pathnames, patterns=None, ignore_patterns=None):
  """Filters from a set of paths based on acceptable patterns and
 ignorable patterns."""
  result = []
  if patterns is None:
    patterns = ['*']
  if ignore_patterns is None:
    ignore_patterns = []
  for pathname in pathnames:
    if match_patterns(pathname, patterns) and not match_patterns(pathname,
                                                                 ignore_patterns):
      result.append(pathname)
  return result

def absolute_walker(pathname, recursive):
  if recursive:
    walk = os.walk
  else:
    def walk(_path):
      try:
        return next(os.walk(_path))
      except NameError:
        return os.walk(_path).next()
  for root, directories, filenames in walk(pathname):
    yield root
    for directory in directories:
      yield os.path.abspath(os.path.join(root, directory))
    for filename in filenames:
      yield os.path.abspath(os.path.join(root, filename))

def glob_recursive(pathname, patterns=None, ignore_patterns=None):
  full_paths = []
  for root, _, filenames in os.walk(pathname):
    for filename in filenames:
      full_path = os.path.abspath(os.path.join(root, filename))
      full_paths.append(full_path)
  filepaths = filter_paths(full_paths, patterns, ignore_patterns)
  return filepaths

def check_sum(pathname='.', patterns=None, ignore_patterns=None):
  checksum = 0
  for f in glob_recursive(pathname, patterns, ignore_patterns):
    stats = os.stat(f)
    checksum += stats[stat.ST_SIZE] + stats[stat.ST_MTIME]
  return checksum

if __name__ == "__main__":
  if len(sys.argv) > 1:
    path = sys.argv[1]
  else:
    path = '.'

  if len(sys.argv) > 2:
    command = sys.argv[2]
  else:
    commands = [
      # Build documentation automatically as well as the source code
      # changes.
      "make SPHINXBUILD=../bin/sphinx-build -C docs html",

      # The reports coverage generates all by itself are more
      # user-friendly than the ones which `nosetests --with-coverage`
      # generates. Therefore, we call `coverage` explicitly to
      # generate reports, and to keep the reports in synchronization
      # with the source code, we erase all coverage information
      # before regenerating reports or running `nosetests`.
      "bin/coverage erase",
      "bin/python-tests ./run_tests.py",
      "bin/coverage html",
      ]
    command = '; '.join(commands)

  previous_checksum = 0
  while True:
    calculated_checksum = check_sum(path,
                                    patterns=['*.py', '*.rst', '*.rst.inc'])
    if calculated_checksum != previous_checksum:
      previous_checksum = calculated_checksum
      subprocess.Popen(command, shell=True)
    time.sleep(2)

<commit_msg>Fixes another copyright line.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2010, 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
# Copyright 2012 Google, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
:module: nosy
:synopsis: Rewrite of Jeff Winkler's nosy script tailored to testing watchdog
:platform: OS-independent
"""

import os.path
import sys
import stat
import time
import subprocess
from fnmatch import fnmatch

def match_patterns(pathname, patterns):
  """Returns ``True`` if the pathname matches any of the given patterns."""
  for pattern in patterns:
    if fnmatch(pathname, pattern):
      return True
  return False

def filter_paths(pathnames, patterns=None, ignore_patterns=None):
  """Filters from a set of paths based on acceptable patterns and
 ignorable patterns."""
  result = []
  if patterns is None:
    patterns = ['*']
  if ignore_patterns is None:
    ignore_patterns = []
  for pathname in pathnames:
    if match_patterns(pathname, patterns) and not match_patterns(pathname,
                                                                 ignore_patterns):
      result.append(pathname)
  return result

def absolute_walker(pathname, recursive):
  if recursive:
    walk = os.walk
  else:
    def walk(_path):
      try:
        return next(os.walk(_path))
      except NameError:
        return os.walk(_path).next()
  for root, directories, filenames in walk(pathname):
    yield root
    for directory in directories:
      yield os.path.abspath(os.path.join(root, directory))
    for filename in filenames:
      yield os.path.abspath(os.path.join(root, filename))

def glob_recursive(pathname, patterns=None, ignore_patterns=None):
  full_paths = []
  for root, _, filenames in os.walk(pathname):
    for filename in filenames:
      full_path = os.path.abspath(os.path.join(root, filename))
      full_paths.append(full_path)
  filepaths = filter_paths(full_paths, patterns, ignore_patterns)
  return filepaths

def check_sum(pathname='.', patterns=None, ignore_patterns=None):
  checksum = 0
  for f in glob_recursive(pathname, patterns, ignore_patterns):
    stats = os.stat(f)
    checksum += stats[stat.ST_SIZE] + stats[stat.ST_MTIME]
  return checksum

if __name__ == "__main__":
  if len(sys.argv) > 1:
    path = sys.argv[1]
  else:
    path = '.'

  if len(sys.argv) > 2:
    command = sys.argv[2]
  else:
    commands = [
      # Build documentation automatically as well as the source code
      # changes.
      "make SPHINXBUILD=../bin/sphinx-build -C docs html",

      # The reports coverage generates all by itself are more
      # user-friendly than the ones which `nosetests --with-coverage`
      # generates. Therefore, we call `coverage` explicitly to
      # generate reports, and to keep the reports in synchronization
      # with the source code, we erase all coverage information
      # before regenerating reports or running `nosetests`.
      "bin/coverage erase",
      "bin/python-tests ./run_tests.py",
      "bin/coverage html",
      ]
    command = '; '.join(commands)

  previous_checksum = 0
  while True:
    calculated_checksum = check_sum(path,
                                    patterns=['*.py', '*.rst', '*.rst.inc'])
    if calculated_checksum != previous_checksum:
      previous_checksum = calculated_checksum
      subprocess.Popen(command, shell=True)
    time.sleep(2)

<|endoftext|>


<commit_before>#! /usr/bin/env python2.7

# Python 2 compatibility
from __future__ import division
from __future__ import print_function

# Standard libraries
import os
import sys
import subprocess
import re
from argparse import ArgumentParser
import multiprocessing
import shutil

# More Python 2 compatibility
if sys.version_info.major == 3:
    from itertools import zip
elif sys.version_info.major == 2:
    from itertools import izip as zip

from itertools import permutations, product, chain
from string import Template

# Related modules
import numpy as np

try:
    import cantera as ct
    from cantera import ck2cti
except ImportError:
    print('Error: Cantera must be installed.')
    raise

# Local imports
import utils
from pyJac import create_jacobian
import partially_stirred_reactor as pasr

# Compiler based on language
cmd_compile = dict(c='gcc',
                   icc='icc',
                   cuda='nvcc'
                   )

# Flags based on language
flags = dict(c=['-std=c99', '-O3', '-mtune=native',
                '-fopenmp'],
             icc=['-std=c99', '-O3', '-xhost', '-fp-model', 'precise', '-ipo'],
             cuda=['-O3', '-arch=sm_20',
                   '-I/usr/local/cuda/include/',
                   '-I/usr/local/cuda/samples/common/inc/',
                   '-dc']
             )

libs = dict(c=['-lm', '-std=c99', '-fopenmp'],
            cuda=['-arch=sm_20'],
            icc=['-m64', '-ipo', '-lm', '-std=c99']
            )

def is_pdep(rxn):
    return (isinstance(rxn, ct.ThreeBodyReaction) or
    isinstance(rxn, ct.FalloffReaction) or
    isinstance(rxn, ct.ChemicallyActivatedReaction))

def check_step_file(filename, steplist):
    #checks file for existing data
    #and returns number of runs left to do
    #for each # of does in steplist
    runs = {}
    for step in steplist:
        runs[step] = 0
    if not 'cuda' in filename:
        raise Exception(filename)

    try:
        with open(filename, 'r') as file:
            lines = [line.strip() for line in file.readlines()]
        for line in lines:
            try:
                vals = line.split(',')
                if len(vals) == 2:
                    vals = [float(v) for v in vals]
                    runs[vals[0]] += 1
            except:
                pass
        return runs
    except:
        return runs

def check_file(filename):
        #checks file for existing data
        #and returns number of runs left to do
        try:
            with open(filename, 'r') as file:
                lines = [line.strip() for line in file.readlines()]
            num_completed = 0
            to_find = 2
            for line in lines:
                try:
                    vals = line.split(',')
                    if len(vals) == to_find:
                        i = int(vals[0])
                        f = float(vals[1])
                        num_completed += 1
                except:
                    pass
            return num_completed
        except:
            return 0

def getf(x):
    return x[x.rindex('jacobs') + len('jacobs') + 1:] if 'jacobs' in x else x

def compiler(fstruct):
    args = [cmd_compile[fstruct.lang]]
    args.extend(flags[fstruct.lang])
    include = ['-I./' + d for d in fstruct.i_dirs]
    args.extend(include)
    args.extend([
        '-I.' + os.path.sep + fstruct.build_dir,
        '-c', os.path.join(fstruct.build_dir, fstruct.filename +
                    utils.file_ext[fstruct.lang]),
        '-o', os.path.join(fstruct.test_dir, getf(fstruct.filename) + '.o')
        ])
    args = [val for val in args if val.strip()]
    try:
        subprocess.check_call(args)
    except subprocess.CalledProcessError:
        print('Error: compilation failed for ' + fstruct.filename +
                utils.file_ext[fstruct.lang])
        return -1
    return 0

def linker(lang, temp_lang, test_dir, filelist):
    args = [cmd_compile[temp_lang]]
    args.extend([os.path.join(test_dir, getf(f) + '.o') for f in filelist])
    args.extend(['-o', os.path.join(test_dir, 'speedtest')])
    args.extend(libs[temp_lang])
    if lang == 'tchem':
        if os.getenv('TCHEM_HOME'):
            tchem_home = os.getenv('TCHEM_HOME')
        else:
            raise SystemError('TCHEM_HOME environment variable not set.')
        args.extend(['-L' + os.path.join(tchem_home, 'lib'), '-ltchem'])

    try:
        subprocess.check_call(args)
    except subprocess.CalledProcessError:
        print('Error: linking of test program failed.')
        sys.exit(1)

class file_struct(object):
    def __init__(self, lang, filename, i_dirs, args, build_dir, test_dir):
        self.lang = lang
        self.filename = filename
        self.i_dirs = i_dirs
        self.args = args
        self.build_dir = build_dir
        self.test_dir = test_dir

def performance_tester():
    pdir = 'performance'
    home = os.getcwd()
    build_dir = 'out'
    test_dir = 'test'

    def get_file_list(pmod, gpu=False, FD=False, tchem=False):
        i_dirs = [build_dir]
        if tchem:
            files = ['test', 'read_initial_conditions',
                        'mechanism', 'mass_mole']
            return i_dirs, files
        files = ['chem_utils', 'dydt', 'spec_rates',
             'rxn_rates', 'test', 'read_initial_conditions',
             'mechanism', 'mass_mole'
             ]
        if pmod:
            files += ['rxn_rates_pres_mod']
        if FD:
            files += ['fd_jacob']
        else:
            files += ['jacob']
            if gpu:
                ext = utils.file_ext['cuda']
                flist = 'jac_list_cuda'
            else:
                ext = utils.file_ext['c']
                flist = 'jac_list_c'
            try:
                with open(os.path.join(build_dir, 'jacobs', flist)) as file:
                    vals = file.readline().strip().split(' ')
                    vals = [os.path.join('jacobs', f[:f.index(ext)]) for f in vals]
                    files += vals
                    i_dirs.append(os.path.join(build_dir, 'jacobs'))
            except:
                pass
        if gpu:
            files += ['gpu_memory']

        return i_dirs, files

    #find the mechanisms to test
    mechanism_list = {}
    for name in os.listdir(pdir):
        if os.path.isdir(os.path.join(pdir, name)):
            #check for cti
            files = [f for f in os.listdir(os.path.join(pdir, name)) if
                        os.path.isfile(os.path.join(pdir, name, f))]
            for f in files:
                if f.endswith('.cti'):
                    mechanism_list[name] = {}
                    mechanism_list[name]['mech'] = f
                    mechanism_list[name]['chemkin'] = f.replace('.cti', '.dat')
                    gas = ct.Solution(os.path.join(pdir, name, f))
                    mechanism_list[name]['ns'] = gas.n_species

                    thermo = next((tf for tf in files if 'therm' in tf), None)
                    if thermo is not None:
                        mechanism_list[name]['thermo'] = thermo

    if os.getenv('TCHEM_HOME'):
        tchem_home = os.getenv('TCHEM_HOME')
    else:
        raise SystemError('TCHEM_HOME environment variable not set.')

    cache_opt_base = [False, True]
    shared_base = [True, False]
    finite_diffs_base = [False, True]

    cpu_repeats = 10
    gpu_repeats = 10

    #langs=['c', 'tchem']
    for mech_name, mech_info in sorted(mechanism_list.items(), key=lambda x:x[1]['ns']):
        #get the cantera object
        gas = ct.Solution(os.path.join(home, pdir, mech_name, mech_info['mech']))
        pmod = any([is_pdep(rxn) for rxn in gas.reactions()])

        #ensure directory structure is valid
        os.chdir(os.path.join(home, pdir, mech_name))
        subprocess.check_call(['mkdir', '-p', build_dir])
        subprocess.check_call(['mkdir', '-p', test_dir])

        #clear old data
        with open(os.path.join('data.bin'), 'wb') as file:
            pass

        npy_files = [f for f in os.listdir(os.path.join(home, pdir, mech_name))
                        if f.endswith('.npy')
                        and os.path.isfile(f)]
        num_conditions = 0
        #load PaSR data for different pressures/conditions, and save to binary c file
        for npy in npy_files:
            state_data = np.load(npy)
            state_data = state_data.reshape(state_data.shape[0] *
                                state_data.shape[1],
                                state_data.shape[2]
                                )
            with open(os.path.join('data.bin'), "ab") as file:
                    state_data.tofile(file)

            num_conditions += state_data.shape[0]
            print(num_conditions)

        #figure out gpu steps
        step_size = 1
        steplist = []
        while step_size < num_conditions:
            steplist.append(step_size)
            step_size *= 2
        if step_size / 2 != num_conditions:
            steplist.append(num_conditions)

        the_path = os.getcwd()
        first_run = True

        for lang in langs:
            temp_lang = 'c' if lang != 'cuda' else 'cuda'
            if lang == 'cuda':
                shared = shared_base
                finite_diffs = finite_diffs_base
                cache_opt = cache_opt_base
            elif lang == 'c':
                shared = [False]
                finite_diffs = finite_diffs_base
                cache_opt = cache_opt_base
            elif lang == 'tchem':
                finite_diffs = [False]
                cache_opt = [False]
                shared = [False]

            for FD in finite_diffs:
                if FD:
                    shutil.copy(os.path.join(home, 'fd_jacob{}'.format(utils.file_ext[temp_lang])),
                                os.path.join(build_dir, 'fd_jacob{}'.format(utils.file_ext[temp_lang])))
                    shared = [False]
                    cache_opt = [False]

                for opt in cache_opt:
                    for smem in shared:

                        data_output = '{}_{}_{}_{}_output.txt'.format(lang, 'co' if opt else 'nco',
                                                                    'smem' if smem else 'nosmem',
                                                                    'fd' if FD else 'ajac')
                        data_output = os.path.join(the_path, data_output)
                        if lang != 'cuda':
                            repeats = cpu_repeats
                            num_completed = check_file(data_output)
                            if num_completed >= repeats:
                                continue
                            todo = {num_conditions: repeats - num_completed}
                        else:
                            repeats = gpu_repeats
                            todo = check_step_file(data_output, steplist)
                            if all(todo[x] >= repeats for x in todo):
                                continue
                            for x in todo:
                                todo[x] = repeats - todo[x]

                        if lang != 'tchem':
                            create_jacobian(lang, mech_info['mech'],
                                            optimize_cache=opt,
                                            build_path=build_dir,
                                            no_shared=not smem,
                                            num_blocks=8, num_threads=64,
                                            force_optimize=first_run
                                            )
                            if opt:
                                first_run = False

                        #now we need to write the reader
                        shutil.copy(os.path.join(home, 'static_files', 'read_initial_conditions{}'.format(utils.file_ext[temp_lang])),
                                    os.path.join(os.getcwd(), build_dir, 'read_initial_conditions{}'.format(utils.file_ext[temp_lang])))

                        #write the tester
                        file_data = {'datafile' : os.path.join(the_path, 'data.bin')}
                        if lang == 'c' or lang == 'cuda':
                            with open(os.path.join(home, 'static_files',
                                                    'tester{}.in'.format(utils.file_ext[temp_lang]))
                                                   , 'r') as file:
                                src = Template(file.read())
                            src = src.substitute(file_data)
                        else:
                            file_data['mechfile'] = mech_info['chemkin']
                            file_data['thermofile'] = mech_info['thermo']
                            with open(os.path.join(home, 'static_files',
                                                   'tc_tester.c.in'), 'r') as file:
                                src = Template(file.read())
                            src = src.substitute(file_data)
                        with open(os.path.join(build_dir,
                                              'test{}'.format(utils.file_ext[temp_lang]))
                                              , 'w') as file:
                            file.write(src)

                        #copy timer
                        shutil.copy(os.path.join(home, 'static_files', 'timer.h'),
                                    os.path.join(os.getcwd(), build_dir, 'timer.h'))

                        #get file lists
                        i_dirs, files = get_file_list(pmod, gpu=lang=='cuda', FD=FD, tchem=lang=='tchem')

                        # Compile generated source code
                        structs = [file_struct(temp_lang, f, flags[temp_lang], i_dirs, build_dir, test_dir) for f in files]

                        pool = multiprocessing.Pool()
                        results = pool.map(compiler, structs)
                        pool.close()
                        pool.join()
                        if any(r == -1 for r in results):
                            sys.exit(-1)

                        linker(lang, temp_lang, test_dir, files)

                        if lang == 'tchem':
                            #copy periodic table and mechanisms in
                            shutil.copy(os.path.join(tchem_home, 'data', 'periodictable.dat'),
                                                    'periodictable.dat')

                        with open(data_output, 'a+') as file:
                            for stepsize in todo:
                                for i in range(todo[stepsize]):
                                    print(i, "/", todo[stepsize])
                                    subprocess.check_call([os.path.join(the_path, test_dir, 'speedtest'),
                                    str(num_conditions)], stdout=file)

if __name__=='__main__':
    # command line arguments
    performance_tester()<commit_msg>update the performance tester to accept the broken out rate routines<commit_after>#! /usr/bin/env python2.7

# Python 2 compatibility
from __future__ import division
from __future__ import print_function

# Standard libraries
import os
import sys
import subprocess
import re
from argparse import ArgumentParser
import multiprocessing
import shutil

# More Python 2 compatibility
if sys.version_info.major == 3:
    from itertools import zip
elif sys.version_info.major == 2:
    from itertools import izip as zip

from itertools import permutations, product, chain
from string import Template

# Related modules
import numpy as np

try:
    import cantera as ct
    from cantera import ck2cti
except ImportError:
    print('Error: Cantera must be installed.')
    raise

# Local imports
import utils
from pyJac import create_jacobian
import partially_stirred_reactor as pasr

# Compiler based on language
cmd_compile = dict(c='gcc',
                   icc='icc',
                   cuda='nvcc'
                   )

# Flags based on language
flags = dict(c=['-std=c99', '-O3', '-mtune=native',
                '-fopenmp'],
             icc=['-std=c99', '-O3', '-xhost', '-fp-model', 'precise', '-ipo'],
             cuda=['-O3', '-arch=sm_20',
                   '-I/usr/local/cuda/include/',
                   '-I/usr/local/cuda/samples/common/inc/',
                   '-dc']
             )

libs = dict(c=['-lm', '-std=c99', '-fopenmp'],
            cuda=['-arch=sm_20'],
            icc=['-m64', '-ipo', '-lm', '-std=c99']
            )

def is_pdep(rxn):
    return (isinstance(rxn, ct.ThreeBodyReaction) or
    isinstance(rxn, ct.FalloffReaction) or
    isinstance(rxn, ct.ChemicallyActivatedReaction))

def check_step_file(filename, steplist):
    #checks file for existing data
    #and returns number of runs left to do
    #for each # of does in steplist
    runs = {}
    for step in steplist:
        runs[step] = 0
    if not 'cuda' in filename:
        raise Exception(filename)

    try:
        with open(filename, 'r') as file:
            lines = [line.strip() for line in file.readlines()]
        for line in lines:
            try:
                vals = line.split(',')
                if len(vals) == 2:
                    vals = [float(v) for v in vals]
                    runs[vals[0]] += 1
            except:
                pass
        return runs
    except:
        return runs

def check_file(filename):
        #checks file for existing data
        #and returns number of runs left to do
        try:
            with open(filename, 'r') as file:
                lines = [line.strip() for line in file.readlines()]
            num_completed = 0
            to_find = 2
            for line in lines:
                try:
                    vals = line.split(',')
                    if len(vals) == to_find:
                        i = int(vals[0])
                        f = float(vals[1])
                        num_completed += 1
                except:
                    pass
            return num_completed
        except:
            return 0

def getf(x):
    return os.path.basename(x)

def compiler(fstruct):
    args = [cmd_compile[fstruct.lang]]
    args.extend(flags[fstruct.lang])
    include = ['-I./' + d for d in fstruct.i_dirs]
    args.extend(include)
    args.extend([
        '-I.' + os.path.sep + fstruct.build_dir,
        '-c', os.path.join(fstruct.build_dir, fstruct.filename +
                    utils.file_ext[fstruct.lang]),
        '-o', os.path.join(fstruct.test_dir, getf(fstruct.filename) + '.o')
        ])
    args = [val for val in args if val.strip()]
    try:
        subprocess.check_call(args)
    except subprocess.CalledProcessError:
        print('Error: compilation failed for ' + fstruct.filename +
                utils.file_ext[fstruct.lang])
        return -1
    return 0

def linker(lang, temp_lang, test_dir, filelist):
    args = [cmd_compile[temp_lang]]
    args.extend([os.path.join(test_dir, getf(f) + '.o') for f in filelist])
    args.extend(['-o', os.path.join(test_dir, 'speedtest')])
    args.extend(libs[temp_lang])
    if lang == 'tchem':
        if os.getenv('TCHEM_HOME'):
            tchem_home = os.getenv('TCHEM_HOME')
        else:
            raise SystemError('TCHEM_HOME environment variable not set.')
        args.extend(['-L' + os.path.join(tchem_home, 'lib'), '-ltchem'])

    try:
        subprocess.check_call(args)
    except subprocess.CalledProcessError:
        print('Error: linking of test program failed.')
        sys.exit(1)

class file_struct(object):
    def __init__(self, lang, filename, i_dirs, args, build_dir, test_dir):
        self.lang = lang
        self.filename = filename
        self.i_dirs = i_dirs
        self.args = args
        self.build_dir = build_dir
        self.test_dir = test_dir

def performance_tester():
    pdir = 'performance'
    home = os.getcwd()
    build_dir = 'out'
    test_dir = 'test'

    def get_file_list(pmod, gpu=False, FD=False, tchem=False):
        i_dirs = [build_dir]
        if tchem:
            files = ['test', 'read_initial_conditions',
                        'mechanism', 'mass_mole']
            return i_dirs, files
        files = ['chem_utils', 'dydt', 'spec_rates',
             'rxn_rates', 'test', 'read_initial_conditions',
             'mechanism', 'mass_mole'
             ]
        if pmod:
            files += ['rxn_rates_pres_mod']
        if FD:
            files += ['fd_jacob']
        else:
            files += ['jacob']
            test_lang = 'c' if not gpu else 'cuda'
            flists = [('rates', 'rate_list_{}'), ('jacobs', 'jac_list_{}')]
            for flist in flists:
                try:
                    with open(os.path.join(build_dir, flist[0], flist[1].format(test_lang))) as file:
                        vals = file.readline().strip().split(' ')
                        vals = [os.path.join(flist[0],
                                    f[:f.index(utils.file_ext[test_lang])]) for f in vals]
                        files += vals
                        i_dirs.append(os.path.join(build_dir, flist[0]))
                except:
                    pass
        if gpu:
            files += ['gpu_memory']

        return i_dirs, files

    #find the mechanisms to test
    mechanism_list = {}
    for name in os.listdir(pdir):
        if os.path.isdir(os.path.join(pdir, name)):
            #check for cti
            files = [f for f in os.listdir(os.path.join(pdir, name)) if
                        os.path.isfile(os.path.join(pdir, name, f))]
            for f in files:
                if f.endswith('.cti'):
                    mechanism_list[name] = {}
                    mechanism_list[name]['mech'] = f
                    mechanism_list[name]['chemkin'] = f.replace('.cti', '.dat')
                    gas = ct.Solution(os.path.join(pdir, name, f))
                    mechanism_list[name]['ns'] = gas.n_species

                    thermo = next((tf for tf in files if 'therm' in tf), None)
                    if thermo is not None:
                        mechanism_list[name]['thermo'] = thermo

    if os.getenv('TCHEM_HOME'):
        tchem_home = os.getenv('TCHEM_HOME')
    else:
        raise SystemError('TCHEM_HOME environment variable not set.')

    cache_opt_base = [False, True]
    shared_base = [True, False]
    finite_diffs_base = [False, True]

    cpu_repeats = 10
    gpu_repeats = 10

    #langs=['c', 'tchem']
    for mech_name, mech_info in sorted(mechanism_list.items(), key=lambda x:x[1]['ns']):
        #get the cantera object
        gas = ct.Solution(os.path.join(home, pdir, mech_name, mech_info['mech']))
        pmod = any([is_pdep(rxn) for rxn in gas.reactions()])

        #ensure directory structure is valid
        os.chdir(os.path.join(home, pdir, mech_name))
        subprocess.check_call(['mkdir', '-p', build_dir])
        subprocess.check_call(['mkdir', '-p', test_dir])

        #clear old data
        with open(os.path.join('data.bin'), 'wb') as file:
            pass

        npy_files = [f for f in os.listdir(os.path.join(home, pdir, mech_name))
                        if f.endswith('.npy')
                        and os.path.isfile(f)]
        num_conditions = 0
        #load PaSR data for different pressures/conditions, and save to binary c file
        for npy in npy_files:
            state_data = np.load(npy)
            state_data = state_data.reshape(state_data.shape[0] *
                                state_data.shape[1],
                                state_data.shape[2]
                                )
            with open(os.path.join('data.bin'), "ab") as file:
                    state_data.tofile(file)

            num_conditions += state_data.shape[0]
            print(num_conditions)

        #figure out gpu steps
        step_size = 1
        steplist = []
        while step_size < num_conditions:
            steplist.append(step_size)
            step_size *= 2
        if step_size / 2 != num_conditions:
            steplist.append(num_conditions)

        the_path = os.getcwd()
        first_run = True

        for lang in langs:
            temp_lang = 'c' if lang != 'cuda' else 'cuda'
            if lang == 'cuda':
                shared = shared_base
                finite_diffs = finite_diffs_base
                cache_opt = cache_opt_base
            elif lang == 'c':
                shared = [False]
                finite_diffs = finite_diffs_base
                cache_opt = cache_opt_base
            elif lang == 'tchem':
                finite_diffs = [False]
                cache_opt = [False]
                shared = [False]

            for FD in finite_diffs:
                if FD:
                    shutil.copy(os.path.join(home, 'fd_jacob{}'.format(utils.file_ext[temp_lang])),
                                os.path.join(build_dir, 'fd_jacob{}'.format(utils.file_ext[temp_lang])))
                    shared = [False]
                    cache_opt = [False]

                for opt in cache_opt:
                    for smem in shared:

                        data_output = '{}_{}_{}_{}_output.txt'.format(lang, 'co' if opt else 'nco',
                                                                    'smem' if smem else 'nosmem',
                                                                    'fd' if FD else 'ajac')
                        data_output = os.path.join(the_path, data_output)
                        if lang != 'cuda':
                            repeats = cpu_repeats
                            num_completed = check_file(data_output)
                            if num_completed >= repeats:
                                continue
                            todo = {num_conditions: repeats - num_completed}
                        else:
                            repeats = gpu_repeats
                            todo = check_step_file(data_output, steplist)
                            if all(todo[x] >= repeats for x in todo):
                                continue
                            for x in todo:
                                todo[x] = repeats - todo[x]

                        if lang != 'tchem':
                            create_jacobian(lang, mech_info['mech'],
                                            optimize_cache=opt,
                                            build_path=build_dir,
                                            no_shared=not smem,
                                            num_blocks=8, num_threads=64,
                                            force_optimize=first_run
                                            )
                            if opt:
                                first_run = False

                        #now we need to write the reader
                        shutil.copy(os.path.join(home, 'static_files', 'read_initial_conditions{}'.format(utils.file_ext[temp_lang])),
                                    os.path.join(os.getcwd(), build_dir, 'read_initial_conditions{}'.format(utils.file_ext[temp_lang])))

                        #write the tester
                        file_data = {'datafile' : os.path.join(the_path, 'data.bin')}
                        if lang == 'c' or lang == 'cuda':
                            with open(os.path.join(home, 'static_files',
                                                    'tester{}.in'.format(utils.file_ext[temp_lang]))
                                                   , 'r') as file:
                                src = Template(file.read())
                            src = src.substitute(file_data)
                        else:
                            file_data['mechfile'] = mech_info['chemkin']
                            file_data['thermofile'] = mech_info['thermo']
                            with open(os.path.join(home, 'static_files',
                                                   'tc_tester.c.in'), 'r') as file:
                                src = Template(file.read())
                            src = src.substitute(file_data)
                        with open(os.path.join(build_dir,
                                              'test{}'.format(utils.file_ext[temp_lang]))
                                              , 'w') as file:
                            file.write(src)

                        #copy timer
                        shutil.copy(os.path.join(home, 'static_files', 'timer.h'),
                                    os.path.join(os.getcwd(), build_dir, 'timer.h'))

                        #get file lists
                        i_dirs, files = get_file_list(pmod, gpu=lang=='cuda', FD=FD, tchem=lang=='tchem')

                        # Compile generated source code
                        structs = [file_struct(temp_lang, f, flags[temp_lang], i_dirs, build_dir, test_dir) for f in files]

                        pool = multiprocessing.Pool()
                        results = pool.map(compiler, structs)
                        pool.close()
                        pool.join()
                        if any(r == -1 for r in results):
                            sys.exit(-1)

                        linker(lang, temp_lang, test_dir, files)

                        if lang == 'tchem':
                            #copy periodic table and mechanisms in
                            shutil.copy(os.path.join(tchem_home, 'data', 'periodictable.dat'),
                                                    'periodictable.dat')

                        with open(data_output, 'a+') as file:
                            for stepsize in todo:
                                for i in range(todo[stepsize]):
                                    print(i, "/", todo[stepsize])
                                    subprocess.check_call([os.path.join(the_path, test_dir, 'speedtest'),
                                    str(num_conditions)], stdout=file)

if __name__=='__main__':
    # command line arguments
    performance_tester()<|endoftext|>


<commit_before>import re

import param
from ipywidgets import SelectMultiple, Button, HBox, VBox, Layout, Text

from .util import named_objs

class CrossSelect(SelectMultiple):
    """
    CrossSelect provides a two-tab multi-selection widget with regex
    text filtering. Items can be transferred with buttons between the
    selected and unselected options.
    """

    def __init__(self, *args, **kwargs):
        # Compute selected and unselected values
        options = kwargs.get('options', {})
        if isinstance(options, list):
            options = named_objs([(opt, opt) for opt in options])
        self._reverse_lookup = {v: k for k, v in options.items()}
        selected = [self._reverse_lookup[v] for v in kwargs.get('value', [])]
        unselected = [k for k in options if k not in selected]

        # Define whitelist and blacklist
        self._lists = {False: SelectMultiple(options=unselected),
                       True: SelectMultiple(options=selected)}

        self._lists[False].observe(self._update_selection, 'value')
        self._lists[True].observe(self._update_selection, 'value')

        # Define buttons
        button_layout = Layout(width='50px')
        self._buttons = {False: Button(description='<<', layout=button_layout),
                         True: Button(description='>>', layout=button_layout)}
        self._buttons[False].on_click(self._apply_selection)
        self._buttons[True].on_click(self._apply_selection)

        # Define search
        self._search = {False: Text(placeholder='Filter available options'),
                        True: Text(placeholder='Filter selected options')}
        self._search[False].observe(self._filter_options, 'value')
        self._search[True].observe(self._filter_options, 'value')

        # Define Layout
        no_margin = Layout(margin='0')
        search_layout = Layout(margin='0', display='flex', justify_content='space-between')
        search_row = HBox([self._search[False], self._search[True]], layout=search_layout)
        button_box = VBox([self._buttons[True], self._buttons[False]],
                          layout=Layout(margin='auto 0'))
        tab_row = HBox([self._lists[False], button_box, self._lists[True]],
                       layout=no_margin)
        self._composite = VBox([search_row, tab_row], layout=no_margin)

        self.observe(self._update_options, 'options')
        self.observe(self._update_value, 'value')

        self._selected = {False: [], True: []}
        self._query = {False: '', True: ''}
        super(CrossSelect, self).__init__(*args, **dict(kwargs, options=options))

    def _update_value(self, event):
        selected = [self._reverse_lookup.get(v, v) for v in event['new']]
        self._lists[True].options = selected
        self._lists[True].value = []
        self._lists[False].options = [o for o in self.options
                                      if o not in selected]

    def _update_options(self, event):
        """
        Updates the options of each of the sublists after the options
        for the whole widget are updated.
        """
        self._reverse_lookup = {v: k for k, v in event['new'].items()}
        options = list(event['new'].keys()) if isinstance(event, dict) else event['new']
        self._selected[False] = []
        self._selected[True] = []
        self._lists[True].options = ['']
        self._lists[True].value = []
        self._lists[False].options = options
        self._lists[False].value = []
        self._apply_filters()

    def _apply_filters(self):
        self._filter_options({'owner': self._search[False]})
        self._filter_options({'owner': self._search[True]})

    def _filter_options(self, event):
        """
        Filters unselected options based on a text query event.
        """
        selected = event['owner'] is self._search[True]
        query = self._query[selected] if 'new' not in event else event['new']
        self._query[selected] = query
        other = self._lists[not selected].options
        options = [o for o in self.options if o not in other]
        if not query:
            self._lists[selected].options = options
            self._lists[selected].value = []
        else:
            try:
                match = re.compile(query)
                matches = filter(match.search, options)
                options = matches + [opt for opt in options if opt not in matches]
            except:
                matches = options
            self._lists[selected].options = options if options else ['']
            self._lists[selected].value = matches

    def _update_selection(self, event):
        """
        Updates the current selection in each list.
        """
        selected = event['owner'] is self._lists[True]
        self._selected[selected] = [v for v in event['new'] if v != '']

    def _apply_selection(self, event):
        """
        Applies the current selection depending on which button was
        pressed.
        """
        selected = event is self._buttons[True]
        new = self._selected[not selected]
        old = self._lists[selected].options
        other = self._lists[not selected].options

        merged = sorted([v for v in list(old) + list(new) if v != ''])
        leftovers = sorted([o for o in other if o not in new and o != ''])
        new_values = merged if selected else leftovers
        self._lists[selected].options = merged if merged else ['']
        self._lists[not selected].options = leftovers if leftovers else ['']
        self.value = [self._options_dict[o] for o in self._lists[True].options if o != '']
        self._apply_filters()

    def _ipython_display_(self, **kwargs):
        """
        Displays the composite widget.
        """
        self._composite._ipython_display_(**kwargs)

    def get_state(self, key=None, drop_defaults=False):
        # HACK: Lets this composite widget pretend to be a regular widget
        # when included into a layout.
        if key in ['value', '_options_labels']:
            return super(CrossSelect, self).get_state(key)
        return self._composite.get_state(key)

<commit_msg>Made CrossSelect widget layout work across ipywidget versions<commit_after>import re

import param
from ipywidgets import SelectMultiple, Button, HBox, VBox, Layout, Text

from .util import named_objs

class CrossSelect(SelectMultiple):
    """
    CrossSelect provides a two-tab multi-selection widget with regex
    text filtering. Items can be transferred with buttons between the
    selected and unselected options.
    """

    def __init__(self, *args, **kwargs):
        # Compute selected and unselected values
        options = kwargs.get('options', {})
        if isinstance(options, list):
            options = named_objs([(opt, opt) for opt in options])
        self._reverse_lookup = {v: k for k, v in options.items()}
        selected = [self._reverse_lookup[v] for v in kwargs.get('value', [])]
        unselected = [k for k in options if k not in selected]

        # Define whitelist and blacklist
        self._lists = {False: SelectMultiple(options=unselected),
                       True: SelectMultiple(options=selected)}

        self._lists[False].observe(self._update_selection, 'value')
        self._lists[True].observe(self._update_selection, 'value')

        # Define buttons
        button_layout = Layout(width='50px')
        self._buttons = {False: Button(description='<<', layout=button_layout),
                         True: Button(description='>>', layout=button_layout)}
        self._buttons[False].on_click(self._apply_selection)
        self._buttons[True].on_click(self._apply_selection)

        # Define search
        self._search = {False: Text(placeholder='Filter available options'),
                        True: Text(placeholder='Filter selected options')}
        self._search[False].observe(self._filter_options, 'value')
        self._search[True].observe(self._filter_options, 'value')

        # Define Layout
        no_margin = Layout(margin='0')
        row_layout = Layout(margin='0', display='flex', justify_content='space-between')

        search_row = HBox([self._search[False], self._search[True]])
        search_row.layout = row_layout
        button_box = VBox([self._buttons[True], self._buttons[False]],
                          layout=Layout(margin='auto 0'))
        tab_row = HBox([self._lists[False], button_box, self._lists[True]])
        tab_row.layout = row_layout
        self._composite = VBox([search_row, tab_row], layout=no_margin)

        self.observe(self._update_options, 'options')
        self.observe(self._update_value, 'value')

        self._selected = {False: [], True: []}
        self._query = {False: '', True: ''}
        super(CrossSelect, self).__init__(*args, **dict(kwargs, options=options))

    def _update_value(self, event):
        selected = [self._reverse_lookup.get(v, v) for v in event['new']]
        self._lists[True].options = selected
        self._lists[True].value = []
        self._lists[False].options = [o for o in self.options
                                      if o not in selected]

    def _update_options(self, event):
        """
        Updates the options of each of the sublists after the options
        for the whole widget are updated.
        """
        self._reverse_lookup = {v: k for k, v in event['new'].items()}
        options = list(event['new'].keys()) if isinstance(event, dict) else event['new']
        self._selected[False] = []
        self._selected[True] = []
        self._lists[True].options = ['']
        self._lists[True].value = []
        self._lists[False].options = options
        self._lists[False].value = []
        self._apply_filters()

    def _apply_filters(self):
        self._filter_options({'owner': self._search[False]})
        self._filter_options({'owner': self._search[True]})

    def _filter_options(self, event):
        """
        Filters unselected options based on a text query event.
        """
        selected = event['owner'] is self._search[True]
        query = self._query[selected] if 'new' not in event else event['new']
        self._query[selected] = query
        other = self._lists[not selected].options
        options = [o for o in self.options if o not in other]
        if not query:
            self._lists[selected].options = options
            self._lists[selected].value = []
        else:
            try:
                match = re.compile(query)
                matches = filter(match.search, options)
                options = matches + [opt for opt in options if opt not in matches]
            except:
                matches = options
            self._lists[selected].options = options if options else ['']
            self._lists[selected].value = matches

    def _update_selection(self, event):
        """
        Updates the current selection in each list.
        """
        selected = event['owner'] is self._lists[True]
        self._selected[selected] = [v for v in event['new'] if v != '']

    def _apply_selection(self, event):
        """
        Applies the current selection depending on which button was
        pressed.
        """
        selected = event is self._buttons[True]
        new = self._selected[not selected]
        old = self._lists[selected].options
        other = self._lists[not selected].options

        merged = sorted([v for v in list(old) + list(new) if v != ''])
        leftovers = sorted([o for o in other if o not in new and o != ''])
        new_values = merged if selected else leftovers
        self._lists[selected].options = merged if merged else ['']
        self._lists[not selected].options = leftovers if leftovers else ['']
        self.value = [self._options_dict[o] for o in self._lists[True].options if o != '']
        self._apply_filters()

    def _ipython_display_(self, **kwargs):
        """
        Displays the composite widget.
        """
        self._composite._ipython_display_(**kwargs)

    def get_state(self, key=None, drop_defaults=False):
        # HACK: Lets this composite widget pretend to be a regular widget
        # when included into a layout.
        if key in ['value', '_options_labels']:
            return super(CrossSelect, self).get_state(key)
        return self._composite.get_state(key)

<|endoftext|>


<commit_before>#===============================================================================
# Copyright (c) 2012 - 2014, GPy authors (see AUTHORS.txt).
# Copyright (c) 2015, Max Zwiessele
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of paramax nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#===============================================================================

from .model import Model
from .parameterized import Parameterized
from .param import Param
from .core.observable_array import ObsAr
from paramz import transformations as constraints
from . import caching, optimization
from . import examples
from cPickle import UnpicklingError

def _unpickle(file_or_path, pickle, strcl, p3kw):
    if isinstance(file_or_path, strcl):
        with open(file_or_path, 'rb') as f:
            m = pickle.load(f, **p3kw)
    else:
        m = pickle.load(file_or_path, **p3kw)
    return m

def load(file_or_path):
    """
    Load a previously pickled model, using `m.pickle('path/to/file.pickle)'`

    :param file_name: path/to/file.pickle
    """
    try:
        import cPickle as pickle
        strcl = basestring
        p3kw = {}
        return _unpickle(file_or_path, pickle, strcl, p3kw)
    except ImportError: #python3
        import pickle
        strcl = str
        p3kw = dict(encoding='latin1')
        return _unpickle(file_or_path, pickle, strcl, p3kw)
    except UnpicklingError:
        import pickle
        return _unpickle(file_or_path, pickle, strcl, p3kw)

    return m<commit_msg>[load] loading of pickle fallback to non c-code<commit_after>#===============================================================================
# Copyright (c) 2012 - 2014, GPy authors (see AUTHORS.txt).
# Copyright (c) 2015, Max Zwiessele
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of paramax nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#===============================================================================

from .model import Model
from .parameterized import Parameterized
from .param import Param
from .core.observable_array import ObsAr
from paramz import transformations as constraints
from . import caching, optimization
from . import examples

def _unpickle(file_or_path, pickle, strcl, p3kw):
    if isinstance(file_or_path, strcl):
        with open(file_or_path, 'rb') as f:
            m = pickle.load(f, **p3kw)
    else:
        m = pickle.load(file_or_path, **p3kw)
    return m

def load(file_or_path):
    """
    Load a previously pickled model, using `m.pickle('path/to/file.pickle)'`

    :param file_name: path/to/file.pickle
    """
    from pickle import UnpicklingError
    try:
        import cPickle as pickle
        strcl = basestring
        p3kw = {}
        return _unpickle(file_or_path, pickle, strcl, p3kw)
    except ImportError: #python3
        import pickle
        strcl = str
        p3kw = dict(encoding='latin1')
        return _unpickle(file_or_path, pickle, strcl, p3kw)
    except UnpicklingError:
        import pickle
        return _unpickle(file_or_path, pickle, strcl, p3kw)

    return m<|endoftext|>