<commit_before>#!/usr/bin/python -t
# Simple attempt at a gui for my ikettle
#
# I'm a perl guy, but they'll laugh at me at work if I wrote this in perl/tk
#
# protocol: http://www.awe.com/mark/blog/20140223.html
#
# things to do:
#    nice interface with shiny buttons and circles
#    sound sample when boiled
#    scan for IP (look for 2000 open, then HELLOAPP stuff)
#    remember last working IP to save above
#    get rid of connect button, just try to connect every so often
#                               or rescan or something else
#
# Mark J Cox 2014

import gtk
import pango
import gobject, socket

class MooseKettle(gtk.Window):

    def __init__(self, ip):
        super(MooseKettle, self).__init__()

        self.kettleconnected = 0
        self.ip = ip
#        self.set_decorated(False)
        self.resize(400,34)
        self.connect("destroy", self.gotofail)

        box = gtk.HBox()
        self.add(box)

        self.bconnect = gtk.Button("connect")
        self.bconnect.connect("clicked", self.clickbconnect)
        box.pack_start(self.bconnect)
        self.bconnect.show()

        self.bboil = gtk.ToggleButton("boil")
        self.bboil.connect("clicked", self.clickboil)
        self.bboil.show()
        box.pack_start(self.bboil)
        self.b100 = gtk.ToggleButton("100")
        self.b100.connect("clicked", self.clicksend, "set sys output 0x80")
        box.pack_start(self.b100)
        self.b100.show()
        self.b95 = gtk.ToggleButton("95")
        self.b95.connect("clicked", self.clicksend, "set sys output 0x2")
        box.pack_start(self.b95)
        self.b95.show()
        self.b80 = gtk.ToggleButton("80")
        self.b80.connect("clicked", self.clicksend, "set sys output 0x4000")
        box.pack_start(self.b80)
        self.b80.show()
        self.b65 = gtk.ToggleButton("65")
        self.b65.connect("clicked", self.clicksend, "set sys output 0x200")
        box.pack_start(self.b65)
        self.b65.show()
        self.bwarm = gtk.ToggleButton("warm")
        self.bwarm.connect("clicked", self.clicksend, "set sys output 0x8")
        box.pack_start(self.bwarm)
        self.bwarm.show()
        self.setbuttons(False) # initially not connected

        button = gtk.Button("quit")
        button.connect("clicked", self.gotofail)
        box.pack_start(button)
        button.show()

        box.show()

        self.show_all()

    def setbuttons(self, status):
        self.bboil.set_sensitive(status)
        self.b100.set_sensitive(status)
        self.b95.set_sensitive(status)
        self.b80.set_sensitive(status)
        self.b65.set_sensitive(status)
        self.bwarm.set_sensitive(status)

    def clickboil(self, widget, data=None):
        if widget.get_active():
            self.kettlesend("set sys output 0x4")
        else:
            self.kettlesend("set sys output 0x0")

    def clicksend(self, widget, data=None):
        widget.handler_block_by_func(self.clicksend)
        if widget.get_active():
            self.kettlesend(data)
            widget.set_active(False)
        else:
            widget.set_active(True)
        widget.handler_unblock_by_func(self.clicksend)

    def kettlesend(self, data):
#        print data
        self.sock.send(data+"\n")

    def clickbconnect(self, widget):
#        print "Connecting...."
        self.kconnect()

    def gotofail(self, widget):
#        print "Closing"
        try:
            self.sock.close()
        except:
            pass
        gtk.main_quit()

    def expose(self, widget, event):
        # redundant function to draw a nice circle for the GUI
        w, h = widget.window.get_size()
        xgc = widget.window.new_gc()

        xgc.set_rgb_fg_color(gtk.gdk.color_parse("green"))
        xgc.line_width = 3
        widget.window.draw_arc(xgc, False, 5, 5, 80, 80, 0*64, 360*64)

        context = self.create_pango_context()
        self.layout = self.create_pango_layout("moose")
        desc = pango.FontDescription('Sans 14')
        self.layout.set_font_description(desc)

        widget.window.draw_layout(xgc, 20, 35, layout=self.layout)

    def kconnect(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((self.ip,2000))
        self.sock.send("HELLOKETTLE\n")
        gobject.io_add_watch(self.sock, gobject.IO_IN, self.handler)

    def setbutton(self, button, status):
        button.handler_block_by_func(self.clicksend)
        button.set_active(status)
        button.handler_unblock_by_func(self.clicksend)

    def handler(self, conn, *args):
        line = conn.recv(4096)
        if not len(line):
#            print "Connection closed."
            self.bconnect.set_sensitive(True)
            self.setbuttons(False) # not connected
            return False
        else:
            for myline in line.splitlines():
#                print "got a line: "+myline
                if (myline.startswith("HELLOAPP")):
                    self.kettleconnected = 1
                    self.setbuttons(True) # connected
                    self.bconnect.set_sensitive(False)
                    conn.send("get sys status\n");
                if (myline.startswith("sys status key=")):
                    if (len(myline)<16):
                        key = 0
                    else:
                        key = ord(myline[15]) & 0x3f
                        if (key == 10):
                            key = 0
                    self.setbutton(self.b100,key&0x20)
                    self.setbutton(self.b95,key&0x10)
                    self.setbutton(self.b80,key&0x8)
                    self.setbutton(self.b65,key&0x4)
                    self.setbutton(self.bwarm,key&0x2)
                    self.bboil.handler_block_by_func(self.clickboil)
                    self.bboil.set_active(key&0x1)
                    self.bboil.handler_unblock_by_func(self.clickboil)
#                    print "key = %d"%key
                if (myline == "sys status 0x100"):
                    self.setbutton(self.b100,True)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b80,False)
                    self.setbutton(self.b65,False)
                elif (myline == "sys status 0x95"):
                    self.setbutton(self.b95,True)
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b80,False)
                    self.setbutton(self.b65,False)
                elif (myline == "sys status 0x80"):
                    self.setbutton(self.b80,True)
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b65,False)
                elif (myline == "sys status 0x65"):
                    self.setbutton(self.b65,True)
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b80,False)
                elif (myline == "sys status 0x11"):
                    self.setbutton(self.bwarm,True)
                elif (myline == "sys status 0x10"):
                    self.setbutton(self.bwarm,False)
                elif (myline == "sys status 0x5"):
                    self.bboil.handler_block_by_func(self.clickboil)
                    self.bboil.set_active(True)
                    self.bboil.handler_unblock_by_func(self.clickboil)
                elif (myline == "sys status 0x0"):
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b80,False)
                    self.setbutton(self.b65,False)
                    self.setbutton(self.bwarm,False)
                    self.bboil.handler_block_by_func(self.clickboil)
                    self.bboil.set_active(False)
                    self.bboil.handler_unblock_by_func(self.clickboil)
                elif (myline == "sys status 0x3"):
                    message = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_INFO,
                                                buttons=gtk.BUTTONS_NONE,message_format=None);

                    message.set_markup("Your Kettle Has Boiled")
                    message.show()
                elif (myline == "sys status 0x1"):
                    message = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_INFO,
                                                buttons=gtk.BUTTONS_NONE,message_format=None);
                    message.set_markup("Your Kettle Was Removed")
                    message.show()

            return True

import argparse
parser = argparse.ArgumentParser(description="Simple GUI interface to the ikettle")
parser.add_argument('-i','--ip',help='IP address for kettle', required=True)
args = vars(parser.parse_args())
MooseKettle(args['ip'])
gtk.main()
<commit_msg>Windows users don't like to use arguments, so if no arguments are given or we can't connect then pop up a box and ask for the IP address.  Or get it from the command line or a local config file. If we connect to a working kettle then create or update the config file with the address we found it at.  We probably should add a SCAN<commit_after>#!/usr/bin/python -t
# Simple attempt at a gui for my ikettle
#
# I'm a perl guy, but they'll laugh at me at work if I wrote this in perl/tk
#
# protocol: http://www.awe.com/mark/blog/20140223.html
#
# things to do:
#    nice interface with shiny buttons and circles
#    sound sample when boiled
#    scan for IP (look for 2000 open, then HELLOAPP stuff)
#    remember last working IP to save above
#    get rid of connect button, just try to connect every so often
#                               or rescan or something else
#
# Mark J Cox 2014

import gtk
import pango
import gobject, socket

cnf = "moosekettle.cfg"

class MooseKettle(gtk.Window):

    def __init__(self, configip, argip):
        super(MooseKettle, self).__init__()

        self.kettleconnected = 0
        self.configip = configip
        self.ip = configip
        if (argip):
            self.ip = argip
#        self.set_decorated(False)
        self.resize(400,34)
        self.connect("destroy", self.gotofail)

        box = gtk.HBox()
        self.add(box)

        self.bconnect = gtk.Button("connect")
        self.bconnect.connect("clicked", self.clickbconnect)
        box.pack_start(self.bconnect)
        self.bconnect.show()

        self.bboil = gtk.ToggleButton("boil")
        self.bboil.connect("clicked", self.clickboil)
        self.bboil.show()
        box.pack_start(self.bboil)
        self.b100 = gtk.ToggleButton("100")
        self.b100.connect("clicked", self.clicksend, "set sys output 0x80")
        box.pack_start(self.b100)
        self.b100.show()
        self.b95 = gtk.ToggleButton("95")
        self.b95.connect("clicked", self.clicksend, "set sys output 0x2")
        box.pack_start(self.b95)
        self.b95.show()
        self.b80 = gtk.ToggleButton("80")
        self.b80.connect("clicked", self.clicksend, "set sys output 0x4000")
        box.pack_start(self.b80)
        self.b80.show()
        self.b65 = gtk.ToggleButton("65")
        self.b65.connect("clicked", self.clicksend, "set sys output 0x200")
        box.pack_start(self.b65)
        self.b65.show()
        self.bwarm = gtk.ToggleButton("warm")
        self.bwarm.connect("clicked", self.clicksend, "set sys output 0x8")
        box.pack_start(self.bwarm)
        self.bwarm.show()
        self.setbuttons(False) # initially not connected

        button = gtk.Button("quit")
        button.connect("clicked", self.gotofail)
        box.pack_start(button)
        button.show()

        box.show()

        self.show_all()
        self.kconnect()

    def setbuttons(self, status):
        self.bboil.set_sensitive(status)
        self.b100.set_sensitive(status)
        self.b95.set_sensitive(status)
        self.b80.set_sensitive(status)
        self.b65.set_sensitive(status)
        self.bwarm.set_sensitive(status)

    def clickboil(self, widget, data=None):
        if widget.get_active():
            self.kettlesend("set sys output 0x4")
        else:
            self.kettlesend("set sys output 0x0")

    def clicksend(self, widget, data=None):
        widget.handler_block_by_func(self.clicksend)
        if widget.get_active():
            self.kettlesend(data)
            widget.set_active(False)
        else:
            widget.set_active(True)
        widget.handler_unblock_by_func(self.clicksend)

    def kettlesend(self, data):
#        print data
        self.sock.send(data+"\n")

    def clickbconnect(self, widget):
#        print "Connecting...."
        self.kconnect()

    def gotofail(self, widget):
#        print "Closing"
        try:
            self.sock.close()
        except:
            pass
        gtk.main_quit()

    def expose(self, widget, event):
        # redundant function to draw a nice circle for the GUI
        w, h = widget.window.get_size()
        xgc = widget.window.new_gc()

        xgc.set_rgb_fg_color(gtk.gdk.color_parse("green"))
        xgc.line_width = 3
        widget.window.draw_arc(xgc, False, 5, 5, 80, 80, 0*64, 360*64)

        context = self.create_pango_context()
        self.layout = self.create_pango_layout("moose")
        desc = pango.FontDescription('Sans 14')
        self.layout.set_font_description(desc)

        widget.window.draw_layout(xgc, 20, 35, layout=self.layout)

    def kconnect(self):
        if (not self.ip):
            self.ip = self.get_ip("Enter IP Address of Kettle","127.0.0.1")
            if (not self.ip):
                return
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            self.sock.connect((self.ip,2000))
            self.sock.send("HELLOKETTLE\n")
            gobject.io_add_watch(self.sock, gobject.IO_IN, self.handler)
            gobject.timeout_add(5000, self.check_connected)
        except:
            message = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_INFO,
                                        buttons=gtk.BUTTONS_NONE,message_format=None);
            message.set_markup("Failed to connect to kettle")
            message.show()
            self.ip=""  # the one given didnt work

    def get_ip(self, text, default=''):
        msgd = gtk.MessageDialog(self, gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT,
                              gtk.MESSAGE_QUESTION, gtk.BUTTONS_OK_CANCEL, text)
        entry = gtk.Entry()
        entry.set_text(default)
        entry.show()
        msgd.vbox.pack_end(entry)
        entry.connect('activate', lambda _: msgd.response(gtk.RESPONSE_OK))
        msgd.set_default_response(gtk.RESPONSE_OK)

        msgr = msgd.run()
        text = entry.get_text().decode('utf8')
        msgd.destroy()
        if (msgr == gtk.RESPONSE_OK):
            return text
        else:
            return None

    def check_connected(self):
        if (self.kettleconnected == 0):
            try:
                self.sock.close()
            except:
                pass
            message = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_INFO,
                                        buttons=gtk.BUTTONS_NONE,message_format=None);
            message.set_markup("Failed to connect to kettle")
            message.show()

    def setbutton(self, button, status):
        button.handler_block_by_func(self.clicksend)
        button.set_active(status)
        button.handler_unblock_by_func(self.clicksend)

    def handler(self, conn, *args):
        line = conn.recv(4096)
        if not len(line):  # "Connection closed."
            self.bconnect.set_sensitive(True)
            self.setbuttons(False) # not connected
            self.kconnect()
            return False
        else:
            for myline in line.splitlines():
#                print "got a line: "+myline
                if (myline.startswith("HELLOAPP")):
                    self.kettleconnected = 1
                    self.setbuttons(True) # connected
                    self.bconnect.set_sensitive(False)
                    self.writeconfig(self.ip)
                    conn.send("get sys status\n")
                if (myline.startswith("sys status key=")):
                    if (len(myline)<16):
                        key = 0
                    else:
                        key = ord(myline[15]) & 0x3f
                    self.setbutton(self.b100,key&0x20)
                    self.setbutton(self.b95,key&0x10)
                    self.setbutton(self.b80,key&0x8)
                    self.setbutton(self.b65,key&0x4)
                    self.setbutton(self.bwarm,key&0x2)
                    self.bboil.handler_block_by_func(self.clickboil)
                    self.bboil.set_active(key&0x1)
                    self.bboil.handler_unblock_by_func(self.clickboil)
                if (myline == "sys status 0x100"):
                    self.setbutton(self.b100,True)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b80,False)
                    self.setbutton(self.b65,False)
                elif (myline == "sys status 0x95"):
                    self.setbutton(self.b95,True)
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b80,False)
                    self.setbutton(self.b65,False)
                elif (myline == "sys status 0x80"):
                    self.setbutton(self.b80,True)
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b65,False)
                elif (myline == "sys status 0x65"):
                    self.setbutton(self.b65,True)
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b80,False)
                elif (myline == "sys status 0x11"):
                    self.setbutton(self.bwarm,True)
                elif (myline == "sys status 0x10"):
                    self.setbutton(self.bwarm,False)
                elif (myline == "sys status 0x5"):
                    self.bboil.handler_block_by_func(self.clickboil)
                    self.bboil.set_active(True)
                    self.bboil.handler_unblock_by_func(self.clickboil)
                elif (myline == "sys status 0x0"):
                    self.setbutton(self.b100,False)
                    self.setbutton(self.b95,False)
                    self.setbutton(self.b80,False)
                    self.setbutton(self.b65,False)
                    self.setbutton(self.bwarm,False)
                    self.bboil.handler_block_by_func(self.clickboil)
                    self.bboil.set_active(False)
                    self.bboil.handler_unblock_by_func(self.clickboil)
                elif (myline == "sys status 0x3"):
                    message = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_INFO,
                                                buttons=gtk.BUTTONS_NONE,message_format=None);
                    message.set_markup("Your Kettle Has Boiled")
                    message.show()
                elif (myline == "sys status 0x1"):
                    message = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_INFO,
                                                buttons=gtk.BUTTONS_NONE,message_format=None);
                    message.set_markup("Your Kettle Was Removed")
                    message.show()

            return True

    def writeconfig(self,ip):
        if (ip == self.configip):
            return
        try:
            config.add_section('Main')
        except:
            pass
        config.set('Main','ip',ip)
        with open(cnf, 'wb') as configfile:
            config.write(configfile)

import argparse
import ConfigParser

config = ConfigParser.RawConfigParser()
configip = None
try:
    config.read(cnf)
    configip = config.get('Main','ip')
except:
    pass

parser = argparse.ArgumentParser(description="Simple GUI interface to the ikettle")
parser.add_argument('-i','--ip',help='IP address for kettle', required=False)
args = vars(parser.parse_args())

MooseKettle(configip,args['ip'])
gtk.main()
<|endoftext|>


<commit_before># Copyright 2016 Kevin Murray <spam@kdmurray.id.au>
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import screed
import numpy as np
import scipy as sp
from scipy import sparse
from scipy.sparse import linalg
import yaml

class TransitionCounter(object):

    def __init__(self, k, alphabet=set("ACGT")):
        self.bitmask = 2**(2*k)-1  # Set lowest 2*k bits
        self.k = k
        self.alphabet = set(alphabet)
        self.n = len(alphabet) ** k
        self.transition_counts = np.zeros((self.n, len(alphabet)))
        self._transitions = None
        self._P = None

    def _clear(self):
        self._transitions = None
        self._P = None

    def save(self, filename):
        data = {
            'alphabet': list(sorted(self.alphabet)),
            'k': self.k,
            'transitions': self.transitions.tolist(),
        }
        with open(filename, 'w') as fh:
            yaml.dump(data, fh)

    def load(self, filename):
        with open(filename, 'r') as fh:
            data = yaml.load(fh)
        datakeys = list(sorted(data.keys()))
        if datakeys != ['alphabet', 'k', 'transitions']:
            exc = ValueError("Data file is not valid")
            exc.keys = datakeys
            raise exc
        self.__init__(data['k'], data['alphabet'])
        self._transitions = np.array(data['transitions'])
        self.transition_counts = np.array(data['transitions'])

    def consume(self, sequence):
        self._clear()
        for subseq in sequence.split('N'):
            if not subseq or len(subseq) < self.k:
                continue
            kmers = iter_kmers(subseq, self.k)
            # pop the first kmer
            fr = next(kmers)
            for to in kmers:
                self.transition_counts[fr, to & 3] += 1
                fr = to

    def consume_file(self, filename):
        self._clear()
        with screed.open(filename) as sequences:
            for seq in sequences:
                self.consume(seq['sequence'])

    @property
    def transitions(self):
        if self._transitions is not None:
            return self._transitions
        transitions = self.transition_counts
        transitions /= transitions.sum(1)[:, np.newaxis]
        self._transitions = transitions
        return transitions

    @property
    def P(self):
        if self._P is not None:
            return self._P
        sparse_P = sparse.lil_matrix((self.n, self.n))
        num_kmers, alpha_size = self.transitions.shape
        for fr in range(num_kmers):
            for a in range(alpha_size):
                to = (fr << 2 | a) & self.bitmask
                sparse_P[fr, to] = self.transitions[fr, a]
        self._P = sparse_P
        return sparse_P

    @property
    def steady_state(self):
        v, w = linalg.eigs(self.P.transpose(), which='LR')
        ssf = np.real(w[:, v.argmax()])
        ssf /= ssf.sum()
        return ssf
<commit_msg>Add missing import to counter<commit_after># Copyright 2016 Kevin Murray <spam@kdmurray.id.au>
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import screed
import numpy as np
import scipy as sp
from scipy import sparse
from scipy.sparse import linalg
import yaml

from .util import (
    iter_kmers,
)

class TransitionCounter(object):

    def __init__(self, k, alphabet=set("ACGT")):
        self.bitmask = 2**(2*k)-1  # Set lowest 2*k bits
        self.k = k
        self.alphabet = set(alphabet)
        self.n = len(alphabet) ** k
        self.transition_counts = np.zeros((self.n, len(alphabet)))
        self._transitions = None
        self._P = None

    def _clear(self):
        self._transitions = None
        self._P = None

    def save(self, filename):
        data = {
            'alphabet': list(sorted(self.alphabet)),
            'k': self.k,
            'transitions': self.transitions.tolist(),
        }
        with open(filename, 'w') as fh:
            yaml.dump(data, fh)

    def load(self, filename):
        with open(filename, 'r') as fh:
            data = yaml.load(fh)
        datakeys = list(sorted(data.keys()))
        if datakeys != ['alphabet', 'k', 'transitions']:
            exc = ValueError("Data file is not valid")
            exc.keys = datakeys
            raise exc
        self.__init__(data['k'], data['alphabet'])
        self._transitions = np.array(data['transitions'])
        self.transition_counts = np.array(data['transitions'])

    def consume(self, sequence):
        self._clear()
        for subseq in sequence.split('N'):
            if not subseq or len(subseq) < self.k:
                continue
            kmers = iter_kmers(subseq, self.k)
            # pop the first kmer
            fr = next(kmers)
            for to in kmers:
                self.transition_counts[fr, to & 3] += 1
                fr = to

    def consume_file(self, filename):
        self._clear()
        with screed.open(filename) as sequences:
            for seq in sequences:
                self.consume(seq['sequence'])

    @property
    def transitions(self):
        if self._transitions is not None:
            return self._transitions
        transitions = self.transition_counts
        transitions /= transitions.sum(1)[:, np.newaxis]
        self._transitions = transitions
        return transitions

    @property
    def P(self):
        if self._P is not None:
            return self._P
        sparse_P = sparse.lil_matrix((self.n, self.n))
        num_kmers, alpha_size = self.transitions.shape
        for fr in range(num_kmers):
            for a in range(alpha_size):
                to = (fr << 2 | a) & self.bitmask
                sparse_P[fr, to] = self.transitions[fr, a]
        self._P = sparse_P
        return sparse_P

    @property
    def steady_state(self):
        v, w = linalg.eigs(self.P.transpose(), which='LR')
        ssf = np.real(w[:, v.argmax()])
        ssf /= ssf.sum()
        return ssf
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import pandas as pd

def get_value(qmcout,keyword='Madelung',delimiter='=',val_type=float,val_loc=-1):
    """ find the value of the line 'keyword = value' in qmcout """
    from mmap import mmap

    # open file
    fhandle = open(qmcout,'r+')
    mm = mmap(fhandle.fileno(),0)

    # find first line with the keyword
    idx = mm.find(keyword)
    if idx == -1:
        raise IOError(keyword + ' not found')
    # end

    # go to line and read
    mm.seek(idx)
    line = mm.readline().strip('\n')
    val_text = line.split(delimiter)[val_loc]

    # try to obtain the value of keyword
    val = val_type(val_text) # rely on internal exception

    # close file
    fhandle.close()

    return val
# end def get_madelung

def retrieve_psig(h5_file,only_occupied=False,occupations=None):
    """ return a list dictionaries of DFT orbital coefficients in PW basis by reading an hdf5 file written by pw2qmcpack. If only_occupied=True and a database of occupied orbitals are given, then only read orbitals that are occupied. """
    import h5py
    if only_occupied and (occupations is None):
        raise NotImplementedError("no occupation database is given")
    # end if

    ha = 27.21138602 # ev from 2014 CODATA

    orbitals = []

    h5handle = h5py.File(h5_file)
    electron = h5handle['electrons']

    kpt_labels = []
    for key in electron.keys():
        if key.startswith('kpoint'):
            kpt_labels.append(key)
        # end if
    # end for key

    nk = electron['number_of_kpoints'].value
    assert nk==len(kpt_labels)

    for label in kpt_labels:

        # get kpoint index
        kpt_idx = int( label.split('_')[-1] )

        # get plane wave wave numbers
        if kpt_idx == 0:
            mypath = os.path.join(label,'gvectors')
            gvecs = electron[mypath].value
        # end if

        # verify eigenstates at this kpoint
        kpt_ptr = electron[os.path.join(label,'spin_0')]
        nbnd = kpt_ptr['number_of_states'].value

        evals = kpt_ptr['eigenvalues'].value

        # compare to nscf output (eigenvalues and occupation)
        if occupations is not None:
            mydf = occupations[occupations['ik']==kpt_idx]
            myval= mydf['eval'].values[0]
            myocc= mydf['occ'].values[0]
            assert nbnd == len(myval), "expect %d bands, nscf has %d bands" % (nbnd,len(myval))
            assert np.allclose(evals*ha,myval,atol=1e-4), str(evals*ha-myval)
        # end if

        for iband in range(nbnd):
            if only_occupied and (np.isclose(myocc[iband],0.0)):
                continue
            # end if
            psig  = kpt_ptr['state_%d/psi_g'%iband].value
            entry = {'ik':kpt_idx,'iband':iband,'eval':evals[iband],'psig':psig}
            orbitals.append(entry)
        # end for iband

    # end for label

    h5handle.close()
    return orbitals
# end def retrieve_psig

def retrieve_system(h5_file):
    import h5py
    h5handle = h5py.File(h5_file)
    lattice  = h5handle['supercell/primitive_vectors'].value
    #elem     = h5handle['atoms/species_ids'].value # why so complicated?
    pos      = h5handle['atoms/positions'].value
    gvecs    = h5handle['electrons/kpoint_0/gvectors'].value
    h5handle.close()
    return {'axes':lattice,'pos':pos,'gvecs':gvecs}
# end def

def epl_val_err(epl_out):
    """ epl_out is expected to be an output of energy.pl from QMCPACK
     It simply has to have the format {name:22c}={val:17.3f} +/- {err:26.4f}.
     rows with forces will be recognized with 'force_prefix' """
    tab = pd.read_table(epl_out,delimiter='=',names=['name','text'])
    tab = tab.dropna()

    def text2val(text):
	tokens = text.split('+/-')
	if len(tokens) != 2:
	    raise NotImplementedError('unrecognized value '+text)
	# end if
	val,err = map(float,tokens)
	return pd.Series({'val':val,'err':err})
    # end def text2val

    df = pd.concat( [tab.drop('text',axis=1),tab['text'].apply(text2val)],axis=1 )
    return df
# end def epl_val_err

def sk_from_fs_out(fs_out):
    """ extract fluctuating S(k) from qmcfinitesize output
      returns: kmag,sk,vk,spk,spsk
       kmag: magnitude of kvectors, sk: raw fluc. S(k), vk: long-range potential after break-up
       spk: kmags for splined S(k), spsk: splined S(k) """

    import reader
    bint = reader.BlockInterpreter()
    sfile = reader.SearchableFile(fs_out)

    # read raw data
    block_text = sfile.block_text('#SK_RAW_START#','#SK_RAW_STOP#')
    kmag,sk,vk = bint.matrix(block_text[block_text.find('\n')+1:]).T

    # read splined S(k)
    block_text = sfile.block_text('#SK_SPLINE_START#','#SK_SPLINE_STOP#')
    spk,spsk = bint.matrix(block_text[block_text.find('\n')+1:]).T

    return kmag,sk,vk,spk,spsk
# end def

import numpy as np
from copy import deepcopy
def read_jastrows(jas_node):
    """ 'jas_node' should be an xml node containing bspline jastrows
     put coefficients and attributes into a list of dictionaries """

    if (jas_node.attrib["type"] != "Two-Body"): # works for one-body! miracle!
        pass#raise TypeError("input is not a two-body Jastrow xml node")
    elif (jas_node.attrib["function"].lower() != "bspline"):
        raise NotImplementedError("can only handle bspline Jastrows for now")
    # end if

    data = []
    for corr in jas_node.xpath('./correlation'):
        coeff = corr.xpath('./coefficients')[0]
        entry = deepcopy( corr.attrib )
        entry.update(coeff.attrib)
        entry['coeff'] = np.array(coeff.text.split(),dtype=float)
        entry['type']  = jas_node.attrib['type']
        data.append(entry)
    # end for corr

    return data
# end def read_jastrows

from lxml import etree
def extract_jastrows(qmcpack_input,json_name='jas.json',warn=True,force_refresh=False):
    """ given a QMCPACK input that contains linear optimization, extract all printed Jastrows and store in a local database
     1. parse 'qmcpack_input' for the qmc[@metho="linear"] section
     2. for each *.opt.xml, parse if it exists
     3. parse each opt.xml and make local database """

    failed = False

    subdir = os.path.dirname(qmcpack_input)
    target_json = os.path.join(subdir,json_name)
    if os.path.isfile(target_json) and (not force_refresh):
        if warn:
            print "skipping %s" % subdir
        # end if
        return 0 # skip ths file
    # end if

    parser = etree.XMLParser(remove_blank_text=True)

    # get prefix
    xml = etree.parse(qmcpack_input,parser)
    proj = xml.xpath("//project")[0]
    prefix = proj.attrib['id']

    # determine number of optimization loops
    all_qmc_sections = xml.xpath('.//qmc[@method="linear"]')
    all_iopt = 0 # track multiple 'linear' sections
    data = []
    for qmc_section in all_qmc_sections:
        # for each linear optimization:

        # find the number of loops
        nopt = 1
        loop = qmc_section.getparent()
        if loop.tag == 'loop':
            nopt = int(loop.attrib['max'])
        # end if

        # collect all jastrow coefficients
        for iopt in range(nopt):
            # get optimization file
            opt_file = prefix + ".s%s.opt.xml" % str(iopt).zfill(3)
            opt_xml  = os.path.join(subdir,opt_file)
            if not os.path.isfile(opt_xml):
                if warn:
                    print "skipping %d in %s" % (all_iopt,subdir)
                # end if
                continue
            # end if

            # parse optimization file
            opt = etree.parse(opt_xml,parser)
            jnodes = opt.xpath('//jastrow')
            for jas_node in jnodes:
                entries = read_jastrows(jas_node)
                for entry in entries:
                    entry['iopt'] = all_iopt
                # end for entry
                data.append(entry)
            # end for
            all_iopt += 1
        # end for iopt
    # end for qmc_section
    if len(data) == 0:
        failed = True
    else:
        df = pd.DataFrame( data )
        df.to_json(target_json)
    # end if

    return failed
# end def extract_jastrows

def extract_best_jastrow_set(opt_input,opt_json='opt_scalar.json',nequil='auto',force_refresh=False):
    import nexus_addon as na
    subdir = os.path.dirname(opt_input)

    # locally create jas.json
    extract_jastrows(opt_input,force_refresh=force_refresh)

    # locally create opt_scalar.json
    scalar_json = os.path.join(subdir,opt_json)
    if (not os.path.isfile(scalar_json)) or force_refresh:
        # initialize analyzer
        from qmca import QBase
        options = {"equilibration":nequil}
        QBase.options.transfer_from(options)

        entry = na.scalars_from_input(opt_input)
        pd.DataFrame(entry).to_json(scalar_json)
    # end if

    # get best jastrow set
    best_jas = collect_best_jastrow_set(subdir)
    return best_jas
# end def extract_best_jastrow_set

def collect_best_jastrow_set(subdir,jas_json='jas.json',opt_json='opt_scalar.json'
        ,rval_weight=0.75,rerr_weight=0.25):
    """ find best set of jastrows in 'subdir', assume files:
     1. jas.json: a database of QMCPACK bspline jastrows with 'iopt' column
     2. opt_scalar.json: a database of QMCPACK scalars including 'LocalEnergy_mean', 'LocalEnergy_error', 'Variance_mean', and 'Variance_error' """
    from dmc_database_analyzer import div_columns

    jfile = os.path.join(subdir,jas_json)
    if not os.path.isfile(jfile):
        raise RuntimeError('%s not found in %s' % (jfile,subdir))
    # end if

    ofile = os.path.join(subdir,opt_json)
    if not os.path.isfile(ofile):
        raise RuntimeError('%s not found in %s' % (ofile,subdir))
    # end if

    jdf = pd.read_json(jfile) # jastrows
    sdf = pd.read_json(ofile) # scalars

    # same units for stddev and LocalEnergy
    sdf['stddev_mean'] = sdf['Variance_mean'].apply(np.sqrt)
    sdf['stddev_error'] = sdf['Variance_error'].apply(np.sqrt)

    # make ratios
    ratio_mean, ratio_error = div_columns(['stddev','LocalEnergy'],sdf)

    # take both value and error into account
    rv_cost = ratio_mean/ratio_mean.mean()
    re_cost = ratio_error/ratio_error.mean()

    # make a cost function
    cost = rv_cost*rval_weight + re_cost*rerr_weight

    # minimize cost function
    idx  = np.argmin(cost)

    # grab winner jastrows
    best_jas = jdf[jdf['iopt']==idx].copy()
    return best_jas
# end def collect_best_jastrow_set
<commit_msg>use all_iopt<commit_after>#!/usr/bin/env python
import os
import pandas as pd

def get_value(qmcout,keyword='Madelung',delimiter='=',val_type=float,val_loc=-1):
    """ find the value of the line 'keyword = value' in qmcout """
    from mmap import mmap

    # open file
    fhandle = open(qmcout,'r+')
    mm = mmap(fhandle.fileno(),0)

    # find first line with the keyword
    idx = mm.find(keyword)
    if idx == -1:
        raise IOError(keyword + ' not found')
    # end

    # go to line and read
    mm.seek(idx)
    line = mm.readline().strip('\n')
    val_text = line.split(delimiter)[val_loc]

    # try to obtain the value of keyword
    val = val_type(val_text) # rely on internal exception

    # close file
    fhandle.close()

    return val
# end def get_madelung

def retrieve_psig(h5_file,only_occupied=False,occupations=None):
    """ return a list dictionaries of DFT orbital coefficients in PW basis by reading an hdf5 file written by pw2qmcpack. If only_occupied=True and a database of occupied orbitals are given, then only read orbitals that are occupied. """
    import h5py
    if only_occupied and (occupations is None):
        raise NotImplementedError("no occupation database is given")
    # end if

    ha = 27.21138602 # ev from 2014 CODATA

    orbitals = []

    h5handle = h5py.File(h5_file)
    electron = h5handle['electrons']

    kpt_labels = []
    for key in electron.keys():
        if key.startswith('kpoint'):
            kpt_labels.append(key)
        # end if
    # end for key

    nk = electron['number_of_kpoints'].value
    assert nk==len(kpt_labels)

    for label in kpt_labels:

        # get kpoint index
        kpt_idx = int( label.split('_')[-1] )

        # get plane wave wave numbers
        if kpt_idx == 0:
            mypath = os.path.join(label,'gvectors')
            gvecs = electron[mypath].value
        # end if

        # verify eigenstates at this kpoint
        kpt_ptr = electron[os.path.join(label,'spin_0')]
        nbnd = kpt_ptr['number_of_states'].value

        evals = kpt_ptr['eigenvalues'].value

        # compare to nscf output (eigenvalues and occupation)
        if occupations is not None:
            mydf = occupations[occupations['ik']==kpt_idx]
            myval= mydf['eval'].values[0]
            myocc= mydf['occ'].values[0]
            assert nbnd == len(myval), "expect %d bands, nscf has %d bands" % (nbnd,len(myval))
            assert np.allclose(evals*ha,myval,atol=1e-4), str(evals*ha-myval)
        # end if

        for iband in range(nbnd):
            if only_occupied and (np.isclose(myocc[iband],0.0)):
                continue
            # end if
            psig  = kpt_ptr['state_%d/psi_g'%iband].value
            entry = {'ik':kpt_idx,'iband':iband,'eval':evals[iband],'psig':psig}
            orbitals.append(entry)
        # end for iband

    # end for label

    h5handle.close()
    return orbitals
# end def retrieve_psig

def retrieve_system(h5_file):
    import h5py
    h5handle = h5py.File(h5_file)
    lattice  = h5handle['supercell/primitive_vectors'].value
    #elem     = h5handle['atoms/species_ids'].value # why so complicated?
    pos      = h5handle['atoms/positions'].value
    gvecs    = h5handle['electrons/kpoint_0/gvectors'].value
    h5handle.close()
    return {'axes':lattice,'pos':pos,'gvecs':gvecs}
# end def

def epl_val_err(epl_out):
    """ epl_out is expected to be an output of energy.pl from QMCPACK
     It simply has to have the format {name:22c}={val:17.3f} +/- {err:26.4f}.
     rows with forces will be recognized with 'force_prefix' """
    tab = pd.read_table(epl_out,delimiter='=',names=['name','text'])
    tab = tab.dropna()

    def text2val(text):
	tokens = text.split('+/-')
	if len(tokens) != 2:
	    raise NotImplementedError('unrecognized value '+text)
	# end if
	val,err = map(float,tokens)
	return pd.Series({'val':val,'err':err})
    # end def text2val

    df = pd.concat( [tab.drop('text',axis=1),tab['text'].apply(text2val)],axis=1 )
    return df
# end def epl_val_err

def sk_from_fs_out(fs_out):
    """ extract fluctuating S(k) from qmcfinitesize output
      returns: kmag,sk,vk,spk,spsk
       kmag: magnitude of kvectors, sk: raw fluc. S(k), vk: long-range potential after break-up
       spk: kmags for splined S(k), spsk: splined S(k) """

    import reader
    bint = reader.BlockInterpreter()
    sfile = reader.SearchableFile(fs_out)

    # read raw data
    block_text = sfile.block_text('#SK_RAW_START#','#SK_RAW_STOP#')
    kmag,sk,vk = bint.matrix(block_text[block_text.find('\n')+1:]).T

    # read splined S(k)
    block_text = sfile.block_text('#SK_SPLINE_START#','#SK_SPLINE_STOP#')
    spk,spsk = bint.matrix(block_text[block_text.find('\n')+1:]).T

    return kmag,sk,vk,spk,spsk
# end def

import numpy as np
from copy import deepcopy
def read_jastrows(jas_node):
    """ 'jas_node' should be an xml node containing bspline jastrows
     put coefficients and attributes into a list of dictionaries """

    if (jas_node.attrib["type"] != "Two-Body"): # works for one-body! miracle!
        pass#raise TypeError("input is not a two-body Jastrow xml node")
    elif (jas_node.attrib["function"].lower() != "bspline"):
        raise NotImplementedError("can only handle bspline Jastrows for now")
    # end if

    data = []
    for corr in jas_node.xpath('./correlation'):
        coeff = corr.xpath('./coefficients')[0]
        entry = deepcopy( corr.attrib )
        entry.update(coeff.attrib)
        entry['coeff'] = np.array(coeff.text.split(),dtype=float)
        entry['type']  = jas_node.attrib['type']
        data.append(entry)
    # end for corr

    return data
# end def read_jastrows

from lxml import etree
def extract_jastrows(qmcpack_input,json_name='jas.json',warn=True,force_refresh=False):
    """ given a QMCPACK input that contains linear optimization, extract all printed Jastrows and store in a local database
     1. parse 'qmcpack_input' for the qmc[@metho="linear"] section
     2. for each *.opt.xml, parse if it exists
     3. parse each opt.xml and make local database """

    failed = False

    subdir = os.path.dirname(qmcpack_input)
    target_json = os.path.join(subdir,json_name)
    if os.path.isfile(target_json) and (not force_refresh):
        if warn:
            print "skipping %s" % subdir
        # end if
        return 0 # skip ths file
    # end if

    parser = etree.XMLParser(remove_blank_text=True)

    # get prefix
    xml = etree.parse(qmcpack_input,parser)
    proj = xml.xpath("//project")[0]
    prefix = proj.attrib['id']

    # determine number of optimization loops
    all_qmc_sections = xml.xpath('.//qmc[@method="linear"]')
    all_iopt = 0 # track multiple 'linear' sections
    data = []
    for qmc_section in all_qmc_sections:
        # for each linear optimization:

        # find the number of loops
        nopt = 1
        loop = qmc_section.getparent()
        if loop.tag == 'loop':
            nopt = int(loop.attrib['max'])
        # end if

        # collect all jastrow coefficients
        for iopt in range(nopt):
            # get optimization file
            opt_file = prefix + ".s%s.opt.xml" % str(all_iopt).zfill(3)
            opt_xml  = os.path.join(subdir,opt_file)
            if not os.path.isfile(opt_xml):
                if warn:
                    print "skipping %d in %s" % (all_iopt,subdir)
                # end if
                continue
            # end if

            # parse optimization file
            opt = etree.parse(opt_xml,parser)
            jnodes = opt.xpath('//jastrow')
            for jas_node in jnodes:
                entries = read_jastrows(jas_node)
                for entry in entries:
                    entry['iopt'] = all_iopt
                # end for entry
                data.append(entry)
            # end for
            all_iopt += 1
        # end for iopt
    # end for qmc_section
    if len(data) == 0:
        failed = True
    else:
        df = pd.DataFrame( data )
        df.to_json(target_json)
    # end if

    return failed
# end def extract_jastrows

def extract_best_jastrow_set(opt_input,opt_json='opt_scalar.json',nequil='auto',force_refresh=False):
    import nexus_addon as na
    subdir = os.path.dirname(opt_input)

    # locally create jas.json
    extract_jastrows(opt_input,force_refresh=force_refresh)

    # locally create opt_scalar.json
    scalar_json = os.path.join(subdir,opt_json)
    if (not os.path.isfile(scalar_json)) or force_refresh:
        # initialize analyzer
        from qmca import QBase
        options = {"equilibration":nequil}
        QBase.options.transfer_from(options)

        entry = na.scalars_from_input(opt_input)
        pd.DataFrame(entry).to_json(scalar_json)
    # end if

    # get best jastrow set
    best_jas = collect_best_jastrow_set(subdir)
    return best_jas
# end def extract_best_jastrow_set

def collect_best_jastrow_set(subdir,jas_json='jas.json',opt_json='opt_scalar.json'
        ,rval_weight=0.75,rerr_weight=0.25):
    """ find best set of jastrows in 'subdir', assume files:
     1. jas.json: a database of QMCPACK bspline jastrows with 'iopt' column
     2. opt_scalar.json: a database of QMCPACK scalars including 'LocalEnergy_mean', 'LocalEnergy_error', 'Variance_mean', and 'Variance_error' """
    from dmc_database_analyzer import div_columns

    jfile = os.path.join(subdir,jas_json)
    if not os.path.isfile(jfile):
        raise RuntimeError('%s not found in %s' % (jfile,subdir))
    # end if

    ofile = os.path.join(subdir,opt_json)
    if not os.path.isfile(ofile):
        raise RuntimeError('%s not found in %s' % (ofile,subdir))
    # end if

    jdf = pd.read_json(jfile) # jastrows
    sdf = pd.read_json(ofile) # scalars

    # same units for stddev and LocalEnergy
    sdf['stddev_mean'] = sdf['Variance_mean'].apply(np.sqrt)
    sdf['stddev_error'] = sdf['Variance_error'].apply(np.sqrt)

    # make ratios
    ratio_mean, ratio_error = div_columns(['stddev','LocalEnergy'],sdf)

    # take both value and error into account
    rv_cost = ratio_mean/ratio_mean.mean()
    re_cost = ratio_error/ratio_error.mean()

    # make a cost function
    cost = rv_cost*rval_weight + re_cost*rerr_weight

    # minimize cost function
    idx  = np.argmin(cost)

    # grab winner jastrows
    best_jas = jdf[jdf['iopt']==idx].copy()
    return best_jas
# end def collect_best_jastrow_set
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
#第 0001 题：**做为 Apple Store App 独立开发者，你要搞限时促销，为你的应用**生成激活码**（或者优惠券），使用 Python 如何生成 200 个激活码（或者优惠券）
#by DockerFan random.choice
import random, string

def random_series(count,len=10):
	str='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
	series_set =[]

	for i in range(0,count):
		series = ''
		for j in range(0,len):
			series += random.choice(str)
			if series not in series_set:
				series_set.append(series)
		print (series)

random_series(20)

<commit_msg>Update<commit_after># -*- coding:utf-8 -*-
#第 0001 题：**做为 Apple Store App 独立开发者，你要搞限时促销，为你的应用**生成激活码**（或者优惠券），使用 Python 如何生成 200 个激活码（或者优惠券）

import random, string

forSelect = string.ascii_letters + "0123456789"

def generate(count, length):
    # count = 200
    # length = 20

    for x in range(count):
        Re = ""
        for y in range(length):
            Re += random.choice(forSelect)
        print(Re)

if __name__ == "__main__":
    generate(200, 20)
<|endoftext|>


<commit_before>"""
mshpScraper.py
Missouri State Highway Patrol scraper for crash data!
[azide0x37] Alexander Templeton (nepenthe.me)

Arguments:
    Production:
        optional:
            date:
            name:
            county:
            severity:
            age_of_occupant:

    Devel:
        optional:
            url: of webpage

Returns: pandas Data.Frame object

Class usage template
myScrape = mshpScraper()
myScrape()
"""

import urllib2
import geopy
import pandas as pd
from bs4 import BeautifulSoup
from collections import OrderedDict
from datetime import datetime

class mshpScraper:

    def __init__(
                self,
                url = 'http://www.mshp.dps.missouri.gov/HP68/SearchAction',
                colNames = ['Name', 'Age', 'Hometown', 'Severity', 'Date', 'Time', 'County', 'Location', 'Troop'],
                county = 'Jefferson'):

        self.url = url
        self.colNames = colNames

        #Use a proper useragent to evade the anti-hack software
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]

        #Cache our response
        #TODO: Add __call__ option to redownload
        self.response = opener.open(self.url)

        #Set better formatting for testing console print
        pd.set_option('display.max_rows', 500)
        pd.set_option('display.max_columns', 500)
        pd.set_option('display.width', 200)

    def __str__(self):
        try:
            return self.url
        except:
            print "Unable to print url."

    #Setting this default method allows one to call an instance like a function. NEATO!
    def __call__(self):
        webpageSouped = BeautifulSoup(self.response.read())

        #This is the particular table with the data we need.
        #TODO: Find a better way to pick the right one?
        table = webpageSouped.find('table', summary="Table listing details of the accident.")
        rows = table.findAll('tr')

        #TODO: Pull colNames from the table headngs
        dataset = []

        for tr in rows:
            cols = tr.findAll('td')
            row_data = OrderedDict()
            counter = 0

            #TODO: the zeroth element of cols is null; why? Fix.
            for td in cols[1:]:
                text = ''.join(td.find(text=True))
                try:
                    row_data[self.colNames[counter]] = text
                    counter += 1
                except:
                    counter = 0
                    continue
            dataset.append(row_data)

        #Convert dataset (List of Dicts) to pandas DataFrame
        #Extract date and time colums to column of dateTime objects, then drop

        def datetimeCoerce(row):
            return datetime(row['Date'], axis = 1)

        returnData = pd.DataFrame(dataset)
        #returnData['DateTime'] = returnData.apply(lambda row: datetimeCoerce(row))

        #FIXME drop these unused columns
        #returnData.drop('Date', 1, inplace = True)

        #FIXME this doesn't work at all

        #Location coersion
        #geolocator = geopy.geocoders.OpenMapQuest()

        #location = geolocator.geocode(returnData['Location'])

        #returnData['Location'] = returnData.apply(lambda row: geolocator.geocode(row.Location + ", " + row.County + " County, MO"))
        #returnData['Latitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).latitude)
        #returnData['Longitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).longitude)

        #Ship it!
        return returnData[pd.notnull(returnData['Name'])]

myScrape = mshpScraper()
print myScrape()
<commit_msg>started implementation of sklearn-pandas:wq<commit_after>"""
mshpScraper.py
Missouri State Highway Patrol scraper for crash data!
[azide0x37] Alexander Templeton (nepenthe.me)

Arguments:
    Production:
        optional:
            date:
            name:
            county:
            severity:
            age_of_occupant:

    Devel:
        optional:
            url: of webpage

Returns: pandas Data.Frame object

Class usage template
myScrape = mshpScraper()
myScrape()
"""

import urllib2
import geopy
import pandas as pd
from sklearn_pandas import DataFrameMapper
from bs4 import BeautifulSoup
from collections import OrderedDict
from datetime import datetime

class mshpScraper:

    def __init__(
                self,
                url = 'http://www.mshp.dps.missouri.gov/HP68/SearchAction',
                colNames = ['Name', 'Age', 'Hometown', 'Severity', 'Date', 'Time', 'County', 'Location', 'Troop'],
                county = 'Jefferson'):

        self.url = url
        self.colNames = colNames

        #Use a proper useragent to evade the anti-hack software
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]

        #Cache our response
        #TODO: Add __call__ option to redownload
        self.response = opener.open(self.url)

        #Set better formatting for testing console print
        pd.set_option('display.max_rows', 500)
        pd.set_option('display.max_columns', 500)
        pd.set_option('display.width', 200)

    def __str__(self):
        try:
            return self.url
        except:
            print "Unable to print url."

    #Setting this default method allows one to call an instance like a function. NEATO!
    def __call__(self):
        webpageSouped = BeautifulSoup(self.response.read())

        #This is the particular table with the data we need.
        #TODO: Find a better way to pick the right one?
        table = webpageSouped.find('table', summary="Table listing details of the accident.")
        rows = table.findAll('tr')

        #TODO: Pull colNames from the table headngs
        dataset = []

        for tr in rows:
            cols = tr.findAll('td')
            row_data = OrderedDict()
            counter = 0

            #TODO: the zeroth element of cols is null; why? Fix.
            for td in cols[1:]:
                text = ''.join(td.find(text=True))
                try:
                    row_data[self.colNames[counter]] = text
                    counter += 1
                except:
                    counter = 0
                    continue
            dataset.append(row_data)

        #Convert dataset (List of Dicts) to pandas DataFrame
        #Extract date and time colums to column of dateTime objects, then drop

        def datetimeCoerce(row):
            return datetime(row['Date'], axis = 1)

        returnData = pd.DataFrame(dataset)
        #returnData['DateTime'] = returnData.apply(lambda row: datetimeCoerce(row))

        #FIXME drop these unused columns
        #returnData.drop('Date', 1, inplace = True)

        #FIXME this doesn't work at all

        #Location coersion
        #geolocator = geopy.geocoders.OpenMapQuest()

        #location = geolocator.geocode(returnData['Location'])

        #returnData['Location'] = returnData.apply(lambda row: geolocator.geocode(row.Location + ", " + row.County + " County, MO"))
        #returnData['Latitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).latitude)
        #returnData['Longitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).longitude)

        #Ship it!
        return returnData[pd.notnull(returnData['Name'])]

myScrape = mshpScraper()
data = myScrape()
mapper = DataFrameMapper([
    '', sklearn.preprocessing.
<|endoftext|>


<commit_before>from conans.model.conan_file import ConanFile
from conans import CMake
import os

class DefaultNameConan(ConanFile):
    name = "DefaultName"
    version = "0.1"
    settings = "os", "compiler", "arch", "build_type"
    generators = "cmake"
    requires = "eigen/3.2@jslee02/testing"

    def build(self):
        cmake = CMake(self.settings)
        self.run('cmake . %s' % cmake.command_line)
        self.run("cmake --build . %s" % cmake.build_config)

    def imports(self):
        self.copy(pattern="*.dll", dst="bin", src="bin")
        self.copy(pattern="*.dylib", dst="bin", src="lib")

    def test(self):
        self.run("cd bin && .%smytest" % os.sep)

<commit_msg>Test stable channel instead of testing channel for eigen<commit_after>from conans.model.conan_file import ConanFile
from conans import CMake
import os

class DefaultNameConan(ConanFile):
    name = "DefaultName"
    version = "0.1"
    settings = "os", "compiler", "arch", "build_type"
    generators = "cmake"
    requires = "eigen/3.2@jslee02/stable"

    def build(self):
        cmake = CMake(self.settings)
        self.run('cmake . %s' % cmake.command_line)
        self.run("cmake --build . %s" % cmake.build_config)

    def imports(self):
        self.copy(pattern="*.dll", dst="bin", src="bin")
        self.copy(pattern="*.dylib", dst="bin", src="lib")

    def test(self):
        self.run("cd bin && .%smytest" % os.sep)

<|endoftext|>


<commit_before># -*-coding:Utf-8 -*
# ====================================================================
# MIT License
#
# Copyright (c) 2014-2017 The muLAn project team
# Copyright (c) 2014-2017 Clément Ranc & Arnaud Cassan
#
# See the LICENCE file
# ====================================================================

# --------------------------------------------------------------------
# Packages
# --------------------------------------------------------------------
import os
import sys
import importlib
import muLAn
import muLAn.packages.sortmodels as mulansort
import muLAn.packages.sequential as sequential
# --------------------------------------------------------------------
# Fonctions
# --------------------------------------------------------------------
def getlogo():
    logo =\
            "                 _       ___        \n"\
          + "                | |     / _ \       \n"\
          + " _ __ ___  _   _| |    / /_\ \_ __  \n"\
          + "| '_ ` _ \| | | | |    |  _  | '_ \ \n"\
          + "| | | | | | |_| | |____| | | | | | |\n"\
          + "|_| |_| |_|\__,_\_____/\_| |_/_| |_|\n"
    return logo
# --------------------------------------------------------------------
def check_slash(path):
    if len(path) > 0:
        if path[-1] != '/':
            path = '{:s}/'.format(path)
    return path
# --------------------------------------------------------------------
def gettxt_inifile(fn, kw, sep=':'):
    try:
        file = open(fn, 'r')
        result = [line.strip().split(sep)[1].strip() for line in
                file if line.strip().split(sep)[0].strip() == kw][-1]
        file.close()
    except IOError:
        sys.exit('File {:s} not found.'.format(fn))

    return result
# --------------------------------------------------------------------
def getpath_event():
    """Return the path of the event directory.

    :return: path of event directory
    :rtype: string
    """
    ici = os.getcwd()
    path_event = check_slash(os.path.realpath(ici))
    return path_event
# --------------------------------------------------------------------
def getint_verbose(path_event):
    fn_as = '{:s}setup.ini'.format(path_event)
    v = int(gettxt_inifile(fn_as, 'Verbose'))
    return v
# --------------------------------------------------------------------
def print_welcome(verbose=1):
        logo = getlogo()
        legend = '\033[90mMicro-Lensing Analysis software\033[0m\n'
        version = '\033[90mVersion {:s}'.format(muLAn.__version__)

        welcome = '\033[1m\033[34m{:s}\033[0m{:s}{:s}\033[0m'.format(logo, legend, version)
        print welcome
# ----------------------------------------------------------------------
def set_default_options(options):
    options.update({
        'sortno': False
        })
# ----------------------------------------------------------------------
def check_options(options):
    if not 'sortno' in options:
        options.update({'sortno': False})
    if not 'plot' in options:
        options.update({'plot': None})
    if not 'fit' in options:
        options.update({'fit': False})
    if not 'archive' in options:
        options.update({'archive': None})
    if not 'ncores' in options:
        options.update({'ncores': -1})
    if not 'nchains' in options:
        options.update({'nchains': -1})
    if not 'resume' in options:
        options.update({'resume': False})
    if not 'verbose' in options:
        options.update({'verbose': -1})
    if not 'optimize' in options:
        options.update({'optimize': False})
# --------------------------------------------------------------------
def run_mulan(path_event, options):
    sequential.run_sequence(path_event, options)
# ----------------------------------------------------------------------
def sortmodels():
    mulansort.McmcFiles().sort()
# ----------------------------------------------------------------------
def stop():
    # Safe Emergency Stop
    ici = os.getcwd()
    path_event = os.path.realpath(ici)+'/'
    if os.path.exists(path_event + '.emergencystop'):
        os.remove(path_event + '.emergencystop')
        file = open(path_event + '.emergencystop', 'w')
        file.write('1')
        file.close()
    fn_lock = '{:s}.lock'.format(path_event) # Remove .lock file
    if os.path.exists(fn_lock):
        os.remove(fn_lock)
# ----------------------------------------------------------------------
def run(options=None):

    # Welcoming message depending on verbose option
    ici = os.getcwd()
    path_event = check_slash(os.path.realpath(ici))
    verbose = getint_verbose(path_event)
    print_welcome(verbose=verbose)

    # Use a default options dictionary if not provided by user
    if options == None: options = dict()
    check_options(options)

    # Run the modeling code
    run_mulan(path_event, options)

<commit_msg>Remove function set_default_options not used.<commit_after># -*-coding:Utf-8 -*
# ====================================================================
# MIT License
#
# Copyright (c) 2014-2017 The muLAn project team
# Copyright (c) 2014-2017 Clément Ranc & Arnaud Cassan
#
# See the LICENCE file
# ====================================================================

# --------------------------------------------------------------------
# Packages
# --------------------------------------------------------------------
import os
import sys
import importlib
import muLAn
import muLAn.packages.sortmodels as mulansort
import muLAn.packages.sequential as sequential
# --------------------------------------------------------------------
# Fonctions
# --------------------------------------------------------------------
def getlogo():
    logo =\
            "                 _       ___        \n"\
          + "                | |     / _ \       \n"\
          + " _ __ ___  _   _| |    / /_\ \_ __  \n"\
          + "| '_ ` _ \| | | | |    |  _  | '_ \ \n"\
          + "| | | | | | |_| | |____| | | | | | |\n"\
          + "|_| |_| |_|\__,_\_____/\_| |_/_| |_|\n"
    return logo
# --------------------------------------------------------------------
def check_slash(path):
    if len(path) > 0:
        if path[-1] != '/':
            path = '{:s}/'.format(path)
    return path
# --------------------------------------------------------------------
def gettxt_inifile(fn, kw, sep=':'):
    try:
        file = open(fn, 'r')
        result = [line.strip().split(sep)[1].strip() for line in
                file if line.strip().split(sep)[0].strip() == kw][-1]
        file.close()
    except IOError:
        sys.exit('File {:s} not found.'.format(fn))

    return result
# --------------------------------------------------------------------
def getpath_event():
    """Return the path of the event directory.

    :return: path of event directory
    :rtype: string
    """
    ici = os.getcwd()
    path_event = check_slash(os.path.realpath(ici))
    return path_event
# --------------------------------------------------------------------
def getint_verbose(path_event):
    fn_as = '{:s}setup.ini'.format(path_event)
    v = int(gettxt_inifile(fn_as, 'Verbose'))
    return v
# --------------------------------------------------------------------
def print_welcome(verbose=1):
        logo = getlogo()
        legend = '\033[90mMicro-Lensing Analysis software\033[0m\n'
        version = '\033[90mVersion {:s}'.format(muLAn.__version__)

        welcome = '\033[1m\033[34m{:s}\033[0m{:s}{:s}\033[0m'.format(logo, legend, version)
        print welcome
# ----------------------------------------------------------------------
def check_options(options):
    if not 'sortno' in options:
        options.update({'sortno': False})
    if not 'plot' in options:
        options.update({'plot': None})
    if not 'fit' in options:
        options.update({'fit': False})
    if not 'archive' in options:
        options.update({'archive': None})
    if not 'ncores' in options:
        options.update({'ncores': -1})
    if not 'nchains' in options:
        options.update({'nchains': -1})
    if not 'resume' in options:
        options.update({'resume': False})
    if not 'verbose' in options:
        options.update({'verbose': -1})
    if not 'optimize' in options:
        options.update({'optimize': False})
# --------------------------------------------------------------------
def run_mulan(path_event, options):
    sequential.run_sequence(path_event, options)
# ----------------------------------------------------------------------
def sortmodels():
    mulansort.McmcFiles().sort()
# ----------------------------------------------------------------------
def stop():
    # Safe Emergency Stop
    ici = os.getcwd()
    path_event = os.path.realpath(ici)+'/'
    if os.path.exists(path_event + '.emergencystop'):
        os.remove(path_event + '.emergencystop')
        file = open(path_event + '.emergencystop', 'w')
        file.write('1')
        file.close()
    fn_lock = '{:s}.lock'.format(path_event) # Remove .lock file
    if os.path.exists(fn_lock):
        os.remove(fn_lock)
# ----------------------------------------------------------------------
def run(options=None):

    # Welcoming message depending on verbose option
    ici = os.getcwd()
    path_event = check_slash(os.path.realpath(ici))
    verbose = getint_verbose(path_event)
    print_welcome(verbose=verbose)

    # Use a default options dictionary if not provided by user
    if options == None: options = dict()
    check_options(options)

    # Run the modeling code
    run_mulan(path_event, options)

<|endoftext|>


<commit_before>import os
import argparse

import numpy as np
import scipy.ndimage as nd
import tensorflow as tf

from tensorflow.examples.tutorials.mnist import input_data

def show_image(image):
    import matplotlib.pyplot as plt
    plt.imshow(image, cmap="gray", vmin=0.0, vmax=1.0)
    plt.show()

def read_image(path, max_intensity):
    image = nd.imread(path, mode="L")
    image = np.asarray(image, dtype=np.float32) / 255.0
    img_min, img_max = image.min(), image.max()

    if img_min != img_max:
        if img_min > 0.0:
            image -= img_min
        if img_max > 0.0:
            image /= img_max
        if max_intensity < 1.0:
            image *= max_intensity
    else:
        if img_max > max_intensity:
            image = np.ones_like(image) * max_intensity

    return image

def crop_non_empty(image):
    non_zero_row_ids = np.array(np.nonzero(np.sum(image, axis=0)))[0]
    non_zero_col_ids = np.array(np.nonzero(np.sum(image, axis=1)))[0]
    x_start, x_end = non_zero_row_ids[0], non_zero_row_ids[-1]
    y_start, y_end = non_zero_col_ids[0], non_zero_col_ids[-1]

    return image[y_start:y_end+1, x_start:x_end+1]

def overlaps(x, y, w, h, positions, boxes, gap):
    for i in range(len(positions) // 2):
        p, b = positions[i*2:(i+1)*2], boxes[i*2:(i+1)*2]
        l1x, l1y, r1x, r1y = x-gap, y-gap, x+w+gap-1, y+h+gap-1
        l2x, l2y, r2x, r2y = p[0], p[1], p[0]+b[0]-1, p[1]+b[1]-1

        if l1x <= r2x and l2x <= r1x:
            return True
        if l1y >= r2y and l2y >= r1y:
            return True

    return False

def generate_multi_image(single_images, num_images, image_dim, canvas_dim, bg=None,
                         min_w=1.0, max_w=1.0, min_h=1.0, max_h=1.0, min_ang=0.0, max_ang=0.0,
                         gap=0, margin=0):
    ready = False
    while not ready:
        canvas = np.zeros(
            [canvas_dim, canvas_dim],
            dtype=single_images[0].dtype
        )

        placed_image_ids = []
        placed_image_positions = []
        placed_image_boxes = []

        if num_digits == 0:
            break

        try:
            for i in range(num_images):
                idx = np.random.randint(len(single_images))
                image = np.reshape(single_images[idx], [image_dim, image_dim])
                image = crop_non_empty(image)

                if min_w != 1.0 or max_w != 1.0 or min_h != 1.0 or max_h != 1.0:
                    new_width = np.random.uniform(min_w, max_w)
                    new_height = np.random.uniform(min_h, max_h)

                    image = nd.affine_transform(
                        image,
                        matrix=np.array([[1.0 / new_height, 0.0], [0.0, 1.0 / new_width]]),
                        output_shape=(int(image_dim * new_height), int(image_dim * new_width)),
                        order=5
                    )

                    image = np.clip(image, 0.0, 1.0)
                    image = np.where(image >= 0.05, image, np.zeros_like(image))
                    image = crop_non_empty(image)

                if min_ang != 0.0 or max_ang != 0.0:
                    new_angle = np.random.uniform(min_ang, max_ang)

                    image = nd.rotate(image, new_angle, order=5)

                    image = np.clip(image, 0.0, 1.0)
                    image = np.where(image >= 0.05, image, np.zeros_like(image))
                    image = crop_non_empty(image)

                h, w = image.shape
                position_find_attempts = 0
                position_found = False

                while position_find_attempts < 100:
                    x = np.random.randint(margin, canvas_dim - w - margin + 1)
                    y = np.random.randint(margin, canvas_dim - h - margin + 1)

                    if i == 0 or not overlaps(x, y, w, h, placed_image_positions, placed_image_boxes, gap):
                        position_found = True
                        break

                    position_find_attempts += 1

                if position_found:
                    canvas[y:y+h, x:x+w] += image

                    placed_image_positions.extend([x, y])
                    placed_image_boxes.extend([w, h])
                    placed_image_ids.append(idx)

                    if i == num_images - 1:
                        ready = True
                else:
                    break

        except IndexError:
            pass

    if bg is not None:
        canvas = np.clip(canvas + bg, 0.0, 1.0)

    return canvas, placed_image_ids, placed_image_positions, placed_image_boxes

def write_to_records(filename, images, indices, positions, boxes, labels, digits):
    def _int64_list_feature(value):
        return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

    def _bytes_feature(value):
        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

    rows, cols = images[0].shape

    writer = tf.python_io.TFRecordWriter(filename + ".tfrecords")
    for index in range(len(images)):
        example = tf.train.Example(
            features=tf.train.Features(
                feature={
                    "height": _int64_list_feature([rows]),
                    "width": _int64_list_feature([cols]),
                    "digits": _int64_list_feature([digits[index]]),
                    "indices": _bytes_feature(np.asarray(indices[index], dtype=np.int32).tostring()),
                    "positions": _bytes_feature(np.asarray(positions[index], dtype=np.int32).tostring()),
                    "boxes": _bytes_feature(np.asarray(boxes[index], dtype=np.int32).tostring()),
                    "labels": _bytes_feature(np.asarray(labels[index], dtype=np.int32).tostring()),
                    "image": _bytes_feature(np.ravel(images[index]).tostring())
                }
            )
        )
        writer.write(example.SerializeToString())
    writer.close()

def shuffle_lists(*lists):
    shuffled = []

    perm = np.random.permutation(
        range(len(lists[0]))
    )

    for l in lists:
        shuffled.append([l[i] for i in perm])

    return shuffled

def read_and_decode(fqueue, batch_size, canvas_size, num_threads):
    reader = tf.TFRecordReader()
    key, value = reader.read(fqueue)

    features = tf.parse_single_example(
        value,
        features={
            'image': tf.FixedLenFeature([], tf.string),
            'digits': tf.FixedLenFeature([], tf.int64)
        }
    )

    batch = tf.train.shuffle_batch(
        [
            tf.reshape(tf.decode_raw(features['image'], tf.float32), [canvas_size * canvas_size]),
            tf.cast(features['digits'], tf.int32)
        ],
        batch_size=batch_size,
        capacity=10000+batch_size*10,
        min_after_dequeue=10000,
        num_threads=num_threads
    )

    return batch

def read_test_data(filename):
    record_iterator = tf.python_io.tf_record_iterator(path=filename)

    images_list, digits_list = [], []
    for string_record in record_iterator:
        example = tf.train.Example()
        example.ParseFromString(string_record)

        images_list.append(np.fromstring(example.features.feature['image'].bytes_list.value[0], dtype=np.float32))
        digits_list.append(int(example.features.feature['digits'].int64_list.value[0]))

    return np.array(images_list), np.array(digits_list)

if __name__ == "__main__":

    DEFAULT_MAX_DIGITS = 3
    DEFAULT_MAX_IN_COMMON = 2
    DEFAULT_IMAGES_PER_DIGIT = 20000
    DEFAULT_TEST_SET_SIZE = 1000

    MNIST_FOLDER = "mnist_data/"
    MULTI_MNIST_FOLDER = "multi_mnist_data/"

    CANVAS_SIZE = 50
    IMAGE_SIZE = 28

    parser = argparse.ArgumentParser()
    parser.add_argument("--max-digits", type=int, choices=list(range(7)), default=DEFAULT_MAX_DIGITS)
    parser.add_argument("--max-in-common", type=int, choices=list(range(7)), default=DEFAULT_MAX_IN_COMMON)
    parser.add_argument("--images-per-digit", type=int, default=DEFAULT_IMAGES_PER_DIGIT)
    parser.add_argument("--test-set-size", type=int, default=DEFAULT_TEST_SET_SIZE)
    parser.add_argument("--digit-box-gap", type=int, default=0)
    parser.add_argument("--canvas-margin", type=int, default=0)
    parser.add_argument("--bg-path", default="")
    parser.add_argument("--bg-max-intensity", type=float, default=1.0)
    parser.add_argument("--min-width-scale", type=float, default=1.0)
    parser.add_argument("--max-width-scale", type=float, default=1.0)
    parser.add_argument("--min-height-scale", type=float, default=1.0)
    parser.add_argument("--max-height-scale", type=float, default=1.0)
    parser.add_argument("--min-rotation-angle", type=float, default=0.0)
    parser.add_argument("--max-rotation-angle", type=float, default=0.0)
    args = parser.parse_args()

    if not os.path.exists(MULTI_MNIST_FOLDER):
        os.makedirs(MULTI_MNIST_FOLDER)

    background = read_image(args.bg_path, args.bg_max_intensity) if args.bg_path != "" else None

    dataset = input_data.read_data_sets(MNIST_FOLDER, validation_size=0)

    common_images, common_indices, common_positions = [], [], []
    common_boxes, common_labels, common_digits = [], [], []

    np.random.seed(0)

    used_digit_ids = set([])
    for num_digits in range(args.max_digits + 1):
        strata_images, strata_indices = [], []
        strata_positions, strata_boxes = [], []
        strata_labels = []

        print()
        print("Generating {} digit images...".format(num_digits))
        for item in range(args.images_per_digit):
            img, ids, pos, box = generate_multi_image(
                dataset.train.images, num_digits, IMAGE_SIZE, CANVAS_SIZE, bg=background,
                min_w=args.min_width_scale, max_w=args.max_width_scale,
                min_h=args.min_height_scale, max_h=args.max_height_scale,
                min_ang=args.min_rotation_angle, max_ang=args.max_rotation_angle,
                gap=args.digit_box_gap, margin=args.canvas_margin
            )

            if num_digits <= args.max_in_common:
                for digit_id in ids:
                    used_digit_ids.add(digit_id)

            strata_images.append(img)
            strata_indices.append(ids)
            strata_positions.append(pos)
            strata_boxes.append(box)
            strata_labels.append(list(dataset.train.labels[ids]))

            if (item + 1) % 1000 == 0:
                print("{0} done".format(item + 1))
        print()

        strata_digits = [num_digits] * args.images_per_digit

        if num_digits <= args.max_in_common:
            common_images.extend(strata_images)
            common_indices.extend(strata_indices)
            common_positions.extend(strata_positions)
            common_boxes.extend(strata_boxes)
            common_labels.extend(strata_labels)
            common_digits.extend(strata_digits)

        print("Writing {} digit images... ".format(num_digits), end="", flush=True)
        write_to_records(MULTI_MNIST_FOLDER + str(num_digits), strata_images, strata_indices,
                         strata_positions, strata_boxes, strata_labels, strata_digits)
        print("done")

        if num_digits == args.max_in_common:
            common_images, common_indices, common_positions, \
              common_boxes, common_labels, common_digits = shuffle_lists(
                common_images, common_indices, common_positions, common_boxes, common_labels, common_digits
              )

            print()
            print("{0} MNIST digits used for 0-{1} digit images".format(len(used_digit_ids), args.max_in_common))
            print("Writing 0-{} digit images to common file... ".format(args.max_in_common), end="", flush=True)
            write_to_records(MULTI_MNIST_FOLDER + "common",
                             common_images[args.test_set_size:], common_indices[args.test_set_size:],
                             common_positions[args.test_set_size:], common_boxes[args.test_set_size:],
                             common_labels[args.test_set_size:], common_digits[args.test_set_size:])
            print("done")

            print("Writing 0-{} digit images to test file... ".format(args.max_in_common), end="", flush=True)
            write_to_records(MULTI_MNIST_FOLDER + "test",
                             common_images[:args.test_set_size], common_indices[:args.test_set_size],
                             common_positions[:args.test_set_size], common_boxes[:args.test_set_size],
                             common_labels[:args.test_set_size], common_digits[:args.test_set_size])
            print("done")
<commit_msg>The new algorithm of picking random MNIST digits to be placed on a multi-MNIST canvas has been implemented to improve the coverage of MNIST digits. Now instead of randomly sampling each image from MNIST, the whole dataset is randomly permuted and then its elements are tried in order. When elements are used, MNIST is shuffled and used in order again.<commit_after>import os
import argparse

import numpy as np
import scipy.ndimage as nd
import tensorflow as tf

from tensorflow.examples.tutorials.mnist import input_data

def show_image(image):
    import matplotlib.pyplot as plt
    plt.imshow(image, cmap="gray", vmin=0.0, vmax=1.0)
    plt.show()

def read_image(path, max_intensity):
    image = nd.imread(path, mode="L")
    image = np.asarray(image, dtype=np.float32) / 255.0
    img_min, img_max = image.min(), image.max()

    if img_min != img_max:
        if img_min > 0.0:
            image -= img_min
        if img_max > 0.0:
            image /= img_max
        if max_intensity < 1.0:
            image *= max_intensity
    else:
        if img_max > max_intensity:
            image = np.ones_like(image) * max_intensity

    return image

def crop_non_empty(image):
    non_zero_row_ids = np.array(np.nonzero(np.sum(image, axis=0)))[0]
    non_zero_col_ids = np.array(np.nonzero(np.sum(image, axis=1)))[0]
    x_start, x_end = non_zero_row_ids[0], non_zero_row_ids[-1]
    y_start, y_end = non_zero_col_ids[0], non_zero_col_ids[-1]

    return image[y_start:y_end+1, x_start:x_end+1]

def overlaps(x, y, w, h, positions, boxes, gap):
    for i in range(len(positions) // 2):
        p, b = positions[i*2:(i+1)*2], boxes[i*2:(i+1)*2]
        l1x, l1y, r1x, r1y = x-gap, y-gap, x+w+gap-1, y+h+gap-1
        l2x, l2y, r2x, r2y = p[0], p[1], p[0]+b[0]-1, p[1]+b[1]-1

        if l1x <= r2x and l2x <= r1x:
            return True
        if l1y >= r2y and l2y >= r1y:
            return True

    return False

def generate_multi_image(single_images, num_images, image_dim, canvas_dim, bg=None,
                         min_w=1.0, max_w=1.0, min_h=1.0, max_h=1.0, min_ang=0.0, max_ang=0.0,
                         gap=0, margin=0):

    global digit_ids, next_digit_id, used_digit_ids

    ready = False
    while not ready:
        canvas = np.zeros(
            [canvas_dim, canvas_dim],
            dtype=single_images[0].dtype
        )

        placed_image_ids = []
        placed_image_positions = []
        placed_image_boxes = []

        if num_digits == 0:
            break

        try:
            for i in range(num_images):
                idx = digit_ids[next_digit_id]
                next_digit_id += 1
                if next_digit_id >= len(digit_ids):
                    digit_ids = np.random.permutation(digit_ids)
                    next_digit_id = 0

                image = np.reshape(single_images[idx], [image_dim, image_dim])
                image = crop_non_empty(image)

                if min_w != 1.0 or max_w != 1.0 or min_h != 1.0 or max_h != 1.0:
                    new_width = np.random.uniform(min_w, max_w)
                    new_height = np.random.uniform(min_h, max_h)

                    image = nd.affine_transform(
                        image,
                        matrix=np.array([[1.0 / new_height, 0.0], [0.0, 1.0 / new_width]]),
                        output_shape=(int(image_dim * new_height), int(image_dim * new_width)),
                        order=5
                    )

                    image = np.clip(image, 0.0, 1.0)
                    image = np.where(image >= 0.05, image, np.zeros_like(image))
                    image = crop_non_empty(image)

                if min_ang != 0.0 or max_ang != 0.0:
                    new_angle = np.random.uniform(min_ang, max_ang)

                    image = nd.rotate(image, new_angle, order=5)

                    image = np.clip(image, 0.0, 1.0)
                    image = np.where(image >= 0.05, image, np.zeros_like(image))
                    image = crop_non_empty(image)

                h, w = image.shape
                position_find_attempts = 0
                position_found = False

                while position_find_attempts < 100:
                    x = np.random.randint(margin, canvas_dim - w - margin + 1)
                    y = np.random.randint(margin, canvas_dim - h - margin + 1)

                    if i == 0 or not overlaps(x, y, w, h, placed_image_positions, placed_image_boxes, gap):
                        position_found = True
                        break

                    position_find_attempts += 1

                if position_found:
                    canvas[y:y+h, x:x+w] += image

                    placed_image_positions.extend([x, y])
                    placed_image_boxes.extend([w, h])
                    placed_image_ids.append(idx)

                    if i == num_images - 1:
                        ready = True
                else:
                    break

        except IndexError:
            pass

    if bg is not None:
        canvas = np.clip(canvas + bg, 0.0, 1.0)

    return canvas, placed_image_ids, placed_image_positions, placed_image_boxes

def write_to_records(filename, images, indices, positions, boxes, labels, digits):
    def _int64_list_feature(value):
        return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

    def _bytes_feature(value):
        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

    rows, cols = images[0].shape

    writer = tf.python_io.TFRecordWriter(filename + ".tfrecords")
    for index in range(len(images)):
        example = tf.train.Example(
            features=tf.train.Features(
                feature={
                    "height": _int64_list_feature([rows]),
                    "width": _int64_list_feature([cols]),
                    "digits": _int64_list_feature([digits[index]]),
                    "indices": _bytes_feature(np.asarray(indices[index], dtype=np.int32).tostring()),
                    "positions": _bytes_feature(np.asarray(positions[index], dtype=np.int32).tostring()),
                    "boxes": _bytes_feature(np.asarray(boxes[index], dtype=np.int32).tostring()),
                    "labels": _bytes_feature(np.asarray(labels[index], dtype=np.int32).tostring()),
                    "image": _bytes_feature(np.ravel(images[index]).tostring())
                }
            )
        )
        writer.write(example.SerializeToString())
    writer.close()

def shuffle_lists(*lists):
    shuffled = []

    perm = np.random.permutation(
        range(len(lists[0]))
    )

    for l in lists:
        shuffled.append(np.array([l[i] for i in perm]))

    return shuffled

def read_and_decode(fqueue, batch_size, canvas_size, num_threads):
    reader = tf.TFRecordReader()
    key, value = reader.read(fqueue)

    features = tf.parse_single_example(
        value,
        features={
            'image': tf.FixedLenFeature([], tf.string),
            'digits': tf.FixedLenFeature([], tf.int64)
        }
    )

    batch = tf.train.shuffle_batch(
        [
            tf.reshape(tf.decode_raw(features['image'], tf.float32), [canvas_size * canvas_size]),
            tf.cast(features['digits'], tf.int32)
        ],
        batch_size=batch_size,
        capacity=10000+batch_size*10,
        min_after_dequeue=10000,
        num_threads=num_threads
    )

    return batch

def read_test_data(filename):
    record_iterator = tf.python_io.tf_record_iterator(path=filename)

    images_list, digits_list = [], []
    for string_record in record_iterator:
        example = tf.train.Example()
        example.ParseFromString(string_record)

        images_list.append(np.fromstring(example.features.feature['image'].bytes_list.value[0], dtype=np.float32))
        digits_list.append(int(example.features.feature['digits'].int64_list.value[0]))

    return np.array(images_list), np.array(digits_list)

if __name__ == "__main__":

    DEFAULT_MAX_DIGITS = 3
    DEFAULT_MAX_IN_COMMON = 2
    DEFAULT_IMAGES_PER_DIGIT = 20000
    DEFAULT_TEST_SET_SIZE = 1000

    MNIST_FOLDER = "mnist_data/"
    MULTI_MNIST_FOLDER = "multi_mnist_data/"

    CANVAS_SIZE = 50
    IMAGE_SIZE = 28

    parser = argparse.ArgumentParser()
    parser.add_argument("--max-digits", type=int, choices=list(range(7)), default=DEFAULT_MAX_DIGITS)
    parser.add_argument("--max-in-common", type=int, choices=list(range(7)), default=DEFAULT_MAX_IN_COMMON)
    parser.add_argument("--images-per-digit", type=int, default=DEFAULT_IMAGES_PER_DIGIT)
    parser.add_argument("--test-set-size", type=int, default=DEFAULT_TEST_SET_SIZE)
    parser.add_argument("--digit-box-gap", type=int, default=0)
    parser.add_argument("--canvas-margin", type=int, default=0)
    parser.add_argument("--bg-path", default="")
    parser.add_argument("--bg-max-intensity", type=float, default=1.0)
    parser.add_argument("--min-width-scale", type=float, default=1.0)
    parser.add_argument("--max-width-scale", type=float, default=1.0)
    parser.add_argument("--min-height-scale", type=float, default=1.0)
    parser.add_argument("--max-height-scale", type=float, default=1.0)
    parser.add_argument("--min-rotation-angle", type=float, default=0.0)
    parser.add_argument("--max-rotation-angle", type=float, default=0.0)
    args = parser.parse_args()

    if not os.path.exists(MULTI_MNIST_FOLDER):
        os.makedirs(MULTI_MNIST_FOLDER)

    background = read_image(args.bg_path, args.bg_max_intensity) if args.bg_path != "" else None

    dataset = input_data.read_data_sets(MNIST_FOLDER, validation_size=0)

    common_images, common_indices, common_positions = [], [], []
    common_boxes, common_labels, common_digits = [], [], []

    np.random.seed(0)

    next_digit_id = 0
    used_digit_ids = set([])
    digit_ids = [i for i in range(len(dataset.train.images))]
    digit_ids = np.random.permutation(digit_ids)

    for num_digits in range(args.max_digits + 1):
        strata_images, strata_indices = [], []
        strata_positions, strata_boxes = [], []
        strata_labels = []

        print()
        print("Generating {} digit images...".format(num_digits))
        for item in range(args.images_per_digit):
            img, ids, pos, box = generate_multi_image(
                dataset.train.images, num_digits, IMAGE_SIZE, CANVAS_SIZE, bg=background,
                min_w=args.min_width_scale, max_w=args.max_width_scale,
                min_h=args.min_height_scale, max_h=args.max_height_scale,
                min_ang=args.min_rotation_angle, max_ang=args.max_rotation_angle,
                gap=args.digit_box_gap, margin=args.canvas_margin
            )

            if num_digits <= args.max_in_common:
                for digit_id in ids:
                    used_digit_ids.add(digit_id)

            strata_images.append(img)
            strata_indices.append(ids)
            strata_positions.append(pos)
            strata_boxes.append(box)
            strata_labels.append(list(dataset.train.labels[ids]))

            if (item + 1) % 1000 == 0:
                print("{0} done".format(item + 1))
        print()

        strata_digits = [num_digits] * args.images_per_digit

        if num_digits <= args.max_in_common:
            common_images.extend(strata_images)
            common_indices.extend(strata_indices)
            common_positions.extend(strata_positions)
            common_boxes.extend(strata_boxes)
            common_labels.extend(strata_labels)
            common_digits.extend(strata_digits)

        print("Writing {} digit images... ".format(num_digits), end="", flush=True)
        write_to_records(MULTI_MNIST_FOLDER + str(num_digits), strata_images, strata_indices,
                         strata_positions, strata_boxes, strata_labels, strata_digits)
        print("done")

        if num_digits == args.max_in_common:
            common_images, common_indices, common_positions, \
              common_boxes, common_labels, common_digits = shuffle_lists(
                common_images, common_indices, common_positions, common_boxes, common_labels, common_digits
              )

            print()
            print("{0} MNIST digits used for 0-{1} digit images".format(len(used_digit_ids), args.max_in_common))
            print("Writing 0-{} digit images to common file... ".format(args.max_in_common), end="", flush=True)
            write_to_records(MULTI_MNIST_FOLDER + "common",
                             common_images[args.test_set_size:], common_indices[args.test_set_size:],
                             common_positions[args.test_set_size:], common_boxes[args.test_set_size:],
                             common_labels[args.test_set_size:], common_digits[args.test_set_size:])
            print("done")

            print("Writing 0-{} digit images to test file... ".format(args.max_in_common), end="", flush=True)
            write_to_records(MULTI_MNIST_FOLDER + "test",
                             common_images[:args.test_set_size], common_indices[:args.test_set_size],
                             common_positions[:args.test_set_size], common_boxes[:args.test_set_size],
                             common_labels[:args.test_set_size], common_digits[:args.test_set_size])
            print("done")
<|endoftext|>