<commit_before>#!/usr/bin/env python
# -*- coding: utf-8; py-indent-offset:4 -*-
###############################################################################
#
# Copyright (C) 2016 Bitquant Research Laboratories (Asia) Limited
#
# Released under Simplified BSD License
#
###############################################################################
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import datetime
import time
import spstore
import io
from backtrader.feeds import feed
from backtrader.utils import date2num
from backtrader.utils.py3 import with_metaclass

class MetaSharpPointData(feed.MetaCSVDataBase):
    def __init__(cls, name, bases, dct):
        '''Class has already been created ... register'''
        # Initialize the class
        super(MetaSharpPointData, cls).__init__(name, bases, dct)
        spstore.SharpPointStore.DataCls = cls

class SharpPointCSVData(with_metaclass(MetaSharpPointData, feed.CSVDataBase)):
    '''
    Parses a self-defined CSV Data used for testing.

    Specific parameters:

      - ``dataname``: The filename to parse or a file-like object
      - ``datasource`` : Product id
    '''
    params = (('nullvalue', float('NaN')),
              ('tickersource', None),
              ('newdata', False),
              ('keepalive', False),
              ('debug', True),
              ('streaming', False))

    def __init__(self, **kwargs):
        super(SharpPointCSVData, self).__init__()
        self.o = spstore.SharpPointStore(**kwargs)

    def start(self):
        if self.f is None:
            if self.p.tickersource is None:
                dataname = self.p.dataname
            else:
                dataname = self.p.tickersource
            if hasattr(dataname, 'readline'):
                self.f =dataname
            else:
                # Let an exception propagate to let the caller know
                self.f = io.open(dataname, 'r')

        super(SharpPointCSVData, self).start()
        if self.p.newdata:
            self.f.seek(0, 2)
            self.forward()
            dt = datetime.datetime.now()
            self.lines.datetime[0] = date2num(dt)
            self.lines.open[0] = self.p.nullvalue
            self.lines.high[0] = self.p.nullvalue
            self.lines.low[0] = self.p.nullvalue
            self.lines.close[0] = self.p.nullvalue
            self.lines.volume[0] = self.p.nullvalue
            self.lines.openinterest[0] = self.p.nullvalue
        if self.p.streaming:
            print("start price streaming")
            self.o.streaming_prices(self.p.dataname)

    def _load(self):
        if self.f is None:
            return False

        # Let an exception propagate to let the caller know
        while True:
            line = self.f.readline()
            if not line:
                if not self.p.keepalive:
                    return False
                else:
                    time.sleep(0.1)
                    continue
            else:
                line = line.rstrip('\n')
                if self.p.debug:
                    print(line)
                linetokens = line.split(self.separator)
                if linetokens[4] == self.p.dataname:
                    return self._loadline(linetokens)

    def islive(self):
        return self.p.keepalive

    def _loadline(self, linetokens):
        itoken = iter(linetokens)

        price = float(next(itoken))
        volume = float(next(itoken))
        dtnum = float(next(itoken))
        dt = datetime.datetime.fromtimestamp(dtnum)
        self.lines.datetime[0] = date2num(dt)
        self.lines.open[0] = price
        self.lines.high[0] = price
        self.lines.low[0] = price
        self.lines.close[0] = price
        self.lines.volume[0] = volume
        self.lines.openinterest[0] = 0.0
        return True

class SharpPointCSV(feed.CSVFeedBase):
    DataCls = SharpPointCSVData
<commit_msg>default spfeed debug to false<commit_after>#!/usr/bin/env python
# -*- coding: utf-8; py-indent-offset:4 -*-
###############################################################################
#
# Copyright (C) 2016 Bitquant Research Laboratories (Asia) Limited
#
# Released under Simplified BSD License
#
###############################################################################
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import datetime
import time
import spstore
import io
from backtrader.feeds import feed
from backtrader.utils import date2num
from backtrader.utils.py3 import with_metaclass

class MetaSharpPointData(feed.MetaCSVDataBase):
    def __init__(cls, name, bases, dct):
        '''Class has already been created ... register'''
        # Initialize the class
        super(MetaSharpPointData, cls).__init__(name, bases, dct)
        spstore.SharpPointStore.DataCls = cls

class SharpPointCSVData(with_metaclass(MetaSharpPointData, feed.CSVDataBase)):
    '''
    Parses a self-defined CSV Data used for testing.

    Specific parameters:

      - ``dataname``: The filename to parse or a file-like object
      - ``datasource`` : Product id
    '''
    params = (('nullvalue', float('NaN')),
              ('tickersource', None),
              ('newdata', False),
              ('keepalive', False),
              ('debug', False),
              ('streaming', False))

    def __init__(self, **kwargs):
        super(SharpPointCSVData, self).__init__()
        self.o = spstore.SharpPointStore(**kwargs)

    def start(self):
        if self.f is None:
            if self.p.tickersource is None:
                dataname = self.p.dataname
            else:
                dataname = self.p.tickersource
            if hasattr(dataname, 'readline'):
                self.f =dataname
            else:
                # Let an exception propagate to let the caller know
                self.f = io.open(dataname, 'r')

        super(SharpPointCSVData, self).start()
        if self.p.newdata:
            self.f.seek(0, 2)
            self.forward()
            dt = datetime.datetime.now()
            self.lines.datetime[0] = date2num(dt)
            self.lines.open[0] = self.p.nullvalue
            self.lines.high[0] = self.p.nullvalue
            self.lines.low[0] = self.p.nullvalue
            self.lines.close[0] = self.p.nullvalue
            self.lines.volume[0] = self.p.nullvalue
            self.lines.openinterest[0] = self.p.nullvalue
        if self.p.streaming:
            print("start price streaming")
            self.o.streaming_prices(self.p.dataname)

    def _load(self):
        if self.f is None:
            return False

        # Let an exception propagate to let the caller know
        while True:
            line = self.f.readline()
            if not line:
                if not self.p.keepalive:
                    return False
                else:
                    time.sleep(0.1)
                    continue
            else:
                line = line.rstrip('\n')
                if self.p.debug:
                    print(line)
                linetokens = line.split(self.separator)
                if linetokens[4] == self.p.dataname:
                    return self._loadline(linetokens)

    def islive(self):
        return self.p.keepalive

    def _loadline(self, linetokens):
        itoken = iter(linetokens)

        price = float(next(itoken))
        volume = float(next(itoken))
        dtnum = float(next(itoken))
        dt = datetime.datetime.fromtimestamp(dtnum)
        self.lines.datetime[0] = date2num(dt)
        self.lines.open[0] = price
        self.lines.high[0] = price
        self.lines.low[0] = price
        self.lines.close[0] = price
        self.lines.volume[0] = volume
        self.lines.openinterest[0] = 0.0
        return True

class SharpPointCSV(feed.CSVFeedBase):
    DataCls = SharpPointCSVData
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os
import time
import sys
import subprocess

import cereal.messaging as messaging
from common.basedir import BASEDIR
from common.params import Params
from selfdrive.test.helpers import set_params_enabled

def cputime_total(ct):
  return ct.cpuUser + ct.cpuSystem + ct.cpuChildrenUser + ct.cpuChildrenSystem

def print_cpu_usage(first_proc, last_proc):
  procs = [
    ("selfdrive.controls.controlsd", 45.0),
    ("./loggerd", 33.90),
    ("selfdrive.locationd.locationd", 29.5),
    ("selfdrive.controls.plannerd", 11.84),
    ("selfdrive.locationd.paramsd", 10.5),
    ("./_modeld", 7.12),
    ("selfdrive.controls.radard", 9.54),
    ("./camerad", 7.07),
    ("./_sensord", 6.17),
    ("./_ui", 5.82),
    ("./boardd", 3.63),
    ("./_dmonitoringmodeld", 2.67),
    ("selfdrive.logmessaged", 2.71),
    ("selfdrive.thermald.thermald", 2.41),
    ("selfdrive.locationd.calibrationd", 2.0),
    ("selfdrive.monitoring.dmonitoringd", 1.90),
    ("./proclogd", 1.54),
    ("./_gpsd", 0.09),
    ("./clocksd", 0.02),
    ("./ubloxd", 0.02),
    ("selfdrive.tombstoned", 0),
    ("./logcatd", 0),
  ]

  r = True
  dt = (last_proc.logMonoTime - first_proc.logMonoTime) / 1e9
  result = "------------------------------------------------\n"
  for proc_name, normal_cpu_usage in procs:
    try:
      first = [p for p in first_proc.procLog.procs if proc_name in p.cmdline][0]
      last = [p for p in last_proc.procLog.procs if proc_name in p.cmdline][0]
      cpu_time = cputime_total(last) - cputime_total(first)
      cpu_usage = cpu_time / dt * 100.
      if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0):
        result += f"Warning {proc_name} using more CPU than normal\n"
        r = False
      elif cpu_usage < min(normal_cpu_usage * 0.65, max(normal_cpu_usage - 1.0, 0.0)):
        result += f"Warning {proc_name} using less CPU than normal\n"
        r = False
      result += f"{proc_name.ljust(35)}  {cpu_usage:.2f}%\n"
    except IndexError:
      result += f"{proc_name.ljust(35)}  NO METRICS FOUND\n"
      r = False
  result += "------------------------------------------------\n"
  print(result)
  return r

def test_cpu_usage():
  cpu_ok = False

  # start manager
  manager_path = os.path.join(BASEDIR, "selfdrive/manager.py")
  manager_proc = subprocess.Popen(["python", manager_path])
  try:
    proc_sock = messaging.sub_sock('procLog', conflate=True, timeout=2000)

    # wait until everything's started
    start_time = time.monotonic()
    while time.monotonic() - start_time < 210:
      if Params().get("CarParams") is not None:
        break
      time.sleep(2)

    # take first sample
    time.sleep(5)
    first_proc = messaging.recv_sock(proc_sock, wait=True)
    if first_proc is None:
      raise Exception("\n\nTEST FAILED: progLog recv timed out\n\n")

    # run for a minute and get last sample
    time.sleep(60)
    last_proc = messaging.recv_sock(proc_sock, wait=True)
    cpu_ok = print_cpu_usage(first_proc, last_proc)
  finally:
    manager_proc.terminate()
    ret = manager_proc.wait(20)
    if ret is None:
      manager_proc.kill()
  return cpu_ok

if __name__ == "__main__":
  set_params_enabled()
  Params().delete("CarParams")

  passed = False
  try:
    passed = test_cpu_usage()
  finally:
    sys.exit(int(not passed))
<commit_msg>Update radard cpu usage<commit_after>#!/usr/bin/env python3
import os
import time
import sys
import subprocess

import cereal.messaging as messaging
from common.basedir import BASEDIR
from common.params import Params
from selfdrive.test.helpers import set_params_enabled

def cputime_total(ct):
  return ct.cpuUser + ct.cpuSystem + ct.cpuChildrenUser + ct.cpuChildrenSystem

def print_cpu_usage(first_proc, last_proc):
  procs = [
    ("selfdrive.controls.controlsd", 45.0),
    ("./loggerd", 33.90),
    ("selfdrive.locationd.locationd", 29.5),
    ("selfdrive.controls.plannerd", 11.84),
    ("selfdrive.locationd.paramsd", 10.5),
    ("./_modeld", 7.12),
    ("./camerad", 7.07),
    ("./_sensord", 6.17),
    ("./_ui", 5.82),
    ("selfdrive.controls.radard", 5.67),
    ("./boardd", 3.63),
    ("./_dmonitoringmodeld", 2.67),
    ("selfdrive.logmessaged", 2.71),
    ("selfdrive.thermald.thermald", 2.41),
    ("selfdrive.locationd.calibrationd", 2.0),
    ("selfdrive.monitoring.dmonitoringd", 1.90),
    ("./proclogd", 1.54),
    ("./_gpsd", 0.09),
    ("./clocksd", 0.02),
    ("./ubloxd", 0.02),
    ("selfdrive.tombstoned", 0),
    ("./logcatd", 0),
  ]

  r = True
  dt = (last_proc.logMonoTime - first_proc.logMonoTime) / 1e9
  result = "------------------------------------------------\n"
  for proc_name, normal_cpu_usage in procs:
    try:
      first = [p for p in first_proc.procLog.procs if proc_name in p.cmdline][0]
      last = [p for p in last_proc.procLog.procs if proc_name in p.cmdline][0]
      cpu_time = cputime_total(last) - cputime_total(first)
      cpu_usage = cpu_time / dt * 100.
      if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0):
        result += f"Warning {proc_name} using more CPU than normal\n"
        r = False
      elif cpu_usage < min(normal_cpu_usage * 0.65, max(normal_cpu_usage - 1.0, 0.0)):
        result += f"Warning {proc_name} using less CPU than normal\n"
        r = False
      result += f"{proc_name.ljust(35)}  {cpu_usage:.2f}%\n"
    except IndexError:
      result += f"{proc_name.ljust(35)}  NO METRICS FOUND\n"
      r = False
  result += "------------------------------------------------\n"
  print(result)
  return r

def test_cpu_usage():
  cpu_ok = False

  # start manager
  manager_path = os.path.join(BASEDIR, "selfdrive/manager.py")
  manager_proc = subprocess.Popen(["python", manager_path])
  try:
    proc_sock = messaging.sub_sock('procLog', conflate=True, timeout=2000)

    # wait until everything's started
    start_time = time.monotonic()
    while time.monotonic() - start_time < 210:
      if Params().get("CarParams") is not None:
        break
      time.sleep(2)

    # take first sample
    time.sleep(5)
    first_proc = messaging.recv_sock(proc_sock, wait=True)
    if first_proc is None:
      raise Exception("\n\nTEST FAILED: progLog recv timed out\n\n")

    # run for a minute and get last sample
    time.sleep(60)
    last_proc = messaging.recv_sock(proc_sock, wait=True)
    cpu_ok = print_cpu_usage(first_proc, last_proc)
  finally:
    manager_proc.terminate()
    ret = manager_proc.wait(20)
    if ret is None:
      manager_proc.kill()
  return cpu_ok

if __name__ == "__main__":
  set_params_enabled()
  Params().delete("CarParams")

  passed = False
  try:
    passed = test_cpu_usage()
  finally:
    sys.exit(int(not passed))
<|endoftext|>


<commit_before>import requests
import hmac
import hashlib
import uuid
import json
from log import log

from requests.adapters import HTTPAdapter
from urllib3.poolmanager import PoolManager
import ssl

class SSLAdapter(HTTPAdapter):
    '''An HTTPS Transport Adapter that uses an arbitrary SSL version.'''
    def __init__(self, ssl_version=None, **kwargs):
        self.ssl_version = ssl_version

        super(SSLAdapter, self).__init__(**kwargs)

    def init_poolmanager(self, connections, maxsize, block=False):
        self.poolmanager = PoolManager(num_pools=connections,
                                       maxsize=maxsize,
                                       block=block,
                                       ssl_version=self.ssl_version)

def report_status(server, apikey, secret, verify, **kwargs):
    """
    Reports status via the HTTPS API. Given the server, API key and
    secret, this method generates the hash_result of a nonce.

    Pass hostname, status, and info in the kwargs. Raises an exception
    when a non-200 result code is returned.

    Keyword arguments:
        server -- The server to report status to. Can be host:port
        apikey -- The API key to use
        secret -- The API secret associated with this API key
        verify -- Whether or not to verify the SSL connection
        hostname -- The hostname to identify this server by
        status -- OK or ERROR
        info -- Dictionary of more information
    """
    raw_secret = secret.decode('hex')
    nonce = str(uuid.uuid4())

    hash_result = hmac.new(raw_secret, nonce, hashlib.sha256).hexdigest()

    log.debug("Got body: {}".format(kwargs))

    # Requests doesn't handle TLSv1 by default
    s = requests.Session()
    s.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))

    resp = s.post('https://{}/update/{}/{}'.format(server, apikey, hash_result),
                data=json.dumps(kwargs),
                headers={
                    'Content-Type':'application/json',
                    'X-Nonce':nonce},
                verify=False)

    resp.raise_for_status()
<commit_msg>Revert "Change package for status, hopefully fixes bug"<commit_after>import requests
import hmac
import hashlib
import uuid
import json
from log import log

from requests.adapters import HTTPAdapter
from requests.packages.urllib3.poolmanager import PoolManager
import ssl

class SSLAdapter(HTTPAdapter):
    '''An HTTPS Transport Adapter that uses an arbitrary SSL version.'''
    def __init__(self, ssl_version=None, **kwargs):
        self.ssl_version = ssl_version

        super(SSLAdapter, self).__init__(**kwargs)

    def init_poolmanager(self, connections, maxsize, block=False):
        self.poolmanager = PoolManager(num_pools=connections,
                                       maxsize=maxsize,
                                       block=block,
                                       ssl_version=self.ssl_version)

def report_status(server, apikey, secret, verify, **kwargs):
    """
    Reports status via the HTTPS API. Given the server, API key and
    secret, this method generates the hash_result of a nonce.

    Pass hostname, status, and info in the kwargs. Raises an exception
    when a non-200 result code is returned.

    Keyword arguments:
        server -- The server to report status to. Can be host:port
        apikey -- The API key to use
        secret -- The API secret associated with this API key
        verify -- Whether or not to verify the SSL connection
        hostname -- The hostname to identify this server by
        status -- OK or ERROR
        info -- Dictionary of more information
    """
    raw_secret = secret.decode('hex')
    nonce = str(uuid.uuid4())

    hash_result = hmac.new(raw_secret, nonce, hashlib.sha256).hexdigest()

    log.debug("Got body: {}".format(kwargs))

    # Requests doesn't handle TLSv1 by default
    s = requests.Session()
    s.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))

    resp = s.post('https://{}/update/{}/{}'.format(server, apikey, hash_result),
                data=json.dumps(kwargs),
                headers={
                    'Content-Type':'application/json',
                    'X-Nonce':nonce},
                verify=False)

    resp.raise_for_status()
<|endoftext|>


<commit_before>import chainer
from chainer import cuda
import chainer.functions as F
import chainer.links as L

import numpy as np
from crop import crop

class RAM(chainer.Chain):
    def __init__(self, n_e=128, n_h=256, g_size=8, n_step=6,
                 scale=1, var=0.01, use_lstm=False):
        self.n_h = n_h
        self.g_size = g_size
        self.n_step = n_step
        self.scale = scale
        self.var = var
        self.use_lstm = use_lstm

        if self.use_lstm:
            super(RAM, self).__init__(
                emb_l=L.Linear(2, n_e), # embed location
                emb_x=L.Linear(g_size*g_size*scale, n_e), # embed image
                fc_lg=L.Linear(n_e, n_h), # loc to glimpse
                fc_xg=L.Linear(n_e, n_h), # image to glimpse
                core_lstm=L.LSTM(n_h, n_h), # core LSTM
                fc_ha=L.Linear(n_h, 10), # core to action
                fc_hl=L.Linear(n_h, 2), # core to loc
                fc_hb=L.Linear(n_h, 1), # core to baseline
            )
        else:
            super(RAM, self).__init__(
                emb_l=L.Linear(2, n_e), # embed location
                emb_x=L.Linear(g_size*g_size*scale, n_e), # embed image
                fc_lg=L.Linear(n_e, n_h), # loc to glimpse
                fc_xg=L.Linear(n_e, n_h), # image to glimpse
                core_hh=L.Linear(n_h, n_h), # core rnn
                core_gh=L.Linear(n_h, n_h), # glimpse to core
                fc_ha=L.Linear(n_h, 10), # core to action
                fc_hl=L.Linear(n_h, 2), # core to loc
                fc_hb=L.Linear(n_h, 1), # core to baseline
            )

    def clear(self, bs, train):
        self.loss = None
        self.accuracy = None

        # init internal state of core RNN h
        if self.use_lstm:
            self.core_lstm.reset_state()
        else:
            self.h = chainer.Variable(
                self.xp.zeros(shape=(bs,self.n_h), dtype=self.xp.float32),
                volatile=not train)

    def __call__(self, x, t, train=True):
        bs = x.data.shape[0] # batch size
        self.clear(bs, train)
        mean_ln_p = 0

        # init mean location
        l = chainer.Variable(
            self.xp.asarray(
                np.random.uniform(-1, 1, size=(bs,2)).astype(self.xp.float32)),
            volatile=not train)

        # forward n_steps times
        for i in range(self.n_step - 1):
            l, ln_p = self.forward(x, l, train, action=False)[:2]
            if train:
                mean_ln_p += ln_p
        y, b = self.forward(x, l, train, action=True)[2:4]
        mean_ln_p /= self.n_step

        # loss with softmax cross entropy
        self.loss_action = F.softmax_cross_entropy(y, t)
        self.loss = self.loss_action
        self.accuracy = F.accuracy(y, t)

        if train:
            # reward
            r = (self.xp.where(self.xp.argmax(y.data,axis=1)==t.data,1.,0.)
                ).astype(self.xp.float32)
            # SE between reward and baseline
            self.loss_base = F.mean_squared_error(r, b)
            self.loss += self.loss_base
            # loss with reinforce rule
            self.loss_reinforce = 0.01 * F.sum(-mean_ln_p * (r-b.data))/bs
            self.loss += self.loss_reinforce
        return self.loss

    def forward(self, x, l, train, action):
        x.volatile = 'on'
        # Retina Encoding
        if self.xp == np:
            loc = l.data
        else:
            loc = self.xp.asnumpy(l.data)
        hg = crop(x, center=loc, size=self.g_size)
        # multi-scale glimpse
        for k in range(1, self.scale):
            s = np.power(2, k)
            patch = crop(x, center=loc, size=self.g_size*s)
            patch = F.average_pooling_2d(patch, ksize=s)
            hg = F.concat((hg, patch), axis=1)
        if train: hg.volatile = 'off'
        hg = F.relu(self.emb_x(hg))

        # Location Encoding
        hl = F.relu(self.emb_l(l))

        # Glimpse Net
        g = F.relu(self.fc_lg(hl) + self.fc_xg(hg))

        # Core Net
        if self.use_lstm:
            self.h = self.core_lstm(g)
        else:
            self.h = F.relu(self.core_hh(self.h) + self.core_gh(g))

        # Location Net: unchain h
        unchained_h = chainer.Variable(self.h.data, volatile=not train)
        m = F.tanh(self.fc_hl(unchained_h))

        if train:
            # generate sample from Normal(mean,var)
            eps = (self.xp.random.normal(0, 1, size=m.data.shape)
                  ).astype(self.xp.float32)
            l = m.data + np.sqrt(self.var)*eps # do not backward via l

            # get log(location policy)
            ln_p = -0.5 * F.sum(F.square(l-m), axis=1) / self.var

            l = chainer.Variable(l, volatile=not train)

        if action:
            # Action Net
            y = self.fc_ha(self.h)

            if train:
                # Baseline: unchain h
                b = self.fc_hb(unchained_h)
                b = F.reshape(b, (-1,))
                return l, ln_p, y, b
            else:
                return m, None, y, None
        else:
            if train:
                return l, ln_p, None, None
            else:
                return m, None, None, None

    def infer(self, x, init_l):
        train = False
        bs = 1 # batch size
        self.clear(bs, train)

        ys = self.xp.zeros(shape=(self.n_step,10), dtype=self.xp.float32)
        locs = self.xp.zeros(shape=(self.n_step,2), dtype=self.xp.float32)
        locs[0] = np.array(init_l)

        # forward
        l = chainer.Variable(
            self.xp.asarray(init_l).reshape(bs,2).astype(self.xp.float32),
            volatile=not train)

        for i in range(self.n_step - 1):
            l, _, y = self.forward(x, l, False, action=True)[0:3]
            locs[i+1] = l.data[0]
            ys[i] = F.softmax(y).data[0]

        y = self.forward(x, l, False, action=True)[2]
        ys[i+1] = F.softmax(y).data[0]
        y = self.xp.argmax(ys[i+1])

        if self.xp != np:
            ys = self.xp.asnumpy(ys)
            locs = self.xp.asnumpy(locs)

        return y, ys, locs
<commit_msg>backward reinforce grad to core<commit_after>import chainer
from chainer import cuda
import chainer.functions as F
import chainer.links as L
import numpy as np
from crop import crop

class RAM(chainer.Chain):
    def __init__(self, n_e=128, n_h=256, g_size=8, n_step=6,
                 scale=1, var=0.01, use_lstm=False):
        self.n_h = n_h
        self.g_size = g_size
        self.n_step = n_step
        self.scale = scale
        self.var = var
        self.use_lstm = use_lstm

        if self.use_lstm:
            super(RAM, self).__init__(
                emb_l=L.Linear(2, n_e), # embed location
                emb_x=L.Linear(g_size*g_size*scale, n_e), # embed image
                fc_lg=L.Linear(n_e, n_h), # loc to glimpse
                fc_xg=L.Linear(n_e, n_h), # image to glimpse
                core_lstm=L.LSTM(n_h, n_h), # core LSTM
                fc_ha=L.Linear(n_h, 10), # core to action
                fc_hl=L.Linear(n_h, 2), # core to loc
                fc_hb=L.Linear(n_h, 1), # core to baseline
            )
        else:
            super(RAM, self).__init__(
                emb_l=L.Linear(2, n_e), # embed location
                emb_x=L.Linear(g_size*g_size*scale, n_e), # embed image
                fc_lg=L.Linear(n_e, n_h), # loc to glimpse
                fc_xg=L.Linear(n_e, n_h), # image to glimpse
                core_hh=L.Linear(n_h, n_h), # core rnn
                core_gh=L.Linear(n_h, n_h), # glimpse to core
                fc_ha=L.Linear(n_h, 10), # core to action
                fc_hl=L.Linear(n_h, 2), # core to loc
                fc_hb=L.Linear(n_h, 1), # core to baseline
            )

    def clear(self, bs, train):
        self.loss = None
        self.accuracy = None

        # init internal state of core RNN h
        if self.use_lstm:
            self.core_lstm.reset_state()
        else:
            self.h = chainer.Variable(
                self.xp.zeros(shape=(bs,self.n_h), dtype=self.xp.float32),
                volatile=not train)

    def __call__(self, x, t, train=True):
        bs = x.data.shape[0] # batch size
        self.clear(bs, train)
        mean_ln_p = 0

        # init mean location
        l = chainer.Variable(
            self.xp.asarray(
                np.random.uniform(-1, 1, size=(bs,2)).astype(self.xp.float32)),
            volatile=not train)

        # forward n_steps times
        for i in range(self.n_step - 1):
            l, ln_p = self.forward(x, l, train, action=False)[:2]
            if train:
                mean_ln_p += ln_p
        y, b = self.forward(x, l, train, action=True)[2:4]
        mean_ln_p /= self.n_step

        # loss with softmax cross entropy
        self.loss_action = F.softmax_cross_entropy(y, t)
        self.loss = self.loss_action
        self.accuracy = F.accuracy(y, t)

        if train:
            # reward
            r = (self.xp.where(self.xp.argmax(y.data,axis=1)==t.data,1.,0.)
                ).astype(self.xp.float32)

            # SE between reward and baseline
            self.loss_base = F.mean_squared_error(r, b)
            self.loss += self.loss_base

            # loss with reinforce rule
            self.loss_reinforce = 0.01 * F.sum(-mean_ln_p * (r-b.data))/bs
            self.loss += self.loss_reinforce

        return self.loss

    def forward(self, x, l, train, action):
        # Retina Encoding
        x.volatile = 'on' # do not backward
        if self.xp == np:
            loc = l.data
        else:
            loc = self.xp.asnumpy(l.data)
        hg = crop(x, center=loc, size=self.g_size)

        # multi-scale glimpse
        for k in range(1, self.scale):
            s = np.power(2, k)
            patch = crop(x, center=loc, size=self.g_size*s)
            patch = F.average_pooling_2d(patch, ksize=s)
            hg = F.concat((hg, patch), axis=1)
        if train: hg.volatile = 'off' # backward upto link emb_x
        hg = F.relu(self.emb_x(hg))

        # Location Encoding
        hl = F.relu(self.emb_l(l))

        # Glimpse Net
        g = F.relu(self.fc_lg(hl) + self.fc_xg(hg))

        # Core Net
        if self.use_lstm:
            self.h = self.core_lstm(g)
        else:
            self.h = F.relu(self.core_hh(self.h) + self.core_gh(g))

        # Location Net
        m = F.tanh(self.fc_hl(self.h))

        if train:
            # sample from Normal(mean,var)
            eps = (self.xp.random.normal(0, 1, size=m.data.shape)
                  ).astype(self.xp.float32)
            l = m.data + np.sqrt(self.var)*eps

            # log(location policy)
            ln_p = -0.5 * F.sum(F.square(l-m), axis=1) / self.var
            l = chainer.Variable(l, volatile=not train) # do not backward via l
        else:
            l = m
            ln_p = None

        # Action Net
        if action:
            y = self.fc_ha(self.h)
        else:
            y = None

        # Baseline
        if train and action:
            b = F.sigmoid(self.fc_hb(self.h))
            b = F.reshape(b, (-1,))
        else:
            b = None

        return l, ln_p, y, b

    def infer(self, x, init_l):
        train = False
        bs = 1 # batch size
        self.clear(bs, train)

        ys = self.xp.zeros(shape=(self.n_step,10), dtype=self.xp.float32)
        locs = self.xp.zeros(shape=(self.n_step,2), dtype=self.xp.float32)
        locs[0] = np.array(init_l)

        # forward
        l = chainer.Variable(
            self.xp.asarray(init_l).reshape(bs,2).astype(self.xp.float32),
            volatile=not train)

        for i in range(self.n_step - 1):
            l, _, y = self.forward(x, l, False, action=True)[0:3]
            locs[i+1] = l.data[0]
            ys[i] = F.softmax(y).data[0]

        y = self.forward(x, l, False, action=True)[2]
        ys[i+1] = F.softmax(y).data[0]
        y = self.xp.argmax(ys[i+1])

        if self.xp != np:
            ys = self.xp.asnumpy(ys)
            locs = self.xp.asnumpy(locs)

        return y, ys, locs
<|endoftext|>


<commit_before>"""Recurrent Models of Visual Attention V. Mnih et al."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import tensorflow as tf
import numpy as np

from glimpseNetwork import GlimpseNet
from locationNetwork import LocNet
from glimpseSensor import GlimpseSensor

from utils import weight_variable, bias_variable, loglikelihood
from config import Config

from tensorflow.examples.tutorials.mnist import input_data

logging.getLogger().setLevel(logging.INFO)

rnn_cell = tf.contrib.rnn
seq2seq = tf.contrib.legacy_seq2seq

mnist = input_data.read_data_sets('MNIST_data', one_hot=False)

config = Config()
n_steps = config.step

loc_mean_arr = []
sampled_loc_arr = []

# placeholders
images_ph = tf.placeholder(tf.float32,
                           [None, config.original_size * config.original_size *
                            config.num_channels])
labels_ph = tf.placeholder(tf.int64, [None])

# Build the aux nets.
with tf.variable_scope('glimpse_net'):
    gl_sensor = GlimpseSensor(config)
    gl = GlimpseNet(gl_sensor, config, images_ph)
with tf.variable_scope('loc_net'):
    loc_net = LocNet(config)

def get_next_input(output, i):
    loc, loc_mean = loc_net(output)
    gl_next = gl(loc)
    loc_mean_arr.append(loc_mean)
    sampled_loc_arr.append(loc)
    return gl_next

# number of examples
N = tf.shape(images_ph)[0]

init_loc = tf.random_uniform((N, 2), minval=-1, maxval=1)
init_glimpse = gl(init_loc)
# Core network.
lstm_cell = rnn_cell.LSTMCell(config.cell_size, state_is_tuple=True)
init_state = lstm_cell.zero_state(N, tf.float32)
inputs = [init_glimpse]
inputs.extend([0] * (config.num_glimpses))
outputs, _ = seq2seq.rnn_decoder(
    inputs, init_state, lstm_cell, loop_function=get_next_input)

# Time independent baselines
with tf.variable_scope('baseline'):
    w_baseline = weight_variable((config.cell_output_size, 1))
    b_baseline = bias_variable((1,))
baselines = []
for t, output in enumerate(outputs[1:]):
    baseline_t = tf.nn.xw_plus_b(output, w_baseline, b_baseline)
    baseline_t = tf.squeeze(baseline_t)
    baselines.append(baseline_t)
baselines = tf.stack(baselines)  # [timesteps, batch_sz]
baselines = tf.transpose(baselines)  # [batch_sz, timesteps]

# Take the last step only.
output = outputs[-1]
# Build classification network.
with tf.variable_scope('cls'):
    w_logit = weight_variable((config.cell_output_size, config.num_classes))
    b_logit = bias_variable((config.num_classes,))
logits = tf.nn.xw_plus_b(output, w_logit, b_logit)
softmax = tf.nn.softmax(logits)

# cross-entropy.
xent = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels_ph)
xent = tf.reduce_mean(xent)
pred_labels = tf.argmax(logits, 1)
# 0/1 reward.
reward = tf.cast(tf.equal(pred_labels, labels_ph), tf.float32)
rewards = tf.expand_dims(reward, 1)  # [batch_sz, 1]
rewards = tf.tile(rewards, (1, config.num_glimpses))  # [batch_sz, timesteps]
logll = loglikelihood(loc_mean_arr, sampled_loc_arr, config.loc_std)
advs = rewards - tf.stop_gradient(baselines)
logllratio = tf.reduce_mean(logll * advs)
reward = tf.reduce_mean(reward)

baselines_mse = tf.reduce_mean(tf.square((rewards - baselines)))
var_list = tf.trainable_variables()
# hybrid loss
loss = -logllratio + xent + baselines_mse  # `-` for minimize
grads = tf.gradients(loss, var_list)
grads, _ = tf.clip_by_global_norm(grads, config.max_grad_norm)

# learning rate
global_step = tf.get_variable(
    'global_step', [], initializer=tf.constant_initializer(0), trainable=False)
training_steps_per_epoch = mnist.train.num_examples // config.batch_size
starter_learning_rate = config.lr_start
# decay per training epoch
learning_rate = tf.train.exponential_decay(
    starter_learning_rate,
    global_step,
    training_steps_per_epoch,
    0.97,
    staircase=True)
learning_rate = tf.maximum(learning_rate, config.lr_min)
opt = tf.train.AdamOptimizer(learning_rate)
train_op = opt.apply_gradients(zip(grads, var_list), global_step=global_step)

with tf.Session() as sess:
  sess.run(tf.global_variables_initializer())
  for i in range(n_steps):
    images, labels = mnist.train.next_batch(config.batch_size)
    # duplicate M times, see Eqn (2)
    images = np.tile(images, [config.M, 1])
    labels = np.tile(labels, [config.M])
    loc_net.samping = True
    adv_val, baselines_mse_val, xent_val, logllratio_val, \
        reward_val, loss_val, lr_val, _ = sess.run(
            [advs, baselines_mse, xent, logllratio,
             reward, loss, learning_rate, train_op],
            feed_dict={
                images_ph: images,
                labels_ph: labels
            })
    if i and i % 100 == 0:
      logging.info('step {}: lr = {:3.6f}'.format(i, lr_val))
      logging.info(
          'step {}: reward = {:3.4f}\tloss = {:3.4f}\txent = {:3.4f}'.format(
              i, reward_val, loss_val, xent_val))
      logging.info('llratio = {:3.4f}\tbaselines_mse = {:3.4f}'.format(
          logllratio_val, baselines_mse_val))

    if i and i % training_steps_per_epoch == 0:
      # Evaluation
      for dataset in [mnist.validation, mnist.test]:
        steps_per_epoch = dataset.num_examples // config.eval_batch_size
        correct_cnt = 0
        num_samples = steps_per_epoch * config.batch_size
        loc_net.sampling = True
        for test_step in range(steps_per_epoch):
          images, labels = dataset.next_batch(config.batch_size)
          labels_bak = labels
          # Duplicate M times
          images = np.tile(images, [config.M, 1])
          labels = np.tile(labels, [config.M])
          softmax_val = sess.run(softmax,
                                 feed_dict={
                                     images_ph: images,
                                     labels_ph: labels
                                 })
          softmax_val = np.reshape(softmax_val,
                                   [config.M, -1, config.num_classes])
          softmax_val = np.mean(softmax_val, 0)
          pred_labels_val = np.argmax(softmax_val, 1)
          pred_labels_val = pred_labels_val.flatten()
          correct_cnt += np.sum(pred_labels_val == labels_bak)
        acc = correct_cnt / num_samples
        if dataset == mnist.validation:
          logging.info('valid accuracy = {}'.format(acc))
        else:
          logging.info('test accuracy = {}'.format(acc))
<commit_msg>follow pep08<commit_after>"""Recurrent Models of Visual Attention V. Mnih et al."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import tensorflow as tf
import numpy as np

from glimpseNetwork import GlimpseNet
from locationNetwork import LocNet
from glimpseSensor import GlimpseSensor

from utils import weight_variable, bias_variable, loglikelihood
from config import Config

from tensorflow.examples.tutorials.mnist import input_data

logging.getLogger().setLevel(logging.INFO)

rnn_cell = tf.contrib.rnn
seq2seq = tf.contrib.legacy_seq2seq

mnist = input_data.read_data_sets('MNIST_data', one_hot=False)

config = Config()
n_steps = config.step

loc_mean_arr = []
sampled_loc_arr = []

# placeholders
images_ph = tf.placeholder(tf.float32,
                           [None, config.original_size * config.original_size *
                            config.num_channels])
labels_ph = tf.placeholder(tf.int64, [None])

# Build the aux nets.
with tf.variable_scope('glimpse_net'):
    gl_sensor = GlimpseSensor(config)
    gl = GlimpseNet(gl_sensor, config, images_ph)
with tf.variable_scope('loc_net'):
    loc_net = LocNet(config)

def get_next_input(output, i):
    loc, loc_mean = loc_net(output)
    gl_next = gl(loc)
    loc_mean_arr.append(loc_mean)
    sampled_loc_arr.append(loc)
    return gl_next

# number of examples
N = tf.shape(images_ph)[0]

init_loc = tf.random_uniform((N, 2), minval=-1, maxval=1)
init_glimpse = gl(init_loc)
# Core network.
lstm_cell = rnn_cell.LSTMCell(config.cell_size, state_is_tuple=True)
init_state = lstm_cell.zero_state(N, tf.float32)
inputs = [init_glimpse]
inputs.extend([0] * (config.num_glimpses))
outputs, _ = seq2seq.rnn_decoder(
    inputs, init_state, lstm_cell, loop_function=get_next_input)

# Time independent baselines
with tf.variable_scope('baseline'):
    w_baseline = weight_variable((config.cell_output_size, 1))
    b_baseline = bias_variable((1,))
baselines = []
for t, output in enumerate(outputs[1:]):
    baseline_t = tf.nn.xw_plus_b(output, w_baseline, b_baseline)
    baseline_t = tf.squeeze(baseline_t)
    baselines.append(baseline_t)
baselines = tf.stack(baselines)  # [timesteps, batch_sz]
baselines = tf.transpose(baselines)  # [batch_sz, timesteps]

# Take the last step only.
output = outputs[-1]
# Build classification network.
with tf.variable_scope('cls'):
    w_logit = weight_variable((config.cell_output_size, config.num_classes))
    b_logit = bias_variable((config.num_classes,))

logits = tf.nn.xw_plus_b(output, w_logit, b_logit)
softmax = tf.nn.softmax(logits)

# cross-entropy.
xent = tf.nn.sparse_softmax_cross_entropy_with_logits(
    logits=logits, labels=labels_ph
)
xent = tf.reduce_mean(xent)
pred_labels = tf.argmax(logits, 1)
# 0/1 reward.
reward = tf.cast(tf.equal(pred_labels, labels_ph), tf.float32)
rewards = tf.expand_dims(reward, 1)  # [batch_sz, 1]
rewards = tf.tile(rewards, (1, config.num_glimpses))  # [batch_sz, timesteps]
logll = loglikelihood(loc_mean_arr, sampled_loc_arr, config.loc_std)
advs = rewards - tf.stop_gradient(baselines)
logllratio = tf.reduce_mean(logll * advs)
reward = tf.reduce_mean(reward)

baselines_mse = tf.reduce_mean(tf.square((rewards - baselines)))
var_list = tf.trainable_variables()
# hybrid loss
loss = -logllratio + xent + baselines_mse  # `-` for minimize
grads = tf.gradients(loss, var_list)
grads, _ = tf.clip_by_global_norm(grads, config.max_grad_norm)

# learning rate
global_step = tf.get_variable(
    'global_step', [], initializer=tf.constant_initializer(0), trainable=False)
training_steps_per_epoch = mnist.train.num_examples // config.batch_size
starter_learning_rate = config.lr_start
# decay per training epoch
learning_rate = tf.train.exponential_decay(
    starter_learning_rate,
    global_step,
    training_steps_per_epoch,
    0.97,
    staircase=True)
learning_rate = tf.maximum(learning_rate, config.lr_min)
opt = tf.train.AdamOptimizer(learning_rate)
train_op = opt.apply_gradients(zip(grads, var_list), global_step=global_step)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for i in range(n_steps):
        images, labels = mnist.train.next_batch(config.batch_size)
        # duplicate M times, see Eqn (2)
        images = np.tile(images, [config.M, 1])
        labels = np.tile(labels, [config.M])
        loc_net.samping = True

        feed_dict = {images_ph: images, labels_ph: labels}
        runnables = [
            advs, baselines_mse, xent, logllratio,
            reward, loss, learning_rate, train_op
        ]
        adv_val, baselines_mse_val, xent_val, logllratio_val, reward_val, \
            loss_val, lr_val, _ = sess.run(runnables, feed_dict=feed_dict)

        if i and i % 100 == 0:
            logging.info('step {}: lr = {:3.6f}'.format(i, lr_val))
            info = 'step {}: reward = {:3.4f}\tloss = {:3.4f}\txent = {:3.4f}'
            logging.info(
                info.format(i, reward_val, loss_val, xent_val)
            )
            logging.info('llratio = {:3.4f}\tbaselines_mse = {:3.4f}'.format(
                logllratio_val, baselines_mse_val)
            )

        if i and i % training_steps_per_epoch == 0:
            # Evaluation
            for dataset in [mnist.validation, mnist.test]:
                steps_per_epoch = dataset.num_examples // config.eval_batch_size
                correct_cnt = 0
                num_samples = steps_per_epoch * config.batch_size
                loc_net.sampling = True
                for test_step in range(steps_per_epoch):
                    images, labels = dataset.next_batch(config.batch_size)
                    labels_bak = labels
                    # Duplicate M times
                    images = np.tile(images, [config.M, 1])
                    labels = np.tile(labels, [config.M])
                    feed_dict = {images_ph: images, labels_ph: labels}
                    softmax_val = sess.run(softmax, feed_dict=feed_dict)
                    softmax_val = np.reshape(
                        softmax_val, [config.M, -1, config.num_classes]
                    )
                    softmax_val = np.mean(softmax_val, 0)
                    pred_labels_val = np.argmax(softmax_val, 1)
                    pred_labels_val = pred_labels_val.flatten()
                    correct_cnt += np.sum(pred_labels_val == labels_bak)
                acc = correct_cnt / num_samples
                if dataset == mnist.validation:
                    logging.info('valid accuracy = {}'.format(acc))
                else:
                    logging.info('test accuracy = {}'.format(acc))
<|endoftext|>


<commit_before>'''
Parse messages into logical python args, kwargs
according to the various rfcs
'''
#  http://tools.ietf.org/html/rfc2812
import re

RE_IRCLINE = re.compile(
    """
    ^
    (:(?P<prefix>[^ ]+) +)?    # Optional prefix (src, nick!host, etc)
                               # Prefix matches all non-space characters
                               # Must start with a ':' character

    (?P<command>[^ ]+)         # Command is required (JOIN, 001, 403)
                               # Command matches all non-space characters

    (?P<params>( +[^:][^ ]*)*) # Optional params after command
                               # Must have at least one leading space
                               # Params end at first ':' which starts message

    (?: +:(?P<message>.*))?    # Optional message starts after first ':'
                               # Must have at least one leading space
    $
    """, re.VERBOSE)

RAW_COMMANDS = set([
    #  3.1 Connection Registration
    "PASS", "NICK", "USER", "OPER", "MODE", "SERVICE", "QUIT", "SQUIT",
    #  3.2 Channel Operations
    "JOIN", "PART", "MODE", "TOPIC", "NAMES", "LIST", "INVITE", "KICK",
    #  3.3 Sending Messages
    "PRIVMSG", "NOTICE",
    #  3.4 Server Queries and Commands
    "MOTD", "LUSERS", "VERSION", "STATS", "LINKS", "TIME",
    "CONNECT", "TRACE", "ADMIN", "INFO",
    #  3.5 Service Query and Commands
    "SERVLIST", "SQUERY",
    #  3.6 User Based Queries
    "WHO", "WHOIS", "WHOWAS",
    #  3.7 Miscellaneous Messages
    "KILL", "PING", "PONG", "ERROR",
    #  4.0 Optional Features
    "AWAY", "REHASH", "DIE", "RESTART", "SUMMON", "USERS",
    "WALLOPS", "USERHOST", "ISON",
    #  5.0 Replies
    ("001", "RPL_WELCOME"),
    ("002", "RPL_YOURHOST"),
    ("003", "RPL_CREATED"),
    ("004", "RPL_MYINFO"),
    ("005", "RPL_BOUNCE"),
    ("302", "RPL_USERHOST"),
    ("303", "RPL_ISON"),
    ("301", "RPL_AWAY"),
    ("305", "RPL_UNAWAY"),
    ("306", "RPL_NOWAWAY"),
    ("311", "RPL_WHOISUSER"),
    ("312", "RPL_WHOISSERVER"),
    ("313", "RPL_WHOISOPERATOR"),
    ("317", "RPL_WHOISIDLE"),
    ("318", "RPL_ENDOFWHOIS"),
    ("319", "RPL_WHOISCHANNELS"),
    ("314", "RPL_WHOWASUSER"),
    ("369", "RPL_ENDOFWHOWAS"),
    ("321", "RPL_LISTSTART"),
    ("322", "RPL_LIST"),
    ("323", "RPL_LISTEND"),
    ("325", "RPL_UNIQOPIS"),
    ("324", "RPL_CHANNELMODEIS"),
    ("331", "RPL_NOTOPIC"),
    ("332", "RPL_TOPIC"),
    ("341", "RPL_INVITING"),
    ("342", "RPL_SUMMONING"),
    ("346", "RPL_INVITELIST"),
    ("347", "RPL_ENDOFINVITELIST"),
    ("348", "RPL_EXCEPTLIST"),
    ("349", "RPL_ENDOFEXCEPTLIST"),
    ("351", "RPL_VERSION"),
    ("352", "RPL_WHOREPLY"),
    ("315", "RPL_ENDOFWHO"),
    ("353", "RPL_NAMREPLY"),
    ("366", "RPL_ENDOFNAMES"),
    ("364", "RPL_LINKS"),
    ("365", "RPL_ENDOFLINKS"),
    ("367", "RPL_BANLIST"),
    ("368", "RPL_ENDOFBANLIST"),
    ("371", "RPL_INFO"),
    ("374", "RPL_ENDOFINFO"),
    ("375", "RPL_MOTDSTART"),
    ("372", "RPL_MOTD"),
    ("376", "RPL_ENDOFMOTD"),
    ("381", "RPL_YOUREOPER"),
    ("382", "RPL_REHASHING"),
    ("383", "RPL_YOURESERVICE"),
    ("391", "RPL_TIME"),
    ("392", "RPL_USERSSTART"),
    ("393", "RPL_USERS"),
    ("394", "RPL_ENDOFUSERS"),
    ("395", "RPL_NOUSERS"),
    ("200", "RPL_TRACELINK"),
    ("201", "RPL_TRACECONNECTING"),
    ("202", "RPL_TRACEHANDSHAKE"),
    ("203", "RPL_TRACEUNKNOWN"),
    ("204", "RPL_TRACEOPERATOR"),
    ("205", "RPL_TRACEUSER"),
    ("206", "RPL_TRACESERVER"),
    ("207", "RPL_TRACESERVICE"),
    ("208", "RPL_TRACENEWTYPE"),
    ("209", "RPL_TRACECLASS"),
    ("210", "RPL_TRACERECONNECT"),
    ("261", "RPL_TRACELOG"),
    ("262", "RPL_TRACEEND"),
    ("211", "RPL_STATSLINKINFO"),
    ("212", "RPL_STATSCOMMANDS"),
    ("219", "RPL_ENDOFSTATS"),
    ("242", "RPL_STATSUPTIME"),
    ("243", "RPL_STATSOLINE"),
    ("221", "RPL_UMODEIS"),
    ("234", "RPL_SERVLIST"),
    ("235", "RPL_SERVLISTEND"),
    ("251", "RPL_LUSERCLIENT"),
    ("252", "RPL_LUSEROP"),
    ("253", "RPL_LUSERUNKNOWN"),
    ("254", "RPL_LUSERCHANNELS"),
    ("255", "RPL_LUSERME"),
    ("256", "RPL_ADMINME"),
    ("257", "RPL_ADMINLOC1"),
    ("258", "RPL_ADMINLOC2"),
    ("259", "RPL_ADMINEMAIL"),
    ("263", "RPL_TRYAGAIN"),
    ("401", "ERR_NOSUCHNICK"),
    ("402", "ERR_NOSUCHSERVER"),
    ("403", "ERR_NOSUCHCHANNEL"),
    ("404", "ERR_CANNOTSENDTOCHAN"),
    ("405", "ERR_TOOMANYCHANNELS"),
    ("406", "ERR_WASNOSUCHNICK"),
    ("407", "ERR_TOOMANYTARGETS"),
    ("408", "ERR_NOSUCHSERVICE"),
    ("409", "ERR_NOORIGIN"),
    ("411", "ERR_NORECIPIENT"),
    ("412", "ERR_NOTEXTTOSEND"),
    ("413", "ERR_NOTOPLEVEL"),
    ("414", "ERR_WILDTOPLEVEL"),
    ("415", "ERR_BADMASK"),
    ("421", "ERR_UNKNOWNCOMMAND"),
    ("422", "ERR_NOMOTD"),
    ("423", "ERR_NOADMININFO"),
    ("424", "ERR_FILEERROR"),
    ("431", "ERR_NONICKNAMEGIVEN"),
    ("432", "ERR_ERRONEUSNICKNAME"),
    ("433", "ERR_NICKNAMEINUSE"),
    ("436", "ERR_NICKCOLLISION"),
    ("437", "ERR_UNAVAILRESOURCE"),
    ("441", "ERR_USERNOTINCHANNEL"),
    ("442", "ERR_NOTONCHANNEL"),
    ("443", "ERR_USERONCHANNEL"),
    ("444", "ERR_NOLOGIN"),
    ("445", "ERR_SUMMONDISABLED"),
    ("446", "ERR_USERSDISABLED"),
    ("451", "ERR_NOTREGISTERED"),
    ("461", "ERR_NEEDMOREPARAMS"),
    ("462", "ERR_ALREADYREGISTRED"),
    ("463", "ERR_NOPERMFORHOST"),
    ("464", "ERR_PASSWDMISMATCH"),
    ("465", "ERR_YOUREBANNEDCREEP"),
    ("466", "ERR_YOUWILLBEBANNED"),
    ("467", "ERR_KEYSET"),
    ("471", "ERR_CHANNELISFULL"),
    ("472", "ERR_UNKNOWNMODE"),
    ("473", "ERR_INVITEONLYCHAN"),
    ("474", "ERR_BANNEDFROMCHAN"),
    ("475", "ERR_BADCHANNELKEY"),
    ("476", "ERR_BADCHANMASK"),
    ("477", "ERR_NOCHANMODES"),
    ("478", "ERR_BANLISTFULL"),
    ("481", "ERR_NOPRIVILEGES"),
    ("482", "ERR_CHANOPRIVSNEEDED"),
    ("483", "ERR_CANTKILLSERVER"),
    ("484", "ERR_RESTRICTED"),
    ("485", "ERR_UNIQOPPRIVSNEEDED"),
    ("491", "ERR_NOOPERHOST"),
    ("501", "ERR_UMODEUNKNOWNFLAG"),
    ("502", "ERR_USERSDONTMATCH"),

    #  5.3 Reserved Numerics
    ("231", "RPL_SERVICEINFO"),
    ("232", "RPL_ENDOFSERVICES"),
    ("233", "RPL_SERVICE"),
    ("300", "RPL_NONE"),
    ("316", "RPL_WHOISCHANOP"),
    ("361", "RPL_KILLDONE"),
    ("362", "RPL_CLOSING"),
    ("363", "RPL_CLOSEEND"),
    ("373", "RPL_INFOSTART"),
    ("384", "RPL_MYPORTIS"),
    ("213", "RPL_STATSCLINE"),
    ("214", "RPL_STATSNLINE"),
    ("215", "RPL_STATSILINE"),
    ("216", "RPL_STATSKLINE"),
    ("217", "RPL_STATSQLINE"),
    ("218", "RPL_STATSYLINE"),
    ("240", "RPL_STATSVLINE"),
    ("241", "RPL_STATSLLINE"),
    ("244", "RPL_STATSHLINE"),
    ("244", "RPL_STATSSLINE"),
    ("246", "RPL_STATSPING"),
    ("247", "RPL_STATSBLINE"),
    ("250", "RPL_STATSDLINE"),
    ("492", "ERR_NOSERVICEHOST"),
])

# Dict maps from raw string to de-duped command type
# This makes it unambiguous to talk about the command "RPL_WELCOME" even though
# it can be represented by either "RPL_WELCOME" or "001"
COMMANDS = {}
WIRE_COMMANDS = {}
for entry in RAW_COMMANDS:
    # Single representation
    if isinstance(entry, str):
        COMMANDS[entry] = entry
        WIRE_COMMANDS[entry] = entry
    # Numeric <--> str pair
    elif isinstance(entry, tuple):
        assert len(entry) == 2
        number, string = entry
        COMMANDS[number] = string
        COMMANDS[string] = string
        WIRE_COMMANDS[number] = number
        WIRE_COMMANDS[string] = number
    else:
        raise ValueError(
            "Unexpected entry in RAW_COMMANDS: '{}'".format(entry))

def unique_command(command):
    ''' Return the unique string that a command maps to '''
    try:
        return COMMANDS[command.upper()]
    except KeyError:
        raise ValueError("Unknown command '{}'".format(command))

def wire_command(command):
    ''' Return the representation a command uses on the wire '''
    try:
        return WIRE_COMMANDS[command.upper()]
    except KeyError:
        raise ValueError("Unknown command '{}'".format(command))

def parse(msg):
    raise NotImplementedError
<commit_msg>Update rfc docstrings<commit_after>''' Parse messages into python args, kwargs according to rfc 2812 '''
#  http://tools.ietf.org/html/rfc2812
import re

RE_IRCLINE = re.compile(
    """
    ^
    (:(?P<prefix>[^ ]+) +)?    # Optional prefix (src, nick!host, etc)
                               # Prefix matches all non-space characters
                               # Must start with a ':' character

    (?P<command>[^ ]+)         # Command is required (JOIN, 001, 403)
                               # Command matches all non-space characters

    (?P<params>( +[^:][^ ]*)*) # Optional params after command
                               # Must have at least one leading space
                               # Params end at first ':' which starts message

    (?: +:(?P<message>.*))?    # Optional message starts after first ':'
                               # Must have at least one leading space
    $
    """, re.VERBOSE)

RAW_COMMANDS = set([
    #  3.1 Connection Registration
    "PASS", "NICK", "USER", "OPER", "MODE", "SERVICE", "QUIT", "SQUIT",
    #  3.2 Channel Operations
    "JOIN", "PART", "MODE", "TOPIC", "NAMES", "LIST", "INVITE", "KICK",
    #  3.3 Sending Messages
    "PRIVMSG", "NOTICE",
    #  3.4 Server Queries and Commands
    "MOTD", "LUSERS", "VERSION", "STATS", "LINKS", "TIME",
    "CONNECT", "TRACE", "ADMIN", "INFO",
    #  3.5 Service Query and Commands
    "SERVLIST", "SQUERY",
    #  3.6 User Based Queries
    "WHO", "WHOIS", "WHOWAS",
    #  3.7 Miscellaneous Messages
    "KILL", "PING", "PONG", "ERROR",
    #  4.0 Optional Features
    "AWAY", "REHASH", "DIE", "RESTART", "SUMMON", "USERS",
    "WALLOPS", "USERHOST", "ISON",
    #  5.0 Replies
    ("001", "RPL_WELCOME"),
    ("002", "RPL_YOURHOST"),
    ("003", "RPL_CREATED"),
    ("004", "RPL_MYINFO"),
    ("005", "RPL_BOUNCE"),
    ("302", "RPL_USERHOST"),
    ("303", "RPL_ISON"),
    ("301", "RPL_AWAY"),
    ("305", "RPL_UNAWAY"),
    ("306", "RPL_NOWAWAY"),
    ("311", "RPL_WHOISUSER"),
    ("312", "RPL_WHOISSERVER"),
    ("313", "RPL_WHOISOPERATOR"),
    ("317", "RPL_WHOISIDLE"),
    ("318", "RPL_ENDOFWHOIS"),
    ("319", "RPL_WHOISCHANNELS"),
    ("314", "RPL_WHOWASUSER"),
    ("369", "RPL_ENDOFWHOWAS"),
    ("321", "RPL_LISTSTART"),
    ("322", "RPL_LIST"),
    ("323", "RPL_LISTEND"),
    ("325", "RPL_UNIQOPIS"),
    ("324", "RPL_CHANNELMODEIS"),
    ("331", "RPL_NOTOPIC"),
    ("332", "RPL_TOPIC"),
    ("341", "RPL_INVITING"),
    ("342", "RPL_SUMMONING"),
    ("346", "RPL_INVITELIST"),
    ("347", "RPL_ENDOFINVITELIST"),
    ("348", "RPL_EXCEPTLIST"),
    ("349", "RPL_ENDOFEXCEPTLIST"),
    ("351", "RPL_VERSION"),
    ("352", "RPL_WHOREPLY"),
    ("315", "RPL_ENDOFWHO"),
    ("353", "RPL_NAMREPLY"),
    ("366", "RPL_ENDOFNAMES"),
    ("364", "RPL_LINKS"),
    ("365", "RPL_ENDOFLINKS"),
    ("367", "RPL_BANLIST"),
    ("368", "RPL_ENDOFBANLIST"),
    ("371", "RPL_INFO"),
    ("374", "RPL_ENDOFINFO"),
    ("375", "RPL_MOTDSTART"),
    ("372", "RPL_MOTD"),
    ("376", "RPL_ENDOFMOTD"),
    ("381", "RPL_YOUREOPER"),
    ("382", "RPL_REHASHING"),
    ("383", "RPL_YOURESERVICE"),
    ("391", "RPL_TIME"),
    ("392", "RPL_USERSSTART"),
    ("393", "RPL_USERS"),
    ("394", "RPL_ENDOFUSERS"),
    ("395", "RPL_NOUSERS"),
    ("200", "RPL_TRACELINK"),
    ("201", "RPL_TRACECONNECTING"),
    ("202", "RPL_TRACEHANDSHAKE"),
    ("203", "RPL_TRACEUNKNOWN"),
    ("204", "RPL_TRACEOPERATOR"),
    ("205", "RPL_TRACEUSER"),
    ("206", "RPL_TRACESERVER"),
    ("207", "RPL_TRACESERVICE"),
    ("208", "RPL_TRACENEWTYPE"),
    ("209", "RPL_TRACECLASS"),
    ("210", "RPL_TRACERECONNECT"),
    ("261", "RPL_TRACELOG"),
    ("262", "RPL_TRACEEND"),
    ("211", "RPL_STATSLINKINFO"),
    ("212", "RPL_STATSCOMMANDS"),
    ("219", "RPL_ENDOFSTATS"),
    ("242", "RPL_STATSUPTIME"),
    ("243", "RPL_STATSOLINE"),
    ("221", "RPL_UMODEIS"),
    ("234", "RPL_SERVLIST"),
    ("235", "RPL_SERVLISTEND"),
    ("251", "RPL_LUSERCLIENT"),
    ("252", "RPL_LUSEROP"),
    ("253", "RPL_LUSERUNKNOWN"),
    ("254", "RPL_LUSERCHANNELS"),
    ("255", "RPL_LUSERME"),
    ("256", "RPL_ADMINME"),
    ("257", "RPL_ADMINLOC1"),
    ("258", "RPL_ADMINLOC2"),
    ("259", "RPL_ADMINEMAIL"),
    ("263", "RPL_TRYAGAIN"),
    ("401", "ERR_NOSUCHNICK"),
    ("402", "ERR_NOSUCHSERVER"),
    ("403", "ERR_NOSUCHCHANNEL"),
    ("404", "ERR_CANNOTSENDTOCHAN"),
    ("405", "ERR_TOOMANYCHANNELS"),
    ("406", "ERR_WASNOSUCHNICK"),
    ("407", "ERR_TOOMANYTARGETS"),
    ("408", "ERR_NOSUCHSERVICE"),
    ("409", "ERR_NOORIGIN"),
    ("411", "ERR_NORECIPIENT"),
    ("412", "ERR_NOTEXTTOSEND"),
    ("413", "ERR_NOTOPLEVEL"),
    ("414", "ERR_WILDTOPLEVEL"),
    ("415", "ERR_BADMASK"),
    ("421", "ERR_UNKNOWNCOMMAND"),
    ("422", "ERR_NOMOTD"),
    ("423", "ERR_NOADMININFO"),
    ("424", "ERR_FILEERROR"),
    ("431", "ERR_NONICKNAMEGIVEN"),
    ("432", "ERR_ERRONEUSNICKNAME"),
    ("433", "ERR_NICKNAMEINUSE"),
    ("436", "ERR_NICKCOLLISION"),
    ("437", "ERR_UNAVAILRESOURCE"),
    ("441", "ERR_USERNOTINCHANNEL"),
    ("442", "ERR_NOTONCHANNEL"),
    ("443", "ERR_USERONCHANNEL"),
    ("444", "ERR_NOLOGIN"),
    ("445", "ERR_SUMMONDISABLED"),
    ("446", "ERR_USERSDISABLED"),
    ("451", "ERR_NOTREGISTERED"),
    ("461", "ERR_NEEDMOREPARAMS"),
    ("462", "ERR_ALREADYREGISTRED"),
    ("463", "ERR_NOPERMFORHOST"),
    ("464", "ERR_PASSWDMISMATCH"),
    ("465", "ERR_YOUREBANNEDCREEP"),
    ("466", "ERR_YOUWILLBEBANNED"),
    ("467", "ERR_KEYSET"),
    ("471", "ERR_CHANNELISFULL"),
    ("472", "ERR_UNKNOWNMODE"),
    ("473", "ERR_INVITEONLYCHAN"),
    ("474", "ERR_BANNEDFROMCHAN"),
    ("475", "ERR_BADCHANNELKEY"),
    ("476", "ERR_BADCHANMASK"),
    ("477", "ERR_NOCHANMODES"),
    ("478", "ERR_BANLISTFULL"),
    ("481", "ERR_NOPRIVILEGES"),
    ("482", "ERR_CHANOPRIVSNEEDED"),
    ("483", "ERR_CANTKILLSERVER"),
    ("484", "ERR_RESTRICTED"),
    ("485", "ERR_UNIQOPPRIVSNEEDED"),
    ("491", "ERR_NOOPERHOST"),
    ("501", "ERR_UMODEUNKNOWNFLAG"),
    ("502", "ERR_USERSDONTMATCH"),

    #  5.3 Reserved Numerics
    ("231", "RPL_SERVICEINFO"),
    ("232", "RPL_ENDOFSERVICES"),
    ("233", "RPL_SERVICE"),
    ("300", "RPL_NONE"),
    ("316", "RPL_WHOISCHANOP"),
    ("361", "RPL_KILLDONE"),
    ("362", "RPL_CLOSING"),
    ("363", "RPL_CLOSEEND"),
    ("373", "RPL_INFOSTART"),
    ("384", "RPL_MYPORTIS"),
    ("213", "RPL_STATSCLINE"),
    ("214", "RPL_STATSNLINE"),
    ("215", "RPL_STATSILINE"),
    ("216", "RPL_STATSKLINE"),
    ("217", "RPL_STATSQLINE"),
    ("218", "RPL_STATSYLINE"),
    ("240", "RPL_STATSVLINE"),
    ("241", "RPL_STATSLLINE"),
    ("244", "RPL_STATSHLINE"),
    ("244", "RPL_STATSSLINE"),
    ("246", "RPL_STATSPING"),
    ("247", "RPL_STATSBLINE"),
    ("250", "RPL_STATSDLINE"),
    ("492", "ERR_NOSERVICEHOST"),
])

# Dict maps from raw string to de-duped command type
# This makes it unambiguous to talk about the command "RPL_WELCOME" even though
# it can be represented by either "RPL_WELCOME" or "001"
COMMANDS = {}
WIRE_COMMANDS = {}
for entry in RAW_COMMANDS:
    # Single representation
    if isinstance(entry, str):
        COMMANDS[entry] = entry
        WIRE_COMMANDS[entry] = entry
    # Numeric <--> str pair
    elif isinstance(entry, tuple):
        assert len(entry) == 2
        number, string = entry
        COMMANDS[number] = string
        COMMANDS[string] = string
        WIRE_COMMANDS[number] = number
        WIRE_COMMANDS[string] = number
    else:
        raise ValueError(
            "Unexpected entry in RAW_COMMANDS: '{}'".format(entry))

def unique_command(command):
    ''' Return the unique string that a command maps to '''
    try:
        return COMMANDS[command.upper()]
    except KeyError:
        raise ValueError("Unknown command '{}'".format(command))

def wire_command(command):
    ''' Return the representation a command uses on the wire '''
    try:
        return WIRE_COMMANDS[command.upper()]
    except KeyError:
        raise ValueError("Unknown command '{}'".format(command))

def parse(msg):
    ''' Parse messages into python args, kwargs according to rfc 2812 '''
    raise NotImplementedError
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import itertools
import statsmodels.api as sm
import sklearn
import sklearn.ensemble
import palettable

sns.set(style='darkgrid', palette='muted', font_scale=1.5)

__all__ = ['computeROC',
           'computeCVROC',
           'plotROC',
           'plotProb',
           'plot2Prob',
           'lassoVarSelect',
           'smLogisticRegression']

def computeROC(df, model, outcomeVar, predVars):
    """Apply model to df and return performance metrics.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : str
    predVars : ndarray or list
        Predictor variables in the model.

    Returns
    -------
    fpr : np.ndarray
        False-positive rate
    tpr : np.ndarray
        True-positive rate
    auc : float
        Area under the ROC curve
    acc : float
        Accuracy score
    results : returned by model.fit()
        Model results object for test prediction in CV
    prob : pd.Series
        Predicted probabilities with index from df"""

    if not type(predVars) is list:
        predVars = list(predVars)
    tmp = df[[outcomeVar] + predVars].dropna()

    try:
        results = model.fit(X=tmp[predVars], y=tmp[outcomeVar])
        if hasattr(results, 'predict_proba'):
            prob = results.predict_proba(tmp[predVars])[:,1]
        else:
            prob = results.predict(tmp[predVars])
            results.predict_proba = results.predict
        fpr, tpr, thresholds = sklearn.metrics.roc_curve(tmp[outcomeVar].values, prob)

        acc = sklearn.metrics.accuracy_score(tmp[outcomeVar].values, np.round(prob))
        auc = sklearn.metrics.auc(fpr, tpr)
        tpr[0], tpr[-1] = 0,1
    except sm.tools.sm_exceptions.PerfectSeparationError:
        print 'PerfectSeparationError: %s (N = %d; %d predictors)' % (outcomeVar, tmp.shape[0], len(predVars))
        acc = 1.
        fpr = np.zeros(5)
        tpr = np.ones(5)
        tpr[0], tpr[-1] = 0,1
        prob = tmp[outcomeVar].values.astype(float)
        auc = 1.
        results = None
    return fpr, tpr, auc, acc, results, pd.Series(prob, index=tmp.index, name='Prob')

def computeCVROC(df, model, outcomeVar, predVars, LOO=False, nFolds=10):
    """Apply model to df and return performance metrics in a cross-validation framework.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : str
    predVars : ndarray or list
        Predictor variables in the model.
    LOO : bool
        Leave-one-out cross-validation?
    nFolds : int
        N-fold cross-validation (not required for LOO)

    Returns
    -------
    fpr : np.ndarray
        Pre-specified vector of FPR thresholds for interpolation
        fpr = np.linspace(0, 1, 100)
    meanTPR : np.ndarray
        Mean true-positive rate in test fraction.
    auc : float
        Area under the mean ROC curve.
    acc : float
        Mean accuracy score in test fraction.
    results : returned by model.fit()
        Training model results object for each fold
    prob : pd.Series
        Mean predicted probabilities on test data with index from df"""

    if not type(predVars) is list:
        predVars = list(predVars)
    tmp = df[[outcomeVar] + predVars].dropna()
    if LOO:
        cv = sklearn.cross_validation.LeaveOneOut(n=tmp.shape[0])
        nFolds = tmp.shape[0]
    else:
        cv = sklearn.cross_validation.KFold(n=tmp.shape[0],
                                            n_folds=nFolds,
                                            shuffle=True,
                                            random_state=110820)
    fpr = np.linspace(0, 1, 100)
    tpr = np.nan * np.zeros((fpr.shape[0], nFolds))
    acc = 0
    counter = 0
    results = []
    prob = []
    for i, (trainInd, testInd) in enumerate(cv):
        trainDf = tmp.iloc[trainInd]
        testDf = tmp.iloc[testInd]
        trainFPR, trainTPR, trainAUC, trainACC, res, trainProb = computeROC(trainDf,
                                                                            model,
                                                                            outcomeVar,
                                                                            predVars)
        if not res is None:
            counter += 1
            testProb = res.predict_proba(testDf[predVars])[:,1]
            testFPR, testTPR, _ = sklearn.metrics.roc_curve(testDf[outcomeVar].values, testProb)
            tpr[:,i] = np.interp(fpr, testFPR, testTPR)
            acc += sklearn.metrics.accuracy_score(testDf[outcomeVar].values, np.round(testProb))
            results.append(res)
            prob.append(pd.Series(testProb, index=testDf.index))
    if counter != nFolds:
        print 'ROC: did not finish all folds (%d of %d)' % (counter, nFolds)
    if counter >=1:
        meanTPR = np.nanmean(tpr, axis=1)
        meanTPR[0], meanTPR[-1] = 0,1
        meanACC = acc / counter
        meanAUC = sklearn.metrics.auc(fpr, meanTPR)
        """Compute mean probability over test predictions in CV"""
        probS = pd.concat(prob).groupby(level=0).agg(np.mean)
        probS.name = 'Prob'
    else:
        meanTPR = np.nan * fpr
        meanTPR[0], meanTPR[-1] = 0,1
        meanACC = np.nan
        meanAUC = np.nan
        """Compute mean probability over test predictions in CV"""
        probS = np.nan

    return fpr, meanTPR, meanAUC, meanACC, results, probS

def plotROC(df, model, outcomeVar, predictorsList, predictorLabels=None, rocFunc=computeCVROC, **rocKwargs):
    """Plot of multiple ROC curves using same model and same outcomeVar with
    different sets of predictors.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : str
    predictorsList : list
        List of lists of predictor variables for each model.
    predictorLabels : list
        List of labels for the models (optional)
    LOO : bool
        Leave-one-out cross-vlaidation?
    rocFunc : computeCVROC or computeROC
        Function for computing the ROC
    rocKwargs : kwargs
        Additional arguments for rocFunc"""

    if predictorLabels is None:
        predictorLabels = [' + '.join(predVars) for predVars in predictorsList]

    colors = palettable.colorbrewer.qualitative.Set1_8.mpl_colors

    plt.clf()
    plt.gca().set_aspect('equal')
    for predVarsi,predVars in enumerate(predictorsList):
        fpr, tpr, auc, acc, res, probS = rocFunc(df,
                                                 model,
                                                 outcomeVar,
                                                 predVars,
                                                 **rocKwargs)

        label = '%s (AUC = %0.2f; ACC = %0.2f)' % (predictorLabels[predVarsi], auc, acc)
        plt.plot(fpr, tpr, color=colors[predVarsi], lw=2, label=label)
    plt.plot([0, 1], [0, 1], '--', color='gray', label='Chance')
    plt.xlim([-0.05, 1.05])
    plt.ylim([-0.05, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for %s' % outcomeVar)
    plt.legend(loc="lower right")
    plt.show()

def plotProb(df, outcomeVar, prob, **kwargs):
    """Scatter plot of probabilities for one ourcome.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    outcomeVar : str
    prob : pd.Series
        Predicted probabilities returned from computeROC or computeCVROC"""

    colors = palettable.colorbrewer.qualitative.Set1_3.mpl_colors

    tmp = df.join(prob, how='inner')
    tmp = tmp.sort_values(by='Prob')
    tmp['x'] = np.arange(tmp.shape[0])

    plt.clf()
    for color,val in zip(colors, tmp[outcomeVar].unique()):
        ind = tmp[outcomeVar] == val
        lab = '%s = %1.0f (%d)' % (outcomeVar, val, ind.sum())
        plt.scatter(tmp.x.loc[ind], tmp.Prob.loc[ind], label=lab, color=color, **kwargs)
    plt.plot([0,tmp.shape[0]],[0.5, 0.5], 'k--', lw=1)
    plt.legend(loc='upper left')
    plt.ylabel('Predicted Pr(%s)' % outcomeVar)
    plt.ylim((-0.05, 1.05))
    plt.xlim(-1, tmp.shape[0])
    plt.show()

def plot2Prob(df, outcomeVar, prob, **kwargs):
    """Scatter plot of probabilities for two outcomes.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : list
        Contains two outcomeVar for comparison
    prob : list
        Contains two pd.Series with predicted probabilities
        from computeROC or computeCVROC"""
    labels = {(0,0):'Neither',
              (1,1):'Both',
              (0,1):'%s only' % outcomeVar[1],
              (1,0):'%s only' % outcomeVar[0]}
    colors = palettable.colorbrewer.qualitative.Set1_5.mpl_colors
    tmp = df.join(prob[0], how='inner').join(prob[1], how='inner', rsuffix='_Y')

    plt.clf()
    plt.gca().set_aspect('equal')
    prodIter = itertools.product(tmp[outcomeVar[0]].unique(), tmp[outcomeVar[1]].unique())
    for color,val in zip(colors, prodIter):
        valx, valy = val
        ind = (tmp[outcomeVar[0]] == valx) & (tmp[outcomeVar[1]] == valy)
        lab = labels[val] + ' (%d)' % ind.sum()
        plt.scatter(tmp.Prob.loc[ind], tmp.Prob_Y.loc[ind], label=lab, color=color, **kwargs)
    plt.plot([0.5,0.5], [0,1], 'k--', lw=1)
    plt.plot([0,1], [0.5,0.5], 'k--', lw=1)
    plt.ylim((-0.05, 1.05))
    plt.xlim((-0.05, 1.05))
    plt.legend(loc=0)
    plt.ylabel('Predicted Pr(%s)' % outcomeVar[1])
    plt.xlabel('Predicted Pr(%s)' % outcomeVar[0])
    plt.show()

def lassoVarSelect(df, outcomeVar, predVars, nFolds=10, alpha=None):
    """Apply LASSO to df and return performance metrics,
    optionally in a cross-validation framework to select alpha.

    ROC metrics computed on all data.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    outcomeVar : str
    predVars : ndarray or list
        Predictor variables in the model.
    nFolds : int
        N-fold cross-validation (not required for LOO)
    alpha : float
        Constant that multiplies the L1 term (aka lambda)
        Defaults to 1.0
        alpha = 0 is equivalent to OLS
        Use None to set to maximum value given by:
            abs(X.T.dot(Y)).max() / X.shape[0]

    Returns
    -------
    fpr : np.ndarray
        False-positive rate.
    meanTPR : np.ndarray
        True-positive rate.
    auc : float
        Area under the ROC curve.
    acc : float
        Sccuracy score
    results : returned by Lasso.fit()
        Model results object
    prob : pd.Series
        Predicted probabilities with index from df
    varList : list
        Variables with non-zero coefficients
    alpha : float
        Optimal alpha value using coordinate descent path"""
    if not type(predVars) is list:
        predVars = list(predVars)
    tmp = df[[outcomeVar] + predVars].dropna()
    if nFolds == 1 or not alpha is None:
        """Pre-specify alpha, no CV needed"""
        if alpha is None:
            """Use the theoretical max alpha (not sure this is right though)"""
            alpha = np.abs(tmp[predVars].T.dot(tmp[outcomeVar])).max() / tmp.shape[0]
        model = sklearn.linear_model.Lasso(alpha=alpha)
    else:
        model = sklearn.linear_model.LassoCV(cv=nFolds)# , alphas=np.linspace(0.001,0.1,50))
    results = model.fit(y=tmp[outcomeVar], X=tmp[predVars])

    if hasattr(model,'alpha_'):
        optimalAlpha = model.alpha_
    else:
        optimalAlpha = model.alpha

    prob = results.predict(tmp[predVars])
    fpr, tpr, thresholds = sklearn.metrics.roc_curve(tmp[outcomeVar].values, prob)
    acc = sklearn.metrics.accuracy_score(tmp[outcomeVar].values, np.round(prob))
    auc = sklearn.metrics.auc(fpr, tpr)
    varList = np.array(predVars)[results.coef_ != 0].tolist()
    probS = pd.Series(prob, index=tmp.index, name='Prob')
    return fpr, tpr, auc, acc, results, probS, varList, optimalAlpha

class smLogisticRegression(object):
    """A wrapper of statsmodels.GLM to use with sklearn interface"""
    def __init__(self, fit_intercept=True):
        self.fit_intercept = fit_intercept

    def fit(self, X, y):
        if self.fit_intercept:
            exog = sm.add_constant(X)
        else:
            exog = X
        self.res = sm.GLM(endog=y, exog=exog, family=sm.families.Binomial()).fit()
        return self

    def predict_proba(self, X):
        prob = np.zeros((X.shape[0],2))
        prob[:,0] = 1 - self.predict(X)
        prob[:,1] = self.predict(X)
        return prob

    def predict(self, X):
        if self.fit_intercept:
            exog = sm.add_constant(X)
        else:
            exog = X
        pred = self.res.predict(exog)
        return pred
<commit_msg>Added basic ROC stats.<commit_after>import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import itertools
import statsmodels.api as sm
import sklearn
import sklearn.ensemble
import palettable

sns.set(style='darkgrid', palette='muted', font_scale=1.5)

__all__ = ['computeROC',
           'computeCVROC',
           'plotROC',
           'plotProb',
           'plot2Prob',
           'lassoVarSelect',
           'smLogisticRegression',
           'rocStats']

def computeROC(df, model, outcomeVar, predVars):
    """Apply model to df and return performance metrics.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : str
    predVars : ndarray or list
        Predictor variables in the model.

    Returns
    -------
    fpr : np.ndarray
        False-positive rate
    tpr : np.ndarray
        True-positive rate
    auc : float
        Area under the ROC curve
    acc : float
        Accuracy score
    results : returned by model.fit()
        Model results object for test prediction in CV
    prob : pd.Series
        Predicted probabilities with index from df"""

    if not type(predVars) is list:
        predVars = list(predVars)
    tmp = df[[outcomeVar] + predVars].dropna()

    try:
        results = model.fit(X=tmp[predVars], y=tmp[outcomeVar])
        if hasattr(results, 'predict_proba'):
            prob = results.predict_proba(tmp[predVars])[:,1]
        else:
            prob = results.predict(tmp[predVars])
            results.predict_proba = results.predict
        fpr, tpr, thresholds = sklearn.metrics.roc_curve(tmp[outcomeVar].values, prob)

        acc = sklearn.metrics.accuracy_score(tmp[outcomeVar].values, np.round(prob))
        auc = sklearn.metrics.auc(fpr, tpr)
        tpr[0], tpr[-1] = 0,1
    except sm.tools.sm_exceptions.PerfectSeparationError:
        print 'PerfectSeparationError: %s (N = %d; %d predictors)' % (outcomeVar, tmp.shape[0], len(predVars))
        acc = 1.
        fpr = np.zeros(5)
        tpr = np.ones(5)
        tpr[0], tpr[-1] = 0,1
        prob = tmp[outcomeVar].values.astype(float)
        auc = 1.
        results = None
    return fpr, tpr, auc, acc, results, pd.Series(prob, index=tmp.index, name='Prob')

def computeCVROC(df, model, outcomeVar, predVars, LOO=False, nFolds=10):
    """Apply model to df and return performance metrics in a cross-validation framework.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : str
    predVars : ndarray or list
        Predictor variables in the model.
    LOO : bool
        Leave-one-out cross-validation?
    nFolds : int
        N-fold cross-validation (not required for LOO)

    Returns
    -------
    fpr : np.ndarray
        Pre-specified vector of FPR thresholds for interpolation
        fpr = np.linspace(0, 1, 100)
    meanTPR : np.ndarray
        Mean true-positive rate in test fraction.
    auc : float
        Area under the mean ROC curve.
    acc : float
        Mean accuracy score in test fraction.
    results : returned by model.fit()
        Training model results object for each fold
    prob : pd.Series
        Mean predicted probabilities on test data with index from df"""

    if not type(predVars) is list:
        predVars = list(predVars)
    tmp = df[[outcomeVar] + predVars].dropna()
    if LOO:
        cv = sklearn.cross_validation.LeaveOneOut(n=tmp.shape[0])
        nFolds = tmp.shape[0]
    else:
        cv = sklearn.cross_validation.KFold(n=tmp.shape[0],
                                            n_folds=nFolds,
                                            shuffle=True,
                                            random_state=110820)
    fpr = np.linspace(0, 1, 100)
    tpr = np.nan * np.zeros((fpr.shape[0], nFolds))
    acc = 0
    counter = 0
    results = []
    prob = []
    for i, (trainInd, testInd) in enumerate(cv):
        trainDf = tmp.iloc[trainInd]
        testDf = tmp.iloc[testInd]
        trainFPR, trainTPR, trainAUC, trainACC, res, trainProb = computeROC(trainDf,
                                                                            model,
                                                                            outcomeVar,
                                                                            predVars)
        if not res is None:
            counter += 1
            testProb = res.predict_proba(testDf[predVars])[:,1]
            testFPR, testTPR, _ = sklearn.metrics.roc_curve(testDf[outcomeVar].values, testProb)
            tpr[:,i] = np.interp(fpr, testFPR, testTPR)
            acc += sklearn.metrics.accuracy_score(testDf[outcomeVar].values, np.round(testProb))
            results.append(res)
            prob.append(pd.Series(testProb, index=testDf.index))
    if counter != nFolds:
        print 'ROC: did not finish all folds (%d of %d)' % (counter, nFolds)
    if counter >=1:
        meanTPR = np.nanmean(tpr, axis=1)
        meanTPR[0], meanTPR[-1] = 0,1
        meanACC = acc / counter
        meanAUC = sklearn.metrics.auc(fpr, meanTPR)
        """Compute mean probability over test predictions in CV"""
        probS = pd.concat(prob).groupby(level=0).agg(np.mean)
        probS.name = 'Prob'
    else:
        """If we get all PerfectSeparation errors then just report model fit to all data"""
        print "Returning metrics from fitting complete dataset"
        testFPR, testTPR, meanAUC, meanACC, res, probS = computeROC(tmp,
                                                                    model,
                                                                    outcomeVar,
                                                                    predVars)
        meanTPR = np.interp(fpr, testFPR, testTPR)
        meanTPR[0], meanTPR[-1] = 0,1
        results = [res]
        '''
        meanTPR = np.nan * fpr
        meanTPR[0], meanTPR[-1] = 0,1
        meanACC = np.nan
        meanAUC = np.nan
        """Compute mean probability over test predictions in CV"""
        probS = np.nan
        '''

    return fpr, meanTPR, meanAUC, meanACC, results, probS

def plotROC(df, model, outcomeVar, predictorsList, predictorLabels=None, rocFunc=computeCVROC, **rocKwargs):
    """Plot of multiple ROC curves using same model and same outcomeVar with
    different sets of predictors.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : str
    predictorsList : list
        List of lists of predictor variables for each model.
    predictorLabels : list
        List of labels for the models (optional)
    LOO : bool
        Leave-one-out cross-vlaidation?
    rocFunc : computeCVROC or computeROC
        Function for computing the ROC
    rocKwargs : kwargs
        Additional arguments for rocFunc"""

    if predictorLabels is None:
        predictorLabels = [' + '.join(predVars) for predVars in predictorsList]

    colors = palettable.colorbrewer.qualitative.Set1_8.mpl_colors

    plt.clf()
    plt.gca().set_aspect('equal')
    for predVarsi,predVars in enumerate(predictorsList):
        fpr, tpr, auc, acc, res, probS = rocFunc(df,
                                                 model,
                                                 outcomeVar,
                                                 predVars,
                                                 **rocKwargs)

        label = '%s (AUC = %0.2f; ACC = %0.2f)' % (predictorLabels[predVarsi], auc, acc)
        plt.plot(fpr, tpr, color=colors[predVarsi], lw=2, label=label)
    plt.plot([0, 1], [0, 1], '--', color='gray', label='Chance')
    plt.xlim([-0.05, 1.05])
    plt.ylim([-0.05, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for %s' % outcomeVar)
    plt.legend(loc="lower right")
    plt.show()

def plotProb(df, outcomeVar, prob, **kwargs):
    """Scatter plot of probabilities for one ourcome.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    outcomeVar : str
    prob : pd.Series
        Predicted probabilities returned from computeROC or computeCVROC"""

    colors = palettable.colorbrewer.qualitative.Set1_3.mpl_colors

    tmp = df.join(prob, how='inner')
    tmp = tmp.sort_values(by=[outcomeVar,'Prob'])
    tmp['x'] = np.arange(tmp.shape[0])

    plt.clf()
    for color,val in zip(colors, tmp[outcomeVar].unique()):
        ind = tmp[outcomeVar] == val
        lab = '%s = %1.0f (%d)' % (outcomeVar, val, ind.sum())
        plt.scatter(tmp.x.loc[ind], tmp.Prob.loc[ind], label=lab, color=color, **kwargs)
    plt.plot([0,tmp.shape[0]],[0.5, 0.5], 'k--', lw=1)
    plt.legend(loc='upper left')
    plt.ylabel('Predicted Pr(%s)' % outcomeVar)
    plt.ylim((-0.05, 1.05))
    plt.xlim(-1, tmp.shape[0])
    plt.show()

def plot2Prob(df, outcomeVar, prob, **kwargs):
    """Scatter plot of probabilities for two outcomes.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    model : sklearn or other model
        Model must have fit and predict methods.
    outcomeVar : list
        Contains two outcomeVar for comparison
    prob : list
        Contains two pd.Series with predicted probabilities
        from computeROC or computeCVROC"""
    labels = {(0,0):'Neither',
              (1,1):'Both',
              (0,1):'%s only' % outcomeVar[1],
              (1,0):'%s only' % outcomeVar[0]}
    colors = palettable.colorbrewer.qualitative.Set1_5.mpl_colors
    tmp = df.join(prob[0], how='inner').join(prob[1], how='inner', rsuffix='_Y')

    plt.clf()
    plt.gca().set_aspect('equal')
    prodIter = itertools.product(tmp[outcomeVar[0]].unique(), tmp[outcomeVar[1]].unique())
    for color,val in zip(colors, prodIter):
        valx, valy = val
        ind = (tmp[outcomeVar[0]] == valx) & (tmp[outcomeVar[1]] == valy)
        lab = labels[val] + ' (%d)' % ind.sum()
        plt.scatter(tmp.Prob.loc[ind], tmp.Prob_Y.loc[ind], label=lab, color=color, **kwargs)
    plt.plot([0.5,0.5], [0,1], 'k--', lw=1)
    plt.plot([0,1], [0.5,0.5], 'k--', lw=1)
    plt.ylim((-0.05, 1.05))
    plt.xlim((-0.05, 1.05))
    plt.legend(loc=0)
    plt.ylabel('Predicted Pr(%s)' % outcomeVar[1])
    plt.xlabel('Predicted Pr(%s)' % outcomeVar[0])
    plt.show()

def lassoVarSelect(df, outcomeVar, predVars, nFolds=10, alpha=None):
    """Apply LASSO to df and return performance metrics,
    optionally in a cross-validation framework to select alpha.

    ROC metrics computed on all data.

    Parameters
    ----------
    df : pd.DataFrame
        Must contain outcome and predictor variables.
    outcomeVar : str
    predVars : ndarray or list
        Predictor variables in the model.
    nFolds : int
        N-fold cross-validation (not required for LOO)
    alpha : float
        Constant that multiplies the L1 term (aka lambda)
        Defaults to 1.0
        alpha = 0 is equivalent to OLS
        Use None to set to maximum value given by:
            abs(X.T.dot(Y)).max() / X.shape[0]

    Returns
    -------
    fpr : np.ndarray
        False-positive rate.
    meanTPR : np.ndarray
        True-positive rate.
    auc : float
        Area under the ROC curve.
    acc : float
        Sccuracy score
    results : returned by Lasso.fit()
        Model results object
    prob : pd.Series
        Predicted probabilities with index from df
    varList : list
        Variables with non-zero coefficients
    alpha : float
        Optimal alpha value using coordinate descent path"""
    if not type(predVars) is list:
        predVars = list(predVars)
    tmp = df[[outcomeVar] + predVars].dropna()
    if nFolds == 1 or not alpha is None:
        """Pre-specify alpha, no CV needed"""
        if alpha is None:
            """Use the theoretical max alpha (not sure this is right though)"""
            alpha = np.abs(tmp[predVars].T.dot(tmp[outcomeVar])).max() / tmp.shape[0]
        model = sklearn.linear_model.Lasso(alpha=alpha)
    else:
        model = sklearn.linear_model.LassoCV(cv=nFolds)# , alphas=np.linspace(0.001,0.1,50))
    results = model.fit(y=tmp[outcomeVar], X=tmp[predVars])

    if hasattr(model,'alpha_'):
        optimalAlpha = model.alpha_
    else:
        optimalAlpha = model.alpha

    prob = results.predict(tmp[predVars])
    fpr, tpr, thresholds = sklearn.metrics.roc_curve(tmp[outcomeVar].values, prob)
    acc = sklearn.metrics.accuracy_score(tmp[outcomeVar].values, np.round(prob))
    auc = sklearn.metrics.auc(fpr, tpr)
    varList = np.array(predVars)[results.coef_ != 0].tolist()
    probS = pd.Series(prob, index=tmp.index, name='Prob')
    return fpr, tpr, auc, acc, results, probS, varList, optimalAlpha

class smLogisticRegression(object):
    """A wrapper of statsmodels.GLM to use with sklearn interface"""
    def __init__(self, fit_intercept=True):
        self.fit_intercept = fit_intercept

    def fit(self, X, y):
        if self.fit_intercept:
            exog = sm.add_constant(X)
        else:
            exog = X
        self.res = sm.GLM(endog=y, exog=exog, family=sm.families.Binomial()).fit()
        return self

    def predict_proba(self, X):
        prob = np.zeros((X.shape[0],2))
        prob[:,0] = 1 - self.predict(X)
        prob[:,1] = self.predict(X)
        return prob

    def predict(self, X):
        if self.fit_intercept:
            exog = sm.add_constant(X)
        else:
            exog = X
        pred = self.res.predict(exog)
        return pred
def rocStats(obs, pred):
    """Compute stats for a 2x2 table derived from
    observed and predicted data vectors

    Parameters
    ----------
    obs,pred : np.ndarray or pd.Series of shape (n,)

    Returns
    -------
    sens : float
        Sensitivity (1 - false-negative rate)
    spec : float
        Specificity (1 - false-positive rate)
    ppv : float
        Positive predictive value (1 - false-discovery rate)
    npv : float
        Negative predictive value
    acc : float
        Accuracy
    OR : float
        Odds-ratio of the observed event in the two predicted groups.
    rr : float
        Relative rate of the observed event in the two predicted groups.
    nnt : float
        Number needed to treat, to prevent one case.
        (assuming all predicted positives were "treated")"""

    assert obs.shape[0] == pred.shape[0]
    n = obs.shape
    a = (obs & pred).sum()
    b = (obs & (~pred)).sum()
    c = ((~obs) & pred).sum()
    d = ((~obs) & (~pred)).sum()

    sens = a / (a+b)
    spec = d / (c+d)
    ppv = a / (a+c)
    npv = d / (b+d)
    nnt = 1 / (a/(a+c) - b/(b+d))
    acc = (a + d)/n
    rr = (a / (a+c)) / (b / (b+d))
    OR = (a/b) / (c/d)
    return sens, spec, ppv, npv, acc, OR, rr, nnt

<|endoftext|>