<commit_before>from __future__ import division

import ctypes, os, shutil
import numpy as np

# constants
c = 299792458
eps0 = 8.854187817e-12
e = 1.602176565e-19
m_e = 9.10938291e-31
hbar = 1.054571726e-34
h = 2*np.pi*hbar
a0 = 4*np.pi*eps0*hbar**2/m_e/e**2
Ry = h*c * m_e*e**4/8/eps0**2/h**3/c

# find and import shared object / DLL
rootdir = os.path.dirname(os.path.realpath(__file__))

if os.name=="posix":
  lewenstein_so = ctypes.CDLL(os.path.join(rootdir,'lewenstein.so'))
elif os.name=="nt":
  bits = ctypes.sizeof(ctypes.c_voidp)*8
  archdirectory = os.path.join(rootdir,'dll' + str(bits))

  rootdirdll = os.path.join(rootdir,'lewenstein.dll')
  correctdll = os.path.join(archdirectory,'lewenstein.dll')
  if not os.path.exists(rootdirdll) or os.path.getsize(rootdirdll)!=os.path.getsize(correctdll):
    for filename in os.listdir(archdirectory): # for dependencies
      shutil.copy(os.path.join(archdirectory,filename), rootdir)

  lewenstein_so = ctypes.CDLL(os.path.join(rootdir,'lewenstein'))

# Note: explicitly setting lewenstein_so.*.argtypes/restype is necessary to prevent segfault on 64 bit:
# http://stackoverflow.com/questions/17240621/wrapping-simple-c-example-with-ctypes-segmentation-fault

# base class for dipole elements
class dipole_elements(object):
  dims = None
  pointer = None

  def __del__(self):
    raise NotImplementedError('override in subclasses')

# wrap H dipole elements
lewenstein_so.dipole_elements_H_double.argtypes = [ctypes.c_int, ctypes.c_double]
lewenstein_so.dipole_elements_H_double.restype = ctypes.c_void_p
lewenstein_so.dipole_elements_H_double_destroy.argtypes = [ctypes.c_int, ctypes.c_void_p]
lewenstein_so.dipole_elements_H_double_destroy.restype = None

class dipole_elements_H(dipole_elements):
  def __init__(self, dims, ip, wavelength=None):
    if wavelength:
      ip = sau_convert(ip, 'U', 'SAU', wavelength)

    alpha = 2*ip

    self.dims = dims
    self.pointer = lewenstein_so.dipole_elements_H_double(dims, alpha)

  def __del__(self):
    lewenstein_so.dipole_elements_H_double_destroy(self.dims, self.pointer)

# wrap symmetric interpolated dipole elements
lewenstein_so.dipole_elements_symmetric_interpolate_double.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_double, ctypes.c_void_p, ctypes.c_void_p]
lewenstein_so.dipole_elements_symmetric_interpolate_double.restype = ctypes.c_void_p
lewenstein_so.dipole_elements_symmetric_interpolate_double_destroy.argtypes = [ctypes.c_int, ctypes.c_void_p]
lewenstein_so.dipole_elements_symmetric_interpolate_double_destroy.restype = None

class dipole_elements_symmetric_interpolate(dipole_elements):
  # TODO: untested!

  _dr = None
  _di = None

  def __init__(self, dims, p, d, wavelength=None):
     if wavelength is not None:
       p = sau_convert(p, 'p', 'SAU', wavelength)
       d = sau_convert(d, 'd', 'SAU', wavelength)

     self.dims = dims

     N = p.size
     assert d.size==N

     dp = np.min(np.diff(p))
     assert np.isclose(dp, np.max(np.diff(p)), atol=0)

     # dr and di must not be garbage collected until destructor is called, so make it a property of this class
     self._dr = np.require(np.copy(d.real), np.double, ['C', 'A'])
     self._di = np.require(np.copy(d.imag), np.double, ['C', 'A'])

     self.pointer = lewenstein_so.dipole_elements_symmetric_interpolate_double(dims, N, dp, self._dr.ctypes.data, self._di.ctypes.data)

  def __del__(self):
     lewenstein_so.dipole_elements_symmetric_interpolate_double_destroy(self.dims, self.pointer)

# helper function to generate weights
def get_weights(tau,T=2*np.pi,periods_one=1,periods_soft=.5):
  interval_points = sum(tau-tau[0]<=periods_one*T)
  window_points = sum(tau-tau[0]<=periods_soft*T)

  r = np.ones(interval_points+window_points)
  r[-window_points:] = np.cos(np.pi/2 * np.arange(window_points)/window_points)**2

  return r

# helper function for unit conversion
def sau_convert(value, quantity, target, wavelength):
  # scaled atomic unit quantities expressed in SI units
  unit = {}
  unit['t'] = wavelength / c / (2*np.pi)
  unit['omega'] = 1/unit['t']
  unit['U'] = hbar * unit['omega'] # hbar*omega
  unit['q'] = e
  unit['s'] = a0 * np.sqrt(2*Ry/unit['U']) # [concluded from (23) in Lewenstein paper]
  unit['E'] = unit['U'] / unit['q'] / unit['s'] # [concluded from (2) in Lewenstein paper]
  unit['d'] = unit['q'] * unit['s'] # dipole element
  unit['m'] = m_e
  unit['p'] = unit['m'] * unit['s']/unit['t'] # momentum

  if target=="SI":
    return value * unit[quantity]
  elif target=="SAU":
    return value / unit[quantity]
  else:
    raise ValueError('target must be SI or SAU')

# wrap lewenstein function
lewenstein_so.lewenstein_double.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_double, ctypes.c_double, ctypes.c_void_p, ctypes.c_void_p]
lewenstein_so.lewenstein_double.restype = None

def lewenstein(t,Et,ip,wavelength=None,weights=None,at=None,dipole_elements=None,epsilon_t=1e-4):
  # default value for weights
  if weights is None and wavelength is None:
    weights = get_weights(t)
  elif weights is None and wavelength is not None:
    weights = get_weights(t, wavelength/c)

  # unit conversion
  if wavelength is not None:
    t = sau_convert(t, 't', 'SAU', wavelength)
    Et = sau_convert(Et, 'E', 'SAU', wavelength)
    ip = sau_convert(ip, 'U', 'SAU', wavelength)

  # default value for ground state amplitude
  if at is None: at = np.ones_like(t)

  # allocate memory for output
  output = np.empty_like(Et)

  # make sure t axis starts at zero
  t = t - t[0]

  # make sure we have appropriate memory layout before passing to C code
  t = np.require(t, np.double, ['C', 'A'])
  Et = np.require(Et, np.double, ['C', 'A'])
  weights = np.require(weights, np.double, ['C', 'A'])
  at = np.require(at, np.double, ['C', 'A'])
  output = np.require(output, np.double, ['C', 'A', 'W'])

  # get dimensions
  N = t.size
  dims = Et.shape[1] if len(Et.shape)>1 else 1
  weights_length = weights.size

  # check dimensions
  assert at.size==N
  assert Et.shape[0]==N
  assert dims in [1,2,3]
  assert Et.size==N*dims

  # default value for dipole elements
  if dipole_elements is None: dipole_elements = dipole_elements_H(dims, ip=ip)

  # call C function
  lewenstein_so.lewenstein_double(dims, N, t.ctypes.data, Et.ctypes.data, weights_length, weights.ctypes.data, at.ctypes.data, ip, epsilon_t, dipole_elements.pointer, output.ctypes.data)

  # unit conversion
  if wavelength is not None:
    output = sau_convert(output, 'd', 'SI', wavelength)

  return output

if __name__=="__main__":
  import pylab

  # compare to reference, in scaled atomic units
  n = 5

  t = np.arange(n)
  Et = t
  weights = np.ones_like(t)
  ip = 1

  d = lewenstein(t,Et,ip,None,weights)
  reference_d = [0.00000, 0.00000, -2.18318, -2.95464, -1.23753] # computed by Matlab/octave module
  assert np.allclose(d, reference_d, atol=1e-4)
  print "Test passed"

  # plot dipole response for pulse (using SI units)
  wavelength = 1000e-9
  T = wavelength/c
  t = np.linspace(-20*T,20*T,200*40+1)

  fwhm = 30e-15
  tau = fwhm/2/np.sqrt(np.log(np.sqrt(2)))
  Et = np.exp(-(t/tau)**2) * np.cos(2*np.pi/T*t) * np.sqrt(2*1e18/c/eps0)

  ip = 12.13*e # Xe

  d = lewenstein(t,Et,ip,wavelength)

  pylab.semilogy(np.fft.fftfreq(len(t), t[1]-t[0])/(1/T), abs(np.fft.fft(d))**2)
  pylab.xlim((0,100))
  pylab.show()
<commit_msg>prevent potential segfault in pylewenstein<commit_after>from __future__ import division

import ctypes, os, shutil
import numpy as np

# constants
c = 299792458
eps0 = 8.854187817e-12
e = 1.602176565e-19
m_e = 9.10938291e-31
hbar = 1.054571726e-34
h = 2*np.pi*hbar
a0 = 4*np.pi*eps0*hbar**2/m_e/e**2
Ry = h*c * m_e*e**4/8/eps0**2/h**3/c

# find and import shared object / DLL
rootdir = os.path.dirname(os.path.realpath(__file__))

if os.name=="posix":
  lewenstein_so = ctypes.CDLL(os.path.join(rootdir,'lewenstein.so'))
elif os.name=="nt":
  bits = ctypes.sizeof(ctypes.c_voidp)*8
  archdirectory = os.path.join(rootdir,'dll' + str(bits))

  rootdirdll = os.path.join(rootdir,'lewenstein.dll')
  correctdll = os.path.join(archdirectory,'lewenstein.dll')
  if not os.path.exists(rootdirdll) or os.path.getsize(rootdirdll)!=os.path.getsize(correctdll):
    for filename in os.listdir(archdirectory): # for dependencies
      shutil.copy(os.path.join(archdirectory,filename), rootdir)

  lewenstein_so = ctypes.CDLL(os.path.join(rootdir,'lewenstein'))

# Note: explicitly setting lewenstein_so.*.argtypes/restype is necessary to prevent segfault on 64 bit:
# http://stackoverflow.com/questions/17240621/wrapping-simple-c-example-with-ctypes-segmentation-fault

# base class for dipole elements
class dipole_elements(object):
  dims = None
  pointer = None

  def __del__(self):
    raise NotImplementedError('override in subclasses')

# wrap H dipole elements
lewenstein_so.dipole_elements_H_double.argtypes = [ctypes.c_int, ctypes.c_double]
lewenstein_so.dipole_elements_H_double.restype = ctypes.c_void_p
lewenstein_so.dipole_elements_H_double_destroy.argtypes = [ctypes.c_int, ctypes.c_void_p]
lewenstein_so.dipole_elements_H_double_destroy.restype = None

class dipole_elements_H(dipole_elements):
  def __init__(self, dims, ip, wavelength=None):
    if wavelength:
      ip = sau_convert(ip, 'U', 'SAU', wavelength)

    alpha = 2*ip

    self.dims = dims
    self.pointer = lewenstein_so.dipole_elements_H_double(dims, alpha)

  def __del__(self):
    lewenstein_so.dipole_elements_H_double_destroy(self.dims, self.pointer)

# wrap symmetric interpolated dipole elements
lewenstein_so.dipole_elements_symmetric_interpolate_double.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_double, ctypes.c_void_p, ctypes.c_void_p]
lewenstein_so.dipole_elements_symmetric_interpolate_double.restype = ctypes.c_void_p
lewenstein_so.dipole_elements_symmetric_interpolate_double_destroy.argtypes = [ctypes.c_int, ctypes.c_void_p]
lewenstein_so.dipole_elements_symmetric_interpolate_double_destroy.restype = None

class dipole_elements_symmetric_interpolate(dipole_elements):
  # TODO: untested!

  _dr = None
  _di = None

  def __init__(self, dims, p, d, wavelength=None):
     if wavelength is not None:
       p = sau_convert(p, 'p', 'SAU', wavelength)
       d = sau_convert(d, 'd', 'SAU', wavelength)

     self.dims = dims

     N = p.size
     assert d.size==N

     dp = np.min(np.diff(p))
     assert np.isclose(dp, np.max(np.diff(p)), atol=0)

     # dr and di must not be garbage collected until destructor is called, so make it a property of this class
     self._dr = np.require(np.copy(d.real), np.double, ['C', 'A'])
     self._di = np.require(np.copy(d.imag), np.double, ['C', 'A'])

     self.pointer = lewenstein_so.dipole_elements_symmetric_interpolate_double(dims, N, dp, self._dr.ctypes.data, self._di.ctypes.data)

  def __del__(self):
     lewenstein_so.dipole_elements_symmetric_interpolate_double_destroy(self.dims, self.pointer)

# helper function to generate weights
def get_weights(tau,T=2*np.pi,periods_one=1,periods_soft=.5):
  interval_points = sum(tau-tau[0]<=periods_one*T)
  window_points = sum(tau-tau[0]<=periods_soft*T)

  r = np.ones(interval_points+window_points)
  r[-window_points:] = np.cos(np.pi/2 * np.arange(window_points)/window_points)**2

  return r

# helper function for unit conversion
def sau_convert(value, quantity, target, wavelength):
  # scaled atomic unit quantities expressed in SI units
  unit = {}
  unit['t'] = wavelength / c / (2*np.pi)
  unit['omega'] = 1/unit['t']
  unit['U'] = hbar * unit['omega'] # hbar*omega
  unit['q'] = e
  unit['s'] = a0 * np.sqrt(2*Ry/unit['U']) # [concluded from (23) in Lewenstein paper]
  unit['E'] = unit['U'] / unit['q'] / unit['s'] # [concluded from (2) in Lewenstein paper]
  unit['d'] = unit['q'] * unit['s'] # dipole element
  unit['m'] = m_e
  unit['p'] = unit['m'] * unit['s']/unit['t'] # momentum

  if target=="SI":
    return value * unit[quantity]
  elif target=="SAU":
    return value / unit[quantity]
  else:
    raise ValueError('target must be SI or SAU')

# wrap lewenstein function
lewenstein_so.lewenstein_double.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_double, ctypes.c_double, ctypes.c_void_p, ctypes.c_void_p]
lewenstein_so.lewenstein_double.restype = None

def lewenstein(t,Et,ip,wavelength=None,weights=None,at=None,dipole_elements=None,epsilon_t=1e-4):
  # default value for weights
  if weights is None and wavelength is None:
    weights = get_weights(t)
  elif weights is None and wavelength is not None:
    weights = get_weights(t, wavelength/c)

  # unit conversion
  if wavelength is not None:
    t = sau_convert(t, 't', 'SAU', wavelength)
    Et = sau_convert(Et, 'E', 'SAU', wavelength)
    ip = sau_convert(ip, 'U', 'SAU', wavelength)

  # default value for ground state amplitude
  if at is None: at = np.ones_like(t)

  # allocate memory for output
  output = np.empty_like(Et)

  # make sure t axis starts at zero
  t = t - t[0]

  # make sure we have appropriate memory layout before passing to C code
  t = np.require(t, np.double, ['C', 'A'])
  Et = np.require(Et, np.double, ['C', 'A'])
  weights = np.require(weights, np.double, ['C', 'A'])
  at = np.require(at, np.double, ['C', 'A'])
  output = np.require(output, np.double, ['C', 'A', 'W'])

  # get dimensions
  N = t.size
  dims = Et.shape[1] if len(Et.shape)>1 else 1
  weights_length = weights.size

  # check dimensions
  assert at.size==N
  assert Et.shape[0]==N
  assert dims in [1,2,3]
  assert Et.size==N*dims

  # default value for dipole elements
  if dipole_elements is None: dipole_elements = dipole_elements_H(dims, ip=ip)

  # call C function
  assert dipole_elements.dims==dims
  lewenstein_so.lewenstein_double(dims, N, t.ctypes.data, Et.ctypes.data, weights_length, weights.ctypes.data, at.ctypes.data, ip, epsilon_t, dipole_elements.pointer, output.ctypes.data)

  # unit conversion
  if wavelength is not None:
    output = sau_convert(output, 'd', 'SI', wavelength)

  return output

if __name__=="__main__":
  import pylab

  # compare to reference, in scaled atomic units
  n = 5

  t = np.arange(n)
  Et = t
  weights = np.ones_like(t)
  ip = 1

  d = lewenstein(t,Et,ip,None,weights)
  reference_d = [0.00000, 0.00000, -2.18318, -2.95464, -1.23753] # computed by Matlab/octave module
  assert np.allclose(d, reference_d, atol=1e-4)
  print "Test passed"

  # plot dipole response for pulse (using SI units)
  wavelength = 1000e-9
  T = wavelength/c
  t = np.linspace(-20*T,20*T,200*40+1)

  fwhm = 30e-15
  tau = fwhm/2/np.sqrt(np.log(np.sqrt(2)))
  Et = np.exp(-(t/tau)**2) * np.cos(2*np.pi/T*t) * np.sqrt(2*1e18/c/eps0)

  ip = 12.13*e # Xe

  d = lewenstein(t,Et,ip,wavelength)

  pylab.semilogy(np.fft.fftfreq(len(t), t[1]-t[0])/(1/T), abs(np.fft.fft(d))**2)
  pylab.xlim((0,100))
  pylab.show()
<|endoftext|>


<commit_before># This file is part of Maker Keeper Framework.
#
# Copyright (C) 2017-2018 reverendus
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import asyncio
import threading
from typing import Tuple

import eth_keyfile
from eth_utils import coerce_return_to_text, encode_hex
from ethereum import utils
from ethereum.tester import k0
from ethereum.utils import int_to_bytes
from secp256k1 import PrivateKey
from web3 import Web3
from web3.eth import Eth

from pymaker.numeric import Wad

def chain(web3: Web3) -> str:
    block_0 = web3.eth.getBlock(0)['hash']
    if block_0 == "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3":
        return "ethlive"
    elif block_0 == "0xa3c565fc15c7478862d50ccd6561e3c06b24cc509bf388941c25ea985ce32cb9":
        return "kovan"
    elif block_0 == "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d":
        return "ropsten"
    elif block_0 == "0x0cd786a2425d16f152c658316c423e6ce1181e15c3295826d7c9904cba9ce303":
        return "morden"
    else:
        return "unknown"

def http_response_summary(response) -> str:
    return f"{response.status_code} {response.reason}"

def synchronize(futures) -> list:
    if len(futures) > 0:
        loop = asyncio.new_event_loop()
        try:
            return loop.run_until_complete(asyncio.gather(*futures, loop=loop))
        finally:
            loop.close()
    else:
        return []

def eth_balance(web3: Web3, address) -> Wad:
    return Wad(web3.eth.getBalance(address.address))

def int_to_bytes32(value: int) -> bytes:
    assert(isinstance(value, int))
    return value.to_bytes(32, byteorder='big')

def bytes_to_int(value) -> int:
    if isinstance(value, bytes) or isinstance(value, bytearray):
        return int.from_bytes(value, byteorder='big')
    elif isinstance(value, str):
        b = bytearray()
        b.extend(map(ord, value))
        return int.from_bytes(b, byteorder='big')
    else:
        raise AssertionError

def bytes_to_hexstring(value) -> str:
    assert(isinstance(value, bytes) or isinstance(value, bytearray))
    return "0x" + "".join(map(lambda b: format(b, "02x"), value))

def hexstring_to_bytes(value: str) -> bytes:
    assert(isinstance(value, str))
    assert(value.startswith("0x"))
    return bytes.fromhex(value.replace("0x", ""))

class AsyncCallback:
    """Decouples callback invocation from the web3.py filter.

    Decouples callback invocation from the web3.py filter by executing the callback
    in a dedicated thread. If we make web3.py trigger the callback directly, and the callback
    execution takes more than 60 seconds, the `eth_getFilterChanges` call also will not
    get called for 60 seconds and more which will make the filter expire in Parity side.
    It's 60 seconds for Parity, this could be a different value for other nodes,
    but the filter will eventually expire sooner or later anyway.

    Invoking the callback logic in a separate thread allows the web3.py Filter thread
    to keep calling `eth_getFilterChanges` regularly, so the filter stays active.

    Attributes:
        callback: The callback function to be invoked in a separate thread.
    """
    def __init__(self, callback):
        self.callback = callback
        self.thread = None

    def trigger(self, on_start=None, on_finish=None) -> bool:
        """Invokes the callback in a separate thread, unless one is already running.

        If callback isn't currently running, invokes it in a separate thread and returns `True`.
        If the previous callback invocation still hasn't finished, doesn't do anything
        and returns `False`.

        Arguments:
            on_start: Optional method to be called before the actual callback. Can be `None`.
            on_finish: Optional method to be called after the actual callback. Can be `None`.

        Returns:
            `True` if callback has been invoked. `False` otherwise.
        """
        if self.thread is None or not self.thread.is_alive():
            def thread_target():
                if on_start is not None:
                    on_start()
                self.callback()
                if on_finish is not None:
                    on_finish()

            self.thread = threading.Thread(target=thread_target)
            self.thread.start()
            return True
        else:
            return False

    def wait(self):
        """Waits for the currently running callback to finish.

        If the callback isn't running or hasn't even been invoked once, returns instantly."""
        if self.thread is not None:
            self.thread.join()
<commit_msg>Revert "Shorter outpyut of "http_response_summary""<commit_after># This file is part of Maker Keeper Framework.
#
# Copyright (C) 2017-2018 reverendus
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import asyncio
import threading
from typing import Tuple

import eth_keyfile
from eth_utils import coerce_return_to_text, encode_hex
from ethereum import utils
from ethereum.tester import k0
from ethereum.utils import int_to_bytes
from secp256k1 import PrivateKey
from web3 import Web3
from web3.eth import Eth

from pymaker.numeric import Wad

def chain(web3: Web3) -> str:
    block_0 = web3.eth.getBlock(0)['hash']
    if block_0 == "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3":
        return "ethlive"
    elif block_0 == "0xa3c565fc15c7478862d50ccd6561e3c06b24cc509bf388941c25ea985ce32cb9":
        return "kovan"
    elif block_0 == "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d":
        return "ropsten"
    elif block_0 == "0x0cd786a2425d16f152c658316c423e6ce1181e15c3295826d7c9904cba9ce303":
        return "morden"
    else:
        return "unknown"

def http_response_summary(response) -> str:
    text = response.text.replace('\r', '').replace('\n', '')[:512]
    return f"{response.status_code} {response.reason} ({text})"

def synchronize(futures) -> list:
    if len(futures) > 0:
        loop = asyncio.new_event_loop()
        try:
            return loop.run_until_complete(asyncio.gather(*futures, loop=loop))
        finally:
            loop.close()
    else:
        return []

def eth_balance(web3: Web3, address) -> Wad:
    return Wad(web3.eth.getBalance(address.address))

def int_to_bytes32(value: int) -> bytes:
    assert(isinstance(value, int))
    return value.to_bytes(32, byteorder='big')

def bytes_to_int(value) -> int:
    if isinstance(value, bytes) or isinstance(value, bytearray):
        return int.from_bytes(value, byteorder='big')
    elif isinstance(value, str):
        b = bytearray()
        b.extend(map(ord, value))
        return int.from_bytes(b, byteorder='big')
    else:
        raise AssertionError

def bytes_to_hexstring(value) -> str:
    assert(isinstance(value, bytes) or isinstance(value, bytearray))
    return "0x" + "".join(map(lambda b: format(b, "02x"), value))

def hexstring_to_bytes(value: str) -> bytes:
    assert(isinstance(value, str))
    assert(value.startswith("0x"))
    return bytes.fromhex(value.replace("0x", ""))

class AsyncCallback:
    """Decouples callback invocation from the web3.py filter.

    Decouples callback invocation from the web3.py filter by executing the callback
    in a dedicated thread. If we make web3.py trigger the callback directly, and the callback
    execution takes more than 60 seconds, the `eth_getFilterChanges` call also will not
    get called for 60 seconds and more which will make the filter expire in Parity side.
    It's 60 seconds for Parity, this could be a different value for other nodes,
    but the filter will eventually expire sooner or later anyway.

    Invoking the callback logic in a separate thread allows the web3.py Filter thread
    to keep calling `eth_getFilterChanges` regularly, so the filter stays active.

    Attributes:
        callback: The callback function to be invoked in a separate thread.
    """
    def __init__(self, callback):
        self.callback = callback
        self.thread = None

    def trigger(self, on_start=None, on_finish=None) -> bool:
        """Invokes the callback in a separate thread, unless one is already running.

        If callback isn't currently running, invokes it in a separate thread and returns `True`.
        If the previous callback invocation still hasn't finished, doesn't do anything
        and returns `False`.

        Arguments:
            on_start: Optional method to be called before the actual callback. Can be `None`.
            on_finish: Optional method to be called after the actual callback. Can be `None`.

        Returns:
            `True` if callback has been invoked. `False` otherwise.
        """
        if self.thread is None or not self.thread.is_alive():
            def thread_target():
                if on_start is not None:
                    on_start()
                self.callback()
                if on_finish is not None:
                    on_finish()

            self.thread = threading.Thread(target=thread_target)
            self.thread.start()
            return True
        else:
            return False

    def wait(self):
        """Waits for the currently running callback to finish.

        If the callback isn't running or hasn't even been invoked once, returns instantly."""
        if self.thread is not None:
            self.thread.join()
<|endoftext|>


<commit_before>##############################################################################
# pymbar: A Python Library for MBAR
#
# Copyright 2016-2017 University of Colorado Boulder
# Copyright 2010-2017 Memorial Sloan-Kettering Cancer Center
# Portions of this software are Copyright 2010-2016 University of Virginia
#
# Authors: Michael Shirts, John Chodera
# Contributors: Kyle Beauchamp, Levi Naden
#
# pymbar is free software: you can redistribute it and/or modify
# it under the terms of the MIT License
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# MIT License for more details.
#
# You should have received a copy of the MIT License along with pymbar.
##############################################################################

##############################################################################
# imports
##############################################################################

from six.moves import zip_longest
import warnings
import numpy as np

try:  # numexpr used in logsumexp when available.
    import numexpr
    HAVE_NUMEXPR = True
except ImportError:
    HAVE_NUMEXPR = False

##############################################################################
# functions / classes
##############################################################################

class TypeCastPerformanceWarning(RuntimeWarning):
    pass

def kln_to_kn(kln, N_k = None, cleanup = False):

    """ Convert KxKxN_max array to KxN max array

    if self.N is not initialized, it will be here.

    Parameters
    ----------
    u_kln : np.ndarray, float, shape=(KxLxN_max)
    N_k (optional) : np.array
        the N_k matrix from the previous formatting form
    cleanup (optional) : bool
        optional command to clean up, since u_kln can get very large

    Outputs
    -------
    u_kn: np.ndarray, float, shape=(LxN)
    """

    #print "warning: KxLxN_max arrays deprecated; convering into new preferred KxN shape"

    # rewrite into kn shape
    [K, L, N_max] = np.shape(kln)

    if N_k is None:
        # We assume that all N_k are N_max.
        # Not really an easier way to do this without being given the answer.
        N_k = N_max * np.ones([L], dtype=np.int64)
    N = np.sum(N_k)

    kn = np.zeros([L, N], dtype=np.float64)
    i = 0
    for k in range(K):  # loop through the old K; some might be zero
        for ik in range(N_k[k]):
            kn[:, i] = kln[k, :, ik]
            i += 1
    if cleanup:
        del(kln)  # very big, let's explicitly delete

    return kn

def kn_to_n(kn, N_k = None, cleanup = False):

    """ Convert KxN_max array to N array

    Parameters
    ----------
    u_kn: np.ndarray, float, shape=(KxN_max)
    N_k (optional) : np.array
        the N_k matrix from the previous formatting form
    cleanup (optional) : bool
        optional command to clean up, since u_kln can get very large

    Outputs
    -------
    u_n: np.ndarray, float, shape=(N)
    """

    #print "warning: KxN arrays deprecated; convering into new preferred N shape"
    # rewrite into kn shape

    # rewrite into kn shape
    [K, N_max] = np.shape(kn)

    if N_k is None:
        # We assume that all N_k are N_max.
        # Not really an easier way to do this without being given the answer.
        N_k = N_max*np.ones([K], dtype=np.int64)
    N = np.sum(N_k)

    n = np.zeros([N], dtype=np.float64)
    i = 0
    for k in range(K):  # loop through the old K; some might be zero
        for ik in range(N_k[k]):
            n[i] = kn[k, ik]
            i += 1
    if cleanup:
        del(kn)  # very big, let's explicitly delete
    return n

def ensure_type(val, dtype, ndim, name, length=None, can_be_none=False, shape=None,
                warn_on_cast=True, add_newaxis_on_deficient_ndim=False):
    """Typecheck the size, shape and dtype of a numpy array, with optional
    casting.

    Parameters
    ----------
    val : {np.ndaraay, None}
        The array to check
    dtype : {nd.dtype, str}
        The dtype you'd like the array to have
    ndim : int
        The number of dimensions you'd like the array to have
    name : str
        name of the array. This is used when throwing exceptions, so that
        we can describe to the user which array is messed up.
    length : int, optional
        How long should the array be?
    can_be_none : bool
        Is ``val == None`` acceptable?
    shape : tuple, optional
        What should be shape of the array be? If the provided tuple has
        Nones in it, those will be semantically interpreted as matching
        any length in that dimension. So, for example, using the shape
        spec ``(None, None, 3)`` will ensure that the last dimension is of
        length three without constraining the first two dimensions
    warn_on_cast : bool, default=True
        Raise a warning when the dtypes don't match and a cast is done.
    add_newaxis_on_deficient_ndim : bool, default=True
        Add a new axis to the beginining of the array if the number of
        dimensions is deficient by one compared to your specification. For
        instance, if you're trying to get out an array of ``ndim == 3``,
        but the user provides an array of ``shape == (10, 10)``, a new axis will
        be created with length 1 in front, so that the return value is of
        shape ``(1, 10, 10)``.

    Notes
    -----
    The returned value will always be C-contiguous.

    Returns
    -------
    typechecked_val : np.ndarray, None
        If `val=None` and `can_be_none=True`, then this will return None.
        Otherwise, it will return val (or a copy of val). If the dtype wasn't right,
        it'll be casted to the right shape. If the array was not C-contiguous, it'll
        be copied as well.

    """
    if can_be_none and val is None:
        return None

    if not isinstance(val, np.ndarray):
        # special case: if the user is looking for a 1d array, and
        # they request newaxis upconversion, and provided a scalar
        # then we should reshape the scalar to be a 1d length-1 array
        if add_newaxis_on_deficient_ndim and ndim == 1 and np.isscalar(val):
            val = np.array([val])
        else:
            raise TypeError(("%s must be numpy array. "
                             " You supplied type %s" % (name, type(val))))

    if warn_on_cast and val.dtype != dtype:
        warnings.warn("Casting %s dtype=%s to %s " % (name, val.dtype, dtype),
                      TypeCastPerformanceWarning)

    if not val.ndim == ndim:
        if add_newaxis_on_deficient_ndim and val.ndim + 1 == ndim:
            val = val[np.newaxis, ...]
        else:
            raise ValueError(("%s must be ndim %s. "
                              "You supplied %s" % (name, ndim, val.ndim)))

    val = np.ascontiguousarray(val, dtype=dtype)

    if length is not None and len(val) != length:
        raise ValueError(("%s must be length %s. "
                          "You supplied %s" % (name, length, len(val))))

    if shape is not None:
        # the shape specified given by the user can look like (None, None 3)
        # which indicates that ANY length is accepted in dimension 0 or
        # dimension 1
        sentenel = object()
        error = ValueError(("%s must be shape %s. You supplied  "
                            "%s" % (name, str(shape).replace('None', 'Any'), val.shape)))
        for a, b in zip_longest(val.shape, shape, fillvalue=sentenel):
            if a is sentenel or b is sentenel:
                # if the sentenel was reached, it means that the ndim didn't
                # match or something. this really shouldn't happen
                raise error
            if b is None:
                # if the user's shape spec has a None in it, it matches anything
                continue
            if a != b:
                # check for equality
                raise error

    return val

def _logsum(a_n):
    """Compute the log of a sum of exponentiated terms exp(a_n) in a numerically-stable manner.
    NOTE: this function has been deprecated in favor of logsumexp.

    Parameters
    ----------
    a_n : np.ndarray, shape=(n_samples)
        a_n[n] is the nth exponential argument

    Returns
    -------
    a_n : np.ndarray, shape=(n_samples)
        a_n[n] is the nth exponential argument

    Notes
    -----

    _logsum a_n = max_arg + \log \sum_{n=1}^N \exp[a_n - max_arg]

    where max_arg = max_n a_n.  This is mathematically (but not numerically) equivalent to

    _logsum a_n = \log \sum_{n=1}^N \exp[a_n]

    Example
    -------
    >>> a_n = np.array([0.0, 1.0, 1.2], np.float64)
    >>> print('%.3e' % _logsum(a_n))
    1.951e+00
    """

    # Compute the maximum argument.
    max_log_term = np.max(a_n)

    # Compute the reduced terms.
    terms = np.exp(a_n - max_log_term)

    # Compute the log sum.
    log_sum = np.log(np.sum(terms)) + max_log_term

    return log_sum

def logsumexp(a, axis=None, b=None, use_numexpr=True):
    """Compute the log of the sum of exponentials of input elements.

    Parameters
    ----------
    a : array_like
        Input array.
    axis : None or int, optional, default=None
        Axis or axes over which the sum is taken. By default `axis` is None,
        and all elements are summed.
    b : array-like, optional
        Scaling factor for exp(`a`) must be of the same shape as `a` or
        broadcastable to `a`.
    use_numexpr : bool, optional, default=True
        If True, use the numexpr library to speed up the calculation, which
        can give a 2-4X speedup when working with large arrays.

    Returns
    -------
    res : ndarray
        The result, ``log(sum(exp(a)))`` calculated in a numerically
        more stable way. If `b` is given then ``log(sum(b*exp(a)))``
        is returned.

    See Also
    --------
    numpy.logaddexp, numpy.logaddexp2, scipy.special.logsumexp

    Notes
    -----
    This is based on scipy.special.logsumexp but with optional numexpr
    support for improved performance.
    """

    a = np.asarray(a)

    a_max = np.amax(a, axis=axis, keepdims=True)

    if a_max.ndim > 0:
        a_max[~np.isfinite(a_max)] = 0
    elif not np.isfinite(a_max):
        a_max = 0

    if b is not None:
        b = np.asarray(b)
        if use_numexpr and HAVE_NUMEXPR:
            out = np.log(numexpr.evaluate("b * exp(a - a_max)").sum(axis))
        else:
            out = np.log(np.sum(b * np.exp(a - a_max), axis=axis))
    else:
        if use_numexpr and HAVE_NUMEXPR:
            out = np.log(numexpr.evaluate("exp(a - a_max)").sum(axis))
        else:
            out = np.log(np.sum(np.exp(a - a_max), axis=axis))

    a_max = np.squeeze(a_max, axis=axis)
    out += a_max

    return out

def check_w_normalized(W, N_k, tolerance = 1.0e-4):
    """Check the weight matrix W is properly normalized. The sum over N should be 1, and the sum over k by N_k should aslo be 1

    Parameters
    ----------
    W : np.ndarray, shape=(N, K), dtype='float'
        The normalized weight matrix for snapshots and states.
        W[n, k] is the weight of snapshot n in state k.
    N_k : np.ndarray, shape=(K), dtype='int'
        N_k[k] is the number of samples from state k.
    tolerance : float, optional, default=1.0e-4
        Tolerance for checking equality of sums

    Returns
    -------
    None : NoneType
        Returns a None object if test passes, otherwise raises a ParameterError with appropriate message if W is not normalized within tolerance.
    """

    [N, K] = W.shape

    column_sums = np.sum(W, axis=0)
    badcolumns = (np.abs(column_sums - 1) > tolerance)
    if np.any(badcolumns):
        which_badcolumns = np.arange(K)[badcolumns]
        firstbad = which_badcolumns[0]
        raise ParameterError(
            'Warning: Should have \sum_n W_nk = 1.  Actual column sum for state %d was %f. %d other columns have similar problems' %
            (firstbad, column_sums[firstbad], np.sum(badcolumns)))

    row_sums = np.sum(W * N_k, axis=1)
    badrows = (np.abs(row_sums - 1) > tolerance)
    if np.any(badrows):
        which_badrows = np.arange(N)[badrows]
        firstbad = which_badrows[0]
        raise ParameterError(
            'Warning: Should have \sum_k N_k W_nk = 1.  Actual row sum for sample %d was %f. %d other rows have similar problems' %
            (firstbad, row_sums[firstbad], np.sum(badrows)))

    return

# ============================================================================================
# Exception classes
# =============================================================================================

class ParameterError(Exception):

    """
    An error in the input parameters has been detected.

    """
    pass

class ConvergenceError(Exception):

    """
    Convergence could not be achieved.

    """
    pass

class BoundsError(Exception):

    """
    Could not determine bounds on free energy

    """
    pass

class DataError(Exception):

    """
    Data is inconsistent.

    """
    pass
<commit_msg>Added back in text.<commit_after>##############################################################################
# pymbar: A Python Library for MBAR
#
# Copyright 2016-2017 University of Colorado Boulder
# Copyright 2010-2017 Memorial Sloan-Kettering Cancer Center
# Portions of this software are Copyright 2010-2016 University of Virginia
#
# Authors: Michael Shirts, John Chodera
# Contributors: Kyle Beauchamp, Levi Naden
#
# pymbar is free software: you can redistribute it and/or modify
# it under the terms of the MIT License
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# MIT License for more details.
#
# You should have received a copy of the MIT License along with pymbar.
##############################################################################

##############################################################################
# imports
##############################################################################

from six.moves import zip_longest
import warnings
import numpy as np

try:  # numexpr used in logsumexp when available.
    import numexpr
    HAVE_NUMEXPR = True
except ImportError:
    HAVE_NUMEXPR = False

##############################################################################
# functions / classes
##############################################################################

class TypeCastPerformanceWarning(RuntimeWarning):
    pass

def kln_to_kn(kln, N_k = None, cleanup = False):

    """ Convert KxKxN_max array to KxN max array

    if self.N is not initialized, it will be here.

    Parameters
    ----------
    u_kln : np.ndarray, float, shape=(KxLxN_max)
    N_k (optional) : np.array
        the N_k matrix from the previous formatting form
    cleanup (optional) : bool
        optional command to clean up, since u_kln can get very large

    Outputs
    -------
    u_kn: np.ndarray, float, shape=(LxN)
    """

    #print "warning: KxLxN_max arrays deprecated; convering into new preferred KxN shape"

    # rewrite into kn shape
    [K, L, N_max] = np.shape(kln)

    if N_k is None:
        # We assume that all N_k are N_max.
        # Not really an easier way to do this without being given the answer.
        N_k = N_max * np.ones([L], dtype=np.int64)
    N = np.sum(N_k)

    kn = np.zeros([L, N], dtype=np.float64)
    i = 0
    for k in range(K):  # loop through the old K; some might be zero
        for ik in range(N_k[k]):
            kn[:, i] = kln[k, :, ik]
            i += 1
    if cleanup:
        del(kln)  # very big, let's explicitly delete

    return kn

def kn_to_n(kn, N_k = None, cleanup = False):

    """ Convert KxN_max array to N array

    Parameters
    ----------
    u_kn: np.ndarray, float, shape=(KxN_max)
    N_k (optional) : np.array
        the N_k matrix from the previous formatting form
    cleanup (optional) : bool
        optional command to clean up, since u_kln can get very large

    Outputs
    -------
    u_n: np.ndarray, float, shape=(N)
    """

    #print "warning: KxN arrays deprecated; convering into new preferred N shape"
    # rewrite into kn shape

    # rewrite into kn shape
    [K, N_max] = np.shape(kn)

    if N_k is None:
        # We assume that all N_k are N_max.
        # Not really an easier way to do this without being given the answer.
        N_k = N_max*np.ones([K], dtype=np.int64)
    N = np.sum(N_k)

    n = np.zeros([N], dtype=np.float64)
    i = 0
    for k in range(K):  # loop through the old K; some might be zero
        for ik in range(N_k[k]):
            n[i] = kn[k, ik]
            i += 1
    if cleanup:
        del(kn)  # very big, let's explicitly delete
    return n

def ensure_type(val, dtype, ndim, name, length=None, can_be_none=False, shape=None,
                warn_on_cast=True, add_newaxis_on_deficient_ndim=False):
    """Typecheck the size, shape and dtype of a numpy array, with optional
    casting.

    Parameters
    ----------
    val : {np.ndaraay, None}
        The array to check
    dtype : {nd.dtype, str}
        The dtype you'd like the array to have
    ndim : int
        The number of dimensions you'd like the array to have
    name : str
        name of the array. This is used when throwing exceptions, so that
        we can describe to the user which array is messed up.
    length : int, optional
        How long should the array be?
    can_be_none : bool
        Is ``val == None`` acceptable?
    shape : tuple, optional
        What should be shape of the array be? If the provided tuple has
        Nones in it, those will be semantically interpreted as matching
        any length in that dimension. So, for example, using the shape
        spec ``(None, None, 3)`` will ensure that the last dimension is of
        length three without constraining the first two dimensions
    warn_on_cast : bool, default=True
        Raise a warning when the dtypes don't match and a cast is done.
    add_newaxis_on_deficient_ndim : bool, default=True
        Add a new axis to the beginining of the array if the number of
        dimensions is deficient by one compared to your specification. For
        instance, if you're trying to get out an array of ``ndim == 3``,
        but the user provides an array of ``shape == (10, 10)``, a new axis will
        be created with length 1 in front, so that the return value is of
        shape ``(1, 10, 10)``.

    Notes
    -----
    The returned value will always be C-contiguous.

    Returns
    -------
    typechecked_val : np.ndarray, None
        If `val=None` and `can_be_none=True`, then this will return None.
        Otherwise, it will return val (or a copy of val). If the dtype wasn't right,
        it'll be casted to the right shape. If the array was not C-contiguous, it'll
        be copied as well.

    """
    if can_be_none and val is None:
        return None

    if not isinstance(val, np.ndarray):
        # special case: if the user is looking for a 1d array, and
        # they request newaxis upconversion, and provided a scalar
        # then we should reshape the scalar to be a 1d length-1 array
        if add_newaxis_on_deficient_ndim and ndim == 1 and np.isscalar(val):
            val = np.array([val])
        else:
            raise TypeError(("%s must be numpy array. "
                             " You supplied type %s" % (name, type(val))))

    if warn_on_cast and val.dtype != dtype:
        warnings.warn("Casting %s dtype=%s to %s " % (name, val.dtype, dtype),
                      TypeCastPerformanceWarning)

    if not val.ndim == ndim:
        if add_newaxis_on_deficient_ndim and val.ndim + 1 == ndim:
            val = val[np.newaxis, ...]
        else:
            raise ValueError(("%s must be ndim %s. "
                              "You supplied %s" % (name, ndim, val.ndim)))

    val = np.ascontiguousarray(val, dtype=dtype)

    if length is not None and len(val) != length:
        raise ValueError(("%s must be length %s. "
                          "You supplied %s" % (name, length, len(val))))

    if shape is not None:
        # the shape specified given by the user can look like (None, None 3)
        # which indicates that ANY length is accepted in dimension 0 or
        # dimension 1
        sentenel = object()
        error = ValueError(("%s must be shape %s. You supplied  "
                            "%s" % (name, str(shape).replace('None', 'Any'), val.shape)))
        for a, b in zip_longest(val.shape, shape, fillvalue=sentenel):
            if a is sentenel or b is sentenel:
                # if the sentenel was reached, it means that the ndim didn't
                # match or something. this really shouldn't happen
                raise error
            if b is None:
                # if the user's shape spec has a None in it, it matches anything
                continue
            if a != b:
                # check for equality
                raise error

    return val

def _logsum(a_n):
    """Compute the log of a sum of exponentiated terms exp(a_n) in a numerically-stable manner.
    NOTE: this function has been deprecated in favor of logsumexp.

    Parameters
    ----------
    a_n : np.ndarray, shape=(n_samples)
        a_n[n] is the nth exponential argument

    Returns
    -------
    a_n : np.ndarray, shape=(n_samples)
        a_n[n] is the nth exponential argument

    Notes
    -----

    _logsum a_n = max_arg + \log \sum_{n=1}^N \exp[a_n - max_arg]

    where max_arg = max_n a_n.  This is mathematically (but not numerically) equivalent to

    _logsum a_n = \log \sum_{n=1}^N \exp[a_n]

    Example
    -------
    >>> a_n = np.array([0.0, 1.0, 1.2], np.float64)
    >>> print('%.3e' % _logsum(a_n))
    1.951e+00
    """

    # Compute the maximum argument.
    max_log_term = np.max(a_n)

    # Compute the reduced terms.
    terms = np.exp(a_n - max_log_term)

    # Compute the log sum.
    log_sum = np.log(np.sum(terms)) + max_log_term

    return log_sum

def logsumexp(a, axis=None, b=None, use_numexpr=True):
    """Compute the log of the sum of exponentials of input elements.

    Parameters
    ----------
    a : array_like
        Input array.
    axis : None or int, optional, default=None
        Axis or axes over which the sum is taken. By default `axis` is None,
        and all elements are summed.
    b : array-like, optional
        Scaling factor for exp(`a`) must be of the same shape as `a` or
        broadcastable to `a`.
    use_numexpr : bool, optional, default=True
        If True, use the numexpr library to speed up the calculation, which
        can give a 2-4X speedup when working with large arrays.

    Returns
    -------
    res : ndarray
        The result, ``log(sum(exp(a)))`` calculated in a numerically
        more stable way. If `b` is given then ``log(sum(b*exp(a)))``
        is returned.

    See Also
    --------
    numpy.logaddexp, numpy.logaddexp2, scipy.misc.logsumexp (soon to be replaced with  scipy.special.logsumexp)

    Notes
    -----
    This is based on scipy.misc.logsumexp but with optional numexpr
    support for improved performance.
    """

    a = np.asarray(a)

    a_max = np.amax(a, axis=axis, keepdims=True)

    if a_max.ndim > 0:
        a_max[~np.isfinite(a_max)] = 0
    elif not np.isfinite(a_max):
        a_max = 0

    if b is not None:
        b = np.asarray(b)
        if use_numexpr and HAVE_NUMEXPR:
            out = np.log(numexpr.evaluate("b * exp(a - a_max)").sum(axis))
        else:
            out = np.log(np.sum(b * np.exp(a - a_max), axis=axis))
    else:
        if use_numexpr and HAVE_NUMEXPR:
            out = np.log(numexpr.evaluate("exp(a - a_max)").sum(axis))
        else:
            out = np.log(np.sum(np.exp(a - a_max), axis=axis))

    a_max = np.squeeze(a_max, axis=axis)
    out += a_max

    return out

def check_w_normalized(W, N_k, tolerance = 1.0e-4):
    """Check the weight matrix W is properly normalized. The sum over N should be 1, and the sum over k by N_k should aslo be 1

    Parameters
    ----------
    W : np.ndarray, shape=(N, K), dtype='float'
        The normalized weight matrix for snapshots and states.
        W[n, k] is the weight of snapshot n in state k.
    N_k : np.ndarray, shape=(K), dtype='int'
        N_k[k] is the number of samples from state k.
    tolerance : float, optional, default=1.0e-4
        Tolerance for checking equality of sums

    Returns
    -------
    None : NoneType
        Returns a None object if test passes, otherwise raises a ParameterError with appropriate message if W is not normalized within tolerance.
    """

    [N, K] = W.shape

    column_sums = np.sum(W, axis=0)
    badcolumns = (np.abs(column_sums - 1) > tolerance)
    if np.any(badcolumns):
        which_badcolumns = np.arange(K)[badcolumns]
        firstbad = which_badcolumns[0]
        raise ParameterError(
            'Warning: Should have \sum_n W_nk = 1.  Actual column sum for state %d was %f. %d other columns have similar problems' %
            (firstbad, column_sums[firstbad], np.sum(badcolumns)))

    row_sums = np.sum(W * N_k, axis=1)
    badrows = (np.abs(row_sums - 1) > tolerance)
    if np.any(badrows):
        which_badrows = np.arange(N)[badrows]
        firstbad = which_badrows[0]
        raise ParameterError(
            'Warning: Should have \sum_k N_k W_nk = 1.  Actual row sum for sample %d was %f. %d other rows have similar problems' %
            (firstbad, row_sums[firstbad], np.sum(badrows)))

    return

# ============================================================================================
# Exception classes
# =============================================================================================

class ParameterError(Exception):

    """
    An error in the input parameters has been detected.

    """
    pass

class ConvergenceError(Exception):

    """
    Convergence could not be achieved.

    """
    pass

class BoundsError(Exception):

    """
    Could not determine bounds on free energy

    """
    pass

class DataError(Exception):

    """
    Data is inconsistent.

    """
    pass
<|endoftext|>


<commit_before>class C:
    def __init__(self, x):
        self.x = x

    def m1(self): return "calling m1 from C, x = {}".format(self.x)
    def m2(self): return "calling m2 from C, x = {}".format(self.x)

class Proxy(object):
    def __init__(self, prototype):
        self.prototype = prototype

    def m2(self):
        self.x = 10
        return "calling m2 from Proxy x = {}".format(self.x)

    def __getattr__(self, name):
        if hasattr(self.prototype, name):
            return getattr(self.prototype, name)
        else:
            raise AttributeError

c1 = C(5)
c2 = Proxy(c1)

print("Proxied call: {}".format(c2.m1()))
print("Overridden call: {}".format(c2.m2()))
<commit_msg>Better proxy<commit_after>class C:
    def __init__(self, x):
        self.x = x

    def m1(self): return "calling m1 from C, x = {}".format(self.x)
    def m2(self): return "calling m2 from C, x = {}".format(self.x)

class Proxy(object):
    def __init__(self, prototype):
        self.prototype = prototype

    def m2(self):
        self.x = 10
        return "calling m2 from Proxy x = {}".format(self.x)

    def __getattr__(self, name):
        return getattr(self.prototype, name)

c1 = C(5)
c2 = Proxy(c1)

print("Proxied call: {}".format(c2.m1()))
print("Overridden call: {}".format(c2.m2()))
<|endoftext|>


<commit_before># coding: utf-8
from distutils.core import setup
setup(
  name = 'gherkin3',
  packages = ['gherkin3'],
  version = '3.0.0',
  description = 'Gherkin parser',
  author = 'Björn Rasmusson',
  author_email = 'cukes@googlegroups.com',
  url = 'https://github.com/cucumber/gherkin-python',
  license = 'MIT',
  download_url = 'https://github.com/cucumber/gherkin-ruby/archive/v3.0.0.tar.gz',
  keywords = ['gherkin', 'cucumber', 'bdd'],
  classifiers = [],
)
<commit_msg>Correct the download url for gherkin-python<commit_after># coding: utf-8
from distutils.core import setup
setup(
  name = 'gherkin3',
  packages = ['gherkin3'],
  version = '3.0.0',
  description = 'Gherkin parser',
  author = 'Björn Rasmusson',
  author_email = 'cukes@googlegroups.com',
  url = 'https://github.com/cucumber/gherkin-python',
  license = 'MIT',
  download_url = 'https://github.com/cucumber/gherkin-python/archive/v3.0.0.tar.gz',
  keywords = ['gherkin', 'cucumber', 'bdd'],
  classifiers = [],
)
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

setup(
    name='<%NAME%>',
    version='0.0.1',
    author='Christopher Brown',
    author_email='io@henrian.com',
    url='https://github.com/chbrown/<%NAME%>',
    keywords='',
    description='',
    long_description=open('README.md').read(),
    license=open('LICENSE').read(),
    packages=find_packages(),
    include_package_data=True,
    classifiers=[
        # https://pypi.python.org/pypi?:action=list_classifiers
        'Development Status :: 1 - Alpha',
        'License :: OSI Approved :: MIT License',
    ],
    install_requires=[
    ],
    entry_points={
        'console_scripts': [
        ],
    },
)
<commit_msg>Fix invalid Alpha classifier<commit_after>from setuptools import setup, find_packages

setup(
    name='<%NAME%>',
    version='0.0.1',
    author='Christopher Brown',
    author_email='io@henrian.com',
    url='https://github.com/chbrown/<%NAME%>',
    keywords='',
    description='',
    long_description=open('README.md').read(),
    license=open('LICENSE').read(),
    packages=find_packages(),
    include_package_data=True,
    classifiers=[
        # https://pypi.python.org/pypi?:action=list_classifiers
        'Development Status :: 3 - Alpha',
        'License :: OSI Approved :: MIT License',
    ],
    install_requires=[
    ],
    entry_points={
        'console_scripts': [
        ],
    },
)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

try:
    from setuptools import setup
except ImportError:
    from distutils.core import setup

from os.path import join, dirname

from xix-event-mib.config import NAME, VERSION, LICENSE

readme = open(join(dirname(__file__), 'README.rst')).read()
history = open('HISTORY.rst').read().replace('.. :changelog:', '')

requirements = [
    # 'package1', 'package2'

]

test_requirements = [
    # 'package1', 'package2'
]

setup(
    name=NAME,
    version=VERSION,
    description='Python interface to XIX-EVENT-MIB.',
    long_description=readme + '\n\n' + history,
    author='Markus Juenemann',
    author_email='markus@juenemann.net',
    url='https://github.com/mjuenema/xix-event-mib',
    packages=[
        'xix-event-mib',
    ],
    package_dir={'xix-event-mib':
                 'xix-event-mib'},
    include_package_data=True,
    install_requires=requirements,
    license=LICENSE,
    zip_safe=False,

    keywords='xix-event-mib',
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Natural Language :: English',
        "Programming Language :: Python :: 2",
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
    ],
    test_suite='tests',
    tests_require=test_requirements
)

# References
#
# - https://pythonhosted.org/setuptools/index.html
# - http://foobar.lu/wp/2012/05/13/a-comprehensive-step-through-python-packaging-a-k-a-setup-scripts/
<commit_msg>Fixing errors in python/setup.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

try:
    from setuptools import setup
except ImportError:
    from distutils.core import setup

from os.path import join, dirname

from xixeventlib.config import NAME, VERSION, LICENSE

readme = open(join(dirname(__file__), 'README.rst')).read()
history = open('HISTORY.rst').read().replace('.. :changelog:', '')

requirements = [
    # 'package1', 'package2'

]

test_requirements = [
    # 'package1', 'package2'
]

setup(
    name=NAME,
    version=VERSION,
    description='Python interface to XIX-EVENT-MIB.',
    long_description=readme + '\n\n' + history,
    author='Markus Juenemann',
    author_email='markus@juenemann.net',
    url='https://github.com/mjuenema/xix-event-mib',
    packages=[
        'xixeventlib',
    ],
    package_dir={'xixeventlib':
                 'xixeventlib'},
    include_package_data=True,
    install_requires=requirements,
    license=LICENSE,
    zip_safe=False,

    keywords='xixeventlib',
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Natural Language :: English',
        "Programming Language :: Python :: 2",
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
    ],
    test_suite='tests',
    tests_require=test_requirements
)

# References
#
# - https://pythonhosted.org/setuptools/index.html
# - http://foobar.lu/wp/2012/05/13/a-comprehensive-step-through-python-packaging-a-k-a-setup-scripts/
<|endoftext|>


<commit_before>#
# Copyright 2017 Two Pore Guys, Inc.
# All rights reserved
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted providing that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

import os
import Cython.Compiler.Options
Cython.Compiler.Options.annotate = True
from distutils.core import setup
from Cython.Distutils.extension import Extension
from Cython.Distutils import build_ext

os.environ['CC'] = 'clang'
os.environ.setdefault('DESTDIR', '/')

setup(
    name='librpc',
    version='1.0',
    packages=[''],
    package_data={'': ['*.html', '*.c', 'librpc.pxd']},
    cmdclass={'build_ext': build_ext},
    ext_modules=[
        Extension(
            "librpc",
            ["librpc.pyx"],
            extra_compile_args=["-fblocks"],
            extra_link_args=["-g", "-lrpc"],
        )
    ]
)
<commit_msg>Silence clang warnings on Cython-generated file.<commit_after>#
# Copyright 2017 Two Pore Guys, Inc.
# All rights reserved
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted providing that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

import os
import Cython.Compiler.Options
Cython.Compiler.Options.annotate = True
from distutils.core import setup
from Cython.Distutils.extension import Extension
from Cython.Distutils import build_ext

os.environ['CC'] = 'clang'
os.environ.setdefault('DESTDIR', '/')

setup(
    name='librpc',
    version='1.0',
    packages=[''],
    package_data={'': ['*.html', '*.c', 'librpc.pxd']},
    cmdclass={'build_ext': build_ext},
    ext_modules=[
        Extension(
            "librpc",
            ["librpc.pyx"],
            extra_compile_args=["-fblocks", "-Wno-sometimes-uninitialized"],
            extra_link_args=["-g", "-lrpc"],
        )
    ]
)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from keypool import __version__

try:
   from setuptools import setup
   kw = {}

except ImportError:
   from distutils.core import setup
   kw = {}

setup(
   name = 'keypool',
   packages = ['keypool'],
   version = __version__,
   description = 'keypool: generate dict keys',

   author = 'Charlie Liban',
   author_email = 'charlie@tyrannosaur.ca',
   maintainer='Charlie Liban',
   maintainer_email='charlie@tyrannosaur.ca',

   url = 'https://github.com/tyrannosaur/keypool',
   download_url = 'https://github.com/tyrannosaur/keypool/zipball/master',
   keywords = ['data structures'],
   classifiers = [
      'Programming Language :: Python',
      'License :: OSI Approved :: MIT License',
   ],

   license = 'MIT License',

   long_description = """\
Classes and helpers to generate and maintain a pool of unique integer keys.
""",
   **kw
)     <commit_msg>Changed documentation a bit<commit_after>#!/usr/bin/env python

from keypool import __version__

try:
   from setuptools import setup
   kw = {}

except ImportError:
   from distutils.core import setup
   kw = {}

setup(
   name = 'keypool',
   packages = ['keypool'],
   version = __version__,
   description = 'Classes and helpers to generate and maintain a pool of unique integer keys.',

   author = 'Charlie Liban',
   author_email = 'charlie@tyrannosaur.ca',
   maintainer='Charlie Liban',
   maintainer_email='charlie@tyrannosaur.ca',

   url = 'https://github.com/tyrannosaur/keypool',
   download_url = 'https://github.com/tyrannosaur/keypool/zipball/master',
   keywords = ['data structures'],
   classifiers = [
      'Programming Language :: Python',
      'License :: OSI Approved :: MIT License',
   ],

   license = 'MIT License',

   long_description = """\
Classes and helpers to generate and maintain a pool of unique integer keys.
Priority is given to reusing freed keys rather than generating new ones.

This package is meant for situations where keys for a dict are irrelevant or
arbitrary.

Typical usage:

   from keypool import KeypoolDict
   items = KeypoolDict()

   # Assign a value with a unique, generated key
   items[items.next()] = 'hello, world'

   # Assign a value but capture the key
   key = items.setitem('hello again, world')

   # Assign anything except an integer, like a normal dict
   items['hello'] = 'world'
""",
   **kw
)     <|endoftext|>


<commit_before>#!/usr/bin/env python

import os,sys

from setuptools import setup
from jsbeautifier.__version__ import __version__

from setuptools.command.test import test as TestCommand

DIR='jsbeautifier/tests/'
class PyTest(TestCommand):
    user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.pytest_args = ['--assert=plain'] +[DIR+x for x in os.listdir(DIR) if x.endswith('.py') and x[0] not in '._']

    def run_tests(self):
        #import here, cause outside the eggs aren't loaded
        import pytest
        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

setup(name='jsbeautifier',
      version=__version__,
      description='JavaScript unobfuscator and beautifier.',
      long_description=('Beautify, unpack or deobfuscate JavaScript. '
                        'Handles popular online obfuscators.'),
      author='Einar Lielmanis, Stefano Sanfilippo et al.',
      author_email='einar@jsbeautifier.org',
      url='http://jsbeautifier.org',
      scripts=['js-beautify'],
      packages=['jsbeautifier', 'jsbeautifier.tests', 'jsbeautifier.tests.generated',
                'jsbeautifier.unpackers', 'jsbeautifier.unpackers.tests'],
      install_requires=["six>=1.6.1", "editorconfig>=0.12.0"],
      license='MIT',
      test_suite='pytest.collector',
      cmdclass = {'test': PyTest},

     )
<commit_msg>Updated python setup.py<commit_after>#!/usr/bin/env python

import os,sys

from setuptools import setup
from jsbeautifier.__version__ import __version__

from setuptools.command.test import test as TestCommand

DIR='jsbeautifier/tests/'
class PyTest(TestCommand):
    user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.pytest_args = ['--assert=plain'] +[DIR+x for x in os.listdir(DIR) if x.endswith('.py') and x[0] not in '._']

    def run_tests(self):
        #import here, cause outside the eggs aren't loaded
        import pytest
        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

setup(name='jsbeautifier',
      version=__version__,
      description='JavaScript unobfuscator and beautifier.',
      long_description=('Beautify, unpack or deobfuscate JavaScript. '
                        'Handles popular online obfuscators.'),
      author='Einar Lielmanis, Stefano Sanfilippo et al.',
      author_email='einar@jsbeautifier.org',
      url='http://jsbeautifier.org',
      scripts=['js-beautify'],
      packages=['jsbeautifier',
                'jsbeautifier.tests', 'jsbeautifier.tests.generated',
                'jsbeautifer.core',
                'jsbeautifer.javascript',
                'jsbeautifier.unpackers', 'jsbeautifier.unpackers.tests'],
      packages=['jsbeautifier', 'jsbeautifier.tests', 'jsbeautifier.tests.generated',
                'jsbeautifier.unpackers', 'jsbeautifier.unpackers.tests'],
      install_requires=["six>=1.6.1", "editorconfig>=0.12.0"],
      license='MIT',
      test_suite='pytest.collector',
      cmdclass = {'test': PyTest},

     )
<|endoftext|>


<commit_before>'''This module provides a dummy parser for pythran annotations, aka pythran specs.
    * spec_parser reads the specification from a python module and returns them
'''
import ply.lex as lex
import ply.yacc as yacc
import os.path

class SpecParser:
    """ A parser that scans a file lurking for lines such as the following to generate a function signature
#pythran export a((float,(int,long),str list) list list)
#pythran export a(str)
#pythran export a( (str,str), int, long list list)
"""

    ## lex part
    reserved = {
            'pythran' : 'PYTHRAN',
            'export' : 'EXPORT',
            'list' : 'LIST',
            'str' : 'STR',
            'bool' : 'BOOL',
            'int': 'INT',
            'long': 'LONG',
            'float': 'FLOAT',
            }
    tokens = [ 'IDENTIFIER', 'SHARP', 'COMMA', 'LPAREN', 'RPAREN' ] + list(reserved.values())

    # token <> regexp binding
    t_SHARP     = r'\#'
    t_COMMA     = r','
    t_LPAREN    = r'\('
    t_RPAREN    = r'\)'

    def t_IDENTIFER(self,t):
        r'[a-zA-Z_][a-zA-Z_0-9]*'
        t.type = SpecParser.reserved.get(t.value, 'IDENTIFIER')
        return t

    # skipped characters
    t_ignore    = ' \t\r\n'

    # error handling
    def t_error(self,t):
        t.lexer.skip(1)

    ## yacc part

    def p_exports(self,p):
        '''exports :
                   | export exports'''
        p[0]=self.exports

    def p_export(self,p):
        'export : SHARP PYTHRAN EXPORT IDENTIFIER LPAREN opt_types RPAREN'
        self.exports[p[4]]=p[6]

    def p_opt_types(self,p):
        '''opt_types :
                     | types'''
        p[0] = p[1] if p[1] else []

    def p_types(self,p):
        '''types : type
                 | type COMMA types'''
        p[0] = [p[1]] + ([] if len(p) == 2 else p[3])

    def p_type(self,p):
        '''type : term
                | type LIST
                | LPAREN types RPAREN'''
        if len(p) == 2:
            p[0] = p[1]
        elif len(p) == 3:
            p[0] = [p[1]]
        else:
            p[0] = tuple(p[2])

    def p_term(self,p):
        '''term : STR
                | BOOL
                | INT
                | LONG
                | FLOAT'''
        p[0]=eval(p[1])

    def p_error(self, p):
        if p:
            self.parser.errok()

    def __init__(self, **kwargs):
        self.lexer=lex.lex(module=self, debug=0)
        self.parser=yacc.yacc(module=self, debug=0, tabmodule='pythran.parsetab')

    def __call__(self, path):
        self.exports=dict()
        input_file=None
        if os.path.isfile(path):
            input_file=path
            with file(path) as fd:
                data = fd.read()
        else:
            data=path
        self.parser.parse(data, lexer=self.lexer)
        if not self.exports:
            err = SyntaxError("Pythran spec error: no pythran specification")
            if input_file: err.filename=input_file
            raise err
        return self.exports

def spec_parser(input):
    return SpecParser()(input)
<commit_msg>Fix a  bug in pythran spec parsing.<commit_after>'''This module provides a dummy parser for pythran annotations, aka pythran specs.
    * spec_parser reads the specification from a python module and returns them
'''
import ply.lex as lex
import ply.yacc as yacc
import os.path

class SpecParser:
    """ A parser that scans a file lurking for lines such as the following to generate a function signature
#pythran export a((float,(int,long),str list) list list)
#pythran export a(str)
#pythran export a( (str,str), int, long list list)
"""

    ## lex part
    reserved = {
            'pythran' : 'PYTHRAN',
            'export' : 'EXPORT',
            'list' : 'LIST',
            'str' : 'STR',
            'bool' : 'BOOL',
            'int': 'INT',
            'long': 'LONG',
            'float': 'FLOAT',
            }
    tokens = [ 'IDENTIFIER', 'SHARP', 'COMMA', 'LPAREN', 'RPAREN' ] + list(reserved.values())

    # token <> regexp binding
    t_SHARP     = r'\#'
    t_COMMA     = r','
    t_LPAREN    = r'\('
    t_RPAREN    = r'\)'

    def t_IDENTIFER(self,t):
        r'[a-zA-Z_][a-zA-Z_0-9]*'
        t.type = SpecParser.reserved.get(t.value, 'IDENTIFIER')
        return t

    # skipped characters
    t_ignore    = ' \t\r\n'

    # error handling
    def t_error(self,t):
        t.lexer.skip(1)

    ## yacc part

    def p_exports(self,p):
        '''exports :
                   | export exports'''
        p[0]=self.exports

    def p_export(self,p):
        'export : SHARP PYTHRAN EXPORT IDENTIFIER LPAREN opt_types RPAREN'
        self.exports[p[4]]=p[6]

    def p_opt_types(self,p):
        '''opt_types :
                     | types'''
        p[0] = p[1] if len(p)==2 else []

    def p_types(self,p):
        '''types : type
                 | type COMMA types'''
        p[0] = [p[1]] + ([] if len(p) == 2 else p[3])

    def p_type(self,p):
        '''type : term
                | type LIST
                | LPAREN types RPAREN'''
        if len(p) == 2:
            p[0] = p[1]
        elif len(p) == 3:
            p[0] = [p[1]]
        else:
            p[0] = tuple(p[2])

    def p_term(self,p):
        '''term : STR
                | BOOL
                | INT
                | LONG
                | FLOAT'''
        p[0]=eval(p[1])

    def p_error(self, p):
        if p:
            self.parser.errok()

    def __init__(self, **kwargs):
        self.lexer=lex.lex(module=self, debug=0)
        self.parser=yacc.yacc(module=self, debug=0, tabmodule='pythran.parsetab')

    def __call__(self, path):
        self.exports=dict()
        input_file=None
        if os.path.isfile(path):
            input_file=path
            with file(path) as fd:
                data = fd.read()
        else:
            data=path
        self.parser.parse(data, lexer=self.lexer)
        if not self.exports:
            err = SyntaxError("Pythran spec error: no pythran specification")
            if input_file: err.filename=input_file
            raise err
        return self.exports

def spec_parser(input):
    return SpecParser()(input)
<|endoftext|>


<commit_before>"""DeepLearning(主にKeras)関連。

kerasをimportしてしまうとTensorFlowの初期化が始まって重いので、
importしただけではkerasがimportされないように作っている。

"""
import csv
import pathlib

import numpy as np

def my_callback_factory():
    """色々入りのKerasのCallbackクラスを作って返す。

    TerminateOnNaN+ReduceLROnPlateau+EarlyStopping+CSVLoggerのようなもの。

    lossを監視して学習率を制御して、十分学習できたら終了する。
    ついでにログも出す。(ミニバッチ単位＆エポック単位)

    # 引数

    - log_dir: ログ出力先
    - batch_log_name: ミニバッチ単位のログファイル名
    - epoch_log_name: エポック単位のログファイル名

    - lr_list: epoch毎の学習率のリスト (base_lrと排他)

    - base_lr: 学習率の自動調整のベースとする学習率 (lr_listと排他)
    - max_reduces: 学習率の自動調整時、最大で何回学習率を減らすのか
    - reduce_factor: 学習率の自動調整時、学習率を減らしていく割合
    - beta1: lossを監視する際の指数移動平均の係数
    - beta2: lossを監視する際の指数移動平均の係数
    - margin_iterations: 学習率の自動調整時、このバッチ数分までは誤差を考慮して学習率を下げない

    - verbose: 学習率などをprintするなら1

    """
    import keras
    import keras.backend as K

    class _MyCallback(keras.callbacks.Callback):

        def __init__(self, log_dir='.',
                     lr_list=None,
                     base_lr=None,
                     verbose=1,
                     batch_log_name='batchlog.tsv',
                     epoch_log_name='epochlog.tsv',
                     max_reduces=6, reduce_factor=1 / np.sqrt(10),
                     beta1=0.998, beta2=0.999, margin_iterations=100):
            super().__init__()
            # 設定
            assert (lr_list is None) != (base_lr is None)  # どちらか片方のみ必須
            self.log_dir = log_dir
            self.batch_log_name = batch_log_name
            self.epoch_log_name = epoch_log_name
            self.lr_list = lr_list
            self.base_lr = base_lr
            self.max_reduces = max_reduces
            self.reduce_factor = reduce_factor
            self.beta1 = beta1
            self.beta2 = beta2
            self.margin_iterations = margin_iterations
            self.verbose = verbose
            # あとで使うものたち
            self.batch_log_file = None
            self.epoch_log_file = None
            self.batch_log_writer = None
            self.epoch_log_writer = None
            self.keys = None
            self.iterations = 0
            self.iterations_per_reduce = 0
            self.ema1 = 0
            self.ema2 = 0
            self.reduces = 0
            self.stopped_epoch = 0
            self.epoch = 0
            self.reduce_on_epoch_end = False

        def on_train_begin(self, logs=None):
            # ログファイル作成
            d = pathlib.Path(self.log_dir)
            d.mkdir(parents=True, exist_ok=True)
            self.batch_log_file = d.joinpath(self.batch_log_name).open('w')
            self.epoch_log_file = d.joinpath(self.epoch_log_name).open('w')
            self.batch_log_writer = csv.writer(self.batch_log_file, delimiter='\t')
            self.epoch_log_writer = csv.writer(self.epoch_log_file, delimiter='\t')
            self.batch_log_writer.writerow(['epoch', 'batch', 'loss', 'delta_ema'])
            self.keys = None
            # 学習率の設定(base_lr)
            if self.base_lr is not None:
                K.set_value(self.model.optimizer.lr, float(self.base_lr))
                if self.verbose >= 1:
                    print('lr = {}'.format(float(K.get_value(self.model.optimizer.lr))))
            # 色々初期化
            self.iterations = 0
            self.iterations_per_reduce = 0
            self.ema1 = 0
            self.ema2 = 0
            self.reduces = 0
            self.stopped_epoch = 0

        def on_epoch_begin(self, epoch, logs=None):
            if self.lr_list is not None:
                # 学習率の設定(lr_list)
                lr = self.lr_list[epoch]
                if self.verbose >= 1:
                    if epoch == 0 or lr != self.lr_list[epoch - 1]:
                        print('lr = {}'.format(float(lr)))
                K.set_value(self.model.optimizer.lr, float(lr))
            elif self.reduce_on_epoch_end:
                if self.verbose >= 1:
                    print('lr = {}'.format(float(K.get_value(self.model.optimizer.lr))))
            # 色々初期化
            self.epoch = epoch
            self.reduce_on_epoch_end = False

        def on_batch_begin(self, batch, logs=None):
            pass

        def on_batch_end(self, batch, logs=None):
            logs = logs or {}
            loss = logs.get('loss')

            # nanチェック(一応)
            if loss is not None:
                if np.isnan(loss) or np.isinf(loss):
                    print('Batch %d: Invalid loss, terminating training' % (batch))
                    self.model.stop_training = True

            # lossの指数移動平均の算出
            self.ema1 = loss * (1 - self.beta1) + self.ema1 * self.beta1
            self.ema2 = loss * (1 - self.beta2) + self.ema2 * self.beta2
            # Adam風補正
            self.iterations += 1
            hm1 = self.ema1 / (1 - self.beta1 ** self.iterations)
            hm2 = self.ema2 / (1 - self.beta2 ** self.iterations)
            delta_ema = hm2 - hm1
            if self.base_lr is not None:
                # lossの減少が止まってそうなら次のepochから学習率を減らす。
                self.iterations_per_reduce += 1
                if delta_ema <= 0 and self.margin_iterations <= self.iterations_per_reduce:
                    self.reduce_on_epoch_end = True

            # batchログ出力
            self.batch_log_writer.writerow([self.epoch + 1, batch + 1, loss, delta_ema])

        def on_epoch_end(self, epoch, logs=None):
            # batchログ出力
            self.batch_log_file.flush()
            # epochログ出力
            if not self.keys:
                self.keys = sorted(logs.keys())
                self.epoch_log_writer.writerow(['epoch', 'lr'] + self.keys)
            lr = K.get_value(self.model.optimizer.lr)
            metrics = [logs.get(k) for k in self.keys]
            self.epoch_log_writer.writerow([epoch + 1, lr] + metrics)
            self.epoch_log_file.flush()
            # 学習率を減らす/限界まで下がっていたら学習終了
            if self.reduce_on_epoch_end:
                if self.max_reduces <= self.reduces:
                    # 限界まで下がっていたら学習終了
                    self.stopped_epoch = epoch
                    self.model.stop_training = True
                else:
                    # 学習率を減らす
                    self.reduces += 1
                    lr = self.base_lr * self.reduce_factor ** self.reduces
                    K.set_value(self.model.optimizer.lr, float(lr))
                    self.iterations_per_reduce = 0  # 安全装置のリセット

        def on_train_end(self, logs=None):
            self.batch_log_file.close()
            self.epoch_log_file.close()

    return _MyCallback

def session(config=None, gpu_options=None):
    """TensorFlowのセッションの初期化・後始末。

    # 使い方

    ```
    with tk.dl.session():

        # kerasの処理

    ```

    """
    import keras.backend as K

    class _Scope(object):  # pylint: disable=R0903

        def __init__(self, config=None, gpu_options=None):
            self.config = config or {}
            self.gpu_options = gpu_options or {}

        def __enter__(self):
            if K.backend() == 'tensorflow':
                import tensorflow as tf
                self.config.update({'allow_soft_placement': True})
                self.gpu_options.update({'allow_growth': True})
                K.set_session(
                    tf.Session(
                        config=tf.ConfigProto(
                            **self.config, gpu_options=tf.GPUOptions(**self.gpu_options))))

        def __exit__(self, *exc_info):
            if K.backend() == 'tensorflow':
                K.clear_session()

    return _Scope(config=config, gpu_options=gpu_options)
<commit_msg>LearningCurvePlotterの追加<commit_after>"""DeepLearning(主にKeras)関連。

kerasをimportしてしまうとTensorFlowの初期化が始まって重いので、
importしただけではkerasがimportされないように作っている。

"""
import csv
import pathlib
import warnings

import numpy as np
import pandas as pd

def my_callback_factory():
    """色々入りのKerasのCallbackクラスを作って返す。

    TerminateOnNaN+ReduceLROnPlateau+EarlyStopping+CSVLoggerのようなもの。

    lossを監視して学習率を制御して、十分学習できたら終了する。
    ついでにログも出す。(ミニバッチ単位＆エポック単位)

    # 引数

    - log_dir: ログ出力先
    - batch_log_name: ミニバッチ単位のログファイル名
    - epoch_log_name: エポック単位のログファイル名

    - lr_list: epoch毎の学習率のリスト (base_lrと排他)

    - base_lr: 学習率の自動調整のベースとする学習率 (lr_listと排他)
    - max_reduces: 学習率の自動調整時、最大で何回学習率を減らすのか
    - reduce_factor: 学習率の自動調整時、学習率を減らしていく割合
    - beta1: lossを監視する際の指数移動平均の係数
    - beta2: lossを監視する際の指数移動平均の係数
    - margin_iterations: 学習率の自動調整時、このバッチ数分までは誤差を考慮して学習率を下げない

    - verbose: 学習率などをprintするなら1

    """
    import keras
    import keras.backend as K

    class _MyCallback(keras.callbacks.Callback):

        def __init__(self, log_dir='.',
                     lr_list=None,
                     base_lr=None,
                     verbose=1,
                     batch_log_name='batchlog.tsv',
                     epoch_log_name='epochlog.tsv',
                     max_reduces=6, reduce_factor=1 / np.sqrt(10),
                     beta1=0.998, beta2=0.999, margin_iterations=100):
            super().__init__()
            # 設定
            assert (lr_list is None) != (base_lr is None)  # どちらか片方のみ必須
            self.log_dir = log_dir
            self.batch_log_name = batch_log_name
            self.epoch_log_name = epoch_log_name
            self.lr_list = lr_list
            self.base_lr = base_lr
            self.max_reduces = max_reduces
            self.reduce_factor = reduce_factor
            self.beta1 = beta1
            self.beta2 = beta2
            self.margin_iterations = margin_iterations
            self.verbose = verbose
            # あとで使うものたち
            self.batch_log_file = None
            self.epoch_log_file = None
            self.batch_log_writer = None
            self.epoch_log_writer = None
            self.keys = None
            self.iterations = 0
            self.iterations_per_reduce = 0
            self.ema1 = 0
            self.ema2 = 0
            self.reduces = 0
            self.stopped_epoch = 0
            self.epoch = 0
            self.reduce_on_epoch_end = False

        def on_train_begin(self, logs=None):
            # ログファイル作成
            d = pathlib.Path(self.log_dir)
            d.mkdir(parents=True, exist_ok=True)
            self.batch_log_file = d.joinpath(self.batch_log_name).open('w')
            self.epoch_log_file = d.joinpath(self.epoch_log_name).open('w')
            self.batch_log_writer = csv.writer(self.batch_log_file, delimiter='\t')
            self.epoch_log_writer = csv.writer(self.epoch_log_file, delimiter='\t')
            self.batch_log_writer.writerow(['epoch', 'batch', 'loss', 'delta_ema'])
            self.keys = None
            # 学習率の設定(base_lr)
            if self.base_lr is not None:
                K.set_value(self.model.optimizer.lr, float(self.base_lr))
                if self.verbose >= 1:
                    print('lr = {}'.format(float(K.get_value(self.model.optimizer.lr))))
            # 色々初期化
            self.iterations = 0
            self.iterations_per_reduce = 0
            self.ema1 = 0
            self.ema2 = 0
            self.reduces = 0
            self.stopped_epoch = 0

        def on_epoch_begin(self, epoch, logs=None):
            if self.lr_list is not None:
                # 学習率の設定(lr_list)
                lr = self.lr_list[epoch]
                if self.verbose >= 1:
                    if epoch == 0 or lr != self.lr_list[epoch - 1]:
                        print('lr = {}'.format(float(lr)))
                K.set_value(self.model.optimizer.lr, float(lr))
            elif self.reduce_on_epoch_end:
                if self.verbose >= 1:
                    print('lr = {}'.format(float(K.get_value(self.model.optimizer.lr))))
            # 色々初期化
            self.epoch = epoch
            self.reduce_on_epoch_end = False

        def on_batch_begin(self, batch, logs=None):
            pass

        def on_batch_end(self, batch, logs=None):
            logs = logs or {}
            loss = logs.get('loss')

            # nanチェック(一応)
            if loss is not None:
                if np.isnan(loss) or np.isinf(loss):
                    print('Batch %d: Invalid loss, terminating training' % (batch))
                    self.model.stop_training = True

            # lossの指数移動平均の算出
            self.ema1 = loss * (1 - self.beta1) + self.ema1 * self.beta1
            self.ema2 = loss * (1 - self.beta2) + self.ema2 * self.beta2
            # Adam風補正
            self.iterations += 1
            hm1 = self.ema1 / (1 - self.beta1 ** self.iterations)
            hm2 = self.ema2 / (1 - self.beta2 ** self.iterations)
            delta_ema = hm2 - hm1
            if self.base_lr is not None:
                # lossの減少が止まってそうなら次のepochから学習率を減らす。
                self.iterations_per_reduce += 1
                if delta_ema <= 0 and self.margin_iterations <= self.iterations_per_reduce:
                    self.reduce_on_epoch_end = True

            # batchログ出力
            self.batch_log_writer.writerow([self.epoch + 1, batch + 1, loss, delta_ema])

        def on_epoch_end(self, epoch, logs=None):
            # batchログ出力
            self.batch_log_file.flush()
            # epochログ出力
            if not self.keys:
                self.keys = sorted(logs.keys())
                self.epoch_log_writer.writerow(['epoch', 'lr'] + self.keys)
            lr = K.get_value(self.model.optimizer.lr)
            metrics = [logs.get(k) for k in self.keys]
            self.epoch_log_writer.writerow([epoch + 1, lr] + metrics)
            self.epoch_log_file.flush()
            # 学習率を減らす/限界まで下がっていたら学習終了
            if self.reduce_on_epoch_end:
                if self.max_reduces <= self.reduces:
                    # 限界まで下がっていたら学習終了
                    self.stopped_epoch = epoch
                    self.model.stop_training = True
                else:
                    # 学習率を減らす
                    self.reduces += 1
                    lr = self.base_lr * self.reduce_factor ** self.reduces
                    K.set_value(self.model.optimizer.lr, float(lr))
                    self.iterations_per_reduce = 0  # 安全装置のリセット

        def on_train_end(self, logs=None):
            self.batch_log_file.close()
            self.epoch_log_file.close()

    return _MyCallback

def session(config=None, gpu_options=None):
    """TensorFlowのセッションの初期化・後始末。

    # 使い方

    ```
    with tk.dl.session():

        # kerasの処理

    ```

    """
    import keras.backend as K

    class _Scope(object):  # pylint: disable=R0903

        def __init__(self, config=None, gpu_options=None):
            self.config = config or {}
            self.gpu_options = gpu_options or {}

        def __enter__(self):
            if K.backend() == 'tensorflow':
                import tensorflow as tf
                self.config.update({'allow_soft_placement': True})
                self.gpu_options.update({'allow_growth': True})
                K.set_session(
                    tf.Session(
                        config=tf.ConfigProto(
                            **self.config, gpu_options=tf.GPUOptions(**self.gpu_options))))

        def __exit__(self, *exc_info):
            if K.backend() == 'tensorflow':
                K.clear_session()

    return _Scope(config=config, gpu_options=gpu_options)

def learning_curve_plotter_factory():
    """Learning Curvesの描画を行う。

    # 引数
    - filename: 保存先ファイル名。「{metric}」はmetricの値に置換される。
    - metric: 対象とするmetric名。lossとかaccとか。

    # 「Invalid DISPLAY variable」対策
    最初の方に以下のコードを記述する。
    ```
    import matplotlib as mpl
    mpl.use('Agg')
    ```
    """
    import keras

    class _LearningCurvePlotter(keras.callbacks.Callback):

        def __init__(self, filename, metric='loss'):
            self.filename = filename
            self.metric = metric
            self.met_list = []
            self.val_met_list = []
            super().__init__()

        def on_epoch_end(self, epoch, logs=None):
            try:
                self._plot(logs)
            except:
                import traceback
                warnings.warn(traceback.format_exc(), RuntimeWarning)

        def _plot(self, logs):
            met = logs.get(self.metric)
            if met is None:
                warnings.warn('LearningCurvePlotter requires {} available!'.format(self.metric), RuntimeWarning)
            val_met = logs.get('val_{}'.format(self.metric))

            self.met_list.append(met)
            self.val_met_list.append(val_met)

            if len(self.met_list) > 1:
                df = pd.DataFrame()
                df[self.metric] = self.met_list
                if val_met is not None:
                    df['val_{}'.format(self.metric)] = self.val_met_list

                df.plot()

                import matplotlib.pyplot as plt
                plt.savefig(str(self.filename).format(metric=self.metric))
                plt.close()

    return _LearningCurvePlotter
<|endoftext|>