<commit_before>"""
    Clientside API
"""

import datetime
import requests
import json

class ConnectorException(Exception):
    pass

class MissingAction(ConnectorException):
    pass

class UnknownAction(ConnectorException):
    pass

class NotImplementedAction(ConnectorException):
    pass

class MissingArgument(ConnectorException):
    pass

class Connector(object):
    """
        Responsible for connecting to the Stracks service.
    """
    def session_start(self, sessionid):
        raise NotImplementedAction("session_start")

    def session_end(self, sessionid):
        raise NotImplementedAction("session_end")

    def request(self, sessionid, requestdata):
        raise NotImplementedAction("request")

    def send(self, data):
        """
            Do we need the send() intermediate call?
            Why not just invoke the relevant methods directly?
        """
        action = data.get('action')
        if action is None:
            raise MissingAction()

        try:
            if action == 'session_start':
                self.session_start(data['sessionid'])
            elif action == 'session_end':
                self.session_end(data['sessionid'])
            elif action == 'request':
                self.request(data['sessionid'], data['data'])
            else:
                raise UnknownAction(action)
        except KeyError, e:
            raise MissingArgument(e.message)

class HTTPConnector(Connector):
    """
        Connect to the API synchronously through HTTP
    """
    def __init__(self, url):
        """
            A HTTP connector takes the url of the API / AppInstance as
            argument
        """
        self.url = url
        self.queue = []

    def session_start(self, sessionid):
        data = {}
        data['started'] = datetime.datetime.utcnow().isoformat()
        data['sessionid'] = sessionid
        self.queue.append({'action':'start', 'data':data})
        self.flush()

    def session_end(self, sessionid):
        data = {}
        data['ended'] = datetime.datetime.utcnow().isoformat()
        data['sessionid'] = sessionid
        self.queue.append({'action':'end', 'data':data})
        self.flush()

    def request(self, sessionid, requestdata):
        data = {}
        data['sessionid'] = sessionid
        data['requestdata'] = requestdata
        self.queue.append({'action':'request', 'data':data})
        self.flush()

    def flush(self):
        if self.queue:
            requests.post(self.url + "/", data=json.dumps(self.queue))
            self.queue = []

import threading
import Queue
import atexit

import logging

logger = logging.getLogger("stracks")

class ASyncHTTPConnector(HTTPConnector):
    thread = None
    terminate = object()

    TIMEOUT = 10 # seconds

    def __init__(self, url, debug=False):
        super(ASyncHTTPConnector, self).__init__(url)
        self.thread = None
        self.lock = threading.Lock()
        self.thread_queue = Queue.Queue()
        self.backlog = []
        self._debug = debug

    def debug(self, s):
        if self._debug:
            print s

    def loop(self):
        while True:
            self.debug("Loop")
            try:
                item = self.thread_queue.get(timeout=10)
                self.backlog.append(item)
            except Queue.Empty:
                self.debug("Queue 10 sec timeout")

            self.debug("Queue size: %d, backlog size: %d" % (self.thread_queue.qsize(), len(self.backlog)))

            ##
            ## flush entire backlog if possible. May be just one item
            ## (or none at all if queue.get timed out)
            while self.backlog:
                item = self.backlog[0]
                if item is self.terminate:
                    self.debug("Terminating")
                    return  # break out of two loops

                ## https://github.com/kennethreitz/grequests ?

                error = False
                try:
                    requests.post(self.url + "/", data=json.dumps(item),
                                  timeout=self.TIMEOUT)
                    self.backlog.pop()
                except requests.exceptions.Timeout:
                    self.debug("Connection timed out")
                    error = True
                except requests.exceptions.TooManyRedirects:
                    self.debug("Too many redirects")
                    error = True
                except requests.exceptions.URLRequired:
                    self.debug("Not a valid URL")
                    error = True
                except requests.exceptions.SSLError:
                    self.debug("SSL Error")
                    error = True
                except requests.exceptions.ConnectionError:
                    self.debug("Connection error")
                    error = True
                except requests.exceptions.HTTPError:
                    self.debug("HTTP error")
                    error = True
                except requests.exceptions.RequestException:
                    self.debug("Request exception")
                    error = True
                except Exception, e:
                    self.debug("Unknown exception: " + str(e))
                    error = True
                if error:
                    ## increase the delay, but not infinitely
                    # delay = 10 * min(5, tries)
                    ## log error so we can report it eventually
                    self.debug("Delaying")
                    break

                self.debug("Post" + json.dumps(item))

    def stop(self):
        self.lock.acquire()
        try:
            print "Stopping thread"
            if self.thread is not None:
                self.thread_queue.put_nowait(self.terminate)
                self.thread.join()
                self.thread = None
        finally:
            self.lock.release()
        print "Thread stopped"

    def flush(self):
        """ flush queue to thread """
        print "Flushing", self.queue
        if self.queue:
            if self.thread is None:
                self.lock.acquire()
                try:
                    self.thread = threading.Thread(target=self.loop)
                    self.thread.setDaemon(True)
                    print "Starting thread"
                    self.thread.start()
                    print "Thread started"
                finally:
                    self.lock.release()
                atexit.register(self.stop)

            self.thread_queue.put_nowait(self.queue)
            self.queue = []

class RedisConnector(Connector):
    """
        A connector that simply stores data in redis (or any other keystore?),
        where a separate task is responsible for sending the data to the
        Stracks API
    """
<commit_msg>remove bare print's<commit_after>"""
    Clientside API
"""

import datetime
import requests
import json

class ConnectorException(Exception):
    pass

class MissingAction(ConnectorException):
    pass

class UnknownAction(ConnectorException):
    pass

class NotImplementedAction(ConnectorException):
    pass

class MissingArgument(ConnectorException):
    pass

class Connector(object):
    """
        Responsible for connecting to the Stracks service.
    """
    def session_start(self, sessionid):
        raise NotImplementedAction("session_start")

    def session_end(self, sessionid):
        raise NotImplementedAction("session_end")

    def request(self, sessionid, requestdata):
        raise NotImplementedAction("request")

    def send(self, data):
        """
            Do we need the send() intermediate call?
            Why not just invoke the relevant methods directly?
        """
        action = data.get('action')
        if action is None:
            raise MissingAction()

        try:
            if action == 'session_start':
                self.session_start(data['sessionid'])
            elif action == 'session_end':
                self.session_end(data['sessionid'])
            elif action == 'request':
                self.request(data['sessionid'], data['data'])
            else:
                raise UnknownAction(action)
        except KeyError, e:
            raise MissingArgument(e.message)

class HTTPConnector(Connector):
    """
        Connect to the API synchronously through HTTP
    """
    def __init__(self, url):
        """
            A HTTP connector takes the url of the API / AppInstance as
            argument
        """
        self.url = url
        self.queue = []

    def session_start(self, sessionid):
        data = {}
        data['started'] = datetime.datetime.utcnow().isoformat()
        data['sessionid'] = sessionid
        self.queue.append({'action':'start', 'data':data})
        self.flush()

    def session_end(self, sessionid):
        data = {}
        data['ended'] = datetime.datetime.utcnow().isoformat()
        data['sessionid'] = sessionid
        self.queue.append({'action':'end', 'data':data})
        self.flush()

    def request(self, sessionid, requestdata):
        data = {}
        data['sessionid'] = sessionid
        data['requestdata'] = requestdata
        self.queue.append({'action':'request', 'data':data})
        self.flush()

    def flush(self):
        if self.queue:
            requests.post(self.url + "/", data=json.dumps(self.queue))
            self.queue = []

import threading
import Queue
import atexit

import logging

logger = logging.getLogger("stracks")

class ASyncHTTPConnector(HTTPConnector):
    thread = None
    terminate = object()

    TIMEOUT = 10 # seconds

    def __init__(self, url, debug=False):
        super(ASyncHTTPConnector, self).__init__(url)
        self.thread = None
        self.lock = threading.Lock()
        self.thread_queue = Queue.Queue()
        self.backlog = []
        self._debug = debug

    def debug(self, s):
        if self._debug:
            print s

    def loop(self):
        while True:
            self.debug("Loop")
            try:
                item = self.thread_queue.get(timeout=10)
                self.backlog.append(item)
            except Queue.Empty:
                self.debug("Queue 10 sec timeout")

            self.debug("Queue size: %d, backlog size: %d" % (self.thread_queue.qsize(), len(self.backlog)))

            ##
            ## flush entire backlog if possible. May be just one item
            ## (or none at all if queue.get timed out)
            while self.backlog:
                item = self.backlog[0]
                if item is self.terminate:
                    self.debug("Terminating")
                    return  # break out of two loops

                ## https://github.com/kennethreitz/grequests ?

                error = False
                try:
                    requests.post(self.url + "/", data=json.dumps(item),
                                  timeout=self.TIMEOUT)
                    self.backlog.pop()
                except requests.exceptions.Timeout:
                    self.debug("Connection timed out")
                    error = True
                except requests.exceptions.TooManyRedirects:
                    self.debug("Too many redirects")
                    error = True
                except requests.exceptions.URLRequired:
                    self.debug("Not a valid URL")
                    error = True
                except requests.exceptions.SSLError:
                    self.debug("SSL Error")
                    error = True
                except requests.exceptions.ConnectionError:
                    self.debug("Connection error")
                    error = True
                except requests.exceptions.HTTPError:
                    self.debug("HTTP error")
                    error = True
                except requests.exceptions.RequestException:
                    self.debug("Request exception")
                    error = True
                except Exception, e:
                    self.debug("Unknown exception: " + str(e))
                    error = True
                if error:
                    ## increase the delay, but not infinitely
                    # delay = 10 * min(5, tries)
                    ## log error so we can report it eventually
                    self.debug("Delaying")
                    break

                self.debug("Post" + json.dumps(item))

    def stop(self):
        self.lock.acquire()
        try:
            self.debug("Stopping thread")
            if self.thread is not None:
                self.thread_queue.put_nowait(self.terminate)
                self.thread.join()
                self.thread = None
        finally:
            self.lock.release()
        self.debug("Thread stopped")

    def flush(self):
        """ flush queue to thread """
        self.debug("Flushing " + str(self.queue))
        if self.queue:
            if self.thread is None:
                self.lock.acquire()
                try:
                    self.thread = threading.Thread(target=self.loop)
                    self.thread.setDaemon(True)
                    self.debug("Starting thread")
                    self.thread.start()
                    self.debug("Thread started")
                finally:
                    self.lock.release()
                atexit.register(self.stop)

            self.thread_queue.put_nowait(self.queue)
            self.queue = []

class RedisConnector(Connector):
    """
        A connector that simply stores data in redis (or any other keystore?),
        where a separate task is responsible for sending the data to the
        Stracks API
    """
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os
import sys

# Third-party
from astropy import log as logger
import numpy as np
from scipy.signal import argrelmax, argrelmin

# Project
import gary.dynamics as gd
import gary.integrate as gi

__all__ = ['ws_to_freqs', 'worker']

_shape = (2, 11)  # 6 frequencies, max energy diff, done flag, loop bit, dt, nsteps

def ptp_freqs(t, *args):
    freqs = []
    for x in args:
        ix = argrelmax(x, mode='wrap')[0]
        f_max = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        ix = argrelmin(x, mode='wrap')[0]
        f_min = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        freqs.append(np.mean([f_max, f_min]))
    return np.array(freqs)

def estimate_max_period(t, w):
    if w.ndim < 3:
        w = w[:,np.newaxis]

    norbits = w.shape[1]
    periods = []
    for i in range(norbits):
        loop = gd.classify_orbit(w[:,i])
        if np.any(loop):
            # flip coords
            new_w = gd.align_circulation_with_z(w[:,i], loop[0])

            # convert to cylindrical
            R = np.sqrt(new_w[:,0]**2 + new_w[:,1]**2)
            phi = np.arctan2(new_w[:,1], new_w[:,0])
            z = new_w[:,2]

            T = 2*np.pi / ptp_freqs(t, R, phi, z)
        else:
            T = 2*np.pi / ptp_freqs(t, *w[:,i,:3].T)

        periods.append(T)

    return np.array(periods)

def ws_to_freqs(naff, ws, nintvec=15):

    # first get x,y,z frequencies
    logger.info('Solving for XYZ frequencies...')
    fs = [(ws[:,0,j] + 1j*ws[:,0,j+3]) for j in range(3)]
    try:
        fxyz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
    except:
        fxyz = np.ones(3)*np.nan

    # now get other frequencies
    loop = gd.classify_orbit(ws)
    is_loop = np.any(loop)
    if is_loop:
        # need to flip coordinates until circulation is around z axis
        new_ws = gd.align_circulation_with_z(ws, loop[0])

        fs = gd.poincare_polar(new_ws[:,0])
        try:
            logger.info('Solving for Rφz frequencies...')
            fRphiz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
        except:
            fRphiz = np.ones(3)*np.nan

    else:
        fRphiz = np.ones(3)*np.nan

    return np.append(fxyz, fRphiz), is_loop

def estimate_dt_nsteps(potential, w0, nperiods=100):
    # integrate orbit
    t,ws = potential.integrate_orbit(w0, dt=1., nsteps=20000,
                                     Integrator=gi.DOPRI853Integrator,
                                     Integrator_kwargs=dict(nsteps=256))

    # estimate the maximum period
    max_T = round(estimate_max_period(t, ws).max() * 100, -4)
    dt = round(max_T * 1.5E-5, 0)
    nsteps = int(max_T / dt)

    return dt, nsteps

def worker(task):
    # unpack input argument dictionary
    index = task['index']
    w0_filename = task['w0_filename']
    allfreqs_filename = task['allfreqs_filename']
    potential = task['potential']

    # read out just this initial condition
    w0 = np.load(w0_filename)
    allfreqs_shape = (len(w0),) + _shape
    allfreqs = np.memmap(allfreqs_filename, mode='r', shape=allfreqs_shape, dtype='float64')

    # short-circuit if this orbit is already done
    if allfreqs[index,0,7] == 1.:
        return

    # temporary array for results
    tmp = np.zeros(_shape)

    # automatically estimate dt, nsteps
    try:
        dt, nsteps = estimate_dt_nsteps(potential, w0[index].copy())
    except RuntimeError:
        logger.warning("Failed to integrate orbit when estimating dt,nsteps")
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    logger.info("Orbit {}: initial dt={}, nsteps={}".format(index, dt, nsteps))

    dEmax = 1.
    maxiter = 3  # maximum number of times to refine integration step
    for i in range(maxiter):
        if i > 0:
            # adjust timestep and duration if necessary
            dt /= 2.
            nsteps *= 2
            logger.debug("Refining timestep for orbit {} ({}, {})".format(index, dt, nsteps))

        # integrate orbit
        try:
            t,ws = potential.integrate_orbit(w0[index].copy(), dt=dt, nsteps=nsteps,
                                             Integrator=gi.DOPRI853Integrator,
                                             Integrator_kwargs=dict(nsteps=256))
        except RuntimeError:
            # ODE integration failed
            logger.warning("Orbit integration failed.")
            continue

        logger.debug('Orbit integrated')

        # check energy conservation for the orbit
        E = potential.total_energy(ws[:,0,:3].copy(), ws[:,0,3:].copy())
        dE = np.abs(E[1:] - E[0])
        dEmax = dE.max()

        if dEmax < 1E-9:
            break

    if dEmax > 1E-9:
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    # start finding the frequencies -- do first half then second half
    naff = gd.NAFF(t[:nsteps//2+1])
    freqs1,is_loop = ws_to_freqs(naff, ws[:nsteps//2+1])
    freqs2,is_loop = ws_to_freqs(naff, ws[nsteps//2:])

    # save to output array
    tmp[0,:6] = freqs1
    tmp[1,:6] = freqs2

    tmp[:,6] = dEmax
    tmp[:,7] = 1.
    tmp[:,8] = float(is_loop)
    tmp[:,9] = float(dt)
    tmp[:,10] = nsteps//2

    allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
    allfreqs[index] = tmp
    allfreqs.flush()
<commit_msg>twice as many steps<commit_after># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Third-party
from astropy import log as logger
import numpy as np
from scipy.signal import argrelmax, argrelmin

# Project
import gary.dynamics as gd
import gary.integrate as gi

__all__ = ['ws_to_freqs', 'worker']

_shape = (2, 11)  # 6 frequencies, max energy diff, done flag, loop bit, dt, nsteps

def ptp_freqs(t, *args):
    freqs = []
    for x in args:
        ix = argrelmax(x, mode='wrap')[0]
        f_max = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        ix = argrelmin(x, mode='wrap')[0]
        f_min = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        freqs.append(np.mean([f_max, f_min]))
    return np.array(freqs)

def estimate_max_period(t, w):
    if w.ndim < 3:
        w = w[:,np.newaxis]

    norbits = w.shape[1]
    periods = []
    for i in range(norbits):
        loop = gd.classify_orbit(w[:,i])
        if np.any(loop):
            # flip coords
            new_w = gd.align_circulation_with_z(w[:,i], loop[0])

            # convert to cylindrical
            R = np.sqrt(new_w[:,0]**2 + new_w[:,1]**2)
            phi = np.arctan2(new_w[:,1], new_w[:,0])
            z = new_w[:,2]

            T = 2*np.pi / ptp_freqs(t, R, phi, z)
        else:
            T = 2*np.pi / ptp_freqs(t, *w[:,i,:3].T)

        periods.append(T)

    return np.array(periods)

def ws_to_freqs(naff, ws, nintvec=15):

    # first get x,y,z frequencies
    logger.info('Solving for XYZ frequencies...')
    fs = [(ws[:,0,j] + 1j*ws[:,0,j+3]) for j in range(3)]
    try:
        fxyz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
    except:
        fxyz = np.ones(3)*np.nan

    # now get other frequencies
    loop = gd.classify_orbit(ws)
    is_loop = np.any(loop)
    if is_loop:
        # need to flip coordinates until circulation is around z axis
        new_ws = gd.align_circulation_with_z(ws, loop[0])

        fs = gd.poincare_polar(new_ws[:,0])
        try:
            logger.info('Solving for Rφz frequencies...')
            fRphiz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
        except:
            fRphiz = np.ones(3)*np.nan

    else:
        fRphiz = np.ones(3)*np.nan

    return np.append(fxyz, fRphiz), is_loop

def estimate_dt_nsteps(potential, w0, nperiods=100):
    # integrate orbit
    t,ws = potential.integrate_orbit(w0, dt=1., nsteps=20000,
                                     Integrator=gi.DOPRI853Integrator,
                                     Integrator_kwargs=dict(nsteps=256))

    # estimate the maximum period
    max_T = round(estimate_max_period(t, ws).max() * 200, -4)
    dt = round(max_T * 1.5E-5, 0)
    nsteps = int(max_T / dt)

    return dt, nsteps

def worker(task):
    # unpack input argument dictionary
    index = task['index']
    w0_filename = task['w0_filename']
    allfreqs_filename = task['allfreqs_filename']
    potential = task['potential']

    # read out just this initial condition
    w0 = np.load(w0_filename)
    allfreqs_shape = (len(w0),) + _shape
    allfreqs = np.memmap(allfreqs_filename, mode='r', shape=allfreqs_shape, dtype='float64')

    # short-circuit if this orbit is already done
    if allfreqs[index,0,7] == 1.:
        return

    # temporary array for results
    tmp = np.zeros(_shape)

    # automatically estimate dt, nsteps
    try:
        dt, nsteps = estimate_dt_nsteps(potential, w0[index].copy())
    except RuntimeError:
        logger.warning("Failed to integrate orbit when estimating dt,nsteps")
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    logger.info("Orbit {}: initial dt={}, nsteps={}".format(index, dt, nsteps))

    dEmax = 1.
    maxiter = 3  # maximum number of times to refine integration step
    for i in range(maxiter):
        if i > 0:
            # adjust timestep and duration if necessary
            dt /= 2.
            nsteps *= 2
            logger.debug("Refining timestep for orbit {} ({}, {})".format(index, dt, nsteps))

        # integrate orbit
        try:
            t,ws = potential.integrate_orbit(w0[index].copy(), dt=dt, nsteps=nsteps,
                                             Integrator=gi.DOPRI853Integrator,
                                             Integrator_kwargs=dict(nsteps=256))
        except RuntimeError:
            # ODE integration failed
            logger.warning("Orbit integration failed.")
            continue

        logger.debug('Orbit integrated')

        # check energy conservation for the orbit
        E = potential.total_energy(ws[:,0,:3].copy(), ws[:,0,3:].copy())
        dE = np.abs(E[1:] - E[0])
        dEmax = dE.max()

        if dEmax < 1E-9:
            break

    if dEmax > 1E-9:
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    # start finding the frequencies -- do first half then second half
    naff = gd.NAFF(t[:nsteps//2+1])
    freqs1,is_loop = ws_to_freqs(naff, ws[:nsteps//2+1])
    freqs2,is_loop = ws_to_freqs(naff, ws[nsteps//2:])

    # save to output array
    tmp[0,:6] = freqs1
    tmp[1,:6] = freqs2

    tmp[:,6] = dEmax
    tmp[:,7] = 1.
    tmp[:,8] = float(is_loop)
    tmp[:,9] = float(dt)
    tmp[:,10] = nsteps//2

    allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
    allfreqs[index] = tmp
    allfreqs.flush()
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Third-party
from astropy import log as logger
import numpy as np
from scipy.signal import argrelmax, argrelmin

# Project
import gary.dynamics as gd
import gary.integrate as gi

__all__ = ['ws_to_freqs', 'worker']

_shape = (2, 11)  # 6 frequencies, max energy diff, done flag, loop bit, dt, nsteps

def ptp_freqs(t, *args):
    freqs = []
    for x in args:
        ix = argrelmax(x, mode='wrap')[0]
        f_max = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        ix = argrelmin(x, mode='wrap')[0]
        f_min = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        freqs.append(np.mean([f_max, f_min]))
    return np.array(freqs)

def estimate_max_period(t, w):
    if w.ndim < 3:
        w = w[:,np.newaxis]

    norbits = w.shape[1]
    periods = []
    for i in range(norbits):
        loop = gd.classify_orbit(w[:,i])
        if np.any(loop):
            # flip coords
            new_w = gd.align_circulation_with_z(w[:,i], loop[0])[:,0]

            # convert to cylindrical
            R = np.sqrt(new_w[:,0]**2 + new_w[:,1]**2)
            phi = np.arctan2(new_w[:,1], new_w[:,0])
            z = new_w[:,2]

            T = 2*np.pi / ptp_freqs(t, R, phi, z)
        else:
            T = 2*np.pi / ptp_freqs(t, *w[:,i,:3].T)

        periods.append(T)

    return np.array(periods)

def ws_to_freqs(naff, ws, nintvec=15):

    # first get x,y,z frequencies
    logger.info('Solving for XYZ frequencies...')
    fs = [(ws[:,0,j] + 1j*ws[:,0,j+3]) for j in range(3)]
    try:
        fxyz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
    except:
        fxyz = np.ones(3)*np.nan

    # now get other frequencies
    loop = gd.classify_orbit(ws)
    is_loop = np.any(loop)
    if is_loop:
        # need to flip coordinates until circulation is around z axis
        new_ws = gd.align_circulation_with_z(ws, loop[0])

        fs = gd.poincare_polar(new_ws[:,0])
        try:
            logger.info('Solving for Rφz frequencies...')
            fRphiz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
        except:
            fRphiz = np.ones(3)*np.nan

    else:
        fRphiz = np.ones(3)*np.nan

    return np.append(fxyz, fRphiz), is_loop

def estimate_dt_nsteps(potential, w0, nperiods=100):
    # integrate orbit
    t,ws = potential.integrate_orbit(w0, dt=1., nsteps=20000,
                                     Integrator=gi.DOPRI853Integrator,
                                     Integrator_kwargs=dict(nsteps=256))

    # estimate the maximum period
    max_T = round(estimate_max_period(t, ws).max() * 200, -4)
    dt = round(max_T * 1.E-5, 0)
    nsteps = int(max_T / dt)

    return dt, nsteps

def worker(task):
    # unpack input argument dictionary
    index = task['index']
    w0_filename = task['w0_filename']
    allfreqs_filename = task['allfreqs_filename']
    potential = task['potential']

    # read out just this initial condition
    w0 = np.load(w0_filename)
    allfreqs_shape = (len(w0),) + _shape
    allfreqs = np.memmap(allfreqs_filename, mode='r', shape=allfreqs_shape, dtype='float64')

    # short-circuit if this orbit is already done
    if allfreqs[index,0,7] == 1.:
        return

    # temporary array for results
    tmp = np.zeros(_shape)

    # automatically estimate dt, nsteps
    try:
        dt, nsteps = estimate_dt_nsteps(potential, w0[index].copy())
    except RuntimeError:
        logger.warning("Failed to integrate orbit when estimating dt,nsteps")
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    logger.info("Orbit {}: initial dt={}, nsteps={}".format(index, dt, nsteps))

    dEmax = 1.
    maxiter = 5  # maximum number of times to refine integration step
    for i in range(maxiter+1):
        if i > 0:
            # adjust timestep and duration if necessary
            dt /= 2.
            nsteps *= 2
            logger.debug("Refining orbit {} to: dt,nsteps=({},{}). Max. dE={}"
                         .format(index, dt, nsteps, dEmax))

        # integrate orbit
        try:
            t,ws = potential.integrate_orbit(w0[index].copy(), dt=dt, nsteps=nsteps,
                                             Integrator=gi.DOPRI853Integrator,
                                             Integrator_kwargs=dict(nsteps=8192,atol=1E-14,rtol=1E-9))
        except RuntimeError:
            # ODE integration failed
            logger.warning("Orbit integration failed.")
            continue

        logger.debug('Orbit integrated')

        # check energy conservation for the orbit
        E = potential.total_energy(ws[:,0,:3].copy(), ws[:,0,3:].copy())
        dE = np.abs(E[1:] - E[0])
        dEmax = dE.max() / np.abs(E[0])

        if dEmax < 1E-9:
            break

    if dEmax > 1E-9:
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    # start finding the frequencies -- do first half then second half
    naff = gd.NAFF(t[:nsteps//2+1])
    freqs1,is_loop = ws_to_freqs(naff, ws[:nsteps//2+1])
    freqs2,is_loop = ws_to_freqs(naff, ws[nsteps//2:])

    # save to output array
    tmp[0,:6] = freqs1
    tmp[1,:6] = freqs2

    tmp[:,6] = dEmax
    tmp[:,7] = 1.
    tmp[:,8] = float(is_loop)
    tmp[:,9] = float(dt)
    tmp[:,10] = nsteps//2

    allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
    allfreqs[index] = tmp
    allfreqs.flush()
<commit_msg>change maxiter<commit_after># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Third-party
from astropy import log as logger
import numpy as np
from scipy.signal import argrelmax, argrelmin

# Project
import gary.dynamics as gd
import gary.integrate as gi

__all__ = ['ws_to_freqs', 'worker']

_shape = (2, 11)  # 6 frequencies, max energy diff, done flag, loop bit, dt, nsteps

def ptp_freqs(t, *args):
    freqs = []
    for x in args:
        ix = argrelmax(x, mode='wrap')[0]
        f_max = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        ix = argrelmin(x, mode='wrap')[0]
        f_min = np.mean(2*np.pi / (t[ix[1:]] - t[ix[:-1]]))

        freqs.append(np.mean([f_max, f_min]))
    return np.array(freqs)

def estimate_max_period(t, w):
    if w.ndim < 3:
        w = w[:,np.newaxis]

    norbits = w.shape[1]
    periods = []
    for i in range(norbits):
        loop = gd.classify_orbit(w[:,i])
        if np.any(loop):
            # flip coords
            new_w = gd.align_circulation_with_z(w[:,i], loop[0])[:,0]

            # convert to cylindrical
            R = np.sqrt(new_w[:,0]**2 + new_w[:,1]**2)
            phi = np.arctan2(new_w[:,1], new_w[:,0])
            z = new_w[:,2]

            T = 2*np.pi / ptp_freqs(t, R, phi, z)
        else:
            T = 2*np.pi / ptp_freqs(t, *w[:,i,:3].T)

        periods.append(T)

    return np.array(periods)

def ws_to_freqs(naff, ws, nintvec=15):

    # first get x,y,z frequencies
    logger.info('Solving for XYZ frequencies...')
    fs = [(ws[:,0,j] + 1j*ws[:,0,j+3]) for j in range(3)]
    try:
        fxyz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
    except:
        fxyz = np.ones(3)*np.nan

    # now get other frequencies
    loop = gd.classify_orbit(ws)
    is_loop = np.any(loop)
    if is_loop:
        # need to flip coordinates until circulation is around z axis
        new_ws = gd.align_circulation_with_z(ws, loop[0])

        fs = gd.poincare_polar(new_ws[:,0])
        try:
            logger.info('Solving for Rφz frequencies...')
            fRphiz,d,ixes = naff.find_fundamental_frequencies(fs, nintvec=nintvec)
        except:
            fRphiz = np.ones(3)*np.nan

    else:
        fRphiz = np.ones(3)*np.nan

    return np.append(fxyz, fRphiz), is_loop

def estimate_dt_nsteps(potential, w0, nperiods=100):
    # integrate orbit
    t,ws = potential.integrate_orbit(w0, dt=1., nsteps=20000,
                                     Integrator=gi.DOPRI853Integrator,
                                     Integrator_kwargs=dict(nsteps=256))

    # estimate the maximum period
    max_T = round(estimate_max_period(t, ws).max() * 200, -4)
    dt = round(max_T * 1.E-5, 0)
    nsteps = int(max_T / dt)

    return dt, nsteps

def worker(task):
    # unpack input argument dictionary
    index = task['index']
    w0_filename = task['w0_filename']
    allfreqs_filename = task['allfreqs_filename']
    potential = task['potential']

    # read out just this initial condition
    w0 = np.load(w0_filename)
    allfreqs_shape = (len(w0),) + _shape
    allfreqs = np.memmap(allfreqs_filename, mode='r', shape=allfreqs_shape, dtype='float64')

    # short-circuit if this orbit is already done
    if allfreqs[index,0,7] == 1.:
        return

    # temporary array for results
    tmp = np.zeros(_shape)

    # automatically estimate dt, nsteps
    try:
        dt, nsteps = estimate_dt_nsteps(potential, w0[index].copy())
    except RuntimeError:
        logger.warning("Failed to integrate orbit when estimating dt,nsteps")
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    logger.info("Orbit {}: initial dt={}, nsteps={}".format(index, dt, nsteps))

    dEmax = 1.
    maxiter = 3  # maximum number of times to refine integration step
    for i in range(maxiter+1):
        if i > 0:
            # adjust timestep and duration if necessary
            dt /= 2.
            nsteps *= 2
            logger.debug("Refining orbit {} to: dt,nsteps=({},{}). Max. dE={}"
                         .format(index, dt, nsteps, dEmax))

        # integrate orbit
        try:
            t,ws = potential.integrate_orbit(w0[index].copy(), dt=dt, nsteps=nsteps,
                                             Integrator=gi.DOPRI853Integrator,
                                             Integrator_kwargs=dict(nsteps=8192,atol=1E-14,rtol=1E-9))
        except RuntimeError:
            # ODE integration failed
            logger.warning("Orbit integration failed.")
            continue

        logger.debug('Orbit integrated')

        # check energy conservation for the orbit
        E = potential.total_energy(ws[:,0,:3].copy(), ws[:,0,3:].copy())
        dE = np.abs(E[1:] - E[0])
        dEmax = dE.max() / np.abs(E[0])

        if dEmax < 1E-9:
            break

    if dEmax > 1E-9:
        allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
        tmp[:,:] = np.nan
        tmp[:,7] = 1.
        allfreqs[index] = tmp
        allfreqs.flush()
        return

    # start finding the frequencies -- do first half then second half
    naff = gd.NAFF(t[:nsteps//2+1])
    freqs1,is_loop = ws_to_freqs(naff, ws[:nsteps//2+1])
    freqs2,is_loop = ws_to_freqs(naff, ws[nsteps//2:])

    # save to output array
    tmp[0,:6] = freqs1
    tmp[1,:6] = freqs2

    tmp[:,6] = dEmax
    tmp[:,7] = 1.
    tmp[:,8] = float(is_loop)
    tmp[:,9] = float(dt)
    tmp[:,10] = nsteps//2

    allfreqs = np.memmap(allfreqs_filename, mode='r+', shape=allfreqs_shape, dtype='float64')
    allfreqs[index] = tmp
    allfreqs.flush()
<|endoftext|>


<commit_before># Run with something like
#
#   env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003
#
# Mostly to pick up the Guix GN2_PROFILE and python modules

import requests as req
import sys

print "Mechanical Rob firing up..."

if len(sys.argv)<1:
    raise "Problem with arguments"

url = sys.argv[1]
print url

r = req.get(url)

print r
<commit_msg>Use argparse to handle arguments<commit_after># Run with something like
#
#   env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003
#
# Mostly to pick up the Guix GN2_PROFILE and python modules
from __future__ import print_function
from link_checker import check_links
import argparse

print("Mechanical Rob firing up...")

def run_all(args_obj, parser):
    print("")
    print("Running all tests.")
    check_links(args_obj, parser)
    # TODO: Add other functions as they are created.

def print_help(args_obj, parser):
    print(parser.format_help())

def dummy(args_obj, parser):
    print("Not implemented yet.")

desc = """
This is Mechanical-Rob - an automated web server tester for
                         Genenetwork.org
"""
parser = argparse.ArgumentParser(description=desc)

parser.add_argument("-d", "--database", metavar="DB", type=str
                    , default="db_webqtl_s"
                    , help="Use database (default db_webqtl_s)")

parser.add_argument("host", metavar="HOST", type=str
                    , default="http://localhost:5003"
                    , help="The url to the web server")

parser.add_argument("-a", "--all", dest="accumulate", action="store_const"
                    , const=run_all, default=print_help
                    , help="Runs all tests.")

parser.add_argument("-l", "--link-checker", dest="accumulate"
                    , action='store_const', const=check_links, default=print_help
                    , help="Checks for dead links.")

# parser.add_argument("-n", "--navigation", dest="accumulate"
#                     , action="store_const", const=check_navigation, default=print_help
#                     , help="Checks for navigation.")

# parser.add_argument("-m", "--mapping", dest="accumulate"
#                     , action="store_const", const=check_mapping, default=print_help
#                     , help="Checks for mapping.")

# parser.add_argument("-s", "--skip-broken", dest="accumulate"
#                     , action="store_const", const=dummy, default=print_help
#                     , help="Skip tests that are known to be broken.")

args = parser.parse_args()
# print("The arguments object: ", args)

args.accumulate(args, parser)
<|endoftext|>


<commit_before>from PyQt4 import QtCore, QtGui

import shared
import re
import sys
import inspect
from helper_sql import *
from addresses import decodeAddress
from pyelliptic.openssl import OpenSSL
import time

def accountClass(address):
    if not shared.config.has_section(address):
        subscription = SubscriptionAccount(address)
        if subscription.type != 'subscription':
            return None
    try:
        gateway = shared.config.get(address, "gateway")
        for name, cls in inspect.getmembers(sys.modules[__name__], inspect.isclass):
#            obj = g(address)
            if issubclass(cls, GatewayAccount) and cls.gatewayName == gateway:
                return cls(address)
        # general gateway
        return GatewayAccount(address)
    except:
        pass
    # no gateway
    return BMAccount(address)

class BMAccount(object):
    def __init__(self, address = None):
        self.address = address
        self.type = 'normal'
        if shared.config.has_section(address):
            if shared.safeConfigGetBoolean(self.address, 'chan'):
                self.type = "chan"
            elif shared.safeConfigGetBoolean(self.address, 'mailinglist'):
                self.type = "mailinglist"
        else:
            queryreturn = sqlQuery(
                '''select label from subscriptions where address=?''', address)
            if queryreturn:
                self.type = 'subscription'

    def getLabel(self, address = None):
        if address is None:
            address = self.address
        label = address
        if shared.config.has_section(address):
            label = shared.config.get(address, 'label')
        queryreturn = sqlQuery(
            '''select label from addressbook where address=?''', address)
        if queryreturn != []:
            for row in queryreturn:
                label, = row
        else:
            queryreturn = sqlQuery(
                '''select label from subscriptions where address=?''', address)
            if queryreturn != []:
                for row in queryreturn:
                    label, = row
        return label

    def parseMessage(self, toAddress, fromAddress, subject, message):
        self.toAddress = toAddress
        self.fromAddress = fromAddress
        self.subject = subject
        self.message = message
        self.fromLabel = self.getLabel(fromAddress)
        self.toLabel = self.getLabel(toAddress)

class SubscriptionAccount(BMAccount):
    pass

class GatewayAccount(BMAccount):
    gatewayName = None
    def __init__(self, address):
        super(BMAccount, self).__init__(address)

    def send(self):
        status, addressVersionNumber, streamNumber, ripe = decodeAddress(self.toAddress)
        ackdata = OpenSSL.rand(32)
        t = ()
        sqlExecute(
            '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
            '',
            self.toAddress,
            ripe,
            self.fromAddress,
            self.subject,
            self.message,
            ackdata,
            int(time.time()), # sentTime (this will never change)
            int(time.time()), # lastActionTime
            0, # sleepTill time. This will get set when the POW gets done.
            'msgqueued',
            0, # retryNumber
            'sent', # folder
            2, # encodingtype
            shared.config.getint('bitmessagesettings', 'ttl')
        )

        shared.workerQueue.put(('sendmessage', self.toAddress))

    def parseMessage(self, toAddress, fromAddress, subject, message):
        super(BMAccount, self).parseMessage(toAddress, fromAddress, subject, message)

class MailchuckAccount(GatewayAccount):
    # set "gateway" in keys.dat to this
    gatewayName = "mailchuck"
    registrationAddress = "BM-2cVYYrhaY5Gbi3KqrX9Eae2NRNrkfrhCSA"
    unregistrationAddress = "BM-2cVMAHTRjZHCTPMue75XBK5Tco175DtJ9J"
    relayAddress = "BM-2cWim8aZwUNqxzjMxstnUMtVEUQJeezstf"
    regExpIncoming = re.compile("(.*)MAILCHUCK-FROM::(\S+) \| (.*)")
    regExpOutgoing = re.compile("(\S+) (.*)")
    def __init__(self, address):
        super(GatewayAccount, self).__init__(address)

    def createMessage(self, toAddress, fromAddress, subject, message):
        self.subject = toAddress + " " + subject
        self.toAddress = self.relayAddress
        self.fromAddress = fromAddress
        self.message = message

    def register(self, email):
        self.toAddress = self.registrationAddress
        self.subject = email
        self.message = ""
        self.fromAddress = self.address
        self.send()

    def unregister(self):
        self.toAddress = self.unregistrationAddress
        self.subject = ""
        self.message = ""
        self.fromAddress = self.address
        self.send()

    def parseMessage(self, toAddress, fromAddress, subject, message):
        super(GatewayAccount, self).parseMessage(toAddress, fromAddress, subject, message)
        if fromAddress == self.relayAddress:
            matches = self.regExpIncoming.search(subject)
            if not matches is None:
                self.subject = ""
                if not matches.group(1) is None:
                    self.subject += matches.group(1)
                if not matches.group(3) is None:
                    self.subject += matches.group(3)
                if not matches.group(2) is None:
                    self.fromLabel = matches.group(2)
                    self.fromAddress = matches.group(2)
        if toAddress == self.relayAddress:
            matches = self.regExpOutgoing.search(subject)
            if not matches is None:
                if not matches.group(2) is None:
                    self.subject = matches.group(2)
                if not matches.group(1) is None:
                    self.toLabel = matches.group(1)
                    self.toAddress = matches.group(1)<commit_msg>Add BroadcastAccount class<commit_after>from PyQt4 import QtCore, QtGui

import shared
import re
import sys
import inspect
from helper_sql import *
from addresses import decodeAddress
from pyelliptic.openssl import OpenSSL
from utils import str_broadcast_subscribers
import time

def accountClass(address):
    if not shared.config.has_section(address):
        if address == str_broadcast_subscribers:
            subscription = BroadcastAccount(address)
            if subscription.type != 'broadcast':
                return None
        else:
            subscription = SubscriptionAccount(address)
            if subscription.type != 'subscription':
                return None
        return subscription
    try:
        gateway = shared.config.get(address, "gateway")
        for name, cls in inspect.getmembers(sys.modules[__name__], inspect.isclass):
#            obj = g(address)
            if issubclass(cls, GatewayAccount) and cls.gatewayName == gateway:
                return cls(address)
        # general gateway
        return GatewayAccount(address)
    except:
        pass
    # no gateway
    return BMAccount(address)

class BMAccount(object):
    def __init__(self, address = None):
        self.address = address
        self.type = 'normal'
        if shared.config.has_section(address):
            if shared.safeConfigGetBoolean(self.address, 'chan'):
                self.type = "chan"
            elif shared.safeConfigGetBoolean(self.address, 'mailinglist'):
                self.type = "mailinglist"
        elif self.address == str_broadcast_subscribers:
            self.type = 'broadcast'
        else:
            queryreturn = sqlQuery(
                '''select label from subscriptions where address=?''', self.address)
            if queryreturn:
                self.type = 'subscription'

    def getLabel(self, address = None):
        if address is None:
            address = self.address
        label = address
        if shared.config.has_section(address):
            label = shared.config.get(address, 'label')
        queryreturn = sqlQuery(
            '''select label from addressbook where address=?''', address)
        if queryreturn != []:
            for row in queryreturn:
                label, = row
        else:
            queryreturn = sqlQuery(
                '''select label from subscriptions where address=?''', address)
            if queryreturn != []:
                for row in queryreturn:
                    label, = row
        return label

    def parseMessage(self, toAddress, fromAddress, subject, message):
        self.toAddress = toAddress
        self.fromAddress = fromAddress
        self.subject = subject
        self.message = message
        self.fromLabel = self.getLabel(fromAddress)
        self.toLabel = self.getLabel(toAddress)

class SubscriptionAccount(BMAccount):
    pass

class BroadcastAccount(BMAccount):
    pass

class GatewayAccount(BMAccount):
    gatewayName = None
    def __init__(self, address):
        super(BMAccount, self).__init__(address)

    def send(self):
        status, addressVersionNumber, streamNumber, ripe = decodeAddress(self.toAddress)
        ackdata = OpenSSL.rand(32)
        t = ()
        sqlExecute(
            '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
            '',
            self.toAddress,
            ripe,
            self.fromAddress,
            self.subject,
            self.message,
            ackdata,
            int(time.time()), # sentTime (this will never change)
            int(time.time()), # lastActionTime
            0, # sleepTill time. This will get set when the POW gets done.
            'msgqueued',
            0, # retryNumber
            'sent', # folder
            2, # encodingtype
            shared.config.getint('bitmessagesettings', 'ttl')
        )

        shared.workerQueue.put(('sendmessage', self.toAddress))

    def parseMessage(self, toAddress, fromAddress, subject, message):
        super(BMAccount, self).parseMessage(toAddress, fromAddress, subject, message)

class MailchuckAccount(GatewayAccount):
    # set "gateway" in keys.dat to this
    gatewayName = "mailchuck"
    registrationAddress = "BM-2cVYYrhaY5Gbi3KqrX9Eae2NRNrkfrhCSA"
    unregistrationAddress = "BM-2cVMAHTRjZHCTPMue75XBK5Tco175DtJ9J"
    relayAddress = "BM-2cWim8aZwUNqxzjMxstnUMtVEUQJeezstf"
    regExpIncoming = re.compile("(.*)MAILCHUCK-FROM::(\S+) \| (.*)")
    regExpOutgoing = re.compile("(\S+) (.*)")
    def __init__(self, address):
        super(GatewayAccount, self).__init__(address)

    def createMessage(self, toAddress, fromAddress, subject, message):
        self.subject = toAddress + " " + subject
        self.toAddress = self.relayAddress
        self.fromAddress = fromAddress
        self.message = message

    def register(self, email):
        self.toAddress = self.registrationAddress
        self.subject = email
        self.message = ""
        self.fromAddress = self.address
        self.send()

    def unregister(self):
        self.toAddress = self.unregistrationAddress
        self.subject = ""
        self.message = ""
        self.fromAddress = self.address
        self.send()

    def parseMessage(self, toAddress, fromAddress, subject, message):
        super(GatewayAccount, self).parseMessage(toAddress, fromAddress, subject, message)
        if fromAddress == self.relayAddress:
            matches = self.regExpIncoming.search(subject)
            if not matches is None:
                self.subject = ""
                if not matches.group(1) is None:
                    self.subject += matches.group(1)
                if not matches.group(3) is None:
                    self.subject += matches.group(3)
                if not matches.group(2) is None:
                    self.fromLabel = matches.group(2)
                    self.fromAddress = matches.group(2)
        if toAddress == self.relayAddress:
            matches = self.regExpOutgoing.search(subject)
            if not matches is None:
                if not matches.group(2) is None:
                    self.subject = matches.group(2)
                if not matches.group(1) is None:
                    self.toLabel = matches.group(1)
                    self.toAddress = matches.group(1)<|endoftext|>


<commit_before>import itertools as it
import utils as U
# full precision
PREC = '0.3'

def float2str(n):
    """float2str

    Parameters
    ----------
    n : floating point number

    Returns
    -------
    String notation truncated to PREC
    """
    return '{n:{p}f}'.format(p=PREC, n=n).rstrip('0').rstrip('.')

def cx2str(c, x):
    """cx2str

    Parameters
    ----------
    c : coefficient
    x : varibale string

    Returns
    -------
    String notation, e.g. '2.345*x'

    Notes
    ------
    Returns ''

    """
    cstr = float2str(c)
    if cstr == '0' or cstr == '-0':
        cxstr = ''
    elif cstr == '1':
        cxstr = x
    elif cstr == '-1':
        cxstr = '-' + x
    else:
        cxstr = '{}*{}'.format(cstr, x)
    return cxstr

def linexpr2str(xs, coeffs, b):
    """linexpr2str
    coeffs .* xs + b
    Parameters
    ----------
    xs : x0, x1, x2, ...
    coeffs : c0, c1, c2, ...
    b : float

    Returns
    -------
    c0*x0 + c1*x1 + c2*x2 + ...

    Notes
    ------
    Cleans up the expression whenever ci = 0 or +-1
    """
    b1 = (b, '1')
    cx = it.chain(it.izip(coeffs, xs), [b1])
    ret = ' + '.join((filter(None, (cx2str(c, x) for c, x in cx))))
    # make sure an empty string is not returned. This can happen if
    # the linexpr evaluates to a 0
    if not ret:
        ret = '0'
    return ret
<commit_msg>encapsulated the functions in a class to get precision involved<commit_after>import itertools as it
#import utils as U
import err

class Expr2Str(object):
    PREC = None

    @staticmethod
    def set_prec(prec):
        Expr2Str.PREC = prec

    @staticmethod
    def float2str(n):
        """float2str

        Parameters
        ----------
        n : floating point number

        Returns
        -------
        String notation truncated to PREC
        """
        assert(Expr2Str.PREC is not None)
        return '{n:{p}f}'.format(p=Expr2Str.PREC, n=n).rstrip('0').rstrip('.')

    @staticmethod
    def cx2str(c, x):
        """cx2str

        Parameters
        ----------
        c : coefficient
        x : varibale string

        Returns
        -------
        String notation, e.g. '2.345*x'

        Notes
        ------
        Returns ''

        """
        cstr = Expr2Str.float2str(c)
        if cstr == '0' or cstr == '-0':
            cxstr = ''
        elif cstr == '1':
            cxstr = x
        elif cstr == '-1':
            cxstr = '-' + x
        else:
            cxstr = '{}*{}'.format(cstr, x)
        return cxstr

    @staticmethod
    def linexpr2str(xs, coeffs, b):
        """linexpr2str
        coeffs .* xs + b
        Parameters
        ----------
        xs : x0, x1, x2, ...
        coeffs : c0, c1, c2, ...
        b : float

        Returns
        -------
        c0*x0 + c1*x1 + c2*x2 + ...

        Notes
        ------
        Cleans up the expression whenever ci = 0 or +-1
        """
        b1 = (b, '1')
        cx = it.chain(it.izip(coeffs, xs), [b1])
        ret = ' + '.join((filter(None, (Expr2Str.cx2str(c, x) for c, x in cx))))
        # make sure an empty string is not returned. This can happen if
        # the linexpr evaluates to a 0
        if not ret:
            ret = '0'
        return ret
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

'''
Creates a SAL transition system: DFT
'''
#TODO: rename to saltrans_dft.py

#import sympy as sm
import textwrap as tw
from math import isinf

from bmc.helpers.expr2str import Expr2Str

#import utils as U
import settings

class SALTransError(Exception):
    pass

# Make classes out of every header, prop, init, etc
class SALTransSys(object):

    def __init__(self, module_name, vs, init_cons, prop, prec):
        """
        Parameters
        ----------
        module_name : Module name string
        vs : list of variable names
        init_cons : X0 described as an interval constraint
        prop : Unsafe Set described as an interval constraint
        """

        self.prec = prec
        self.vs = vs
        self.prop_name = 'safety'
        self.init_cons = init_cons
        self.module_name = module_name
        self.transitions = []
        self.prop = prop
        # initialize the class with the prec
        Expr2Str.set_prec(prec)
        return

    def add_transition(self, tran):
        self.transitions.append(tran)

    def __str__(self):
        return self.sal_file

    # XXX
    # This is a bit strange, Some of the declarations like OUTPUT and
    # INITIALIZATION are present but others like LOCAL are absent. The
    # one whcich are mandatory (in the present context) are present
    # and the ones which are optional like LOCAL are specified by the
    # variables. This can be fixed later by making this a class.
    @property
    def sal_file(self):
        s = tw.dedent('''
        {c}
        {tp}
        {pm}
        {ld}
        OUTPUT
        {od}
        INITIALIZATION
        {init}
        TRANSITION
        [
        {trans}
        % Always on transition: to overcome the in-complete pwa model
        % from deadlocking
        {atran}
        ]
        END;
        {monitor}
        system: MODULE = PLANT || MONITOR;
        {prop}
        END
        ''').format(c=self.context,
                    tp=self.type_decls,
                    pm=self.plant_module,
                    ld=self.local_decls,
                    od=self.op_decls,
                    init=self.init_set,
                    trans=self.trans,
                    atran=self.always_true_transition,
                    monitor=self.monitor_module,
                    prop=self.safety_prop)
        return s

    @property
    def always_true_transition(self):
        return '\n[] NOP: TRUE -->\n'

    @property
    def context(self):
        s = tw.dedent('''
        {}: CONTEXT =
        BEGIN
        ''').format(self.module_name)
        return s

    @property
    def type_decls(self):
        # There are no type decls
        return ''

    @property
    def plant_module(self):
        s = tw.dedent('''
        PLANT: MODULE =
        BEGIN''')
        return s

    @property
    def op_decls(self):
        s = ['\n\t{v}:REAL' .format(v=v) for v in self.vs]
        return ','.join(s)

    @property
    def local_decls(self):
        return ''

    @property
    def init_set(self):
        iv = self.init_cons
        s = ['\n\t{v} IN {{ r : REAL | r >=  {l} AND r <= {h} }}'.format(
            v=v, l=Expr2Str.float2str(iv.l[i]), h=Expr2Str.float2str(iv.h[i])) for i, v in enumerate(self.vs)]
        return ';'.join(s)

    # sal description
    @property
    def trans(self):
        ts = '[]\n'.join(('{}'.format(i) for i in self.transitions))
        s = '{}'.format(ts)
        return s

    @property
    def safety_prop(self):
        s = tw.dedent('''
        {prop_name} : THEOREM
        system |- G(NOT unsafe);
        ''').format(prop_name=self.prop_name)
        return s

    @property
    def monitor_module(self):
        prop = self.prop
        expr = "{v}' {gle} {c}"

        ls = [
                expr.format(v=v, gle='>=', c=prop.l[i])
                for i, v in enumerate(self.vs)
                if not isinf(prop.l[i])
             ]
        hs = [
                expr.format(v=v, gle='<=', c=prop.h[i])
                for i, v in enumerate(self.vs)
                if not isinf(prop.h[i])
             ]
        prop_str = ' AND '.join(ls + hs)

        # Remove AND cell' = CE
        assert(settings.CE)
        s = tw.dedent('''
        MONITOR: MODULE =
        BEGIN
        OUTPUT
                unsafe : BOOLEAN
        INPUT
        {}
        INITIALIZATION
                unsafe = FALSE
        TRANSITION
        [
        TRUE -->
        unsafe' IN {{r : BOOLEAN | r <=> ({} AND cell' = CE)}}
        ]
        END;''').format(self.op_decls, prop_str)
        return s

class Transition(object):

    def __init__(self, name, g, r):
        self.g = g
        self.r = r
        self.name = name

    def __str__(self):
        s = '{}:\n{} -->\n{}\n'.format(self.name, self.g, self.r)
        return s

class Guard(object):
    def __init__(self, C, d):
        '''Cx - d <= 0'''
        self.C = C
        self.d = d

        self.ncons, self.nvars = self.C.shape
        assert(self.ncons == d.size)
        self._vs = None

    @property
    def vs(self):
        if self._vs is None:
            return ['x'+str(j) for j in range(self.nvars)]
        else:
            return self._vs

    @vs.setter
    def vs(self, vs):
        self._vs = vs

    def __str__(self):
        # num_constraints , num_dimensions
        #nc, nd = self.C.shape
        #TODO: Remove this
        # num states
        #num_x = nr
        # num ex. inputs
        #num_w = nc - nr
        cons = (Expr2Str.linexpr2str(self.vs, self.C[i, :], -self.d[i]) + ' <= 0'
                for i in range(self.ncons))
        return ' AND '.join(cons)

class Reset(object):
    def __init__(self, A, b, error):
        self.A = A
        self.b = b
        self.e = error

        self.nlhs, self.nrhs = self.A.shape
        assert(self.nlhs == b.size)
        self._vs = None
        self._vs_ = None

        #TODO: Remove this
        # num op, num ip
        #nr, nc = self.A.shape
        # num states
        #self.num_x = nr
        # num ex. inputs
        #self.num_w = nc - nr

    @property
    def vs(self):
        if self._vs is None:
            return ['x'+str(j) for j in range(self.nrhs)]
        else:
            return self._vs

    @property
    def vs_(self):
        if self._vs_ is None:
            return ["x{}".format(i) for i in range(self.nlhs)]
        else:
            return self._vs_

    @vs.setter
    def vs(self, vs):
        self._vs = vs

    @vs_.setter
    def vs_(self, vs_):
        self._vs_ = vs_

    def __str__(self):
        s = []

        delta_h = self.b + self.e.h
        delta_l = self.b + self.e.l
        set_def = ("{xi_}' IN {{ r : REAL| "
                   "r >= {Axi_plus_delta_li} AND "
                   "r <= {Axi_plus_delta_hi} }}")
        for vsi_, Ai, dli, dhi in zip(self.vs_, self.A, delta_l, delta_h):
            #if error is not significant, i.e., it is less than the
            # precision used while dumping sal file, ignore it. Else,
            # include it and make the rhs of the assignment a set.
            if Expr2Str.float2str(dli) == Expr2Str.float2str(dhi):
                assignment_stmt = vsi_ + "' = " + Expr2Str.linexpr2str(self.vs, Ai, dli)
            else:
                assignment_stmt = set_def.format(
                        xi_=vsi_,
                        Axi_plus_delta_li=Expr2Str.linexpr2str(self.vs, Ai, dli),
                        Axi_plus_delta_hi=Expr2Str.linexpr2str(self.vs, Ai, dhi)
                        )
            s.append(assignment_stmt)
        return ';\n'.join(s)

###############################################
# ############# CEMETEREY
###############################################

#     @property
#     def safety_prop(self):
#         prop = self.prop
#         v = self.v
#         expr = '{v}{i} {gle} {c}'

#         ls = [
#                 expr.format(v=v, i=i, gle='>=', c=prop.l[i]) for i in range(prop.dim)
#                 if not isinf(prop.l[i])
#              ]
#         hs = [
#                 expr.format(v=v, i=i, gle='<=', c=prop.h[i]) for i in range(prop.dim)
#                 if not isinf(prop.h[i])
#              ]
#         prop_str = ' AND '.join(ls + hs)

#         s = tw.dedent('''
#         {prop_name} : THEOREM
#         system |- NOT F({prop});
#         END''').format(prop_name=self.prop_name, prop=prop_str)
#         return s

# Older code, [same as new one but with comments]
# class Guard(object):
#     def __init__(self, C, d):
#         '''Cx <= d'''
#         self.C = C
#         self.d = d

#     def __str__(self):
#         # num_constraints , num_dimensions
#         nc, nd = self.C.shape

#         xs = ['x'+str(j) for j in range(nd)]
#         cons = (linexpr2str(xs, self.C[i, :], -self.d[i]) + '<= 0'
#                 for i in range(nc))
# #         cons = ('{Cxi} <= {di}'.format(
# #             Cxi=linexpr2str(self.C[i, :], ('x'+str(j) for j in range(nd))),
# #             di=float2str(self.d[i])) for i in range(nc))
#         return ' AND '.join(cons)
#         #return ' AND '.join('{}*x{} + {} <= 0'.format(self.C[i, j], j, -self.d[j]) for i in range(nc) for j in range(nd))

# #     def __str__(self):
# #         s = []
# #         # num_constraints , num_dimensions
# #         nc, nd = self.C.shape
# #         x_sym = ['x{}'.format(i) for i in range(nd)]
# #         x = sm.Matrix(sm.symbols(x_sym))
# #         Cx = self.C * x
# #         s = [str(Cx[i] - self.d[i] <= 0) for i in range(nc)]
# #         return ' AND '.join(s)

# class Reset(object):
#     def __init__(self, A, b, error=None):
#         self.A = A
#         self.b = b
#         self.e = error
# #     def __str__(self):
# #         s = []
# #         ndo, ndi = self.A.shape
# #         x_sym = ['x{}'.format(i) for i in range(ndi)]
# #         x__sym = ["x{}'".format(i) for i in range(ndo)]
# #         x = sm.Matrix(sm.symbols(x_sym))
# #         x_ = sm.Matrix(sm.symbols(x__sym))
# #         Ax = self.A * x
# #         s = ['{} = {}'.format(x_[i], Ax[i] + self.b[i]) for i in range(ndo)]
# #         return ';\n'.join(s)

#     def __str__(self):
#         s = []
#         ndo, ndi = self.A.shape

#         #xi_s = "x{}'"
#         xi_s = ["x{}'".format(i) for i in range(ndo)]
#         xs = ['x'+str(j) for j in range(ndi)]

#         if self.e is None:
#             s = (xi_s[i] + '=' + linexpr2str(xs, self.A[i, :], self.b[i])
#                  for i in range(ndo))
#             return ';\n'.join(s)
# #             s = ['{xi_} = {Axi} + {bi}'.format(
# #                 xi_=xi_s.format(i),
# #                 Axi=linexpr2str(
# #                     self.A[i, :], ('x'+str(j) for j in range(ndi))
# #                     ),
# #                 bi=float2str(self.b[i])
# #                 ) for i in range(ndo)
# #                 ]

#         else:
#             delta_h = self.b + self.e.h
#             delta_l = self.b + self.e.l

#             s = ('{xi_} IN {{ r : REAL| '
#                  'r >= {Axi_plus_delta_li} AND '
#                  'r <= {Axi_plus_delta_hi} }}'
#                  .format(
#                      xi_=xi_s[i],
#                      Axi_plus_delta_li=linexpr2str(xs, self.A[i, :], delta_l[i]),
#                      Axi_plus_delta_hi=linexpr2str(xs, self.A[i, :], delta_h[i])
#                      ) for i in range(ndo)
#                  )

# #             s = ['{xi_} IN {{ r : REAL| '
# #                  'r >= {Axi} + {delta_li} AND '
# #                  'r <= {Axi} + {delta_hi} }}'
# #                  .format(
# #                     xi_=xi_s.format(i),
# #                     Axi=linexpr2str(
# #                         self.A[i, :], ('x'+str(j) for j in range(ndi))
# #                         ),
# #                     delta_li=float2str(delta_l[i]),
# #                     delta_hi=float2str(delta_h[i])
# #                     ) for i in range(ndo)
# #                  ]
#             return ';\n'.join(s)

# #         for i in range(ndo):
# #             row = ['{}*x{}'.format(self.A[i, j], j) for j in range(ndi)]
# #             # x_ = Ax + b
# #             x__str = "x{}'".format(i)
# #             Ax_str = ' + '.join(row)
# #             b_str = '{}'.format(self.b[i])
# #             eqn_str = '{} = {} + {}'.format(x__str, Ax_str, b_str)
# #             s.append(eqn_str)
# #         return ';\n'.join(s)
<commit_msg>retrieve prec from globalopts<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

'''
Creates a SAL transition system: DFT
'''
#TODO: rename to saltrans_dft.py

#import sympy as sm
import textwrap as tw
from math import isinf

from bmc.helpers.expr2str import Expr2Str
from globalopts import opts as gopts

#import utils as U
import settings

class SALTransError(Exception):
    pass

# Make classes out of every header, prop, init, etc
class SALTransSys(object):

    def __init__(self, module_name, vs, init_cons, prop):
        """
        Parameters
        ----------
        module_name : Module name string
        vs : list of variable names
        init_cons : X0 described as an interval constraint
        prop : Unsafe Set described as an interval constraint
        """

        self.vs = vs
        self.prop_name = 'safety'
        self.init_cons = init_cons
        self.module_name = module_name
        self.transitions = []
        self.prop = prop
        # initialize the class with the prec
        Expr2Str.set_prec(gopts.bmc_prec)
        return

    def add_transition(self, tran):
        self.transitions.append(tran)

    def __str__(self):
        return self.sal_file

    # XXX
    # This is a bit strange, Some of the declarations like OUTPUT and
    # INITIALIZATION are present but others like LOCAL are absent. The
    # one whcich are mandatory (in the present context) are present
    # and the ones which are optional like LOCAL are specified by the
    # variables. This can be fixed later by making this a class.
    @property
    def sal_file(self):
        s = tw.dedent('''
        {c}
        {tp}
        {pm}
        {ld}
        OUTPUT
        {od}
        INITIALIZATION
        {init}
        TRANSITION
        [
        {trans}
        % Always on transition: to overcome the in-complete pwa model
        % from deadlocking
        {atran}
        ]
        END;
        {monitor}
        system: MODULE = PLANT || MONITOR;
        {prop}
        END
        ''').format(c=self.context,
                    tp=self.type_decls,
                    pm=self.plant_module,
                    ld=self.local_decls,
                    od=self.op_decls,
                    init=self.init_set,
                    trans=self.trans,
                    atran=self.always_true_transition,
                    monitor=self.monitor_module,
                    prop=self.safety_prop)
        return s

    @property
    def always_true_transition(self):
        return '\n[] NOP: TRUE -->\n'

    @property
    def context(self):
        s = tw.dedent('''
        {}: CONTEXT =
        BEGIN
        ''').format(self.module_name)
        return s

    @property
    def type_decls(self):
        # There are no type decls
        return ''

    @property
    def plant_module(self):
        s = tw.dedent('''
        PLANT: MODULE =
        BEGIN''')
        return s

    @property
    def op_decls(self):
        s = ['\n\t{v}:REAL' .format(v=v) for v in self.vs]
        return ','.join(s)

    @property
    def local_decls(self):
        return ''

    @property
    def init_set(self):
        iv = self.init_cons
        s = ['\n\t{v} IN {{ r : REAL | r >=  {l} AND r <= {h} }}'.format(
            v=v, l=Expr2Str.float2str(iv.l[i]), h=Expr2Str.float2str(iv.h[i])) for i, v in enumerate(self.vs)]
        return ';'.join(s)

    # sal description
    @property
    def trans(self):
        ts = '[]\n'.join(('{}'.format(i) for i in self.transitions))
        s = '{}'.format(ts)
        return s

    @property
    def safety_prop(self):
        s = tw.dedent('''
        {prop_name} : THEOREM
        system |- G(NOT unsafe);
        ''').format(prop_name=self.prop_name)
        return s

    @property
    def monitor_module(self):
        prop = self.prop
        expr = "{v}' {gle} {c}"

        ls = [
                expr.format(v=v, gle='>=', c=prop.l[i])
                for i, v in enumerate(self.vs)
                if not isinf(prop.l[i])
             ]
        hs = [
                expr.format(v=v, gle='<=', c=prop.h[i])
                for i, v in enumerate(self.vs)
                if not isinf(prop.h[i])
             ]
        prop_str = ' AND '.join(ls + hs)

        # Remove AND cell' = CE
        assert(settings.CE)
        s = tw.dedent('''
        MONITOR: MODULE =
        BEGIN
        OUTPUT
                unsafe : BOOLEAN
        INPUT
        {}
        INITIALIZATION
                unsafe = FALSE
        TRANSITION
        [
        TRUE -->
        unsafe' IN {{r : BOOLEAN | r <=> ({} AND cell' = CE)}}
        ]
        END;''').format(self.op_decls, prop_str)
        return s

class Transition(object):

    def __init__(self, name, g, r):
        self.g = g
        self.r = r
        self.name = name

    def __str__(self):
        s = '{}:\n{} -->\n{}\n'.format(self.name, self.g, self.r)
        return s

class Guard(object):
    def __init__(self, C, d):
        '''Cx - d <= 0'''
        self.C = C
        self.d = d

        self.ncons, self.nvars = self.C.shape
        assert(self.ncons == d.size)
        self._vs = None

    @property
    def vs(self):
        if self._vs is None:
            return ['x'+str(j) for j in range(self.nvars)]
        else:
            return self._vs

    @vs.setter
    def vs(self, vs):
        self._vs = vs

    def __str__(self):
        # num_constraints , num_dimensions
        #nc, nd = self.C.shape
        #TODO: Remove this
        # num states
        #num_x = nr
        # num ex. inputs
        #num_w = nc - nr
        cons = (Expr2Str.linexpr2str(self.vs, self.C[i, :], -self.d[i]) + ' <= 0'
                for i in range(self.ncons))
        return ' AND '.join(cons)

class Reset(object):
    def __init__(self, A, b, error):
        self.A = A
        self.b = b
        self.e = error

        self.nlhs, self.nrhs = self.A.shape
        assert(self.nlhs == b.size)
        self._vs = None
        self._vs_ = None

        #TODO: Remove this
        # num op, num ip
        #nr, nc = self.A.shape
        # num states
        #self.num_x = nr
        # num ex. inputs
        #self.num_w = nc - nr

    @property
    def vs(self):
        if self._vs is None:
            return ['x'+str(j) for j in range(self.nrhs)]
        else:
            return self._vs

    @property
    def vs_(self):
        if self._vs_ is None:
            return ["x{}".format(i) for i in range(self.nlhs)]
        else:
            return self._vs_

    @vs.setter
    def vs(self, vs):
        self._vs = vs

    @vs_.setter
    def vs_(self, vs_):
        self._vs_ = vs_

    def __str__(self):
        s = []

        delta_h = self.b + self.e.h
        delta_l = self.b + self.e.l
        set_def = ("{xi_}' IN {{ r : REAL| "
                   "r >= {Axi_plus_delta_li} AND "
                   "r <= {Axi_plus_delta_hi} }}")
        for vsi_, Ai, dli, dhi in zip(self.vs_, self.A, delta_l, delta_h):
            #if error is not significant, i.e., it is less than the
            # precision used while dumping sal file, ignore it. Else,
            # include it and make the rhs of the assignment a set.
            if Expr2Str.float2str(dli) == Expr2Str.float2str(dhi):
                assignment_stmt = vsi_ + "' = " + Expr2Str.linexpr2str(self.vs, Ai, dli)
            else:
                assignment_stmt = set_def.format(
                        xi_=vsi_,
                        Axi_plus_delta_li=Expr2Str.linexpr2str(self.vs, Ai, dli),
                        Axi_plus_delta_hi=Expr2Str.linexpr2str(self.vs, Ai, dhi)
                        )
            s.append(assignment_stmt)
        return ';\n'.join(s)

###############################################
# ############# CEMETEREY
###############################################

#     @property
#     def safety_prop(self):
#         prop = self.prop
#         v = self.v
#         expr = '{v}{i} {gle} {c}'

#         ls = [
#                 expr.format(v=v, i=i, gle='>=', c=prop.l[i]) for i in range(prop.dim)
#                 if not isinf(prop.l[i])
#              ]
#         hs = [
#                 expr.format(v=v, i=i, gle='<=', c=prop.h[i]) for i in range(prop.dim)
#                 if not isinf(prop.h[i])
#              ]
#         prop_str = ' AND '.join(ls + hs)

#         s = tw.dedent('''
#         {prop_name} : THEOREM
#         system |- NOT F({prop});
#         END''').format(prop_name=self.prop_name, prop=prop_str)
#         return s

# Older code, [same as new one but with comments]
# class Guard(object):
#     def __init__(self, C, d):
#         '''Cx <= d'''
#         self.C = C
#         self.d = d

#     def __str__(self):
#         # num_constraints , num_dimensions
#         nc, nd = self.C.shape

#         xs = ['x'+str(j) for j in range(nd)]
#         cons = (linexpr2str(xs, self.C[i, :], -self.d[i]) + '<= 0'
#                 for i in range(nc))
# #         cons = ('{Cxi} <= {di}'.format(
# #             Cxi=linexpr2str(self.C[i, :], ('x'+str(j) for j in range(nd))),
# #             di=float2str(self.d[i])) for i in range(nc))
#         return ' AND '.join(cons)
#         #return ' AND '.join('{}*x{} + {} <= 0'.format(self.C[i, j], j, -self.d[j]) for i in range(nc) for j in range(nd))

# #     def __str__(self):
# #         s = []
# #         # num_constraints , num_dimensions
# #         nc, nd = self.C.shape
# #         x_sym = ['x{}'.format(i) for i in range(nd)]
# #         x = sm.Matrix(sm.symbols(x_sym))
# #         Cx = self.C * x
# #         s = [str(Cx[i] - self.d[i] <= 0) for i in range(nc)]
# #         return ' AND '.join(s)

# class Reset(object):
#     def __init__(self, A, b, error=None):
#         self.A = A
#         self.b = b
#         self.e = error
# #     def __str__(self):
# #         s = []
# #         ndo, ndi = self.A.shape
# #         x_sym = ['x{}'.format(i) for i in range(ndi)]
# #         x__sym = ["x{}'".format(i) for i in range(ndo)]
# #         x = sm.Matrix(sm.symbols(x_sym))
# #         x_ = sm.Matrix(sm.symbols(x__sym))
# #         Ax = self.A * x
# #         s = ['{} = {}'.format(x_[i], Ax[i] + self.b[i]) for i in range(ndo)]
# #         return ';\n'.join(s)

#     def __str__(self):
#         s = []
#         ndo, ndi = self.A.shape

#         #xi_s = "x{}'"
#         xi_s = ["x{}'".format(i) for i in range(ndo)]
#         xs = ['x'+str(j) for j in range(ndi)]

#         if self.e is None:
#             s = (xi_s[i] + '=' + linexpr2str(xs, self.A[i, :], self.b[i])
#                  for i in range(ndo))
#             return ';\n'.join(s)
# #             s = ['{xi_} = {Axi} + {bi}'.format(
# #                 xi_=xi_s.format(i),
# #                 Axi=linexpr2str(
# #                     self.A[i, :], ('x'+str(j) for j in range(ndi))
# #                     ),
# #                 bi=float2str(self.b[i])
# #                 ) for i in range(ndo)
# #                 ]

#         else:
#             delta_h = self.b + self.e.h
#             delta_l = self.b + self.e.l

#             s = ('{xi_} IN {{ r : REAL| '
#                  'r >= {Axi_plus_delta_li} AND '
#                  'r <= {Axi_plus_delta_hi} }}'
#                  .format(
#                      xi_=xi_s[i],
#                      Axi_plus_delta_li=linexpr2str(xs, self.A[i, :], delta_l[i]),
#                      Axi_plus_delta_hi=linexpr2str(xs, self.A[i, :], delta_h[i])
#                      ) for i in range(ndo)
#                  )

# #             s = ['{xi_} IN {{ r : REAL| '
# #                  'r >= {Axi} + {delta_li} AND '
# #                  'r <= {Axi} + {delta_hi} }}'
# #                  .format(
# #                     xi_=xi_s.format(i),
# #                     Axi=linexpr2str(
# #                         self.A[i, :], ('x'+str(j) for j in range(ndi))
# #                         ),
# #                     delta_li=float2str(delta_l[i]),
# #                     delta_hi=float2str(delta_h[i])
# #                     ) for i in range(ndo)
# #                  ]
#             return ';\n'.join(s)

# #         for i in range(ndo):
# #             row = ['{}*x{}'.format(self.A[i, j], j) for j in range(ndi)]
# #             # x_ = Ax + b
# #             x__str = "x{}'".format(i)
# #             Ax_str = ' + '.join(row)
# #             b_str = '{}'.format(self.b[i])
# #             eqn_str = '{} = {} + {}'.format(x__str, Ax_str, b_str)
# #             s.append(eqn_str)
# #         return ';\n'.join(s)
<|endoftext|>


<commit_before>import pytest
from collections import OrderedDict
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import *
from sqlalchemy.dialects.postgresql import *
from sqlalchemy.orm import sessionmaker, load_only
import pandas as pd
import sys, os
if sys.platform == "darwin":
    rootpath = (
        "/Users/bibsian/Desktop/git/database-development/")
    end = "/"

elif sys.platform == "win32":
    rootpath = (
        "C:\\Users\MillerLab\\Desktop\\database-development\\")
    end = "\\"

@pytest.fixture
def engine():
    ''' engine for creating connection and executing statements'''
    engine = create_engine(
        'postgresql+psycopg2:///',
        echo=True)
    return engine

@pytest.fixture
def metadata(engine):
    ''' metadata from database derived with db engine'''
    metadata = MetaData(bind=engine)
    return metadata

@pytest.fixture
def base():
    base = declarative_base()
    return base

@pytest.fixture
def conn(engine):
    '''connection to database'''
    conn = engine.connect()
    return conn

@pytest.fixture
def lter_table(base, metadata):
    class lter_table(base):
        __table__ = Table('lter_table', metadata, autoload=True)
    return lter_table

@pytest.fixture
def study_site_table(base, metadata):

    class study_site_table(base):
        __table__ = Table('study_site_table', metadata, autoload=True)
    return study_site_table

@pytest.fixture
def project_table(base, metadata):
    class project_table(base):
        __table__ = Table('project_table', metadata, autoload=True)
    return project_table

@pytest.fixture
def site_in_project_table(base, metadata):
    class site_in_project_table(base):
        __table__ = Table('site_in_project_table', metadata, autoload=True)
    return site_in_project_table

@pytest.fixture
def taxa_table(base, metadata):
    class taxa_table(base):
        __table__ = Table('taxa_table', metadata, autoload=True)

    return taxa_table

@pytest.fixture
def biomass_table(base, metadata):
    class biomass_table(base):
        __table__ = Table('biomass_table', metadata, autoload=True)
    return biomass_table

@pytest.fixture
def count_table(base, metadata):
    class count_table(base):
        __table__ = Table('count_table', metadata, autoload=True)
    return count_table

@pytest.fixture
def density_table(base, metadata):
    class density_table(base):
        __table__ = Table('density_table', metadata, autoload=True)
    return density_table

@pytest.fixture
def percent_cover_table(base, metadata):
    class percent_cover_table(base):
        __table__ = Table('percent_cover_table', metadata, autoload=True)
    return percent_cover_table

@pytest.fixture
def replace_numeric_null_with_string():
    def replace_numeric_null_with_string(dataframe):
        ''' Function to take values such as -99999 and convert them
        to NA's '''
        for i in dataframe.columns:
            try:
                dataframe[i].replace(
                    {
                        '-999999': 'NA',
                        '-99999': 'NA',
                        '-9999': 'NA',
                        '-999': 'NA',
                        '-888': 'NA',
                        '-8888': 'NA',
                        '-88888': 'NA',
                        '-888888': 'NA'
                    }, inplace=True)
            except:
                print(i + ' did not convert')
    return replace_numeric_null_with_string

@pytest.fixture
def individual_table(base, metadata):
    class individual_table(base):
        __table__ = Table('individual_table', metadata, autoload=True)
    return individual_table

@pytest.fixture
def count(replace_numeric_null_with_string):
    ''' Raw count data '''
    count = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_1.csv')
    replace_numeric_null_with_string(count)
    return count

@pytest.fixture
def density(replace_numeric_null_with_string):
    ''' Raw density data '''
    density = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_2.csv')
    replace_numeric_null_with_string(density)
    return density

@pytest.fixture
def biomass(replace_numeric_null_with_string):
    biomass = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_3.csv')
    replace_numeric_null_with_string(biomass)
    return biomass

@pytest.fixture
def percent(replace_numeric_null_with_string):
    percent = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_4.csv')
    replace_numeric_null_with_string(percent)
    return percent

@pytest.fixture
def individual(replace_numeric_null_with_string):
    individual = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_5.csv')
    replace_numeric_null_with_string(individual)
    return individual

@pytest.fixture
def site_in_proj_subq_stmt(
        engine, site_in_project_table, study_site_table,
        project_table, lter_table):
    '''
    Subquery 1 (used to perform union on all tables)
    This subquery perform the necessary joins to bring together
    four tables:
    1) site_in_project_table
    2) study_site_table
    3) lter_table
    4) project_table
    '''
    stmt = (
        select(
            [
                site_in_project_table,
                study_site_table,
                lter_table.lat_lter,
                lter_table.lng_lter,
                lter_table.lterid,
                project_table
            ]).
        select_from(
            site_in_project_table.__table__.
            join(study_site_table.__table__).
            join(lter_table.__table__).join(project_table)).alias()
    )
    Session = sessionmaker(bind=engine)
    session = Session()
    stmt_result = session.execute(stmt)
    return stmt_result

@pytest.fixture
def taxa_tbl_subq_stmt(
        site_in_proj_subq_stmt, engine,
        taxa_table):
    '''
    Subquery 2 (used to perform union on all tables)
    This subquery performs the join to bring together
    the four tables from subquery 1 and the taxa table.

    '''
    stmt = (
        select([site_in_proj_subq_stmt, taxa_table]).
        select_from(
            site_in_proj_subq_stmt.
            join(taxa_table)).alias()
    )
    Session = sessionmaker(bind=engine)
    session = Session()
    stmt_result = session.execute(stmt)
    return stmt_result

def test_recover_count_data(
        taxa_tbl_subq_stmt, engine, count_table, count):
    print('what')
    count_tbl_subq_stmt = (
        select([
            taxa_tbl_subq_stmt,
            count_table]).
        select_from(
            taxa_tbl_subq_stmt.
            join(
                count_table,
                onclause=and_(
                    taxa_tbl_subq_stmt.c.taxa_table_key ==
                    count_table.taxa_count_fkey,
                    taxa_tbl_subq_stmt.c.site_in_project_key ==
                    count_table.site_in_project_count_fkey
                )
            )
        ).alias('count join')
    )

    # pretty.pprint(count_tbl_subq_stmt.compile().string)
    Session = sessionmaker(bind=engine)
    session = Session()
    count_tbl_subq_result = session.execute(count_tbl_subq_stmt)
    count_tbl_subq_df = pd.DataFrame(count_tbl_subq_result.fetchall())
    count_tbl_subq_df.columns = count_tbl_subq_result.keys()
    session.close()
    print(count['site'].values.tolist())
    print(count_tbl_subq_df['spatial_replication_level_1'].values.tolist())
    assert (
        count['site'].values.tolist() ==
        count_tbl_subq_df['spatial_replication_level_1'].values.tolist()
    ) == True
<commit_msg>Added more test to the count data recovery block<commit_after>import pytest
from collections import OrderedDict
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import *
from sqlalchemy.dialects.postgresql import *
from sqlalchemy.orm import sessionmaker, load_only
import pandas as pd
import sys, os
if sys.platform == "darwin":
    rootpath = (
        "/Users/bibsian/Desktop/git/database-development/")
    end = "/"

elif sys.platform == "win32":
    rootpath = (
        "C:\\Users\MillerLab\\Desktop\\database-development\\")
    end = "\\"

@pytest.fixture
def engine():
    ''' engine for creating connection and executing statements'''
    engine = create_engine(
        'postgresql+psycopg2:///',
        echo=True)
    return engine

@pytest.fixture
def metadata(engine):
    ''' metadata from database derived with db engine'''
    metadata = MetaData(bind=engine)
    return metadata

@pytest.fixture
def base():
    base = declarative_base()
    return base

@pytest.fixture
def conn(engine):
    '''connection to database'''
    conn = engine.connect()
    return conn

@pytest.fixture
def lter_table(base, metadata):
    class lter_table(base):
        __table__ = Table('lter_table', metadata, autoload=True)
    return lter_table

@pytest.fixture
def study_site_table(base, metadata):

    class study_site_table(base):
        __table__ = Table('study_site_table', metadata, autoload=True)
    return study_site_table

@pytest.fixture
def project_table(base, metadata):
    class project_table(base):
        __table__ = Table('project_table', metadata, autoload=True)
    return project_table

@pytest.fixture
def site_in_project_table(base, metadata):
    class site_in_project_table(base):
        __table__ = Table('site_in_project_table', metadata, autoload=True)
    return site_in_project_table

@pytest.fixture
def taxa_table(base, metadata):
    class taxa_table(base):
        __table__ = Table('taxa_table', metadata, autoload=True)

    return taxa_table

@pytest.fixture
def biomass_table(base, metadata):
    class biomass_table(base):
        __table__ = Table('biomass_table', metadata, autoload=True)
    return biomass_table

@pytest.fixture
def count_table(base, metadata):
    class count_table(base):
        __table__ = Table('count_table', metadata, autoload=True)
    return count_table

@pytest.fixture
def density_table(base, metadata):
    class density_table(base):
        __table__ = Table('density_table', metadata, autoload=True)
    return density_table

@pytest.fixture
def percent_cover_table(base, metadata):
    class percent_cover_table(base):
        __table__ = Table('percent_cover_table', metadata, autoload=True)
    return percent_cover_table

@pytest.fixture
def replace_numeric_null_with_string():
    def replace_numeric_null_with_string(dataframe):
        ''' Function to take values such as -99999 and convert them
        to NA's '''
        for i in dataframe.columns:
            try:
                dataframe[i].replace(
                    {
                        '-999999': 'NA',
                        '-99999': 'NA',
                        '-9999': 'NA',
                        '-999': 'NA',
                        '-888': 'NA',
                        '-8888': 'NA',
                        '-88888': 'NA',
                        '-888888': 'NA'
                    }, inplace=True)
            except:
                print(i + ' did not convert')
    return replace_numeric_null_with_string

@pytest.fixture
def individual_table(base, metadata):
    class individual_table(base):
        __table__ = Table('individual_table', metadata, autoload=True)
    return individual_table

@pytest.fixture
def count(replace_numeric_null_with_string):
    ''' Raw count data '''
    count = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_1.csv')
    replace_numeric_null_with_string(count)
    return count

@pytest.fixture
def density(replace_numeric_null_with_string):
    ''' Raw density data '''
    density = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_2.csv')
    replace_numeric_null_with_string(density)
    return density

@pytest.fixture
def biomass(replace_numeric_null_with_string):
    biomass = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_3.csv')
    replace_numeric_null_with_string(biomass)
    return biomass

@pytest.fixture
def percent(replace_numeric_null_with_string):
    percent = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_4.csv')
    replace_numeric_null_with_string(percent)
    return percent

@pytest.fixture
def individual(replace_numeric_null_with_string):
    individual = pd.read_csv(
        rootpath + 'test' + end + 'Datasets_manual_test' +
        end + 'raw_data_test_5.csv')
    replace_numeric_null_with_string(individual)
    return individual

@pytest.fixture
def site_in_proj_subq_stmt(
        engine, site_in_project_table, study_site_table,
        project_table, lter_table):
    '''
    Subquery 1 (used to perform union on all tables)
    This subquery perform the necessary joins to bring together
    four tables:
    1) site_in_project_table
    2) study_site_table
    3) lter_table
    4) project_table
    '''
    stmt = (
        select(
            [
                site_in_project_table,
                study_site_table,
                lter_table.lat_lter,
                lter_table.lng_lter,
                lter_table.lterid,
                project_table
            ]).
        select_from(
            site_in_project_table.__table__.
            join(study_site_table.__table__).
            join(lter_table.__table__).join(project_table)).alias()
    )
    return stmt

@pytest.fixture
def taxa_tbl_subq_stmt(
        site_in_proj_subq_stmt, engine,
        taxa_table):
    '''
    Subquery 2 (used to perform union on all tables)
    This subquery performs the join to bring together
    the four tables from subquery 1 and the taxa table.

    '''
    stmt = (
        select([site_in_proj_subq_stmt, taxa_table]).
        select_from(
            site_in_proj_subq_stmt.
            join(taxa_table)).alias()
    )
    return stmt

def test_recover_count_data(
        taxa_tbl_subq_stmt, engine, count_table, count):
    print('what')
    count_tbl_subq_stmt = (
        select([
            taxa_tbl_subq_stmt,
            count_table]).
        select_from(
            taxa_tbl_subq_stmt.
            join(
                count_table,
                onclause=and_(
                    taxa_tbl_subq_stmt.c.taxa_table_key ==
                    count_table.taxa_count_fkey,
                    taxa_tbl_subq_stmt.c.site_in_project_key ==
                    count_table.site_in_project_count_fkey
                )
            )
        ).alias('count join')
    )

    # pretty.pprint(count_tbl_subq_stmt.compile().string)
    Session = sessionmaker(bind=engine)
    session = Session()
    count_tbl_subq_result = session.execute(count_tbl_subq_stmt)
    count_tbl_subq_df = pd.DataFrame(count_tbl_subq_result.fetchall())
    count_tbl_subq_df.columns = count_tbl_subq_result.keys()
    session.close()
    print(count['site'].values.tolist())
    print(count_tbl_subq_df['spatial_replication_level_1'].values.tolist())

    assert (
        count['site'].values.tolist() ==
        count_tbl_subq_df['spatial_replication_level_1'].values.tolist()
    ) == True

    assert (
        count['count'].values.tolist() ==
        count_tbl_subq_df['count_observation'].values.tolist()
    ) == True

    assert (
        count['genus'].values.tolist() ==
        count_tbl_subq_df['genus'].values.tolist()
    ) == True

    assert (
        count['species'].values.tolist() ==
        count_tbl_subq_df['species'].values.tolist()
    ) == True

    count_tbl_subq_df['spatial_replication_level_2'] = pd.to_numeric(
        count_tbl_subq_df['spatial_replication_level_2'])

    assert (
        count['transect'].values.tolist() ==
        count_tbl_subq_df['spatial_replication_level_2'].values.tolist()
    ) == True

    count_tbl_subq_df['month'] = count_tbl_subq_df['month'].astype(int)

    assert (
        count['month'].values.tolist() ==
        count_tbl_subq_df['month'].values.tolist()
    ) == True
<|endoftext|>