<commit_before>#!/usr/bin/env python
# -*- coding: utf-8; py-indent-offset:4 -*-
###############################################################################
#
# Copyright (C) 2015, 2016 Daniel Rodriguez
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import collections

import backtrader as bt
import io
import spstore
from backtrader.comminfo import CommInfoBase
from backtrader.order import Order, BuyOrder, SellOrder
from backtrader.position import Position
from backtrader.metabase import MetaParams
from backtrader.utils.py3 import with_metaclass

__all__ = ['BackBroker', 'BrokerBack']

class MetaSharpPointBroker(MetaParams):
    def __init__(cls, name, bases, dct):
        '''Class has already been created ... register'''
        # Initialize the class
        super(MetaSharpPointBroker, cls).__init__(name, bases, dct)
        spstore.SharpPointStore.BrokerCls = cls

class SharpPointBroker(with_metaclass(MetaSharpPointBroker, bt.BrokerBase)):
    '''Broker Simulator

      The simulation supports different order types, checking a submitted order
      cash requirements against current cash, keeping track of cash and value
      for each iteration of ``cerebro`` and keeping the current position on
      different datas.

      *cash* is adjusted on each iteration for instruments like ``futures`` for
       which a price change implies in real brokers the addition/substracion of
       cash.

      Supported order types:

        - ``Market``: to be executed with the 1st tick of the next bar (namely
          the ``open`` price)

        - ``Close``: meant for intraday in which the order is executed with the
          closing price of the last bar of the session

        - ``Limit``: executes if the given limit price is seen during the
          session

        - ``Stop``: executes a ``Market`` order if the given stop price is seen

        - ``StopLimit``: sets a ``Limit`` order in motion if the given stop
          price is seen

      Because the broker is instantiated by ``Cerebro`` and there should be
      (mostly) no reason to replace the broker, the params are not controlled
      by the user for the instance.  To change this there are two options:

        1. Manually create an instance of this class with the desired params
           and use ``cerebro.broker = instance`` to set the instance as the
           broker for the ``run`` execution

        2. Use the ``set_xxx`` to set the value using
           ``cerebro.broker.set_xxx`` where ```xxx`` stands for the name of the
           parameter to set

        .. note::

           ``cerebro.broker`` is a *property* supported by the ``getbroker``
           and ``setbroker`` methods of ``Cerebro``

      Params:

        - ``cash`` (default: ``10000``): starting cash

        - ``commission`` (default: ``CommInfoBase(percabs=True)``)
          base commission scheme which applies to all assets

        - ``checksubmit`` (default: ``True``)
          check margin/cash before accepting an order into the system

        - ``eosbar`` (default: ``False``):
          With intraday bars consider a bar with the same ``time`` as the end
          of session to be the end of the session. This is not usually the
          case, because some bars (final auction) are produced by many
          exchanges for many products for a couple of minutes after the end of
          the session

        - ``eosbar`` (default: ``False``):
          With intraday bars consider a bar with the same ``time`` as the end
          of session to be the end of the session. This is not usually the
          case, because some bars (final auction) are produced by many
          exchanges for many products for a couple of minutes after the end of
          the session

        - ``filler`` (default: ``None``)

          A callable with signature: ``callable(order, price, ago)``

            - ``order``: obviously the order in execution. This provides access
              to the *data* (and with it the *ohlc* and *volume* values), the
              *execution type*, remaining size (``order.executed.remsize``) and
              others.

              Please check the ``Order`` documentation and reference for things
              available inside an ``Order`` instance

            - ``price`` the price at which the order is going to be executed in
              the ``ago`` bar

            - ``ago``: index meant to be used with ``order.data`` for the
              extraction of the *ohlc* and *volume* prices. In most cases this
              will be ``0`` but on a corner case for ``Close`` orders, this
              will be ``-1``.

              In order to get the bar volume (for example) do: ``volume =
              order.data.voluume[ago]``

          The callable must return the *executed size* (a value >= 0)

          The callable may of course be an object with ``__call__`` matching
          the aforementioned signature

          With the default ``None`` orders will be completely executed in a
          single shot

    '''
    params = (
        ('cash', 10000.0),
        ('checksubmit', True),
        ('dataname', None),
        ('eosbar', False),
        ('filler', None),
    )
    f = None

    def __init__(self, **kwargs):
        super(SharpPointBroker, self).__init__()
        self.o = spstore.SharpPointStore(**kwargs)
        self.startingcash = self.cash = self.p.cash

        self.orders = collections.OrderedDict()  # orders by order id
        self.pending = collections.deque()  # popleft and append(right)

        self.positions = collections.defaultdict(Position)
        self.notifs = collections.deque()

        self.submitted = collections.deque()
        if self.f is None:
            if hasattr(self.p.dataname, 'readline'):
                self.f = self.p.dataname
            elif self.p.dataname is not None:
                # Let an exception propagate to let the caller know
                self.f = io.open(self.p.dataname, 'a')

    def start(self):
        super(SharpPointBroker, self).start()
        self.o.start(broker=self)

    def stop(self):
        super(SharpPointBroker, self).stop()
        self.o.stop()
        if self.f is not None:
            self.f.close()
            self.f = None

    def get_notification(self):
        try:
            return self.notifs.popleft()
        except IndexError:
            pass

        return None

    def set_filler(self, filler):
        '''Sets a volume filler for volume filling execution'''
        self.p.filler = filler

    def set_checksubmit(self, checksubmit):
        '''Sets the checksubmit parameter'''
        self.p.checksubmit = checksubmit

    def set_eosbar(self, eosbar):
        '''Sets the eosbar parameter (alias: ``seteosbar``'''
        self.p.eosbar = eosbar

    seteosbar = set_eosbar

    def get_cash(self):
        '''Returns the current cash (alias: ``getcash``)'''
        return self.cash

    getcash = get_cash

    def set_cash(self, cash):
        '''Sets the cash parameter (alias: ``setcash``)'''
        self.startingcash = self.cash = self.p.cash = cash

    setcash = set_cash

    def get_value(self, datas=None):
        '''Returns the portfolio value of the given datas (if datas is ``None``, then
        the total portfolio value will be returned (alias: ``getvalue``)
        '''
        return 0.0

    getvalue = get_value

    def getposition(self, data):
        return self.positions[data]

    def orderstatus(self, order):
        o = self.orders[order.ref]
        return o.status

    def submit(self, order):
        if self.p.checksubmit:
            order.submit()
            self.submitted.append(order)
            self.orders.append(order)
            self.notify(order)
        else:
            self.submit_accept(order)
        return order

    def _submit(self, oref):
        order = self.orders[oref]
        order.submit(self)
        self.notify(order)

    def _reject(self, oref):
        order = self.orders[oref]
        order.reject(self)
        self.notify(order)

    def _accept(self, oref):
        order = self.orders[oref]
        order.accept()
        self.notify(order)

    def _cancel(self, oref):
        order = self.orders[oref]
        order.cancel(self)
        self.notify(order)

    def _expire(self, oref):
        order = self.orders[oref]
        order.expire(self)
        self.notify(order)

    def _fill(self, oref, size, price, **kwargs):
        order = self.orders[oref]

        data = order.data
        pos = self.getposition(data, clone=False)
        psize, pprice, opened, closed = pos.update(size, price)

        comminfo = self.getcommissioninfo(data)

        closedvalue = closedcomm = 0.0
        openedvalue = openedcomm = 0.0
        margin = pnl = 0.0

        order.execute(data.datetime[0], size, price,
                      closed, closedvalue, closedcomm,
                      opened, openedvalue, openedcomm,
                      margin, pnl,
                      psize, pprice)

        if order.executed.remsize:
            order.partial()
        else:
            order.completed()

        self.notify(order)

    def buy(self, owner, data,
            size, price=None, plimit=None,
            exectype=None, valid=None, tradeid=0,
            **kwargs):

        order = BuyOrder(owner=owner, data=data,
                         size=size, price=price, pricelimit=plimit,
                         exectype=exectype, valid=valid, tradeid=tradeid)

        order.addinfo(**kwargs)
        self.orders[order.ref] = order
        return self.o.order_create(order)

    def sell(self, owner, data,
             size, price=None, plimit=None,
             exectype=None, valid=None, tradeid=0,
             **kwargs):

        order = SellOrder(owner=owner, data=data,
                          size=size, price=price, pricelimit=plimit,
                          exectype=exectype, valid=valid, tradeid=tradeid)

        order.addinfo(**kwargs)
        self.orders[order.ref] = order
        return self.o.order_create(order)

    def cancel(self, order):
        o = self.orders[order.ref]
        if order.status == Order.Cancelled:  # already cancelled
            return

        return self.o.order_cancel(order)

    def notify(self, order):
        self.notifs.append(order.clone())

    def get_notification(self):
        if not self.notifs:
            return None
        return self.notifs.popleft()

    def next(self):
        self.notifs.append(None)  # mark notification boundary

# Alias
BrokerSharpPoint = SharpPointBroker

if __name__ == '__main__':
    s = SharpPointBroker()
    s.start()

<commit_msg>remove _fill<commit_after>#!/usr/bin/env python
# -*- coding: utf-8; py-indent-offset:4 -*-
###############################################################################
#
# Copyright (C) 2015, 2016 Daniel Rodriguez
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import collections

import backtrader as bt
import io
import spstore
from backtrader.comminfo import CommInfoBase
from backtrader.order import Order, BuyOrder, SellOrder
from backtrader.position import Position
from backtrader.metabase import MetaParams
from backtrader.utils.py3 import with_metaclass

__all__ = ['BackBroker', 'BrokerBack']

class MetaSharpPointBroker(MetaParams):
    def __init__(cls, name, bases, dct):
        '''Class has already been created ... register'''
        # Initialize the class
        super(MetaSharpPointBroker, cls).__init__(name, bases, dct)
        spstore.SharpPointStore.BrokerCls = cls

class SharpPointBroker(with_metaclass(MetaSharpPointBroker, bt.BrokerBase)):
    '''Broker Simulator

      The simulation supports different order types, checking a submitted order
      cash requirements against current cash, keeping track of cash and value
      for each iteration of ``cerebro`` and keeping the current position on
      different datas.

      *cash* is adjusted on each iteration for instruments like ``futures`` for
       which a price change implies in real brokers the addition/substracion of
       cash.

      Supported order types:

        - ``Market``: to be executed with the 1st tick of the next bar (namely
          the ``open`` price)

        - ``Close``: meant for intraday in which the order is executed with the
          closing price of the last bar of the session

        - ``Limit``: executes if the given limit price is seen during the
          session

        - ``Stop``: executes a ``Market`` order if the given stop price is seen

        - ``StopLimit``: sets a ``Limit`` order in motion if the given stop
          price is seen

      Because the broker is instantiated by ``Cerebro`` and there should be
      (mostly) no reason to replace the broker, the params are not controlled
      by the user for the instance.  To change this there are two options:

        1. Manually create an instance of this class with the desired params
           and use ``cerebro.broker = instance`` to set the instance as the
           broker for the ``run`` execution

        2. Use the ``set_xxx`` to set the value using
           ``cerebro.broker.set_xxx`` where ```xxx`` stands for the name of the
           parameter to set

        .. note::

           ``cerebro.broker`` is a *property* supported by the ``getbroker``
           and ``setbroker`` methods of ``Cerebro``

      Params:

        - ``cash`` (default: ``10000``): starting cash

        - ``commission`` (default: ``CommInfoBase(percabs=True)``)
          base commission scheme which applies to all assets

        - ``checksubmit`` (default: ``True``)
          check margin/cash before accepting an order into the system

        - ``eosbar`` (default: ``False``):
          With intraday bars consider a bar with the same ``time`` as the end
          of session to be the end of the session. This is not usually the
          case, because some bars (final auction) are produced by many
          exchanges for many products for a couple of minutes after the end of
          the session

        - ``eosbar`` (default: ``False``):
          With intraday bars consider a bar with the same ``time`` as the end
          of session to be the end of the session. This is not usually the
          case, because some bars (final auction) are produced by many
          exchanges for many products for a couple of minutes after the end of
          the session

        - ``filler`` (default: ``None``)

          A callable with signature: ``callable(order, price, ago)``

            - ``order``: obviously the order in execution. This provides access
              to the *data* (and with it the *ohlc* and *volume* values), the
              *execution type*, remaining size (``order.executed.remsize``) and
              others.

              Please check the ``Order`` documentation and reference for things
              available inside an ``Order`` instance

            - ``price`` the price at which the order is going to be executed in
              the ``ago`` bar

            - ``ago``: index meant to be used with ``order.data`` for the
              extraction of the *ohlc* and *volume* prices. In most cases this
              will be ``0`` but on a corner case for ``Close`` orders, this
              will be ``-1``.

              In order to get the bar volume (for example) do: ``volume =
              order.data.voluume[ago]``

          The callable must return the *executed size* (a value >= 0)

          The callable may of course be an object with ``__call__`` matching
          the aforementioned signature

          With the default ``None`` orders will be completely executed in a
          single shot

    '''
    params = (
        ('cash', 10000.0),
        ('checksubmit', True),
        ('dataname', None),
        ('eosbar', False),
        ('filler', None),
    )
    f = None

    def __init__(self, **kwargs):
        super(SharpPointBroker, self).__init__()
        self.o = spstore.SharpPointStore(**kwargs)
        self.startingcash = self.cash = self.p.cash

        self.orders = collections.OrderedDict()  # orders by order id
        self.pending = collections.deque()  # popleft and append(right)

        self.positions = collections.defaultdict(Position)
        self.notifs = collections.deque()

        self.submitted = collections.deque()
        if self.f is None:
            if hasattr(self.p.dataname, 'readline'):
                self.f = self.p.dataname
            elif self.p.dataname is not None:
                # Let an exception propagate to let the caller know
                self.f = io.open(self.p.dataname, 'a')

    def start(self):
        super(SharpPointBroker, self).start()
        self.o.start(broker=self)

    def stop(self):
        super(SharpPointBroker, self).stop()
        self.o.stop()
        if self.f is not None:
            self.f.close()
            self.f = None

    def get_notification(self):
        try:
            return self.notifs.popleft()
        except IndexError:
            pass

        return None

    def set_filler(self, filler):
        '''Sets a volume filler for volume filling execution'''
        self.p.filler = filler

    def set_checksubmit(self, checksubmit):
        '''Sets the checksubmit parameter'''
        self.p.checksubmit = checksubmit

    def set_eosbar(self, eosbar):
        '''Sets the eosbar parameter (alias: ``seteosbar``'''
        self.p.eosbar = eosbar

    seteosbar = set_eosbar

    def get_cash(self):
        '''Returns the current cash (alias: ``getcash``)'''
        return self.cash

    getcash = get_cash

    def set_cash(self, cash):
        '''Sets the cash parameter (alias: ``setcash``)'''
        self.startingcash = self.cash = self.p.cash = cash

    setcash = set_cash

    def get_value(self, datas=None):
        '''Returns the portfolio value of the given datas (if datas is ``None``, then
        the total portfolio value will be returned (alias: ``getvalue``)
        '''
        return 0.0

    getvalue = get_value

    def getposition(self, data):
        return self.positions[data]

    def orderstatus(self, order):
        o = self.orders[order.ref]
        return o.status

    def submit(self, order):
        if self.p.checksubmit:
            order.submit()
            self.submitted.append(order)
            self.orders.append(order)
            self.notify(order)
        else:
            self.submit_accept(order)
        return order

    def _submit(self, oref):
        order = self.orders[oref]
        order.submit(self)
        self.notify(order)

    def _reject(self, oref):
        order = self.orders[oref]
        order.reject(self)
        self.notify(order)

    def _accept(self, oref):
        order = self.orders[oref]
        order.accept()
        self.notify(order)

    def _cancel(self, oref):
        order = self.orders[oref]
        order.cancel(self)
        self.notify(order)

    def _expire(self, oref):
        order = self.orders[oref]
        order.expire(self)
        self.notify(order)

    def buy(self, owner, data,
            size, price=None, plimit=None,
            exectype=None, valid=None, tradeid=0,
            **kwargs):

        order = BuyOrder(owner=owner, data=data,
                         size=size, price=price, pricelimit=plimit,
                         exectype=exectype, valid=valid, tradeid=tradeid)

        order.addinfo(**kwargs)
        self.orders[order.ref] = order
        return self.o.order_create(order)

    def sell(self, owner, data,
             size, price=None, plimit=None,
             exectype=None, valid=None, tradeid=0,
             **kwargs):

        order = SellOrder(owner=owner, data=data,
                          size=size, price=price, pricelimit=plimit,
                          exectype=exectype, valid=valid, tradeid=tradeid)

        order.addinfo(**kwargs)
        self.orders[order.ref] = order
        return self.o.order_create(order)

    def cancel(self, order):
        o = self.orders[order.ref]
        if order.status == Order.Cancelled:  # already cancelled
            return

        return self.o.order_cancel(order)

    def notify(self, order):
        self.notifs.append(order.clone())

    def get_notification(self):
        if not self.notifs:
            return None
        return self.notifs.popleft()

    def next(self):
        self.notifs.append(None)  # mark notification boundary

# Alias
BrokerSharpPoint = SharpPointBroker

if __name__ == '__main__':
    s = SharpPointBroker()
    s.start()

<|endoftext|>


<commit_before>import oauth2 as oauth
import time
import json
from splitwise.user import Friend
from splitwise.currency import Currency
from splitwise.group import Group
from splitwise.category import Category
from splitwise.expense import Expense

try:
    from urlparse import parse_qsl #Python 2.x
except ImportError: #Python 3
    from urllib.parse import parse_qsl

class Splitwise(object):
    """ The Splitwise class to make the requests to splitwise server.
    """

    #URLs to make the request
    REQUEST_TOKEN_URL   = "https://secure.splitwise.com/api/v3.0/get_request_token"
    ACCESS_TOKEN_URL    = "https://secure.splitwise.com/api/v3.0/get_access_token"
    AUTHORIZE_URL       = "https://secure.splitwise.com/authorize"
    GET_FRIENDS_URL     = "https://secure.splitwise.com/api/v3.0/get_friends"
    GET_GROUPS_URL      = "https://secure.splitwise.com/api/v3.0/get_groups"
    GET_GROUP_URL       = "https://secure.splitwise.com/api/v3.0/get_group"
    GET_CURRENCY_URL    = "https://secure.splitwise.com/api/v3.0/get_currencies"
    GET_CATEGORY_URL    = "https://secure.splitwise.com/api/v3.0/get_categories"
    GET_EXPENSES_URL    = "https://secure.splitwise.com/api/v3.0/get_expenses"
    GET_EXPENSE_URL     = "https://secure.splitwise.com/api/v3.0/get_expense"

    def __init__(self,consumer_key,consumer_secret,access_token=None,debug=False):
        """ Initializes the splitwise class. Sets consumer and access token

        Args:
            consumer_key (str) : Consumer Key provided by Spliwise
            consumer_secret (str): Consumer Secret provided by Splitwise
            access_token (:obj: `dict`) Access Token is a combination of oauth_token and oauth_token_secret

        Returns:
            A Splitwise Object
        """
        self.consumer = oauth.Consumer(consumer_key, consumer_secret)

        #If access token is present then set the Access token
        if access_token:
            self.setAccessToken(access_token)

    def getAuthorizeURL(self):

        client = oauth.Client(self.consumer)

        #Get the request token
        resp, content = client.request(Splitwise.REQUEST_TOKEN_URL, "POST")

        #Check if the response is correct
        if resp['status'] != '200':
            raise Exception("Invalid response %s. Please check your consumer key and secret." % resp['status'])

        request_token = dict(parse_qsl(content.decode("utf-8")))

        return "%s?oauth_token=%s" % (Splitwise.AUTHORIZE_URL, request_token['oauth_token']), request_token['oauth_token_secret']

    def getAccessToken(self,oauth_token,oauth_token_secret,oauth_verifier):

        token = oauth.Token(oauth_token,oauth_token_secret)
        token.set_verifier(oauth_verifier)
        client = oauth.Client(self.consumer, token)

        resp, content = client.request(Splitwise.ACCESS_TOKEN_URL, "POST")

        #Check if the response is correct
        if resp['status'] != '200':
            raise Exception("Invalid response %s. Please check your consumer key and secret." % resp['status'])

        access_token = dict(parse_qsl(content.decode("utf-8")))

        return access_token

    def setAccessToken(self,access_token):
        self.token = oauth.Token(key=access_token["oauth_token"], secret=access_token["oauth_token_secret"])
        self.client = oauth.Client(self.consumer, self.token)

    def __makeRequest(self,url):

        resp, content = self.client.request(url, "GET")
        #print content
        #Check if the response is correct
        if resp['status'] != '200':
            raise Exception("Invalid response %s. Please check your consumer key and secret." % resp['status'])

        return content

    def __prepareOptionsUrl(self,options={}):
        if not isinstance(options,dict) or len(options) == 0:
            return ""

        optstr = "?"

        optstr += "&".join([k+"="+str(v) for k,v in options.iteritems()])

        return optstr

    def getFriends(self):

        content = self.__makeRequest(Splitwise.GET_FRIENDS_URL)
        content = json.loads(content.decode("utf-8"))

        friends = []
        if "friends" in content:
            for f in content["friends"]:
                friends.append(Friend(f))

        return friends

    def getGroups(self):

        content = self.__makeRequest(Splitwise.GET_GROUPS_URL)
        content = json.loads(content.decode("utf-8"))

        groups = []
        if "groups" in content:
            for g in content["groups"]:
                groups.append(Group(g))

        return groups

    def getCurrencies(self):

        content = self.__makeRequest(Splitwise.GET_CURRENCY_URL)
        content = json.loads(content.decode("utf-8"))

        currencies = []
        if "currencies" in content:
            for c in content["currencies"]:
                currencies.append(Currency(c))

        return currencies

    def getCategories(self):

        content = self.__makeRequest(Splitwise.GET_CATEGORY_URL)
        content = json.loads(content.decode("utf-8"))
        categories = []

        if "categories" in content:
            for c in content["categories"]:
                categories.append(Category(c))

        return categories

    def getGroup(self,id=0):

        content = self.__makeRequest(Splitwise.GET_GROUP_URL+"/"+str(id))
        content = json.loads(content.decode("utf-8"))
        group = None
        if "group" in content:
            group = Group(content["group"])

        return group

    def getExpenses(self,offset=None,limit=None,group_id=None,friendship_id=None,dated_after=None,dated_before=None,updated_after=None,updated_before=None):

        options = {}

        params = locals().keys()
        for param in params:
            if param != 'self' and param != 'options' and locals().get(param) is not None:
                options[param] = locals().get(param)

        url = Splitwise.GET_EXPENSES_URL

        url += self.__prepareOptionsUrl(options)
        content = self.__makeRequest(url)
        content = json.loads(content.decode("utf-8"))
        expenses = []
        if "expenses" in content:
            for e in content["expenses"]:
                expenses.append(Expense(e))

        return expenses

    def getExpense(self,id):
        content = self.__makeRequest(Splitwise.GET_EXPENSE_URL+"/"+str(id))
        content = json.loads(content.decode("utf-8"))
        expense = None
        if "expense" in content:
            expense = Expense(content["expense"])

        return expense
<commit_msg>Added Debug<commit_after>import oauth2 as oauth
import time
import json
from splitwise.user import Friend
from splitwise.currency import Currency
from splitwise.group import Group
from splitwise.category import Category
from splitwise.expense import Expense

try:
    from urlparse import parse_qsl #Python 2.x
except ImportError: #Python 3
    from urllib.parse import parse_qsl

class Splitwise(object):
    """ The Splitwise class to make the requests to splitwise server.
    """

    #URLs to make the request
    REQUEST_TOKEN_URL   = "https://secure.splitwise.com/api/v3.0/get_request_token"
    ACCESS_TOKEN_URL    = "https://secure.splitwise.com/api/v3.0/get_access_token"
    AUTHORIZE_URL       = "https://secure.splitwise.com/authorize"
    GET_FRIENDS_URL     = "https://secure.splitwise.com/api/v3.0/get_friends"
    GET_GROUPS_URL      = "https://secure.splitwise.com/api/v3.0/get_groups"
    GET_GROUP_URL       = "https://secure.splitwise.com/api/v3.0/get_group"
    GET_CURRENCY_URL    = "https://secure.splitwise.com/api/v3.0/get_currencies"
    GET_CATEGORY_URL    = "https://secure.splitwise.com/api/v3.0/get_categories"
    GET_EXPENSES_URL    = "https://secure.splitwise.com/api/v3.0/get_expenses"
    GET_EXPENSE_URL     = "https://secure.splitwise.com/api/v3.0/get_expense"

    debug = False

    def __init__(self,consumer_key,consumer_secret,access_token=None,debug=False):
        """ Initializes the splitwise class. Sets consumer and access token

        Args:
            consumer_key (str) : Consumer Key provided by Spliwise
            consumer_secret (str): Consumer Secret provided by Splitwise
            access_token (:obj: `dict`) Access Token is a combination of oauth_token and oauth_token_secret

        Returns:
            A Splitwise Object
        """
        self.consumer = oauth.Consumer(consumer_key, consumer_secret)

        #If access token is present then set the Access token
        if access_token:
            self.setAccessToken(access_token)

    @classmethod
    def setDebug(cls,debug):
        cls.debug = debug

    @classmethod
    def isDebug(cls):
        return cls.debug

    def getAuthorizeURL(self):

        client = oauth.Client(self.consumer)

        #Get the request token
        resp, content = client.request(Splitwise.REQUEST_TOKEN_URL, "POST")

        if Splitwise.isDebug():
            print(resp, content)

        #Check if the response is correct
        if resp['status'] != '200':
            raise Exception("Invalid response %s. Please check your consumer key and secret." % resp['status'])

        request_token = dict(parse_qsl(content.decode("utf-8")))

        return "%s?oauth_token=%s" % (Splitwise.AUTHORIZE_URL, request_token['oauth_token']), request_token['oauth_token_secret']

    def getAccessToken(self,oauth_token,oauth_token_secret,oauth_verifier):

        token = oauth.Token(oauth_token,oauth_token_secret)
        token.set_verifier(oauth_verifier)
        client = oauth.Client(self.consumer, token)

        resp, content = client.request(Splitwise.ACCESS_TOKEN_URL, "POST")

        if Splitwise.isDebug():
            print(resp, content)

        #Check if the response is correct
        if resp['status'] != '200':
            raise Exception("Invalid response %s. Please check your consumer key and secret." % resp['status'])

        access_token = dict(parse_qsl(content.decode("utf-8")))

        return access_token

    def setAccessToken(self,access_token):
        self.token = oauth.Token(key=access_token["oauth_token"], secret=access_token["oauth_token_secret"])
        self.client = oauth.Client(self.consumer, self.token)

    def __makeRequest(self,url):

        resp, content = self.client.request(url, "GET")

        if Splitwise.isDebug():
            print(resp, content)

        #Check if the response is correct
        if resp['status'] != '200':
            raise Exception("Invalid response %s. Please check your consumer key and secret." % resp['status'])

        return content

    def __prepareOptionsUrl(self,options={}):
        if not isinstance(options,dict) or len(options) == 0:
            return ""

        optstr = "?"

        optstr += "&".join([k+"="+str(v) for k,v in options.iteritems()])

        return optstr

    def getFriends(self):

        content = self.__makeRequest(Splitwise.GET_FRIENDS_URL)
        content = json.loads(content.decode("utf-8"))

        friends = []
        if "friends" in content:
            for f in content["friends"]:
                friends.append(Friend(f))

        return friends

    def getGroups(self):

        content = self.__makeRequest(Splitwise.GET_GROUPS_URL)
        content = json.loads(content.decode("utf-8"))

        groups = []
        if "groups" in content:
            for g in content["groups"]:
                groups.append(Group(g))

        return groups

    def getCurrencies(self):

        content = self.__makeRequest(Splitwise.GET_CURRENCY_URL)
        content = json.loads(content.decode("utf-8"))

        currencies = []
        if "currencies" in content:
            for c in content["currencies"]:
                currencies.append(Currency(c))

        return currencies

    def getCategories(self):

        content = self.__makeRequest(Splitwise.GET_CATEGORY_URL)
        content = json.loads(content.decode("utf-8"))
        categories = []

        if "categories" in content:
            for c in content["categories"]:
                categories.append(Category(c))

        return categories

    def getGroup(self,id=0):

        content = self.__makeRequest(Splitwise.GET_GROUP_URL+"/"+str(id))
        content = json.loads(content.decode("utf-8"))
        group = None
        if "group" in content:
            group = Group(content["group"])

        return group

    def getExpenses(self,offset=None,limit=None,group_id=None,friendship_id=None,dated_after=None,dated_before=None,updated_after=None,updated_before=None):

        options = {}

        params = locals().keys()
        for param in params:
            if param != 'self' and param != 'options' and locals().get(param) is not None:
                options[param] = locals().get(param)

        url = Splitwise.GET_EXPENSES_URL

        url += self.__prepareOptionsUrl(options)
        content = self.__makeRequest(url)
        content = json.loads(content.decode("utf-8"))
        expenses = []
        if "expenses" in content:
            for e in content["expenses"]:
                expenses.append(Expense(e))

        return expenses

    def getExpense(self,id):
        content = self.__makeRequest(Splitwise.GET_EXPENSE_URL+"/"+str(id))
        content = json.loads(content.decode("utf-8"))
        expense = None
        if "expense" in content:
            expense = Expense(content["expense"])

        return expense
<|endoftext|>


<commit_before>import sympy as sp

# Brendan Wood, MRocklin, Amelio Vazquez-Reina, Automatically populating matrix elements in SymPy, http://stackoverflow.com/questions/6877061/automatically-populating-matrix-elements-in-sympy, 2011 Aug 02, (accessed 2015 Dec 01)

M = sp.Matrix(3, 3, lambda i, j: sp.var('M_%d%d' % (i+1,j+1)))

for i in xrange(2):
    for j in xrange(i+1, 3):
        M[i, j] = 0

print(M)
<commit_msg>added main()<commit_after>import sympy as sp

def main():
    # Brendan Wood, MRocklin, Amelio Vazquez-Reina, Automatically populating matrix elements in SymPy,
    # http://stackoverflow.com/questions/6877061/automatically-populating-matrix-elements-in-sympy, 2011 Aug 02,
    # (accessed 2015 Dec 01)
    M = sp.Matrix(3, 3, lambda i, j: sp.var('M_%d%d' % (i+1,j+1)))

    for i in xrange(2):
        for j in xrange(i+1, 3):
            M[i, j] = 0

    print(M)

if "__main__" == __name__:
    main()
<|endoftext|>


<commit_before>#    Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Original implementation by Rene de Jong. Updated by Sascha Bischoff.

import logging
import os
import time

from wlauto import AndroidDevice, Parameter
from wlauto.common.gem5.device import BaseGem5Device
from wlauto.exceptions import DeviceError

class Gem5AndroidDevice(BaseGem5Device, AndroidDevice):
    """
    Implements gem5 Android device.

    This class allows a user to connect WA to a simulation using gem5. The
    connection to the device is made using the telnet connection of the
    simulator, and is used for all commands. The simulator does not have ADB
    support, and therefore we need to fall back to using standard shell
    commands.

    Files are copied into the simulation using a VirtIO 9P device in gem5. Files
    are copied out of the simulated environment using the m5 writefile command
    within the simulated system.

    When starting the workload run, the simulator is automatically started by
    Workload Automation, and a connection to the simulator is established. WA
    will then wait for Android to boot on the simulated system (which can take
    hours), prior to executing any other commands on the device. It is also
    possible to resume from a checkpoint when starting the simulation. To do
    this, please append the relevant checkpoint commands from the gem5
    simulation script to the gem5_discription argument in the agenda.

    Host system requirements:
        * VirtIO support. We rely on diod on the host system. This can be
          installed on ubuntu using the following command:

                sudo apt-get install diod

    Guest requirements:
        * VirtIO support. We rely on VirtIO to move files into the simulation.
          Please make sure that the following are set in the kernel
          configuration:

                CONFIG_NET_9P=y

                CONFIG_NET_9P_VIRTIO=y

                CONFIG_9P_FS=y

                CONFIG_9P_FS_POSIX_ACL=y

                CONFIG_9P_FS_SECURITY=y

                CONFIG_VIRTIO_BLK=y

        * m5 binary. Please make sure that the m5 binary is on the device and
          can by found in the path.
    """

    name = 'gem5_android'
    platform = 'android'

    parameters = [
        Parameter('core_names', default=[], override=True),
        Parameter('core_clusters', default=[], override=True),
    ]

    # Overwritten from Device. For documentation, see corresponding method in
    # Device.

    def __init__(self, **kwargs):
        self.logger = logging.getLogger('Gem5AndroidDevice')
        AndroidDevice.__init__(self, **kwargs)
        BaseGem5Device.__init__(self)

    def login_to_device(self):
        pass

    def wait_for_boot(self):
        self.logger.info("Waiting for Android to boot...")
        while True:
            try:
                booted = (int('0' + self.gem5_shell('getprop sys.boot_completed', check_exit_code=False)) == 1)
                anim_finished = (int('0' + self.gem5_shell('getprop service.bootanim.exit', check_exit_code=False)) == 1)
                if booted and anim_finished:
                    break
            except (DeviceError, ValueError):
                pass
            time.sleep(60)

        self.logger.info("Android booted")

    def install(self, filepath, timeout=3 * 3600):  # pylint: disable=W0221
        """ Install an APK or a normal executable """
        ext = os.path.splitext(filepath)[1].lower()
        if ext == '.apk':
            return self.install_apk(filepath, timeout)
        else:
            return self.install_executable(filepath)

    def install_apk(self, filepath, timeout=3 * 3600):  # pylint: disable=W0221
        """
        Install an APK on the gem5 device

        The APK is pushed to the device. Then the file and folder permissions
        are changed to ensure that the APK can be correctly installed. The APK
        is then installed on the device using 'pm'.
        """
        self._check_ready()
        self.logger.info("Installing {}".format(filepath))
        ext = os.path.splitext(filepath)[1].lower()
        if ext == '.apk':
            filename = os.path.basename(filepath)
            on_device_path = os.path.join('/data/local/tmp', filename)
            self.push_file(filepath, on_device_path)
            # We need to make sure that the folder permissions are set
            # correctly, else the APK does not install correctly.
            self.gem5_shell('chmod 775 /data/local/tmp')
            self.gem5_shell('chmod 774 {}'.format(on_device_path))
            self.logger.debug("Actually installing the APK: {}".format(on_device_path))
            return self.gem5_shell("pm install {}".format(on_device_path))
        else:
            raise DeviceError('Can\'t install {}: unsupported format.'.format(filepath))

    def install_executable(self, filepath, with_name=None):
        """ Install an executable """
        executable_name = os.path.basename(filepath)
        on_device_file = self.path.join(self.working_directory, executable_name)
        on_device_executable = self.path.join(self.binaries_directory, executable_name)
        self.push_file(filepath, on_device_file)
        if self.busybox:
            self.execute('{} cp {} {}'.format(self.busybox, on_device_file, on_device_executable))
        else:
            self.execute('cat {} > {}'.format(on_device_file, on_device_executable))
        self.execute('chmod 0777 {}'.format(on_device_executable))
        return on_device_executable

    def uninstall(self, package):
        self._check_ready()
        self.gem5_shell("pm uninstall {}".format(package))

    def dump_logcat(self, outfile, filter_spec=None):
        """ Extract logcat from simulation """
        self.logger.info("Extracting logcat from the simulated system")
        filename = outfile.split('/')[-1]
        command = 'logcat -d > {}'.format(filename)
        self.gem5_shell(command)
        self.pull_file("{}".format(filename), outfile)

    def clear_logcat(self):
        """Clear (flush) logcat log."""
        if self._logcat_poller:
            return self._logcat_poller.clear_buffer()
        else:
            return self.gem5_shell('logcat -c')

    def disable_selinux(self):
        """ Disable SELinux. Overridden as parent implementation uses ADB """
        api_level = int(self.gem5_shell('getprop ro.build.version.sdk').strip())

        # SELinux was added in Android 4.3 (API level 18). Trying to
        # 'getenforce' in earlier versions will produce an error.
        if api_level >= 18:
            se_status = self.execute('getenforce', as_root=True).strip()
            if se_status == 'Enforcing':
                self.execute('setenforce 0', as_root=True)

    def get_properties(self, context):  # pylint: disable=R0801
        """ Get the property files from the device """
        BaseGem5Device.get_properties(self, context)
        props = self._get_android_properties(context)
        return props

    def disable_screen_lock(self):
        """
        Attempts to disable he screen lock on the device.

        Overridden here as otherwise we have issues with too many backslashes.
        """
        lockdb = '/data/system/locksettings.db'
        sqlcommand = "update locksettings set value=\'0\' where name=\'screenlock.disabled\';"
        self.execute('sqlite3 {} "{}"'.format(lockdb, sqlcommand), as_root=True)

    def capture_screen(self, filepath):
        if BaseGem5Device.capture_screen(self, filepath):
            return

        # If we didn't manage to do the above, call the parent class.
        self.logger.warning("capture_screen: falling back to parent class implementation")
        AndroidDevice.capture_screen(self, filepath)

    def initialize(self, context):
        self.resize_shell()
        self.deploy_m5(context, force=False)
<commit_msg>Gem5AndroidDevice: No longer wait for disabled boot animation<commit_after>#    Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Original implementation by Rene de Jong. Updated by Sascha Bischoff.

import logging
import os
import time

from wlauto import AndroidDevice, Parameter
from wlauto.common.gem5.device import BaseGem5Device
from wlauto.exceptions import DeviceError

class Gem5AndroidDevice(BaseGem5Device, AndroidDevice):
    """
    Implements gem5 Android device.

    This class allows a user to connect WA to a simulation using gem5. The
    connection to the device is made using the telnet connection of the
    simulator, and is used for all commands. The simulator does not have ADB
    support, and therefore we need to fall back to using standard shell
    commands.

    Files are copied into the simulation using a VirtIO 9P device in gem5. Files
    are copied out of the simulated environment using the m5 writefile command
    within the simulated system.

    When starting the workload run, the simulator is automatically started by
    Workload Automation, and a connection to the simulator is established. WA
    will then wait for Android to boot on the simulated system (which can take
    hours), prior to executing any other commands on the device. It is also
    possible to resume from a checkpoint when starting the simulation. To do
    this, please append the relevant checkpoint commands from the gem5
    simulation script to the gem5_discription argument in the agenda.

    Host system requirements:
        * VirtIO support. We rely on diod on the host system. This can be
          installed on ubuntu using the following command:

                sudo apt-get install diod

    Guest requirements:
        * VirtIO support. We rely on VirtIO to move files into the simulation.
          Please make sure that the following are set in the kernel
          configuration:

                CONFIG_NET_9P=y

                CONFIG_NET_9P_VIRTIO=y

                CONFIG_9P_FS=y

                CONFIG_9P_FS_POSIX_ACL=y

                CONFIG_9P_FS_SECURITY=y

                CONFIG_VIRTIO_BLK=y

        * m5 binary. Please make sure that the m5 binary is on the device and
          can by found in the path.
    """

    name = 'gem5_android'
    platform = 'android'

    parameters = [
        Parameter('core_names', default=[], override=True),
        Parameter('core_clusters', default=[], override=True),
    ]

    # Overwritten from Device. For documentation, see corresponding method in
    # Device.

    def __init__(self, **kwargs):
        self.logger = logging.getLogger('Gem5AndroidDevice')
        AndroidDevice.__init__(self, **kwargs)
        BaseGem5Device.__init__(self)

    def login_to_device(self):
        pass

    def wait_for_boot(self):
        """
        Wait for the system to boot

        We monitor the sys.boot_completed and service.bootanim.exit system
        properties to determine when the system has finished booting. In the
        event that we cannot coerce the result of service.bootanim.exit to an
        integer, we assume that the boot animation was disabled and do not wait
        for it to finish.

        """
        self.logger.info("Waiting for Android to boot...")
        while True:
            booted = False
            anim_finished = True  # Assume boot animation was disabled on except
            try:
                booted = (int('0' + self.gem5_shell('getprop sys.boot_completed', check_exit_code=False).strip()) == 1)
                anim_finished = (int(self.gem5_shell('getprop service.bootanim.exit', check_exit_code=False).strip()) == 1)
            except ValueError:
                pass
            if booted and anim_finished:
                break
            time.sleep(60)

        self.logger.info("Android booted")

    def install(self, filepath, timeout=3 * 3600):  # pylint: disable=W0221
        """ Install an APK or a normal executable """
        ext = os.path.splitext(filepath)[1].lower()
        if ext == '.apk':
            return self.install_apk(filepath, timeout)
        else:
            return self.install_executable(filepath)

    def install_apk(self, filepath, timeout=3 * 3600):  # pylint: disable=W0221
        """
        Install an APK on the gem5 device

        The APK is pushed to the device. Then the file and folder permissions
        are changed to ensure that the APK can be correctly installed. The APK
        is then installed on the device using 'pm'.
        """
        self._check_ready()
        self.logger.info("Installing {}".format(filepath))
        ext = os.path.splitext(filepath)[1].lower()
        if ext == '.apk':
            filename = os.path.basename(filepath)
            on_device_path = os.path.join('/data/local/tmp', filename)
            self.push_file(filepath, on_device_path)
            # We need to make sure that the folder permissions are set
            # correctly, else the APK does not install correctly.
            self.gem5_shell('chmod 775 /data/local/tmp')
            self.gem5_shell('chmod 774 {}'.format(on_device_path))
            self.logger.debug("Actually installing the APK: {}".format(on_device_path))
            return self.gem5_shell("pm install {}".format(on_device_path))
        else:
            raise DeviceError('Can\'t install {}: unsupported format.'.format(filepath))

    def install_executable(self, filepath, with_name=None):
        """ Install an executable """
        executable_name = os.path.basename(filepath)
        on_device_file = self.path.join(self.working_directory, executable_name)
        on_device_executable = self.path.join(self.binaries_directory, executable_name)
        self.push_file(filepath, on_device_file)
        if self.busybox:
            self.execute('{} cp {} {}'.format(self.busybox, on_device_file, on_device_executable))
        else:
            self.execute('cat {} > {}'.format(on_device_file, on_device_executable))
        self.execute('chmod 0777 {}'.format(on_device_executable))
        return on_device_executable

    def uninstall(self, package):
        self._check_ready()
        self.gem5_shell("pm uninstall {}".format(package))

    def dump_logcat(self, outfile, filter_spec=None):
        """ Extract logcat from simulation """
        self.logger.info("Extracting logcat from the simulated system")
        filename = outfile.split('/')[-1]
        command = 'logcat -d > {}'.format(filename)
        self.gem5_shell(command)
        self.pull_file("{}".format(filename), outfile)

    def clear_logcat(self):
        """Clear (flush) logcat log."""
        if self._logcat_poller:
            return self._logcat_poller.clear_buffer()
        else:
            return self.gem5_shell('logcat -c')

    def disable_selinux(self):
        """ Disable SELinux. Overridden as parent implementation uses ADB """
        api_level = int(self.gem5_shell('getprop ro.build.version.sdk').strip())

        # SELinux was added in Android 4.3 (API level 18). Trying to
        # 'getenforce' in earlier versions will produce an error.
        if api_level >= 18:
            se_status = self.execute('getenforce', as_root=True).strip()
            if se_status == 'Enforcing':
                self.execute('setenforce 0', as_root=True)

    def get_properties(self, context):  # pylint: disable=R0801
        """ Get the property files from the device """
        BaseGem5Device.get_properties(self, context)
        props = self._get_android_properties(context)
        return props

    def disable_screen_lock(self):
        """
        Attempts to disable he screen lock on the device.

        Overridden here as otherwise we have issues with too many backslashes.
        """
        lockdb = '/data/system/locksettings.db'
        sqlcommand = "update locksettings set value=\'0\' where name=\'screenlock.disabled\';"
        self.execute('sqlite3 {} "{}"'.format(lockdb, sqlcommand), as_root=True)

    def capture_screen(self, filepath):
        if BaseGem5Device.capture_screen(self, filepath):
            return

        # If we didn't manage to do the above, call the parent class.
        self.logger.warning("capture_screen: falling back to parent class implementation")
        AndroidDevice.capture_screen(self, filepath)

    def initialize(self, context):
        self.resize_shell()
        self.deploy_m5(context, force=False)
<|endoftext|>


<commit_before>
# MODEL RUNNER PY

# Currently only supports NT3_TC1
# See __main__ section for usage

# tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
# so we need to create a synthetic argv.
import sys
if not hasattr(sys, 'argv'):
    sys.argv  = ['nt3_tc1']

import json
import os
import numpy as np
import importlib
import runner_utils
import log_tools

logger = None

def import_pkg(framework, model_name):
    if framework == 'keras':
        module_name = "{}_baseline_keras2".format(model_name)
        pkg = importlib.import_module(module_name)
    # elif framework is 'mxnet':
    #     import nt3_baseline_mxnet
    #     pkg = nt3_baseline_keras_baseline_mxnet
    # elif framework is 'neon':
    #     import nt3_baseline_neon
    #     pkg = nt3_baseline_neon
    else:
        raise ValueError("Invalid framework: {}".format(framework))
    return pkg

def run(hyper_parameter_map, obj_param):

    global logger
    logger = log_tools.get_logger(logger, __name__)

    framework = hyper_parameter_map['framework']
    model_name = hyper_parameter_map['model_name']
    pkg = import_pkg(framework, model_name)

    runner_utils.format_params(hyper_parameter_map)

    # params is python dictionary
    params = pkg.initialize_parameters()
    for k,v in hyper_parameter_map.items():
        #if not k in params:
        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
        if(k=="dense"):
            if(type(v) != list):
                v=v.split(" ")
            v = [int(i) for i in v]
        if(k=="dense_feature_layers"):
            if(type(v) != list):
                v=v.split(" ")
            v = [int(i) for i in v]
        if(k=="cell_features"):
            cp_str = v
            v = list()
            v.append(cp_str)
        params[k] = v

    logger.debug("WRITE_PARAMS START")
    runner_utils.write_params(params, hyper_parameter_map)
    logger.debug("WRITE_PARAMS STOP")

    history = pkg.run(params)

    runner_utils.keras_clear_session(framework)

    # use the last validation_loss as the value to minimize
    val_loss = history.history['val_loss']
    result = val_loss[-1]
    print("result: ", result)
    return result

# Usage: see how sys.argv is unpacked below:
if __name__ == '__main__':
    logger = log_tools.get_logger(logger, __name__)
    logger.debug("RUN START")

    ( _, # The Python program name (unused)
      param_string,
      instance_directory,
      framework,
      runid,
      obj_param,
      benchmark_timeout ) = sys.argv

    hyper_parameter_map = runner_utils.init(param_string,
                                            instance_directory,
                                            framework, 'save')
    hyper_parameter_map['model_name']    = os.getenv("MODEL_NAME")
    if hyper_parameter_map['model_name'] == None:
        raise Exception("No MODEL_NAME was in the environment!")
    hyper_parameter_map['experiment_id'] = os.getenv("EXPID")
    hyper_parameter_map['run_id']  = runid
    hyper_parameter_map['timeout'] = benchmark_timeout
    # clear sys.argv so that argparse doesn't object
    sys.argv = ['nt3_tc1_runner']

    # Call to Benchmark!
    result = run(hyper_parameter_map, obj_param)

    runner_utils.write_output(result, instance_directory)
    logger.debug("RUN STOP")
<commit_msg>Support models with no output (ref ECP-CANDLE/Benchmarks#8)<commit_after>
# MODEL RUNNER PY

# Currently only supports NT3_TC1
# See __main__ section for usage

# tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
# so we need to create a synthetic argv.
import sys
if not hasattr(sys, 'argv'):
    sys.argv  = ['nt3_tc1']

import json
import os
import numpy as np
import importlib
import runner_utils
import log_tools

logger = None

def import_pkg(framework, model_name):
    if framework == 'keras':
        module_name = "{}_baseline_keras2".format(model_name)
        pkg = importlib.import_module(module_name)
    # elif framework is 'mxnet':
    #     import nt3_baseline_mxnet
    #     pkg = nt3_baseline_keras_baseline_mxnet
    # elif framework is 'neon':
    #     import nt3_baseline_neon
    #     pkg = nt3_baseline_neon
    else:
        raise ValueError("Invalid framework: {}".format(framework))
    return pkg

def run(hyper_parameter_map, obj_param):

    global logger
    logger = log_tools.get_logger(logger, __name__)

    framework = hyper_parameter_map['framework']
    model_name = hyper_parameter_map['model_name']
    pkg = import_pkg(framework, model_name)

    runner_utils.format_params(hyper_parameter_map)

    # params is python dictionary
    params = pkg.initialize_parameters()
    for k,v in hyper_parameter_map.items():
        #if not k in params:
        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
        if(k=="dense"):
            if(type(v) != list):
                v=v.split(" ")
            v = [int(i) for i in v]
        if(k=="dense_feature_layers"):
            if(type(v) != list):
                v=v.split(" ")
            v = [int(i) for i in v]
        if(k=="cell_features"):
            cp_str = v
            v = list()
            v.append(cp_str)
        params[k] = v

    logger.debug("WRITE_PARAMS START")
    runner_utils.write_params(params, hyper_parameter_map)
    logger.debug("WRITE_PARAMS STOP")

    history = pkg.run(params)

    runner_utils.keras_clear_session(framework)

    # Default result if there is no val_loss (as in infer.py)
    result = 0
    if history != None:
        # use the last validation_loss as the value to minimize
        val_loss = history.history['val_loss']
        result = val_loss[-1]
    print("result: ", result)
    return result

# Usage: see how sys.argv is unpacked below:
if __name__ == '__main__':
    logger = log_tools.get_logger(logger, __name__)
    logger.debug("RUN START")

    ( _, # The Python program name (unused)
      param_string,
      instance_directory,
      framework,
      runid,
      obj_param,
      benchmark_timeout ) = sys.argv

    hyper_parameter_map = runner_utils.init(param_string,
                                            instance_directory,
                                            framework, 'save')
    hyper_parameter_map['model_name']    = os.getenv("MODEL_NAME")
    if hyper_parameter_map['model_name'] == None:
        raise Exception("No MODEL_NAME was in the environment!")
    hyper_parameter_map['experiment_id'] = os.getenv("EXPID")
    hyper_parameter_map['run_id']  = runid
    hyper_parameter_map['timeout'] = benchmark_timeout
    # clear sys.argv so that argparse doesn't object
    sys.argv = ['nt3_tc1_runner']

    # Call to Benchmark!
    result = run(hyper_parameter_map, obj_param)

    runner_utils.write_output(result, instance_directory)
    logger.debug("RUN STOP")
<|endoftext|>


<commit_before>import copy
import time
import math

import numpy as np

from Common import logger
from Object import TransformObject, Model
from Utilities import *
from Common.Constants import *
from App import CoreManager

class ParticleManager(Singleton):
    def __init__(self):
        self.active = True
        self.particles = []
        self.active_particles = []

    def clear(self):
        for particle in self.active_particles:
            particle.destroy()

        self.particles = []
        self.active_particles = []

    def create_particle(self, particle_infos):
        particle = Particle(particle_infos)
        self.particles.append(particle)
        particle.play()

    def destroy_particle(self, particle):
        particle.destroy()
        self.active_particles.remove(particle)

    def update(self, dt):
        for particle in self.active_particles:
            particle.update(dt)

class Particle:
    def __init__(self, name, emitters_infos=[]):
        self.emitters_list = []

        for emitters_info in emitters_infos:
            count = emitter_info.get('count', 1)
            emitter_list = []
            self.emitters_list.append(emitter_list)

            for i in range(count):
                emitter = Emitter(emitters_info)
                emitter_list.append(emitter)

    def get_save_data(self):
        save_data = [emitter.get_save_data() for emitter in self.emitters_list]
        return save_data

    def play(self):
        for emitter_list in self.emitters_list:
            for emitter in emitter_list:
                emitter.play()

    def destroy(self):
        for emitter_list in self.emitters_list:
            for emitter in emitter_list:
                emitter.destroy()
        self.emitters_list = []

    def update(self, dt):
        for emitter_list in self.emitters_list:
            for emitter in emitter_list:
                emitter.update(dt)

class Emitter:
    def __init__(self, **emitter_info):
        self.name = emitter_info.get('name', 'Emitter')
        self.first_time = True
        self.alive = False
        self.elapsed_time = 0.0
        self.has_velocity = False
        self.has_rotation_velocity = False
        self.has_scale_velocity = False
        self.billboard = emitter_info.get('billboard', True)
        self.mesh = None
        self.material_instance = None
        self.transform = TransformObject()
        self.fade = emitter_info.get('fade', 0.0)  # negative is fade out, 0.0 is none, positive is fade in

        # sequence
        self.loop = emitter_info.get('loop', 1)  # -1 is infinite
        self.loop_remain = self.loop
        self.cell_count = emitter_info.get('cell_count', [1, 1])
        self.total_cell_count = self.cell_count[0] * self.cell_count[1]
        self.current_sequence = [1, 1]
        self.prev_sequence_index = -1

        # variance
        self.variance_delay = RangeVariable(emitter_info.get('delay', 0.0))
        self.variance_life_time = RangeVariable(emitter_info.get('life_time', 0.0))
        self.variance_play_speed = RangeVariable(emitter_info.get('play_speed', 0.0))
        self.variance_gravity = RangeVariable(emitter_info.get('gravity', 0.0))
        self.variance_opacity = RangeVariable(emitter_info.get('opacity', 1.0))
        self.variance_velocity = RangeVariable(emitter_info.get('velocity', FLOAT3_ZERO))
        self.variance_rotation_velocity = RangeVariable(emitter_info.get('rotation_velocity', FLOAT3_ZERO))
        self.variance_scale_velocity = RangeVariable(emitter_info.get('scale_velocity', FLOAT3_ZERO))
        self.variance_position = RangeVariable(emitter_info.get('position', FLOAT3_ZERO))
        self.variance_rotation = RangeVariable(emitter_info.get('rotation', FLOAT3_ZERO))
        self.variance_scale = RangeVariable(emitter_info.get('scale', Float3(1.0, 1.0, 1.0)))

        self.delay = 0.0
        self.life_time = 0.0
        self.play_speed = 0.0
        self.gravity = 0.0
        self.opacity = 1.0
        self.velocity = Float3()
        self.rotation_velocity = Float3()
        self.scale_velocity = Float3()

        self.attributes = Attributes()

    def get_save_data(self):
        save_data = dict(
            billboard=self.billboard,
            mesh=self.mesh.name if self.mesh is not None else '',
            material_instance=self.material_instance.name if self.material_instance is not None else '',
            fade=self.fade,
            loop=self.loop,
            cell_count=self.cell_count,
            delay=self.variance_delay.get_save_data(),
            life_time=self.variance_life_time.get_save_data(),
            play_speed=self.variance_play_speed.get_save_data(),
            gravity=self.variance_gravity.get_save_data(),
            opacity=self.variance_opacity.get_save_data(),
            velocity=self.variance_velocity.get_save_data(),
            rotation_velocity=self.variance_rotation_velocity.get_save_data(),
            scale_velocity=self.variance_scale_velocity.get_save_data(),
            position=self.variance_position.get_save_data(),
            rotation=self.variance_rotation.get_save_data(),
            scale=self.variance_scale.get_save_data(),
        )
        return save_data

    def get_attribute(self):
        attributes = self.get_save_data()
        self.attributes.set_attribute('name', self.name)
        for key in attributes:
            self.attributes.set_attribute(key, attributes[key])
        return self.attributes

    def set_attribute(self, attributeName, attributeValue, attribute_index):
        if hasattr(self, "variance_" + attributeName):
            variance_value = getattr("variance_" + attributeName)
            variance_value.set_value(*attributeValue)
        elif hasattr(self, attributeName):
            setattr(self, attributeName, attributeValue)
        self.play()

    def refresh(self):
        self.delay = self.variance_delay.get_value()
        self.life_time = self.variance_life_time.get_value()
        self.play_speed = self.variance_play_speed.get_value()
        self.gravity = self.variance_gravity.get_value()
        self.opacity = self.variance_opacity.get_value()
        self.velocity[...] = self.variance_velocity.get_value()
        self.rotation_velocity[...] = self.variance_rotation_velocity.get_value()
        self.scale_velocity[...] = self.variance_scale_velocity.get_value()

        self.transform.set_pos(self.variance_position.get_value())
        self.transform.set_rotation(self.variance_rotation.get_value())
        self.transform.set_scale(self.variance_scale.get_value())

        self.has_velocity = any([v != 0.0 for v in self.velocity])
        self.has_rotation_velocity = any([v != 0.0 for v in self.rotation_velocity])
        self.has_scale_velocity = any([v != 0.0 for v in self.scale_velocity])

    def play(self):
        if 0.0 == self.life_time or 0 == self.loop:
            self.alive = False
            return

        self.alive = True
        self.loop_remain = self.loop
        self.elapsed_time = 0.0
        self.prev_sequence_index = -1
        self.total_cell_count = self.cell_count[0] * self.cell_count[1]
        self.refresh()

    def destroy(self):
        self.alive = False

    def update_sequence(self, life_ratio):
        if self.total_cell_count > 1 and self.play_speed > 0:
            ratio = (life_ratio * self.play_speed) % 1.0
            index = min(self.total_cell_count, int(math.floor((self.total_cell_count - 1) * ratio)))

            if index == self.prev_sequence_index:
                return

            self.prev_sequence_index = index
            self.current_sequence[0] = index % self.cell_count[0]
            self.current_sequence[1] = self.cell_count[1] - int(index / self.cell_count[0]) - 1

    def update(self, dt):
        if not self.alive:
            return

        if 0.0 < self.delay:
            self.delay -= dt
            if self.delay < 0.0:
                self.delay = 0.0
            else:
                return

        self.elapsed_time += dt

        if self.life_time < self.elapsed_time:
            self.elapsed_time %= self.life_time

            if self.loop_remain > 0:
                self.loop_remain -= 1

            if self.loop_remain == 0:
                self.destroy()
                return

            self.refresh()

        life_ratio = 0.0
        if 0.0 < self.life_time:
            life_ratio = self.elapsed_time / self.life_time

        self.update_sequence(life_ratio)

        if 0.0 != self.gravity:
            self.velocity[1] -= self.gravity * dt

        # update transform
        if self.has_velocity:
            self.transform.move(self.velocity * dt)

        if self.has_rotation_velocity:
            self.transform.rotation(self.rotation_velocity * dt)

        if self.has_scale_velocity:
            self.transform.scaling(self.scale_velocity * dt)

        if 0.0 != self.fade:
            opacity = life_ratio if 0.0 < self.fade else (1.0 - life_ratio)
            self.opacity = math.pow(opacity, abs(self.fade))
<commit_msg>Update Particle.py<commit_after>import copy
import time
import math

import numpy as np

from Common import logger
from Object import TransformObject, Model
from Utilities import *
from Common.Constants import *
from App import CoreManager

class ParticleManager(Singleton):
    def __init__(self):
        self.active = True
        self.particles = []
        self.active_particles = []

    def clear(self):
        for particle in self.active_particles:
            particle.destroy()

        self.particles = []
        self.active_particles = []

    def create_particle(self, particle_infos):
        particle = Particle(particle_infos)
        self.particles.append(particle)
        particle.play()

    def destroy_particle(self, particle):
        particle.destroy()
        self.active_particles.remove(particle)

    def update(self, dt):
        for particle in self.active_particles:
            particle.update(dt)

class Particle:
    def __init__(self, name, emitters_infos=[]):
        self.emitters_list = []

        for emitters_info in emitters_infos:
            count = emitter_info.get('count', 1)
            emitter_list = []
            self.emitters_list.append(emitter_list)

            for i in range(count):
                emitter = Emitter(emitters_info)
                emitter_list.append(emitter)

    def get_save_data(self):
        save_data = [emitter.get_save_data() for emitter in self.emitters_list]
        return save_data

    def play(self):
        for emitter_list in self.emitters_list:
            for emitter in emitter_list:
                emitter.play()

    def destroy(self):
        for emitter_list in self.emitters_list:
            for emitter in emitter_list:
                emitter.destroy()
        self.emitters_list = []

    def update(self, dt):
        for emitter_list in self.emitters_list:
            for emitter in emitter_list:
                emitter.update(dt)

class EmitterInstance:
    def __init__(self, **emitter_info):
        self.name = emitter_info.get('name', 'Emitter')
        self.first_time = True
        self.alive = False
        self.elapsed_time = 0.0
        self.has_velocity = False
        self.has_rotation_velocity = False
        self.has_scale_velocity = False
        self.billboard = emitter_info.get('billboard', True)
        self.mesh = None
        self.material_instance = None
        self.transform = TransformObject()
        self.fade = emitter_info.get('fade', 0.0)  # negative is fade out, 0.0 is none, positive is fade in

        # sequence
        self.loop = emitter_info.get('loop', 1)  # -1 is infinite
        self.loop_remain = self.loop
        self.cell_count = emitter_info.get('cell_count', [1, 1])
        self.total_cell_count = self.cell_count[0] * self.cell_count[1]
        self.current_sequence = [1, 1]
        self.prev_sequence_index = -1

        # variance
        self.variance_delay = RangeVariable(emitter_info.get('delay', 0.0))
        self.variance_life_time = RangeVariable(emitter_info.get('life_time', 0.0))
        self.variance_play_speed = RangeVariable(emitter_info.get('play_speed', 0.0))
        self.variance_gravity = RangeVariable(emitter_info.get('gravity', 0.0))
        self.variance_opacity = RangeVariable(emitter_info.get('opacity', 1.0))
        self.variance_velocity = RangeVariable(emitter_info.get('velocity', FLOAT3_ZERO))
        self.variance_rotation_velocity = RangeVariable(emitter_info.get('rotation_velocity', FLOAT3_ZERO))
        self.variance_scale_velocity = RangeVariable(emitter_info.get('scale_velocity', FLOAT3_ZERO))
        self.variance_position = RangeVariable(emitter_info.get('position', FLOAT3_ZERO))
        self.variance_rotation = RangeVariable(emitter_info.get('rotation', FLOAT3_ZERO))
        self.variance_scale = RangeVariable(emitter_info.get('scale', Float3(1.0, 1.0, 1.0)))

        self.delay = 0.0
        self.life_time = 0.0
        self.play_speed = 0.0
        self.gravity = 0.0
        self.opacity = 1.0
        self.velocity = Float3()
        self.rotation_velocity = Float3()
        self.scale_velocity = Float3()

        self.attributes = Attributes()

    def get_save_data(self):
        save_data = dict(
            billboard=self.billboard,
            mesh=self.mesh.name if self.mesh is not None else '',
            material_instance=self.material_instance.name if self.material_instance is not None else '',
            fade=self.fade,
            loop=self.loop,
            cell_count=self.cell_count,
            delay=self.variance_delay.get_save_data(),
            life_time=self.variance_life_time.get_save_data(),
            play_speed=self.variance_play_speed.get_save_data(),
            gravity=self.variance_gravity.get_save_data(),
            opacity=self.variance_opacity.get_save_data(),
            velocity=self.variance_velocity.get_save_data(),
            rotation_velocity=self.variance_rotation_velocity.get_save_data(),
            scale_velocity=self.variance_scale_velocity.get_save_data(),
            position=self.variance_position.get_save_data(),
            rotation=self.variance_rotation.get_save_data(),
            scale=self.variance_scale.get_save_data(),
        )
        return save_data

    def get_attribute(self):
        attributes = self.get_save_data()
        self.attributes.set_attribute('name', self.name)
        for key in attributes:
            self.attributes.set_attribute(key, attributes[key])
        return self.attributes

    def set_attribute(self, attributeName, attributeValue, attribute_index):
        if hasattr(self, "variance_" + attributeName):
            variance_value = getattr("variance_" + attributeName)
            variance_value.set_value(*attributeValue)
        elif hasattr(self, attributeName):
            setattr(self, attributeName, attributeValue)
        self.play()

    def refresh(self):
        self.delay = self.variance_delay.get_value()
        self.life_time = self.variance_life_time.get_value()
        self.play_speed = self.variance_play_speed.get_value()
        self.gravity = self.variance_gravity.get_value()
        self.opacity = self.variance_opacity.get_value()
        self.velocity[...] = self.variance_velocity.get_value()
        self.rotation_velocity[...] = self.variance_rotation_velocity.get_value()
        self.scale_velocity[...] = self.variance_scale_velocity.get_value()

        self.transform.set_pos(self.variance_position.get_value())
        self.transform.set_rotation(self.variance_rotation.get_value())
        self.transform.set_scale(self.variance_scale.get_value())

        self.has_velocity = any([v != 0.0 for v in self.velocity])
        self.has_rotation_velocity = any([v != 0.0 for v in self.rotation_velocity])
        self.has_scale_velocity = any([v != 0.0 for v in self.scale_velocity])

    def play(self):
        if 0.0 == self.life_time or 0 == self.loop:
            self.alive = False
            return

        self.alive = True
        self.loop_remain = self.loop
        self.elapsed_time = 0.0
        self.prev_sequence_index = -1
        self.total_cell_count = self.cell_count[0] * self.cell_count[1]
        self.refresh()

    def destroy(self):
        self.alive = False

    def update_sequence(self, life_ratio):
        if self.total_cell_count > 1 and self.play_speed > 0:
            ratio = (life_ratio * self.play_speed) % 1.0
            index = min(self.total_cell_count, int(math.floor((self.total_cell_count - 1) * ratio)))

            if index == self.prev_sequence_index:
                return

            self.prev_sequence_index = index
            self.current_sequence[0] = index % self.cell_count[0]
            self.current_sequence[1] = self.cell_count[1] - int(index / self.cell_count[0]) - 1

    def update(self, dt):
        if not self.alive:
            return

        if 0.0 < self.delay:
            self.delay -= dt
            if self.delay < 0.0:
                self.delay = 0.0
            else:
                return

        self.elapsed_time += dt

        if self.life_time < self.elapsed_time:
            self.elapsed_time %= self.life_time

            if self.loop_remain > 0:
                self.loop_remain -= 1

            if self.loop_remain == 0:
                self.destroy()
                return

            self.refresh()

        life_ratio = 0.0
        if 0.0 < self.life_time:
            life_ratio = self.elapsed_time / self.life_time

        self.update_sequence(life_ratio)

        if 0.0 != self.gravity:
            self.velocity[1] -= self.gravity * dt

        # update transform
        if self.has_velocity:
            self.transform.move(self.velocity * dt)

        if self.has_rotation_velocity:
            self.transform.rotation(self.rotation_velocity * dt)

        if self.has_scale_velocity:
            self.transform.scaling(self.scale_velocity * dt)

        if 0.0 != self.fade:
            opacity = life_ratio if 0.0 < self.fade else (1.0 - life_ratio)
            self.opacity = math.pow(opacity, abs(self.fade))

class EmitterInfo:
    def __init__(self, **emitter_info):
        self.name = emitter_info.get('name', 'Emitter')
        self.billboard = emitter_info.get('billboard', True)
        self.mesh = emitter_info.get('mesh', None)
        self.material_instance = emitter_info.get('material_instance', None)
        self.fade = emitter_info.get('fade', 0.0)  # negative is fade out, 0.0 is none, positive is fade in

        # sequence
        self.loop = emitter_info.get('loop', 1)  # -1 is infinite
        self.cell_count = emitter_info.get('cell_count', [1, 1])
        self.total_cell_count = self.cell_count[0] * self.cell_count[1]

        # variance
        self.delay = RangeVariable(emitter_info.get('delay', 0.0))
        self.life_time = RangeVariable(emitter_info.get('life_time', 0.0))
        self.play_speed = RangeVariable(emitter_info.get('play_speed', 0.0))
        self.gravity = RangeVariable(emitter_info.get('gravity', 0.0))
        self.opacity = RangeVariable(emitter_info.get('opacity', 1.0))
        self.velocity = RangeVariable(emitter_info.get('velocity', FLOAT3_ZERO))
        self.rotation_velocity = RangeVariable(emitter_info.get('rotation_velocity', FLOAT3_ZERO))
        self.scale_velocity = RangeVariable(emitter_info.get('scale_velocity', FLOAT3_ZERO))
        self.position = RangeVariable(emitter_info.get('position', FLOAT3_ZERO))
        self.rotation = RangeVariable(emitter_info.get('rotation', FLOAT3_ZERO))
        self.scale = RangeVariable(emitter_info.get('scale', Float3(1.0, 1.0, 1.0)))
        self.attributes = Attributes()

    def get_save_data(self):
        save_data = dict(
            billboard=self.billboard,
            mesh=self.mesh.name if self.mesh is not None else '',
            material_instance=self.material_instance.name if self.material_instance is not None else '',
            fade=self.fade,
            loop=self.loop,
            cell_count=self.cell_count,
            delay=self.variance_delay.get_save_data(),
            life_time=self.variance_life_time.get_save_data(),
            play_speed=self.variance_play_speed.get_save_data(),
            gravity=self.variance_gravity.get_save_data(),
            opacity=self.variance_opacity.get_save_data(),
            velocity=self.variance_velocity.get_save_data(),
            rotation_velocity=self.variance_rotation_velocity.get_save_data(),
            scale_velocity=self.variance_scale_velocity.get_save_data(),
            position=self.variance_position.get_save_data(),
            rotation=self.variance_rotation.get_save_data(),
            scale=self.variance_scale.get_save_data(),
        )
        return save_data

    def get_attribute(self):
        attributes = self.get_save_data()
        self.attributes.set_attribute('name', self.name)
        for key in attributes:
            self.attributes.set_attribute(key, attributes[key])
        return self.attributes

    def set_attribute(self, attributeName, attributeValue, attribute_index):
        if hasattr(self, attributeName):
            setattr(self, attributeName, attributeValue)
<|endoftext|>


<commit_before>from rpython.rtyper.lltypesystem import rffi, lltype
from capi import CConfig
import sys
import os
import capi

testdb = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.db")

def opendb(db_name):
	with rffi.scoped_str2charp(db_name) as db_name, lltype.scoped_alloc(capi.SQLITE3PP.TO, 1) as result:
		errorcode = capi.sqlite3_open(db_name, result)
		assert errorcode == 0
		return rffi.cast(capi.SQLITE3P, result[0])

def prepare(db, query):
	length = len(query)
	with rffi.scoped_str2charp(query) as query, lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as result, lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as unused_buffer:
		errorcode = capi.sqlite3_prepare(db, query, length, result, unused_buffer)
		assert errorcode == 0
		return rffi.cast(capi.VDBEP, result[0])

def allocateCursor(vdbe_struct, p1, nField, iDb, isBtreeCursor):
	return capi.sqlite3_allocateCursor(vdbe_struct, p1, nField, iDb, isBtreeCursor)

def sqlite3VdbeMemIntegerify(pMem):
	return capi.sqlite3_sqlite3VdbeMemIntegerify(pMem)

def mainloop(vdbe_struct):
	pc = 0
	length = vdbe_struct.nOp
	ops = vdbe_struct.aOp
	rc = CConfig.SQLITE_OK
	db = vdbe_struct.db
	p = vdbe_struct
	aMem = p.aMem

	while pc < length:
		pOp = ops[pc]

		if pOp.opcode == CConfig.OP_Init:
			if pOp.p2:
				pc = pOp.p2 - 1
		elif pOp.opcode == CConfig.OP_OpenRead or pOp.opcode == CConfig.OP_OpenWrite:
			assert(pOp.p5 & (CConfig.OPFLAG_P2ISREG | CConfig.OPFLAG_BULKCSR) == pOp.p5)
			assert(pOp.opcode == CConfig.OP_OpenWrite or pOp.p5 == 0)
			assert(p.bIsReader)
			assert(pOp.opcode == CConfig.OP_OpenRead or p.readOnly == 0)

			if (p.expired):
				rc = CConfig.SQLITE_ABORT
				break

			nField = 0
			pKeyInfo = lltype.nullptr(capi.KEYINFO)
			p2 = pOp.p2;
			iDb = pOp.p3

			assert(iDb >= 0 and iDb < db.nDb)
			#   assert( (p->btreeMask & (((yDbMask)1)<<iDb))!=0 );

			pDb = db.aDb[iDb]
			pX = pDb.pBt

			assert(not pX)

			if pOp.opcode == CConfig.OP_OpenWrite:
				wrFlag = 1
				#     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
				if pDb.pSchema.file_format < p.minWriteFileFormat:
					p.minWriteFileFormat = pDb.pSchema.file_format
				else:
					wrFlag = 0

			if pOp.p5 & CConfig.OPFLAG_P2ISREG:
				assert(p2 > 0)
				assert(p2 <= p.nMem - p.nCursor)
				pIn2 = aMem[p2]
				#     assert( memIsValid(pIn2) );
				#     assert( (pIn2->flags & MEM_Int)!=0 );
				sqlite3VdbeMemIntegerify(pIn2)
				p2 = rffi.cast(rffi.INT, pIn2.u.i)
				#     /* The p2 value always comes from a prior OP_CreateTable opcode and
				#     ** that opcode will always set the p2 value to 2 or more or else fail.
				#     ** If there were a failure, the prepared statement would have halted
				#     ** before reaching this instruction. */
				#     if( NEVER(p2<2) ) {
				#       rc = SQLITE_CORRUPT_BKPT;
				#       goto abort_due_to_error;
				#     }
				if pOp.p4type == CConfig.P4_KEYINFO:
					pKeyInfo = pOp.p4.pKeyInfo
					#     assert( pKeyInfo->enc==ENC(db) );
					assert(pKeyInfo.db == db)
					nField = pKeyInfo.nField + pKeyInfo.nXField
				elif pOp.p4type == CConfig.P4_INT32:
					nField = pOp.p4.i

				assert(pOp.p1 >= 0)
				assert(nField >= 0)
				#   testcase( nField==0 );  /* Table with INTEGER PRIMARY KEY and nothing else */
				pCur = allocateCursor(p, pOp.p1, nField, iDb, 1)
				#   if( pCur==0 ) goto no_mem;
				pCur.nullRow = 1
				pCur.isOrdered = 1
				#   rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor);
				pCur.pKeyInfo = pKeyInfo
				#   assert( OPFLAG_BULKCSR==BTREE_BULKLOAD );
				#   sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR));

				#   /* Since it performs no memory allocation or IO, the only value that
				#   ** sqlite3BtreeCursor() may return is SQLITE_OK. */
				assert(rc == CConfig.SQLITE_OK)
				#   /* Set the VdbeCursor.isTable variable. Previous versions of
				#   ** SQLite used to check if the root-page flags were sane at this point
				#   ** and report database corruption if they were not, but this check has
				#   ** since moved into the btree layer.  */
				pCur.isTable = pOp.p4type != CConfig.P4_KEYINFO
				break
		else:
			# raise Exception("unimplemented bytecode %s " % pOp.opcode)
			pass

		pc += 1

	return pc

def run():
	db = opendb(testdb)
	ops = prepare(db, 'select name from contacts;')
	pc = mainloop(ops)

def entry_point(argv):
    run()
    return 0

def target(*args):
    return entry_point, None

if __name__ == "__main__":
    entry_point()
<commit_msg>TyperError: arithmetic not supported on <UINT/INT/UCHAR>, its size is too small.<commit_after>from rpython.rtyper.lltypesystem import rffi, lltype
from capi import CConfig
from rpython.rlib.rarithmetic import intmask
import sys
import os
import capi

testdb = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.db")

def opendb(db_name):
	with rffi.scoped_str2charp(db_name) as db_name, lltype.scoped_alloc(capi.SQLITE3PP.TO, 1) as result:
		errorcode = capi.sqlite3_open(db_name, result)
		assert errorcode == 0
		return rffi.cast(capi.SQLITE3P, result[0])

def prepare(db, query):
	length = len(query)
	with rffi.scoped_str2charp(query) as query, lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as result, lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as unused_buffer:
		errorcode = capi.sqlite3_prepare(db, query, length, result, unused_buffer)
		assert errorcode == 0
		return rffi.cast(capi.VDBEP, result[0])

def allocateCursor(vdbe_struct, p1, nField, iDb, isBtreeCursor):
	return capi.sqlite3_allocateCursor(vdbe_struct, p1, nField, iDb, isBtreeCursor)

def sqlite3VdbeMemIntegerify(pMem):
	return capi.sqlite3_sqlite3VdbeMemIntegerify(pMem)

def sqlite3BtreeCursor(p, iTable, wrFlag, pKeyInfo, pCur):
	return capi.sqlite3_sqlite3BtreeCursor(p, iTable, wrFlag, pKeyInfo, pCur)

def mainloop(vdbe_struct):
	pc = 0
	length = vdbe_struct.nOp
	ops = vdbe_struct.aOp
	rc = CConfig.SQLITE_OK
	db = vdbe_struct.db
	p = vdbe_struct
	aMem = p.aMem

	while pc < length:
		pOp = ops[pc]

		if pOp.opcode == CConfig.OP_Init:
			if pOp.p2:
				pc = pOp.p2 - 1
		elif pOp.opcode == CConfig.OP_OpenRead or pOp.opcode == CConfig.OP_OpenWrite:
			assert(pOp.p5 & (CConfig.OPFLAG_P2ISREG | CConfig.OPFLAG_BULKCSR) == pOp.p5)
			assert(pOp.opcode == CConfig.OP_OpenWrite or pOp.p5 == 0)
			assert(p.bIsReader)
			assert(pOp.opcode == CConfig.OP_OpenRead or p.readOnly == 0)

			if (p.expired):
				rc = CConfig.SQLITE_ABORT
				break

			nField = 0
			pKeyInfo = lltype.nullptr(capi.KEYINFO)
			p2 = pOp.p2;
			iDb = pOp.p3
			wrFlag = 0

			assert(iDb >= 0 and iDb < db.nDb)
			#   assert( (p->btreeMask & (((yDbMask)1)<<iDb))!=0 );

			pDb = db.aDb[iDb]
			pX = pDb.pBt

			assert(not pX)

			if pOp.opcode == CConfig.OP_OpenWrite:
				wrFlag = 1
				#     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
				if pDb.pSchema.file_format < p.minWriteFileFormat:
					p.minWriteFileFormat = pDb.pSchema.file_format
				else:
					wrFlag = 0

			if pOp.p5 & CConfig.OPFLAG_P2ISREG:
				assert(p2 > 0)
				assert(p2 <= p.nMem - p.nCursor)
				pIn2 = aMem[p2]
				#     assert( memIsValid(pIn2) );
				#     assert( (pIn2->flags & MEM_Int)!=0 );
				sqlite3VdbeMemIntegerify(pIn2)
				p2 = rffi.cast(rffi.INT, pIn2.u.i)
				#     /* The p2 value always comes from a prior OP_CreateTable opcode and
				#     ** that opcode will always set the p2 value to 2 or more or else fail.
				#     ** If there were a failure, the prepared statement would have halted
				#     ** before reaching this instruction. */
				#     if( NEVER(p2<2) ) {
				#       rc = SQLITE_CORRUPT_BKPT;
				#       goto abort_due_to_error;
				#     }
				if pOp.p4type == CConfig.P4_KEYINFO:
					pKeyInfo = pOp.p4.pKeyInfo
					#     assert( pKeyInfo->enc==ENC(db) );
					assert(pKeyInfo.db == db)
					nField = intmask(pKeyInfo.nField) + intmask(pKeyInfo.nXField)
				elif pOp.p4type == CConfig.P4_INT32:
					nField = pOp.p4.i

				assert(pOp.p1 >= 0)
				assert(nField >= 0)
				#   testcase( nField==0 );  /* Table with INTEGER PRIMARY KEY and nothing else */
				pCur = allocateCursor(p, pOp.p1, nField, iDb, 1)
				#   if( pCur==0 ) goto no_mem;
				pCur.nullRow = rffi.cast(rffi.UCHAR, 1) #rffi.r_uchar(1)
				pCur.isOrdered = bool(1)
				#   rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor);
				rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur.pCursor)
				pCur.pKeyInfo = pKeyInfo
				#   assert( OPFLAG_BULKCSR==BTREE_BULKLOAD );
				#   sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR));

				#   /* Since it performs no memory allocation or IO, the only value that
				#   ** sqlite3BtreeCursor() may return is SQLITE_OK. */
				assert(rc == CConfig.SQLITE_OK)
				#   /* Set the VdbeCursor.isTable variable. Previous versions of
				#   ** SQLite used to check if the root-page flags were sane at this point
				#   ** and report database corruption if they were not, but this check has
				#   ** since moved into the btree layer.  */
				pCur.isTable = pOp.p4type != CConfig.P4_KEYINFO
				break
		else:
			# raise Exception("unimplemented bytecode %s " % pOp.opcode)
			pass

		pc += 1

	return pc

def run():
	db = opendb(testdb)
	ops = prepare(db, 'select name from contacts;')
	pc = mainloop(ops)

def entry_point(argv):
    run()
    return 0

def target(*args):
    return entry_point, None

if __name__ == "__main__":
    entry_point()
<|endoftext|>