<commit_before># -*- coding: utf-8 -*-

import types
import urlparse

import requests_oauthlib as roauth
import pandas as pd

import utils

BASE_URL = 'https://api.tradeking.com/v1'

# TODO(jkoelker) Would be nice to do a proper DSL
class OptionQuery(object):
    FIELDS = ('strikeprice', 'xdate', 'xmonth', 'xyear', 'put_call', 'unique')
    OPS = {'<': 'lt', 'lt': 'lt',
           '>': 'gt', 'gt': 'gt',
           '>=': 'gte', 'gte': 'gte',
           '<=': 'lte', 'lte': 'lte',
           '=': 'eq', '==': 'eq', 'eq': 'eq'}

    def __init__(self, query):
        if isinstance(query, types.StringTypes):
            query = [query]

        self._query = []

        for part in query:
            field, op, value = part.split()
            field = field.lower()

            if field not in self.FIELDS or op not in self.OPS:
                continue

            self._query.append((field, self.OPS[op], value))

    def __str__(self):
        return ' AND '.join(['%s-%s:%s' % (field, op, value)
                             for field, op, value in self._query])

class API(object):
    def __init__(self, consumer_key, consumer_secret,
                 oauth_token, oauth_secret):
        self._api = roauth.OAuth1Session(client_key=consumer_key,
                                         client_secret=consumer_secret,
                                         resource_owner_key=oauth_token,
                                         resource_owner_secret=oauth_secret)

    def join(self, *paths, **kwargs):
        if len(paths) == 1:
            paths = paths[0]

        if kwargs.get('clean', True):
            paths = [p.rstrip('/') for p in paths]

        return '/'.join(paths)

    def request(self, method, url, format='json', decode=True, **kwargs):
        if format:
            url = '.'.join((url, format))

        r = self._api.request(method, url, **kwargs)

        if decode:
            r = r.json()

        return r

    def get(self, url, format='json', decode=True, **kwargs):
        return self.request('GET', url=url, format=format, decode=decode,
                            **kwargs)

    def post(self, url, format='json', decode=True, **kwargs):
        return self.request('POST', url=url, format=format, decode=decode,
                            **kwargs)

class Account(object):
    def __init__(self, api, account_id):
        self._api = api
        self.account_id = account_id

    def _get(self, what=None, **kwargs):
        params = [BASE_URL, 'accounts', self.account_id]

        if what is not None:
            params.append(what)

        path = self._api.join(params)
        return self._api.get(path, **kwargs)

    def _balances(self, **kwargs):
        return self._get('balances', **kwargs)

    def _history(self, date_range='all', transactions='all', **kwargs):
        params = {'range': date_range, 'transactions': transactions}
        return self._get('history', params=params, **kwargs)

    def _holdings(self, **kwargs):
        return self._get('holdings', **kwargs)

    def _orders(self, **kwargs):
        return self._get('orders', **kwargs)

    @property
    def balances(self):
        r = self._balances()
        return r['response']['accountbalance']

    def history(self, date_range='all', transactions='all'):
        r = self._history(date_range=date_range, transactions=transactions)
        return r['response']['transactions']['transaction']

    @property
    def holdings(self):
        r = self._holdings()
        return r['response']['accountholdings']['holding']

    # TODO(jkoelker)
    def order(self, order, preview=True):
        pass

    @property
    def orders(self):
        r = self._orders()
        return r['response']['orderstatus']

class News(object):
    def __init__(self, api):
        self._api = api

    def _article(self, article_id, **kwargs):
        path = self._api.join(BASE_URL, 'market', 'news', article_id)
        return self._api.get(path, **kwargs)

    def _search(self, keywords=None, symbols=None, maxhits=None,
                startdate=None, enddate=None, **kwargs):
        if not keywords and not symbols:
            raise ValueError('Either keywords or symbols are required')

        data = {}

        if keywords:
            if isinstance(keywords, types.StringTypes):
                keywords = [keywords]

            data['keyworks'] = ','.join(keywords)

        if symbols:
            if isinstance(symbols, types.StringTypes):
                symbols = [symbols]

            data['symbols'] = ','.join(symbols)

        if maxhits:
            data['maxhits'] = maxhits

        # TODO(jkoelker) calculate enddate to be now()
        if (not startdate and enddate) or (not enddate and startdate):
            raise ValueError('Both startdate and endate are required if one '
                             'is specified')

        if startdate and enddate:
            data['startdate'] = startdate
            data['enddate'] = enddate

        path = self._api.join(BASE_URL, 'market', 'news', 'search')
        return self._api.post(path, data=data, **kwargs)

    def article(self, article_id):
        r = self._article(article_id=article_id)
        return r['response']['article']

    def search(self, keywords=None, symbols=None, maxhits=None, startdate=None,
               enddate=None):
        r = self._search(keywords=keywords, symbols=symbols, maxhits=maxhits,
                         startdate=startdate, enddate=enddate)
        return r['response']['articles']['article']

class Options(object):
    def __init__(self, api):
        self._api = api

    @staticmethod
    def symbol(underlying, expiration, call_put, strike):
        call_put = call_put.upper()
        if call_put not in ('C', 'P'):
            raise ValueError("call_put value not one of ('C', 'P'): %s" %
                             call_put)

        expiration = pd.to_datetime(expiration).strftime('%y%m%d')

        strike = str(int(strike * 1000))
        strike = ('0' * (8 - len(strike))) + strike

        return '%s%s%s%s' % (underlying, expiration, call_put, strike)

    def _expirations(self, symbol, **kwargs):
        params = {'symbol': symbol}
        path = self._api.join(BASE_URL, 'market', 'options', 'expirations')
        return self._api.get(path, params=params, **kwargs)

    def _search(self, symbol, query, fields=None, query_is_prepared=False,
                **kwargs):
        if not isinstance(query, OptionQuery) and not query_is_prepared:
            query = OptionQuery(query)

        data = {'symbol': symbol, 'query': query}

        if fields is not None:
            data['fids'] = ','.join(fields)

        path = self._api.join(BASE_URL, 'market', 'options', 'search')
        return self._api.post(path, data=data, **kwargs)

    def _strikes(self, symbol, **kwargs):
        params = {'symbol': symbol}
        path = self._api.join(BASE_URL, 'market', 'options', 'strikes')
        return self._api.get(path, params=params, **kwargs)

    def expirations(self, symbol):
        r = self._expirations(symbol=symbol)
        expirations = r['response']['expirationdates']['date']
        return pd.to_datetime(pd.Series(expirations))

    def search(self, symbol, query, fields=None):
        r = self._search(symbol=symbol, query=query, fields=fields)
        quotes = r['response']['quotes']['quote']
        return pd.DataFrame.from_records(quotes, index='symbol')

    def strikes(self, symbol):
        r = self._strikes(symbol=symbol)
        strikes = r['response']['prices']['price']
        return pd.Series(strikes, dtype=float)

class Market(object):
    def __init__(self, api):
        self._api = api

    def _clock(self, **kwargs):
        path = self._api.join(BASE_URL, 'market', 'clock')
        return self._api.get(path, **kwargs)

    def _quotes(self, symbols, fields=None, **kwargs):
        if isinstance(symbols, (types.ListType, types.TupleType)):
            symbols = ','.join(symbols)

        params = {'symbols': symbols}

        if fields is not None:
            params['fids'] = ','.join(fields)

        path = self._api.join(BASE_URL, 'market', 'ext', 'quotes')
        return self._api.get(path, params=params, **kwargs)

    def _toplist(self, list_type='toppctgainers', **kwargs):
        path = self._api.join(BASE_URL, 'market', 'toplists', list_type)
        return self._api.get(path, **kwargs)

    @property
    def clock(self):
        r = self._clock()
        r = r['response']
        del r['@id']
        return r

    @utils.cached_property(ttl=0)
    def news(self):
        return News(self._api)

    @utils.cached_property(ttl=0)
    def options(self):
        return Options(self._api)

    def quotes(self, symbols, fields=None):
        r = self._quotes(symbols=symbols, fields=fields)
        return r['response']['quotes']['quote']

    def toplist(self, list_type='toppctgainers'):
        r = self._toplist(list_type=list_type)
        return r['response']['quotes']['quote']

    # TODO(jkoelker) market/timesales
    # TODO(jkoelker) market/quotes (iterator)

class TradeKing(object):
    def __init__(self, consumer_key, consumer_secret,
                 oauth_token, oauth_secret):
        self._api = API(consumer_key=consumer_key,
                        consumer_secret=consumer_secret,
                        oauth_token=oauth_token,
                        oauth_secret=oauth_secret)

    def _accounts(self, **kwargs):
        path = urlparse.urljoin(BASE_URL, 'accounts')
        return self._api.get(path, **kwargs)

    def account(self, account_id):
        return Account(self._api, account_id)

    @utils.cached_property(ttl=0)
    def market(self):
        return Market(self._api)

    # TODO(jkoelker) member/profile
    # TODO(jkoelker) utility/status
    # TODO(jkoelker) utility/version
    # TODO(jkoelker) utility/version
    # TODO(jkoelker) watchlists
<commit_msg>Fix typo<commit_after># -*- coding: utf-8 -*-

import types
import urlparse

import requests_oauthlib as roauth
import pandas as pd

import utils

BASE_URL = 'https://api.tradeking.com/v1'

# TODO(jkoelker) Would be nice to do a proper DSL
class OptionQuery(object):
    FIELDS = ('strikeprice', 'xdate', 'xmonth', 'xyear', 'put_call', 'unique')
    OPS = {'<': 'lt', 'lt': 'lt',
           '>': 'gt', 'gt': 'gt',
           '>=': 'gte', 'gte': 'gte',
           '<=': 'lte', 'lte': 'lte',
           '=': 'eq', '==': 'eq', 'eq': 'eq'}

    def __init__(self, query):
        if isinstance(query, types.StringTypes):
            query = [query]

        self._query = []

        for part in query:
            field, op, value = part.split()
            field = field.lower()

            if field not in self.FIELDS or op not in self.OPS:
                continue

            self._query.append((field, self.OPS[op], value))

    def __str__(self):
        return ' AND '.join(['%s-%s:%s' % (field, op, value)
                             for field, op, value in self._query])

class API(object):
    def __init__(self, consumer_key, consumer_secret,
                 oauth_token, oauth_secret):
        self._api = roauth.OAuth1Session(client_key=consumer_key,
                                         client_secret=consumer_secret,
                                         resource_owner_key=oauth_token,
                                         resource_owner_secret=oauth_secret)

    def join(self, *paths, **kwargs):
        if len(paths) == 1:
            paths = paths[0]

        if kwargs.get('clean', True):
            paths = [p.rstrip('/') for p in paths]

        return '/'.join(paths)

    def request(self, method, url, format='json', decode=True, **kwargs):
        if format:
            url = '.'.join((url, format))

        r = self._api.request(method, url, **kwargs)

        if decode:
            r = r.json()

        return r

    def get(self, url, format='json', decode=True, **kwargs):
        return self.request('GET', url=url, format=format, decode=decode,
                            **kwargs)

    def post(self, url, format='json', decode=True, **kwargs):
        return self.request('POST', url=url, format=format, decode=decode,
                            **kwargs)

class Account(object):
    def __init__(self, api, account_id):
        self._api = api
        self.account_id = account_id

    def _get(self, what=None, **kwargs):
        params = [BASE_URL, 'accounts', self.account_id]

        if what is not None:
            params.append(what)

        path = self._api.join(params)
        return self._api.get(path, **kwargs)

    def _balances(self, **kwargs):
        return self._get('balances', **kwargs)

    def _history(self, date_range='all', transactions='all', **kwargs):
        params = {'range': date_range, 'transactions': transactions}
        return self._get('history', params=params, **kwargs)

    def _holdings(self, **kwargs):
        return self._get('holdings', **kwargs)

    def _orders(self, **kwargs):
        return self._get('orders', **kwargs)

    @property
    def balances(self):
        r = self._balances()
        return r['response']['accountbalance']

    def history(self, date_range='all', transactions='all'):
        r = self._history(date_range=date_range, transactions=transactions)
        return r['response']['transactions']['transaction']

    @property
    def holdings(self):
        r = self._holdings()
        return r['response']['accountholdings']['holding']

    # TODO(jkoelker)
    def order(self, order, preview=True):
        pass

    @property
    def orders(self):
        r = self._orders()
        return r['response']['orderstatus']

class News(object):
    def __init__(self, api):
        self._api = api

    def _article(self, article_id, **kwargs):
        path = self._api.join(BASE_URL, 'market', 'news', article_id)
        return self._api.get(path, **kwargs)

    def _search(self, keywords=None, symbols=None, maxhits=None,
                startdate=None, enddate=None, **kwargs):
        if not keywords and not symbols:
            raise ValueError('Either keywords or symbols are required')

        data = {}

        if keywords:
            if isinstance(keywords, types.StringTypes):
                keywords = [keywords]

            data['keywords'] = ','.join(keywords)

        if symbols:
            if isinstance(symbols, types.StringTypes):
                symbols = [symbols]

            data['symbols'] = ','.join(symbols)

        if maxhits:
            data['maxhits'] = maxhits

        # TODO(jkoelker) calculate enddate to be now()
        if (not startdate and enddate) or (not enddate and startdate):
            raise ValueError('Both startdate and endate are required if one '
                             'is specified')

        if startdate and enddate:
            data['startdate'] = startdate
            data['enddate'] = enddate

        path = self._api.join(BASE_URL, 'market', 'news', 'search')
        return self._api.post(path, data=data, **kwargs)

    def article(self, article_id):
        r = self._article(article_id=article_id)
        return r['response']['article']

    def search(self, keywords=None, symbols=None, maxhits=None, startdate=None,
               enddate=None):
        r = self._search(keywords=keywords, symbols=symbols, maxhits=maxhits,
                         startdate=startdate, enddate=enddate)
        return r['response']['articles']['article']

class Options(object):
    def __init__(self, api):
        self._api = api

    @staticmethod
    def symbol(underlying, expiration, call_put, strike):
        call_put = call_put.upper()
        if call_put not in ('C', 'P'):
            raise ValueError("call_put value not one of ('C', 'P'): %s" %
                             call_put)

        expiration = pd.to_datetime(expiration).strftime('%y%m%d')

        strike = str(int(strike * 1000))
        strike = ('0' * (8 - len(strike))) + strike

        return '%s%s%s%s' % (underlying, expiration, call_put, strike)

    def _expirations(self, symbol, **kwargs):
        params = {'symbol': symbol}
        path = self._api.join(BASE_URL, 'market', 'options', 'expirations')
        return self._api.get(path, params=params, **kwargs)

    def _search(self, symbol, query, fields=None, query_is_prepared=False,
                **kwargs):
        if not isinstance(query, OptionQuery) and not query_is_prepared:
            query = OptionQuery(query)

        data = {'symbol': symbol, 'query': query}

        if fields is not None:
            data['fids'] = ','.join(fields)

        path = self._api.join(BASE_URL, 'market', 'options', 'search')
        return self._api.post(path, data=data, **kwargs)

    def _strikes(self, symbol, **kwargs):
        params = {'symbol': symbol}
        path = self._api.join(BASE_URL, 'market', 'options', 'strikes')
        return self._api.get(path, params=params, **kwargs)

    def expirations(self, symbol):
        r = self._expirations(symbol=symbol)
        expirations = r['response']['expirationdates']['date']
        return pd.to_datetime(pd.Series(expirations))

    def search(self, symbol, query, fields=None):
        r = self._search(symbol=symbol, query=query, fields=fields)
        quotes = r['response']['quotes']['quote']
        return pd.DataFrame.from_records(quotes, index='symbol')

    def strikes(self, symbol):
        r = self._strikes(symbol=symbol)
        strikes = r['response']['prices']['price']
        return pd.Series(strikes, dtype=float)

class Market(object):
    def __init__(self, api):
        self._api = api

    def _clock(self, **kwargs):
        path = self._api.join(BASE_URL, 'market', 'clock')
        return self._api.get(path, **kwargs)

    def _quotes(self, symbols, fields=None, **kwargs):
        if isinstance(symbols, (types.ListType, types.TupleType)):
            symbols = ','.join(symbols)

        params = {'symbols': symbols}

        if fields is not None:
            params['fids'] = ','.join(fields)

        path = self._api.join(BASE_URL, 'market', 'ext', 'quotes')
        return self._api.get(path, params=params, **kwargs)

    def _toplist(self, list_type='toppctgainers', **kwargs):
        path = self._api.join(BASE_URL, 'market', 'toplists', list_type)
        return self._api.get(path, **kwargs)

    @property
    def clock(self):
        r = self._clock()
        r = r['response']
        del r['@id']
        return r

    @utils.cached_property(ttl=0)
    def news(self):
        return News(self._api)

    @utils.cached_property(ttl=0)
    def options(self):
        return Options(self._api)

    def quotes(self, symbols, fields=None):
        r = self._quotes(symbols=symbols, fields=fields)
        return r['response']['quotes']['quote']

    def toplist(self, list_type='toppctgainers'):
        r = self._toplist(list_type=list_type)
        return r['response']['quotes']['quote']

    # TODO(jkoelker) market/timesales
    # TODO(jkoelker) market/quotes (iterator)

class TradeKing(object):
    def __init__(self, consumer_key, consumer_secret,
                 oauth_token, oauth_secret):
        self._api = API(consumer_key=consumer_key,
                        consumer_secret=consumer_secret,
                        oauth_token=oauth_token,
                        oauth_secret=oauth_secret)

    def _accounts(self, **kwargs):
        path = urlparse.urljoin(BASE_URL, 'accounts')
        return self._api.get(path, **kwargs)

    def account(self, account_id):
        return Account(self._api, account_id)

    @utils.cached_property(ttl=0)
    def market(self):
        return Market(self._api)

    # TODO(jkoelker) member/profile
    # TODO(jkoelker) utility/status
    # TODO(jkoelker) utility/version
    # TODO(jkoelker) utility/version
    # TODO(jkoelker) watchlists
<|endoftext|>


<commit_before>import sys
import re
import subprocess
import os

log_file = open( "translate_smt.log", 'w' )

smt_cmd = os.getenv("SMT_LOCATION")
smt_proc = subprocess.Popen([smt_cmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE)

vars = {}
int_re = re.compile("^-?([0-9])+(:([0-9]+))?$")
find_ranges = {}

same_len_ops = {
   "add" : "bvadd",
   "sub" : "bvadd",
   "negate" : "bvneg",
   "chose" : "ite",
   "and" : "bvand",
   "or" : "bvor",
   "xor" : "bvxor" }

one_bit_ops = {
    "ltes" : (True, True, True),
    "lteu" : (False, True, True),
    "gtes" : (True, False, True),
    "gteu" : (False, False, True),
    "lts" : (True, True, False),
    "ltu" : (False, True, False),
    "gts" : (True, False, False),
    "gtu" : (False, False, False) }

############# utility methods
def smt_proc_write( val ):
    log_file.write(val)
    smt_proc.stdin.write(val)

def introduce_var(name,rng):
    global vars
    vars[name] = rng
    smt_proc_write("(declare-fun "+name+" () (_ BitVec "+str(rng)+"))\n")

def const_to_bv( match ):
    mag = int(match.group(1))
    size = match.group(3)
    if size == None:
        size = len(bin(mag))-2
    return ( const_bv_literal(mag,int(size)), int(size) )

def arg_to_smt(arg):
    match = int_re.match(arg)
    if match == None:
        return (arg,vars[arg])
    return const_to_bv( match )

def const_bv_literal( num, size ):
    raw = bin(num)[2:]
    raw = ( "0"*(size-len(raw) ) ) + raw
    return "#b"+raw

def const_arg_to_smt( arg ):
    match = int_re.match(arg)
    mag = int( match.group(1) )
    size = match.group(3)
    if size == None:
        size = len(bin(mag))-2
    return ( mag, int(size) )

def assert_smt(statement):
    return "(assert "+statement+")\n"

def assign_smt(statement,out):
    return "(= "+out+" "+statement+")"

########### end utility methods

def comparison(a, b, is_signed, is_lt, is_eq):
    ret = "(bvult "
    arga = arg_to_smt(a)
    argb = arg_to_smt(b)
    if is_lt:
        if is_eq:
            ret += argb[0] + " " + arga[0] + ")"
            ret = "(not "+ret+")"
        else:
            ret += arga[0] + " " + argb[0] + ")"
    else:
        if is_eq:
            ret += argb[0] + " " + arga[0] + ")"
        else:
            ret += arga[0] + " " + argb[0] + ")"
            ret = "(not "+ret+")"
    return ret

def map_ops(op, args):
    retval = "("
    arg_size = 0
    if op in same_len_ops:
        retval += same_len_ops[op]
    else:
        compvals = one_bit_ops[op]
        retval = comparison( args[0], args[1], compvals[0], compvals[1], compvals[2] )
        arg_size = 1
        return ( retval, arg_size )
    for i in xrange(len(args)):
        vals = arg_to_smt(args[i])
        vals_bv_form = vals[0]
        if i == 0 and op == "chose":
            retval += " (= "+vals_bv_form+" #b1)"
        elif i == 1 and op == "sub":
            retval += " (bvneg "+vals_bv_form+")"
        else:
            retval += " "+vals_bv_form
        if op in same_len_ops:
            arg_size = vals[1]
    return ( retval + ")", arg_size )

def sp_shiftl(args):
    arg1val = const_arg_to_smt(args[1])
    if arg1val == None:
        raise Error("shiftl must shift by constant amount")
    arg2val = arg_to_smt(args[0])
    state = "(bvshl "+arg2val[0]+" "+const_bv_literal(arg1val[0],arg1val[1])+")"
    return (state, arg1val[0] + arg2val[1] )

def sp_shiftr(args):
    arg1val = const_arg_to_smt(args[1])
    if arg1val == None:
        raise Error("shiftr must shift by constant amount")
    arg2val = arg_to_smt(args[0])
    state = "(bvshr "+arg2val[0]+" "+const_bv_literal(arg1val[0],arg1val[1])+")"
    return (state, arg2val[1] - arg1val[0] )

#TODO: must support decode operation
def sp_concatls(args):
    first = arg_to_smt(args[0])
    ret = first[0]
    bit_size_accum = first[1]
    for i in xrange(1, len(args) ):
        val = arg_to_smt( args[i] )
        ret = "(concat "+ret+" "+val[0]+")"
        bit_size_accum += val[1]
    return ( ret, bit_size_accum )

def sp_concat(args):
    ret = ""
    bit_size_accum = 0
    num_close = 0
    for i in xrange( len(args) ):
        val = arg_to_smt( args[i] )
        if i != len(args)-1:
            ret += "(concat "+val[0]+" "
            num_close += 1
        else:
            ret += val[0]
        bit_size_accum += val[1]
    ret += ")"*num_close
    return ( ret, bit_size_accum )

def sp_select(args):
    ret = "((_ extract "
    from_arg = const_arg_to_smt(args[1])
    to_arg = const_arg_to_smt( args[2] )
    base_arg = arg_to_smt( args[0] )
    ret += str(from_arg[0]) + " " + str(to_arg[0]) + ") "
    ret += base_arg[0] + ")"
    return ( ret, to_arg[0] - from_arg[0] )

def sp_sextend( args ):
    # TODO: handle signed arguments
    to_arg = const_arg_to_smt( args[1] )
    base_arg = arg_to_smt( args[0] )
    ret = "(concat #b"+("0"*(to_arg[0]-base_arg[1]))+" "+base_arg[0]+")"
    return ( ret, to_arg[0] )

special = { "shiftl" : sp_shiftl,
            "shiftr" : sp_shiftr,
            "concat" : sp_concat,
            "select" : sp_select,
            "sextend" : sp_sextend,
            "trunc" : lambda arg: sp_sextend( [ arg[0], "0:1" ] ),
            "zextend" : sp_sextend,
            "concatls" : sp_concatls }

def map_il_to_smt(op,args,out):
    if op in same_len_ops:
        op_clause = map_ops(op,args)
    elif op in one_bit_ops:
        op_clause = map_ops(op,args)
    elif op in special:
        op_clause = special[op](args)
    introduce_var(out, op_clause[1] )
    if op == "chose":
        find_ranges[out] = op_clause[1]
    assign_clause = op_clause[0]
    if op in one_bit_ops:
        assign_clause = "(ite "+assign_clause+" #b1 #b0)"
    assign_clause = assign_smt(assign_clause,out)
    assert_clause = assert_smt(assign_clause)
    smt_proc_write( assert_clause )

def map_dot_to_smt(dot_op, args):
    global vars
    if dot_op == ".input":
        bit_width = int(args[2])
        introduce_var(args[0], bit_width )
    if dot_op == ".remove":
        vars.pop( args[1] )

def header():
    return """(set-option :global-decls true)
(set-info :smt-lib-version 2.0)
(set-logic QF_BV)
"""

def make_guess(arg,lessthan):
    smt_proc_write( "(push 1)\n" )
    smt_proc_write( assert_smt("(not (bvult "+arg+" "+const_bv_literal(lessthan,vars[arg])+"))") )
    smt_proc_write("(check-sat)\n")
    line = smt_proc.stdout.readline().strip()
    log_file.write( "-> "+line+"\n" )
    if line == "sat":
        smt_proc_write( "(pop 1)\n" )
        return False
    elif line == "" or line == "unsat":
        return True
    else:
        raise Exception("SMT lib exception: "+line)

def search_range(var,end):
    # TODO: experiment with this search. try binary search
    start = 0
    max_val = end
    while max_val > start and not make_guess(var,2**start):
        start = start + 1
    return start

if __name__=="__main__":
    smt_proc_write( header() )
    file = open( sys.argv[1], 'r' )
    for line in file.xreadlines():
        args = line.split()
        if len(args) == 0:
            continue
        if args[0][0] == ".":
            map_dot_to_smt( args[0], args[1:] )
        else:
            map_il_to_smt( args[1], args[2:], args[0] )
    for i in find_ranges:
        rng = search_range( i, find_ranges[i] )
        log_file.write( i +" "+str( rng )+"\n" )

    smt_proc_write( "(exit)\n" )
    smt_proc.wait()
    log_file.close()

<commit_msg>using bit vector functions<commit_after>import sys
import re
import subprocess
import os

log_file = open( "translate_smt.log", 'w' )

smt_cmd = os.getenv("SMT_LOCATION")
smt_proc = subprocess.Popen([smt_cmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE)

vars = {}
int_re = re.compile("^-?([0-9]+)(:([0-9]+))?$")
find_ranges = {}

same_len_ops = {
   "add" : "bvadd",
   "sub" : "bvadd",
   "negate" : "bvneg",
   "chose" : "ite",
   "and" : "bvand",
   "or" : "bvor",
   "xor" : "bvxor",
   "not" : "bvnot" }

one_bit_ops = {
    "ltes" : (True, True, True, False),
    "lteu" : (False, True, True, False),
    "gtes" : (True, False, True, False),
    "gteu" : (False, False, True, False),
    "lts" :  (True, True, False, False),
    "ltu" :  (False, True, False, False),
    "gts" :  (True, False, False, False),
    "gtu" :  (False, False, False, False),
    "equ" :  (False, False, False, True),
    "nequ" : (False, False, True, True) }

############# utility methods
def smt_proc_write( val ):
    log_file.write(val)
    smt_proc.stdin.write(val)

def introduce_var(name,rng):
    global vars
    vars[name] = rng
    smt_proc_write("(declare-fun "+name+" () (_ BitVec "+str(rng)+"))\n")

def const_to_bv( match ):
    mag = int(match.group(1))
    size = match.group(3)
    if size == None:
        size = len(bin(mag))-2
    return ( const_bv_literal(mag,int(size)), int(size) )

def arg_to_smt(arg):
    match = int_re.match(arg)
    if match == None:
        return (arg,vars[arg])
    return const_to_bv( match )

def const_bv_literal( num, size ):
    raw = bin(num)[2:]
    raw = ( "0"*(size-len(raw) ) ) + raw
    return "#b"+raw

def const_arg_to_smt( arg ):
    match = int_re.match(arg)
    mag = int( match.group(1) )
    size = match.group(3)
    if size == None:
        size = len(bin(mag))-2
    return mag
#    return ( mag, int(size) )

def assert_smt(statement):
    return "(assert "+statement+")\n"

def assign_smt(statement,out):
    return "(= "+out+" "+statement+")"

########### end utility methods

def comparison(a, b, params):
    ( is_signed, is_lt, is_eq_comp, is_equ_fun ) = params
    ret = "(bvult "
    arga = arg_to_smt(a)
    argb = arg_to_smt(b)
    if is_equ_fun:
        ret = "(= "+arga[0]+" "+argb[0]+")"
        if not is_eq_comp:
            ret = "(not "+ret+")"
        return ret
    if is_lt:
        if is_eq_comp:
            ret += argb[0] + " " + arga[0] + ")"
            ret = "(not "+ret+")"
        else:
            ret += arga[0] + " " + argb[0] + ")"
    else:
        if is_eq_comp:
            ret += argb[0] + " " + arga[0] + ")"
        else:
            ret += arga[0] + " " + argb[0] + ")"
            ret = "(not "+ret+")"
    return ret

def map_ops(op, args):
    retval = "("
    arg_size = 0
    if op in same_len_ops:
        retval += same_len_ops[op]
    elif op in one_bit_ops:
        compvals = one_bit_ops[op]
        retval = comparison( args[0], args[1], compvals )
        arg_size = 1
        return ( retval, arg_size )
    for i in xrange(len(args)):
        vals = arg_to_smt(args[i])
        vals_bv_form = vals[0]
        if i == 0 and op == "chose":
            retval += " (= "+vals_bv_form+" #b1)"
        elif i == 1 and op == "sub":
            retval += " (bvneg "+vals_bv_form+")"
        else:
            retval += " "+vals_bv_form
        if op in same_len_ops:
            arg_size = vals[1]
    return ( retval + ")", arg_size )

def sp_shiftl(args):
    arg1val = const_arg_to_smt(args[1])
    if arg1val == None:
        raise Error("shiftl must shift by constant amount")
    arg2val = arg_to_smt(args[0])
    state = "(bvshl "+arg2val[0]+" "+const_bv_literal(arg1val,arg1val[1])+")"
    return (state, arg1val[0] + arg2val[1] )

def sp_shiftr(args):
    arg1val = const_arg_to_smt(args[1])
    if arg1val == None:
        raise Error("shiftr must shift by constant amount")
    arg2val = arg_to_smt(args[0])
    state = "(bvshr "+arg2val[0]+" "+const_bv_literal(arg1val,arg1val[1])+")"
    return (state, arg2val[1] - arg1val[0] )

#TODO: must support decode operation
def sp_concatls(args):
    first = arg_to_smt(args[0])
    ret = first[0]
    bit_size_accum = first[1]
    for i in xrange(1, len(args) ):
        val = arg_to_smt( args[i] )
        ret = "(concat "+ret+" "+val[0]+")"
        bit_size_accum += val[1]
    return ( ret, bit_size_accum )

def sp_concat(args):
    ret = ""
    bit_size_accum = 0
    num_close = 0
    for i in xrange( len(args) ):
        val = arg_to_smt( args[i] )
        if i != len(args)-1:
            ret += "(concat "+val[0]+" "
            num_close += 1
        else:
            ret += val[0]
        bit_size_accum += val[1]
    ret += ")"*num_close
    return ( ret, bit_size_accum )

def sp_select(args):
    ret = "((_ extract "
    from_arg = const_arg_to_smt(args[1])
    to_arg = const_arg_to_smt( args[2] )
    base_arg = arg_to_smt( args[0] )
    ret += str(to_arg-1) + " " + str(from_arg) + ") "
    ret += base_arg[0] + ")"
    return ( ret, to_arg - from_arg )

def sp_sextend( args ):
    # TODO: handle signed arguments
    to_arg = const_arg_to_smt( args[1] )
    base_arg = arg_to_smt( args[0] )
    ret = "(concat #b"+("0"*(to_arg-base_arg[1]))+" "+base_arg[0]+")"
    return ( ret, to_arg )

special = { "shiftl" : sp_shiftl,
            "shiftr" : sp_shiftr,
            "concat" : sp_concat,
            "select" : sp_select,
            "sextend" : sp_sextend,
            "trunc" : lambda arg: sp_sextend( [ arg[0], "0:1" ] ),
            "zextend" : sp_sextend,
            "concatls" : sp_concatls }

def map_il_to_smt(op,args,out):
    if op in same_len_ops:
        op_clause = map_ops(op,args)
    elif op in one_bit_ops:
        op_clause = map_ops(op,args)
    elif op in special:
        op_clause = special[op](args)
    else:
        log_file.write("Error: unknown function "+op+"\n")
        raise Exception("unknown function "+op)
    introduce_var(out, op_clause[1] )
    if op == "chose":
        find_ranges[out] = op_clause[1]
    assign_clause = op_clause[0]
    if op in one_bit_ops:
        assign_clause = "(ite "+assign_clause+" #b1 #b0)"
    assign_clause = assign_smt(assign_clause,out)
    assert_clause = assert_smt(assign_clause)
    smt_proc_write( assert_clause )

def map_dot_to_smt(dot_op, args):
    global vars
    if dot_op == ".input":
        bit_width = int(args[2])
        introduce_var(args[0], bit_width )
    # if dot_op == ".remove":
    #     del vars[args[0]]

def header():
    return """(set-option :global-decls true)
(set-info :smt-lib-version 2.0)
(set-logic QF_BV)
"""

def make_guess(arg,lessthan):
    smt_proc_write( "(push 1)\n" )
    smt_proc_write( assert_smt("(not (bvult "+arg+" "+const_bv_literal(lessthan,vars[arg])+"))") )
    smt_proc_write("(check-sat)\n")
    line = smt_proc.stdout.readline().strip()
    log_file.write( "-> "+line+"\n" )
    if line == "sat":
        smt_proc_write( "(pop 1)\n" )
        return False
    elif line == "" or line == "unsat":
        print arg
        return True
    else:
        raise Exception("SMT lib exception: "+line)

def search_range(var,end):
    # TODO: experiment with this search. try binary search
    start = 0
    max_val = end
    while max_val > start and not make_guess(var,2**start):
        start = start + 1
    return start

if __name__=="__main__":
    smt_proc_write( header() )
    file = open( sys.argv[1], 'r' )
    for line in file.xreadlines():
        args = line.split()
        if len(args) == 0:
            continue
        if args[0][0] == ".":
            map_dot_to_smt( args[0], args[1:] )
        else:
            map_il_to_smt( args[1], args[2:], args[0] )
    for i in find_ranges:
        rng = search_range( i, find_ranges[i] )
        log_file.write( i +" "+str( rng )+"\n" )

    smt_proc_write( "(exit)\n" )
    smt_proc.wait()
    log_file.close()
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import functools

import flask_sqlalchemy

db = flask_sqlalchemy.SQLAlchemy()
pk_type = db.Integer
pk = functools.partial(db.Column, pk_type, primary_key=True)

class Namespace(db.Model):
    id = pk()
    name = db.Column(db.String(128))

# Component registry

class Component(db.Model):
    id = pk()
    name = db.Column(db.String(128))
    schemas = db.relationship("Schema", backref="component")
    templates = db.relationship("Template", backref="component")

class Schema(db.Model):
    id = pk()
    name = db.Column(db.String(128))
    component_id = db.Column(pk_type, db.ForeignKey(Component.id))
    namespace_id = db.Column(pk_type, db.ForeignKey(Namespace.id))
    content = db.Column(db.Text)

class Template(db.Model):
    id = pk()
    name = db.Column(db.String(128))
    component_id = db.Column(pk_type, db.ForeignKey(Component.id))
    content = db.Column(db.Text)

# Environment data storage

_environment_components = db.Table(
    'environment_components',
    db.Column('environment_id', pk_type, db.ForeignKey('environment.id')),
    db.Column('component_id', pk_type, db.ForeignKey('component.id')),
)

class Environment(db.Model):
    id = pk()
    components = db.relationship(Component, secondary=_environment_components)

class TemplateInstance(db.Model):
    id = pk()
    template_id = db.Column(pk_type, db.ForeignKey(Template.id))
    environment_id = db.Column(pk_type, db.ForeignKey(Environment.id))
    content = db.Column(db.Text)

class NamespaceInstance(db.Model):
    id = pk()
    namespace_id = db.Column(pk_type, db.ForeignKey(Namespace.id))
    environment_id = db.Column(pk_type, db.ForeignKey(Environment.id))
<commit_msg>Cut unused models for now<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import functools

import flask_sqlalchemy

db = flask_sqlalchemy.SQLAlchemy()
pk_type = db.Integer
pk = functools.partial(db.Column, pk_type, primary_key=True)

class Namespace(db.Model):
    id = pk()
    name = db.Column(db.String(128))

# Component registry

class Component(db.Model):
    id = pk()
    name = db.Column(db.String(128))
    schemas = db.relationship("Schema", backref="component")
    templates = db.relationship("Template", backref="component")

class Schema(db.Model):
    id = pk()
    name = db.Column(db.String(128))
    component_id = db.Column(pk_type, db.ForeignKey(Component.id))
    namespace_id = db.Column(pk_type, db.ForeignKey(Namespace.id))
    content = db.Column(db.Text)

class Template(db.Model):
    id = pk()
    name = db.Column(db.String(128))
    component_id = db.Column(pk_type, db.ForeignKey(Component.id))
    content = db.Column(db.Text)

# Environment data storage

_environment_components = db.Table(
    'environment_components',
    db.Column('environment_id', pk_type, db.ForeignKey('environment.id')),
    db.Column('component_id', pk_type, db.ForeignKey('component.id')),
)

class Environment(db.Model):
    id = pk()
    components = db.relationship(Component, secondary=_environment_components)
<|endoftext|>


<commit_before># Tweepy
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.

from tweepy.error import TweepError
from tweepy.utils import parse_datetime, parse_html_value, parse_a_href, \
        parse_search_datetime, unescape_html

class ResultSet(list):
    """A list like object that holds results from a Twitter API query."""

class Model(object):

    def __init__(self, api=None):
        self._api = api

    def __getstate__(self):
        # pickle
        pickle = dict(self.__dict__)
        try:
            del pickle['_api']  # do not pickle the API reference
        except KeyError:
            pass
        return pickle

    @classmethod
    def parse(cls, api, json):
        """Parse a JSON object into a model instance."""
        raise NotImplementedError

    @classmethod
    def parse_list(cls, api, json_list):
        """Parse a list of JSON objects into a result set of model instances."""
        results = ResultSet()
        for obj in json_list:
            if obj:
                results.append(cls.parse(api, obj))
        return results

class Status(Model):

    @classmethod
    def parse(cls, api, json):
        status = cls(api)
        for k, v in json.items():
            if k == 'user':
                user_model = getattr(api.parser.model_factory, 'user')
                user = user_model.parse(api, v)
                setattr(status, 'author', user)
                setattr(status, 'user', user)  # DEPRECIATED
            elif k == 'created_at':
                setattr(status, k, parse_datetime(v))
            elif k == 'source':
                if '<' in v:
                    setattr(status, k, parse_html_value(v))
                    setattr(status, 'source_url', parse_a_href(v))
                else:
                    setattr(status, k, v)
                    setattr(status, 'source_url', None)
            elif k == 'retweeted_status':
                setattr(status, k, Status.parse(api, v))
            elif k == 'place':
                if v is not None:
                    setattr(status, k, Place.parse(api, v))
                else:
                    setattr(status, k, None)
            else:
                setattr(status, k, v)
        return status

    def destroy(self):
        return self._api.destroy_status(self.id)

    def retweet(self):
        return self._api.retweet(self.id)

    def retweets(self):
        return self._api.retweets(self.id)

    def favorite(self):
        return self._api.create_favorite(self.id)

class User(Model):

    @classmethod
    def parse(cls, api, json):
        user = cls(api)
        for k, v in json.items():
            if k == 'created_at':
                setattr(user, k, parse_datetime(v))
            elif k == 'status':
                setattr(user, k, Status.parse(api, v))
            elif k == 'following':
                # twitter sets this to null if it is false
                if v is True:
                    setattr(user, k, True)
                else:
                    setattr(user, k, False)
            else:
                setattr(user, k, v)
        return user

    @classmethod
    def parse_list(cls, api, json_list):
        if isinstance(json_list, list):
            item_list = json_list
        else:
            item_list = json_list['users']

        results = ResultSet()
        for obj in item_list:
            results.append(cls.parse(api, obj))
        return results

    def timeline(self, **kargs):
        return self._api.user_timeline(user_id=self.id, **kargs)

    def friends(self, **kargs):
        return self._api.friends(user_id=self.id, **kargs)

    def followers(self, **kargs):
        return self._api.followers(user_id=self.id, **kargs)

    def follow(self):
        self._api.create_friendship(user_id=self.id)
        self.following = True

    def unfollow(self):
        self._api.destroy_friendship(user_id=self.id)
        self.following = False

    def lists_memberships(self, *args, **kargs):
        return self._api.lists_memberships(user=self.screen_name, *args, **kargs)

    def lists_subscriptions(self, *args, **kargs):
        return self._api.lists_subscriptions(user=self.screen_name, *args, **kargs)

    def lists(self, *args, **kargs):
        return self._api.lists(user=self.screen_name, *args, **kargs)

    def followers_ids(self, *args, **kargs):
        return self._api.followers_ids(user_id=self.id, *args, **kargs)

class DirectMessage(Model):

    @classmethod
    def parse(cls, api, json):
        dm = cls(api)
        for k, v in json.items():
            if k == 'sender' or k == 'recipient':
                setattr(dm, k, User.parse(api, v))
            elif k == 'created_at':
                setattr(dm, k, parse_datetime(v))
            else:
                setattr(dm, k, v)
        return dm

    def destroy(self):
        return self._api.destroy_direct_message(self.id)

class Friendship(Model):

    @classmethod
    def parse(cls, api, json):
        relationship = json['relationship']

        # parse source
        source = cls(api)
        for k, v in relationship['source'].items():
            setattr(source, k, v)

        # parse target
        target = cls(api)
        for k, v in relationship['target'].items():
            setattr(target, k, v)

        return source, target

class Category(Model):

    @classmethod
    def parse(cls, api, json):
        category = cls(api)
        for k, v in json.items():
            setattr(category, k, v)
        return category

class SavedSearch(Model):

    @classmethod
    def parse(cls, api, json):
        ss = cls(api)
        for k, v in json.items():
            if k == 'created_at':
                setattr(ss, k, parse_datetime(v))
            else:
                setattr(ss, k, v)
        return ss

    def destroy(self):
        return self._api.destroy_saved_search(self.id)

class SearchResult(Model):

    @classmethod
    def parse(cls, api, json):
        result = cls()
        for k, v in json.items():
            if k == 'created_at':
                setattr(result, k, parse_search_datetime(v))
            elif k == 'source':
                setattr(result, k, parse_html_value(unescape_html(v)))
            else:
                setattr(result, k, v)
        return result

    @classmethod
    def parse_list(cls, api, json_list, result_set=None):
        results = ResultSet()
        results.max_id = json_list.get('max_id')
        results.since_id = json_list.get('since_id')
        results.refresh_url = json_list.get('refresh_url')
        results.next_page = json_list.get('next_page')
        results.results_per_page = json_list.get('results_per_page')
        results.page = json_list.get('page')
        results.completed_in = json_list.get('completed_in')
        results.query = json_list.get('query')

        for obj in json_list['results']:
            results.append(cls.parse(api, obj))
        return results

class List(Model):

    @classmethod
    def parse(cls, api, json):
        lst = List(api)
        for k,v in json.items():
            if k == 'user':
                setattr(lst, k, User.parse(api, v))
            elif k == 'created_at':
                setattr(lst, k, parse_datetime(v))
            else:
                setattr(lst, k, v)
        return lst

    @classmethod
    def parse_list(cls, api, json_list, result_set=None):
        results = ResultSet()
        if isinstance(json_list, dict):
            json_list = json_list['lists']
        for obj in json_list:
            results.append(cls.parse(api, obj))
        return results

    def update(self, **kargs):
        return self._api.update_list(self.slug, **kargs)

    def destroy(self):
        return self._api.destroy_list(self.slug)

    def timeline(self, **kargs):
        return self._api.list_timeline(self.user.screen_name, self.slug, **kargs)

    def add_member(self, id):
        return self._api.add_list_member(self.slug, id)

    def remove_member(self, id):
        return self._api.remove_list_member(self.slug, id)

    def members(self, **kargs):
        return self._api.list_members(self.user.screen_name, self.slug, **kargs)

    def is_member(self, id):
        return self._api.is_list_member(self.user.screen_name, self.slug, id)

    def subscribe(self):
        return self._api.subscribe_list(self.user.screen_name, self.slug)

    def unsubscribe(self):
        return self._api.unsubscribe_list(self.user.screen_name, self.slug)

    def subscribers(self, **kargs):
        return self._api.list_subscribers(self.user.screen_name, self.slug, **kargs)

    def is_subscribed(self, id):
        return self._api.is_subscribed_list(self.user.screen_name, self.slug, id)

class Relation(Model):
    @classmethod
    def parse(cls, api, json):
        result = cls(api)
        for k,v in json.items():
            if k == 'value' and json['kind'] in ['Tweet', 'LookedupStatus']:
                setattr(result, k, Status.parse(api, v))
            elif k == 'results':
                setattr(result, k, Relation.parse_list(api, v))
            else:
                setattr(result, k, v)
        return result

class Relationship(Model):
    @classmethod
    def parse(cls, api, json):
        result = cls(api)
        for k,v in json.items():
            if k == 'connections':
            	setattr(result, 'is_following', 'following' in v)
                setattr(result, 'is_followed_by', 'followed_by' in v)
            else:
                setattr(result, k, v)
        return result

class JSONModel(Model):

    @classmethod
    def parse(cls, api, json):
        return json

class IDModel(Model):

    @classmethod
    def parse(cls, api, json):
        if isinstance(json, list):
            return json
        else:
            return json['ids']

class BoundingBox(Model):

    @classmethod
    def parse(cls, api, json):
        result = cls(api)
        if json is not None:
            for k, v in json.items():
                setattr(result, k, v)
        return result

    def origin(self):
        """
        Return longitude, latitude of southwest (bottom, left) corner of
        bounding box, as a tuple.

        This assumes that bounding box is always a rectangle, which
        appears to be the case at present.
        """
        return tuple(self.coordinates[0][0])

    def corner(self):
        """
        Return longitude, latitude of northeast (top, right) corner of
        bounding box, as a tuple.

        This assumes that bounding box is always a rectangle, which
        appears to be the case at present.
        """
        return tuple(self.coordinates[0][2])

class Place(Model):

    @classmethod
    def parse(cls, api, json):
        place = cls(api)
        for k, v in json.items():
            if k == 'bounding_box':
                # bounding_box value may be null (None.)
                # Example: "United States" (id=96683cc9126741d1)
                if v is not None:
                    t = BoundingBox.parse(api, v)
                else:
                    t = v
                setattr(place, k, t)
            elif k == 'contained_within':
                # contained_within is a list of Places.
                setattr(place, k, Place.parse_list(api, v))
            else:
                setattr(place, k, v)
        return place

    @classmethod
    def parse_list(cls, api, json_list):
        if isinstance(json_list, list):
            item_list = json_list
        else:
            item_list = json_list['result']['places']

        results = ResultSet()
        for obj in item_list:
            results.append(cls.parse(api, obj))
        return results

class ModelFactory(object):
    """
    Used by parsers for creating instances
    of models. You may subclass this factory
    to add your own extended models.
    """

    status = Status
    user = User
    direct_message = DirectMessage
    friendship = Friendship
    saved_search = SavedSearch
    search_result = SearchResult
    category = Category
    list = List
    relation = Relation
    relationship = Relationship

    json = JSONModel
    ids = IDModel
    place = Place
    bounding_box = BoundingBox

<commit_msg>Fixed broken User.lists() API call<commit_after># Tweepy
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.

from tweepy.error import TweepError
from tweepy.utils import parse_datetime, parse_html_value, parse_a_href, \
        parse_search_datetime, unescape_html

class ResultSet(list):
    """A list like object that holds results from a Twitter API query."""

class Model(object):

    def __init__(self, api=None):
        self._api = api

    def __getstate__(self):
        # pickle
        pickle = dict(self.__dict__)
        try:
            del pickle['_api']  # do not pickle the API reference
        except KeyError:
            pass
        return pickle

    @classmethod
    def parse(cls, api, json):
        """Parse a JSON object into a model instance."""
        raise NotImplementedError

    @classmethod
    def parse_list(cls, api, json_list):
        """Parse a list of JSON objects into a result set of model instances."""
        results = ResultSet()
        for obj in json_list:
            if obj:
                results.append(cls.parse(api, obj))
        return results

class Status(Model):

    @classmethod
    def parse(cls, api, json):
        status = cls(api)
        for k, v in json.items():
            if k == 'user':
                user_model = getattr(api.parser.model_factory, 'user')
                user = user_model.parse(api, v)
                setattr(status, 'author', user)
                setattr(status, 'user', user)  # DEPRECIATED
            elif k == 'created_at':
                setattr(status, k, parse_datetime(v))
            elif k == 'source':
                if '<' in v:
                    setattr(status, k, parse_html_value(v))
                    setattr(status, 'source_url', parse_a_href(v))
                else:
                    setattr(status, k, v)
                    setattr(status, 'source_url', None)
            elif k == 'retweeted_status':
                setattr(status, k, Status.parse(api, v))
            elif k == 'place':
                if v is not None:
                    setattr(status, k, Place.parse(api, v))
                else:
                    setattr(status, k, None)
            else:
                setattr(status, k, v)
        return status

    def destroy(self):
        return self._api.destroy_status(self.id)

    def retweet(self):
        return self._api.retweet(self.id)

    def retweets(self):
        return self._api.retweets(self.id)

    def favorite(self):
        return self._api.create_favorite(self.id)

class User(Model):

    @classmethod
    def parse(cls, api, json):
        user = cls(api)
        for k, v in json.items():
            if k == 'created_at':
                setattr(user, k, parse_datetime(v))
            elif k == 'status':
                setattr(user, k, Status.parse(api, v))
            elif k == 'following':
                # twitter sets this to null if it is false
                if v is True:
                    setattr(user, k, True)
                else:
                    setattr(user, k, False)
            else:
                setattr(user, k, v)
        return user

    @classmethod
    def parse_list(cls, api, json_list):
        if isinstance(json_list, list):
            item_list = json_list
        else:
            item_list = json_list['users']

        results = ResultSet()
        for obj in item_list:
            results.append(cls.parse(api, obj))
        return results

    def timeline(self, **kargs):
        return self._api.user_timeline(user_id=self.id, **kargs)

    def friends(self, **kargs):
        return self._api.friends(user_id=self.id, **kargs)

    def followers(self, **kargs):
        return self._api.followers(user_id=self.id, **kargs)

    def follow(self):
        self._api.create_friendship(user_id=self.id)
        self.following = True

    def unfollow(self):
        self._api.destroy_friendship(user_id=self.id)
        self.following = False

    def lists_memberships(self, *args, **kargs):
        return self._api.lists_memberships(user=self.screen_name, *args, **kargs)

    def lists_subscriptions(self, *args, **kargs):
        return self._api.lists_subscriptions(user=self.screen_name, *args, **kargs)

    def lists(self, *args, **kargs):
        return self._api.lists_all(user=self.screen_name, *args, **kargs)

    def followers_ids(self, *args, **kargs):
        return self._api.followers_ids(user_id=self.id, *args, **kargs)

class DirectMessage(Model):

    @classmethod
    def parse(cls, api, json):
        dm = cls(api)
        for k, v in json.items():
            if k == 'sender' or k == 'recipient':
                setattr(dm, k, User.parse(api, v))
            elif k == 'created_at':
                setattr(dm, k, parse_datetime(v))
            else:
                setattr(dm, k, v)
        return dm

    def destroy(self):
        return self._api.destroy_direct_message(self.id)

class Friendship(Model):

    @classmethod
    def parse(cls, api, json):
        relationship = json['relationship']

        # parse source
        source = cls(api)
        for k, v in relationship['source'].items():
            setattr(source, k, v)

        # parse target
        target = cls(api)
        for k, v in relationship['target'].items():
            setattr(target, k, v)

        return source, target

class Category(Model):

    @classmethod
    def parse(cls, api, json):
        category = cls(api)
        for k, v in json.items():
            setattr(category, k, v)
        return category

class SavedSearch(Model):

    @classmethod
    def parse(cls, api, json):
        ss = cls(api)
        for k, v in json.items():
            if k == 'created_at':
                setattr(ss, k, parse_datetime(v))
            else:
                setattr(ss, k, v)
        return ss

    def destroy(self):
        return self._api.destroy_saved_search(self.id)

class SearchResult(Model):

    @classmethod
    def parse(cls, api, json):
        result = cls()
        for k, v in json.items():
            if k == 'created_at':
                setattr(result, k, parse_search_datetime(v))
            elif k == 'source':
                setattr(result, k, parse_html_value(unescape_html(v)))
            else:
                setattr(result, k, v)
        return result

    @classmethod
    def parse_list(cls, api, json_list, result_set=None):
        results = ResultSet()
        results.max_id = json_list.get('max_id')
        results.since_id = json_list.get('since_id')
        results.refresh_url = json_list.get('refresh_url')
        results.next_page = json_list.get('next_page')
        results.results_per_page = json_list.get('results_per_page')
        results.page = json_list.get('page')
        results.completed_in = json_list.get('completed_in')
        results.query = json_list.get('query')

        for obj in json_list['results']:
            results.append(cls.parse(api, obj))
        return results

class List(Model):

    @classmethod
    def parse(cls, api, json):
        lst = List(api)
        for k,v in json.items():
            if k == 'user':
                setattr(lst, k, User.parse(api, v))
            elif k == 'created_at':
                setattr(lst, k, parse_datetime(v))
            else:
                setattr(lst, k, v)
        return lst

    @classmethod
    def parse_list(cls, api, json_list, result_set=None):
        results = ResultSet()
        if isinstance(json_list, dict):
            json_list = json_list['lists']
        for obj in json_list:
            results.append(cls.parse(api, obj))
        return results

    def update(self, **kargs):
        return self._api.update_list(self.slug, **kargs)

    def destroy(self):
        return self._api.destroy_list(self.slug)

    def timeline(self, **kargs):
        return self._api.list_timeline(self.user.screen_name, self.slug, **kargs)

    def add_member(self, id):
        return self._api.add_list_member(self.slug, id)

    def remove_member(self, id):
        return self._api.remove_list_member(self.slug, id)

    def members(self, **kargs):
        return self._api.list_members(self.user.screen_name, self.slug, **kargs)

    def is_member(self, id):
        return self._api.is_list_member(self.user.screen_name, self.slug, id)

    def subscribe(self):
        return self._api.subscribe_list(self.user.screen_name, self.slug)

    def unsubscribe(self):
        return self._api.unsubscribe_list(self.user.screen_name, self.slug)

    def subscribers(self, **kargs):
        return self._api.list_subscribers(self.user.screen_name, self.slug, **kargs)

    def is_subscribed(self, id):
        return self._api.is_subscribed_list(self.user.screen_name, self.slug, id)

class Relation(Model):
    @classmethod
    def parse(cls, api, json):
        result = cls(api)
        for k,v in json.items():
            if k == 'value' and json['kind'] in ['Tweet', 'LookedupStatus']:
                setattr(result, k, Status.parse(api, v))
            elif k == 'results':
                setattr(result, k, Relation.parse_list(api, v))
            else:
                setattr(result, k, v)
        return result

class Relationship(Model):
    @classmethod
    def parse(cls, api, json):
        result = cls(api)
        for k,v in json.items():
            if k == 'connections':
            	setattr(result, 'is_following', 'following' in v)
                setattr(result, 'is_followed_by', 'followed_by' in v)
            else:
                setattr(result, k, v)
        return result

class JSONModel(Model):

    @classmethod
    def parse(cls, api, json):
        return json

class IDModel(Model):

    @classmethod
    def parse(cls, api, json):
        if isinstance(json, list):
            return json
        else:
            return json['ids']

class BoundingBox(Model):

    @classmethod
    def parse(cls, api, json):
        result = cls(api)
        if json is not None:
            for k, v in json.items():
                setattr(result, k, v)
        return result

    def origin(self):
        """
        Return longitude, latitude of southwest (bottom, left) corner of
        bounding box, as a tuple.

        This assumes that bounding box is always a rectangle, which
        appears to be the case at present.
        """
        return tuple(self.coordinates[0][0])

    def corner(self):
        """
        Return longitude, latitude of northeast (top, right) corner of
        bounding box, as a tuple.

        This assumes that bounding box is always a rectangle, which
        appears to be the case at present.
        """
        return tuple(self.coordinates[0][2])

class Place(Model):

    @classmethod
    def parse(cls, api, json):
        place = cls(api)
        for k, v in json.items():
            if k == 'bounding_box':
                # bounding_box value may be null (None.)
                # Example: "United States" (id=96683cc9126741d1)
                if v is not None:
                    t = BoundingBox.parse(api, v)
                else:
                    t = v
                setattr(place, k, t)
            elif k == 'contained_within':
                # contained_within is a list of Places.
                setattr(place, k, Place.parse_list(api, v))
            else:
                setattr(place, k, v)
        return place

    @classmethod
    def parse_list(cls, api, json_list):
        if isinstance(json_list, list):
            item_list = json_list
        else:
            item_list = json_list['result']['places']

        results = ResultSet()
        for obj in item_list:
            results.append(cls.parse(api, obj))
        return results

class ModelFactory(object):
    """
    Used by parsers for creating instances
    of models. You may subclass this factory
    to add your own extended models.
    """

    status = Status
    user = User
    direct_message = DirectMessage
    friendship = Friendship
    saved_search = SavedSearch
    search_result = SearchResult
    category = Category
    list = List
    relation = Relation
    relationship = Relationship

    json = JSONModel
    ids = IDModel
    place = Place
    bounding_box = BoundingBox

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    abm_sketch
    ~~~~

    Initial implementation of message passing in an agent graph
"""

from scipy.stats.distributions import norm
from scipy.stats.distributions import uniform
from sklearn.metrics.pairwise import euclidean_distances
from matplotlib.pylab import plt
import numpy as np
from random import choice
from uuid import uuid4
from collections import defaultdict

y_pos_dist = norm(500, 10)

cluster_x_dists = {
    'A': uniform(0, 50),
    'B': uniform(30, 50),
    'C': uniform(60, 50)
}

cluster_sizes = {
    'A': 10,
    'B': 15,
    'C': 12
}

cluster_colors = {
    'A': 'r',
    'B': 'b',
    'C': 'g'
}

MESSAGES = [
    "message uno",
    "message dos",
]

class Population(object):
    """A set of connected Entities. Handles message passing and displaying."""

    def __init__(self):
        self.points = []
        self.path = []
        self.show = True
        self.success_lens = []
        self.connectivity_matrix = None

    def pass_message(self, recipient, task, sender):
        """
        Displays current state and routes the message one connection
        down. Handles the case where the recipient is the target. Calls
        Entity.receive_task(). <recipient> is the index of the recipient
        Entity.
        """

        self.display(recipient, task.target)

        self.path.append(recipient)
        if recipient == task.target:
            # import ipdb ; ipdb.set_trace#z()  # breakpoint 223cdcc4 //

            k = float(len(self.path))
            for ix, point in enumerate(self.path):
                i_value = (task.value / k) + ((k - ix) * 5)
                self.points[point].award(i_value)

            self.success_lens.append(len(self.path))
            self.path = []

        self.points[recipient].receive_task(task, sender)

    def set_connections(self):
        """Initializes each Entity's adjacency list."""

        self.connectivity_matrix = get_connectivity_matrix(self.points)
        for index, point in enumerate(self.points):
            point.set_adjacencies(self.connectivity_matrix[index])

    def display(self, current=None, target=None):
        """
        Plots the state of the task. If <show> = False, doesn't plot
        anything and the simulation can run faster.
        """

        if not self.show:
            return

        # Scatter plot of points, color coded by class
        pts = self.points
        size = 35
        for cluster, color in cluster_colors.iteritems():
            class_points = [x for x in pts if x.cluster == cluster]
            plt.scatter([p.x for p in class_points], [p.y for p in class_points], c=color, s=size)

        # Draw the connections
        if self.connectivity_matrix is not None:
            for start_ix, connections in enumerate(self.connectivity_matrix):
                for connect_ix, connected in enumerate(connections):
                    if connected and connect_ix != start_ix:
                        plt.plot(*zip(
                            (pts[start_ix].x, pts[start_ix].y),
                            (pts[connect_ix].x, pts[connect_ix].y)),
                            c='k', linewidth=0.5)

        # Show where the message is going and where it currently is
        if current and target:
            plt.scatter(pts[current].x, pts[current].y, c='m', s=150)
            plt.scatter(pts[target].x,  pts[target].y,  c='y', s=190)

        plt.show()

    def initiate_task(self, fixed_pair=None):
        """Initializes a task, and calls pass_message()."""

        start, end = [-1, -1] if not fixed_pair else fixed_pair

        if fixed_pair and start == end:
            print("changing fixed pair: they're the same!")

        while start == end:
            [start, end] = np.random.randint(len(self.points), size=2).tolist()

        message = choice(MESSAGES)
        print('new task: %s' % message)
        print('starting at %s and aiming for %s' % (start, end))

        task = Task(end, message)
        self.pass_message(start, task, len(self.points))

    def clear(self):
        """
        Clear every Entity's history. Called when the package is stuck
        in a cycle.
        """

        for point in self.points:
            adjs = point.adjacencies, set(point.adjacencies)
            if len(adjs[0]) != len(adjs[1]):
                print("clearing %s" % point.index + '---' * 20)
                print(point.adjacencies, set(point.adjacencies))
            point.sent = []

class Task(object):
    """Represents a message to be routed."""

    def __init__(self, target, message):
        self.target = target
        self.message = message
        self.id = uuid4()
        self.value = 100

class Entity(object):
    """An entity in our world"""

    def __init__(self, population, index, x, y, cluster):
        self.x = x
        self.y = y
        self.index = index
        self.cluster = cluster
        self.adjacencies = []
        self.population = population
        self.task_attempt_map = defaultdict(lambda: [index])
        self.sent = []
        self.value = 0

    def receive_task(self, task, sender):
        if sender not in self.task_attempt_map[task.id]:
            self.task_attempt_map[task.id].append(sender)
        print(sender, self.sent)
        if sender in self.sent:
            print('popping %s' % self.sent.pop(self.sent.index(sender)))
        if set(self.adjacencies).issubset(set(self.task_attempt_map[task.id])):
            print 'caught in a cycle! bailing!'
            self.population.clear()
            return
        if self.index == task.target:
            print 'message delivered! %s' % task.message
        else:
            self.pass_message(task, sender)

    def pass_message(self, task, sender):
        next_recipient = task.target if task.target in self.adjacencies else self.index

        while next_recipient in self.task_attempt_map[task.id]:
            next_recipient = choice(self.adjacencies)

        self.task_attempt_map[task.id].append(next_recipient)
        print "passing message from %s to %s" % (self.index, next_recipient)
        self.sent.append(next_recipient)
        self.population.pass_message(next_recipient, task, self.index)

    def set_adjacencies(self, connectivity_vector):
        """
        Builds the adjacency list for this Entity. <connectivity_vector>
        is a row from an adjacency matrix.
        """

        for connect_ix, connected in enumerate(connectivity_vector):
            if connected and connect_ix != self.index:
                self.adjacencies.append(connect_ix)

    def award(self, value):
        self.value += value
        print(self.value, self.index, self.adjacencies)

        for adj in self.sent:
            assert adj in self.adjacencies
            u_val = uniform.rvs(0, 100, 1)
            print(u_val, u_val < value)
            if u_val < value:
                self.adjacencies.append(adj)
                # import ipdb ; ipdb.set_trace() #z()  # breakpoint f35d9e23 //

        self.value = 0
        self.sent = []

def make_points(cluster, size):
    """Creates a set of points using y_pos_dist and cluster_x_dists."""

    ys = y_pos_dist.rvs(size)
    xs = cluster_x_dists[cluster].rvs(size)
    return list(zip(xs, ys, [cluster] * size))

def make_population():
    """Creates a Population and sets its connections. Uses make_points."""

    points = []
    for cluster, count in cluster_sizes.iteritems():
        points += make_points(cluster, count)

    population = Population()
    for ix, point in enumerate(points):
        pt = Entity(population, ix, *point)
        population.points.append(pt)

    population.set_connections()
    return population

def get_connectivity_matrix(points):
    """
    Computes the connectivity matrix of <points>. Each point is
    connected to each other within a radius.
    """

    points_arr = np.array([[p.x, p.y] for p in points])
    distance_mat = euclidean_distances(points_arr, points_arr)

    min_nonzero = lambda r: min(r[r > 0])
    # axis=1 is the horizontal axis (row-wise)
    min_neighbor_distances = np.apply_along_axis(min_nonzero, axis=1, arr=distance_mat)

    # WHY 2.2 ?
    neighbor_cutoff = np.mean(min_neighbor_distances) * 2.2
    connectivity_matrix = distance_mat < neighbor_cutoff

    return connectivity_matrix

def run():
    """Creates and sets up a Population and runs initiate_task()."""

    pop = make_population()
    pop.initiate_task()

if __name__ == '__main__':
    run()
<commit_msg>Refactor: making cluster_sizes an argument for run() allows for more simulation freedom if tested interactively.<commit_after># -*- coding: utf-8 -*-
"""
    abm_sketch
    ~~~~

    Initial implementation of message passing in an agent graph
"""

from scipy.stats.distributions import norm
from scipy.stats.distributions import uniform
from sklearn.metrics.pairwise import euclidean_distances
from matplotlib.pylab import plt
import numpy as np
from random import choice
from uuid import uuid4
from collections import defaultdict

y_pos_dist = norm(500, 10)

cluster_x_dists = {
    'A': uniform(0, 50),
    'B': uniform(30, 50),
    'C': uniform(60, 50)
}

cluster_colors = {
    'A': 'r',
    'B': 'b',
    'C': 'g'
}

MESSAGES = [
    "message uno",
    "message dos",
]

class Population(object):
    """A set of connected Entities. Handles message passing and displaying."""

    def __init__(self):
        self.points = []
        self.path = []
        self.show = True
        self.success_lens = []
        self.connectivity_matrix = None

    def pass_message(self, recipient, task, sender):
        """
        Displays current state and routes the message one connection
        down. Handles the case where the recipient is the target. Calls
        Entity.receive_task(). <recipient> is the index of the recipient
        Entity.
        """

        self.display(recipient, task.target)

        self.path.append(recipient)
        if recipient == task.target:
            # import ipdb ; ipdb.set_trace#z()  # breakpoint 223cdcc4 //

            k = float(len(self.path))
            for ix, point in enumerate(self.path):
                i_value = (task.value / k) + ((k - ix) * 5)
                self.points[point].award(i_value)

            self.success_lens.append(len(self.path))
            self.path = []

        self.points[recipient].receive_task(task, sender)

    def set_connections(self):
        """Initializes each Entity's adjacency list."""

        self.connectivity_matrix = get_connectivity_matrix(self.points)
        for index, point in enumerate(self.points):
            point.set_adjacencies(self.connectivity_matrix[index])

    def display(self, current=None, target=None):
        """
        Plots the state of the task. If <show> = False, doesn't plot
        anything and the simulation can run faster.
        """

        if not self.show:
            return

        # Scatter plot of points, color coded by class
        pts = self.points
        size = 35
        for cluster, color in cluster_colors.iteritems():
            class_points = [x for x in pts if x.cluster == cluster]
            plt.scatter([p.x for p in class_points], [p.y for p in class_points], c=color, s=size)

        # Draw the connections
        if self.connectivity_matrix is not None:
            for start_ix, connections in enumerate(self.connectivity_matrix):
                for connect_ix, connected in enumerate(connections):
                    if connected and connect_ix != start_ix:
                        plt.plot(*zip(
                            (pts[start_ix].x, pts[start_ix].y),
                            (pts[connect_ix].x, pts[connect_ix].y)),
                            c='k', linewidth=0.5)

        # Show where the message is going and where it currently is
        if current and target:
            plt.scatter(pts[current].x, pts[current].y, c='m', s=150)
            plt.scatter(pts[target].x,  pts[target].y,  c='y', s=190)

        plt.show()

    def initiate_task(self, fixed_pair=None):
        """Initializes a task, and calls pass_message()."""

        start, end = [-1, -1] if not fixed_pair else fixed_pair

        if fixed_pair and start == end:
            print("changing fixed pair: they're the same!")

        while start == end:
            [start, end] = np.random.randint(len(self.points), size=2).tolist()

        message = choice(MESSAGES)
        print('new task: %s' % message)
        print('starting at %s and aiming for %s' % (start, end))

        task = Task(end, message)
        self.pass_message(start, task, len(self.points))

    def clear(self):
        """
        Clear every Entity's history. Called when the package is stuck
        in a cycle.
        """

        for point in self.points:
            adjs = point.adjacencies, set(point.adjacencies)
            if len(adjs[0]) != len(adjs[1]):
                print("clearing %s" % point.index + '---' * 20)
                print(point.adjacencies, set(point.adjacencies))
            point.sent = []

class Task(object):
    """Represents a message to be routed."""

    def __init__(self, target, message):
        self.target = target
        self.message = message
        self.id = uuid4()
        self.value = 100

class Entity(object):
    """An entity in our world"""

    def __init__(self, population, index, x, y, cluster):
        self.x = x
        self.y = y
        self.index = index
        self.cluster = cluster
        self.adjacencies = []
        self.population = population
        self.task_attempt_map = defaultdict(lambda: [index])
        self.sent = []
        self.value = 0

    def receive_task(self, task, sender):
        if sender not in self.task_attempt_map[task.id]:
            self.task_attempt_map[task.id].append(sender)
        print(sender, self.sent)
        if sender in self.sent:
            print('popping %s' % self.sent.pop(self.sent.index(sender)))
        if set(self.adjacencies).issubset(set(self.task_attempt_map[task.id])):
            print 'caught in a cycle! bailing!'
            self.population.clear()
            return
        if self.index == task.target:
            print 'message delivered! %s' % task.message
        else:
            self.pass_message(task, sender)

    def pass_message(self, task, sender):
        next_recipient = task.target if task.target in self.adjacencies else self.index

        while next_recipient in self.task_attempt_map[task.id]:
            next_recipient = choice(self.adjacencies)

        self.task_attempt_map[task.id].append(next_recipient)
        print "passing message from %s to %s" % (self.index, next_recipient)
        self.sent.append(next_recipient)
        self.population.pass_message(next_recipient, task, self.index)

    def set_adjacencies(self, connectivity_vector):
        """
        Builds the adjacency list for this Entity. <connectivity_vector>
        is a row from an adjacency matrix.
        """

        for connect_ix, connected in enumerate(connectivity_vector):
            if connected and connect_ix != self.index:
                self.adjacencies.append(connect_ix)

    def award(self, value):
        self.value += value
        print(self.value, self.index, self.adjacencies)

        for adj in self.sent:
            assert adj in self.adjacencies
            u_val = uniform.rvs(0, 100, 1)
            print(u_val, u_val < value)
            if u_val < value:
                self.adjacencies.append(adj)
                # import ipdb ; ipdb.set_trace() #z()  # breakpoint f35d9e23 //

        self.value = 0
        self.sent = []

def make_points(cluster, size):
    """Creates a set of points using y_pos_dist and cluster_x_dists."""

    ys = y_pos_dist.rvs(size)
    xs = cluster_x_dists[cluster].rvs(size)
    return list(zip(xs, ys, [cluster] * size))

def make_population(cluster_sizes):
    """Creates a Population and sets its connections. Uses make_points."""

    points = []
    for cluster, count in cluster_sizes.iteritems():
        points += make_points(cluster, count)

    population = Population()
    for ix, point in enumerate(points):
        pt = Entity(population, ix, *point)
        population.points.append(pt)

    population.set_connections()
    return population

def get_connectivity_matrix(points):
    """
    Computes the connectivity matrix of <points>. Each point is
    connected to each other within a radius.
    """

    points_arr = np.array([[p.x, p.y] for p in points])
    distance_mat = euclidean_distances(points_arr, points_arr)

    min_nonzero = lambda r: min(r[r > 0])
    # axis=1 is the horizontal axis (row-wise)
    min_neighbor_distances = np.apply_along_axis(min_nonzero, axis=1, arr=distance_mat)

    # WHY 2.2 ?
    neighbor_cutoff = np.mean(min_neighbor_distances) * 2.2
    connectivity_matrix = distance_mat < neighbor_cutoff

    return connectivity_matrix

def run(cluster_sizes):
    """Creates and sets up a Population and runs initiate_task()."""

    pop = make_population(cluster_sizes)
    pop.initiate_task()

if __name__ == '__main__':
    cluster_sizes = {
        'A': 10,
        'B': 15,
        'C': 12
    }

    run(cluster_sizes)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import tweepy
import os
from time import sleep
from random import randrange
import re
from time import localtime

def file_to_dict(file_name):
	"""opens a file and put tab separated values in a dict"""
	f = open(file_name, 'r')
	d_keys = {}
	for i in f.readlines():
		i = i[:len(i) - 1]		#Removing the \n char
		j = i.split('\t')
		d_keys[j[0]] = j[1]
	return d_keys

def twitter_authentification(keys_file):
	"""return a tweepy.api object connected to your account"""
	d_keys = file_to_dict(keys_file)
	auth = tweepy.OAuthHandler(d_keys["consumer_key"],
		d_keys["consumer_secret"])
	auth.set_access_token(d_keys["access_token"],
		d_keys["access_secret"])
	api = tweepy.API(auth)
	return api

def daemonize():
    try:
        pid = os.fork()
    except OSError, e:
        raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
    if (pid == 0):      # Ayanami Rei
        os.setsid()
        try:
            pid = os.fork()
        except OSError, e:
            raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
        if (pid == 0):      # Langley Soryu Asuka
            os.chdir("/")
            os.umask(0)
        else:
            os._exit(0)
    else:
        os._exit(0)
    os.open("/dev/null",os.O_RDWR)
    os.dup2(0,1)
    os.dup2(0,2)
    print pid
    return(0)

def is_direct_mention(text):
	pattern_direct_mention = re.compile("^@.+")
	return re.match(pattern_direct_mention, text)

def contains_link(text):
	pattern_link = re.compile(".*?http.*?")
	return re.match(pattern_link, text)

def parsing_dm_list(dm_list):
	dm_list_copy = []
	for dm in dm_list:
		if not is_direct_mention(dm.text) and not contains_link(dm.text):
			dm_list_copy.append(dm)
	return dm_list_copy

def display_time():
	time = localtime()
	return str(time[3]) + "h" + str(time[4]) + "min"

def logging_tweets(dm):
	f = open("tweets.log", 'a')
	f.write(dm.text.encode('utf-8') + "\t" +
		dm.sender.screen_name.encode('utf-8') + "\t" + display_time())
	f.close()

def erasing_last_dm_batch(last_dm_id):
	old_dms = api.direct_messages(0, last_dm_id)
	for dm in old_dms:
		api.destroy_direct_message(dm.id)

if __name__ == "__main__":
	#retCode = daemonize()
	cur_dir = os.path.dirname(os.path.realpath(__file__))

	api = twitter_authentification(cur_dir + "/keys.conf")
	dms = api.direct_messages()
	dms = parsing_dm_list(dms)
	#first_dm_id = dms[0].id
	time_dilatation = { 0 : 1800,
						1 : 900,
						2 : 600,
						3 : 600,
						4 : 450,
						5 : 450,
						6 : 300,
						7 : 200,
						8 : 150 }

	while 1:
		try:
			print "Nb de DM : " + str(len(dms))

			if len(dms) != 0:
				print "Picking dm"
				picked_dm = dms[randrange(len(dms))]

				logging_tweets(picked_dm)
				dms.remove(picked_dm)
				api.destroy_direct_message(picked_dm.id)

				print picked_dm.text
				api.update_status(picked_dm.text)

				last_dm_id = dms[len(dms) - 1].id
				first_dm_id = last_dm_id

			print "sleeping before following : " + display_time()
			sleep(time_dilatation[len(dms)])

			print "Follow back"
			followers = api.followers()
			for follower in followers:
				if not follower.following:
					try:
						follower.follow()
					except tweepy.error.TweepError:pass

			print "sleeping before tweeting : " + display_time()
			sleep(time_dilatation[len(dms)])

			try:
				#dms = api.direct_messages(first_dm_id)
				dms = api.direct_messages()
			#	try:
			#		erasing_last_dm_batch(last_dm_id)
			#	except NameError:
			#		last_dm_id = first_dm_id
				dms = parsing_dm_list(dms)

			except tweepy.error.TweepError:pass
		except IndexError:pass

<commit_msg>Time dilatation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import tweepy
import os
from time import sleep
from random import randrange
import re
from time import localtime

def file_to_dict(file_name):
	"""opens a file and put tab separated values in a dict"""
	f = open(file_name, 'r')
	d_keys = {}
	for i in f.readlines():
		i = i[:len(i) - 1]		#Removing the \n char
		j = i.split('\t')
		d_keys[j[0]] = j[1]
	return d_keys

def twitter_authentification(keys_file):
	"""return a tweepy.api object connected to your account"""
	d_keys = file_to_dict(keys_file)
	auth = tweepy.OAuthHandler(d_keys["consumer_key"],
		d_keys["consumer_secret"])
	auth.set_access_token(d_keys["access_token"],
		d_keys["access_secret"])
	api = tweepy.API(auth)
	return api

def daemonize():
    try:
        pid = os.fork()
    except OSError, e:
        raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
    if (pid == 0):      # Ayanami Rei
        os.setsid()
        try:
            pid = os.fork()
        except OSError, e:
            raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
        if (pid == 0):      # Langley Soryu Asuka
            os.chdir("/")
            os.umask(0)
        else:
            os._exit(0)
    else:
        os._exit(0)
    os.open("/dev/null",os.O_RDWR)
    os.dup2(0,1)
    os.dup2(0,2)
    print pid
    return(0)

def is_direct_mention(text):
	pattern_direct_mention = re.compile("^@.+")
	return re.match(pattern_direct_mention, text)

def contains_link(text):
	pattern_link = re.compile(".*?http.*?")
	return re.match(pattern_link, text)

def parsing_dm_list(dm_list):
	dm_list_copy = []
	for dm in dm_list:
		if not is_direct_mention(dm.text) and not contains_link(dm.text):
			dm_list_copy.append(dm)
	return dm_list_copy

def display_time():
	time = localtime()
	return str(time[3]) + "h" + str(time[4]) + "min"

def logging_tweets(dm):
	f = open("tweets.log", 'a')
	f.write(dm.text.encode('utf-8') + "\t" +
		dm.sender.screen_name.encode('utf-8') + "\t" + display_time())
	f.close()

def erasing_last_dm_batch(last_dm_id):
	old_dms = api.direct_messages(0, last_dm_id)
	for dm in old_dms:
		api.destroy_direct_message(dm.id)

if __name__ == "__main__":
	#retCode = daemonize()
	cur_dir = os.path.dirname(os.path.realpath(__file__))

	api = twitter_authentification(cur_dir + "/keys.conf")
	dms = api.direct_messages()
	dms = parsing_dm_list(dms)
	#first_dm_id = dms[0].id
	time_dilatation = { 0 : 1800,
						1 : 900,
						2 : 600,
						3 : 600,
						4 : 450,
						5 : 450,
						6 : 300,
						7 : 200,
						8 : 150 }

	while 1:
		try:

			nb_dms = len(dms)
			print "Nb de DM : " + str(nb_dms)

			if nb_dms != 0:
				print "Picking dm"
				picked_dm = dms[randrange(nb_dms)]

				logging_tweets(picked_dm)
				dms.remove(picked_dm)
				api.destroy_direct_message(picked_dm.id)

				print picked_dm.text
				api.update_status(picked_dm.text)

				last_dm_id = dms[nb_dms - 1].id
				first_dm_id = last_dm_id

			print "sleeping before following : " + display_time()
			if nb_dms >= 8:
				nb_dms = 8
			sleep(time_dilatation[nb_dms])

			print "Follow back"
			followers = api.followers()
			for follower in followers:
				if not follower.following:
					try:
						follower.follow()
					except tweepy.error.TweepError:pass

			print "sleeping before tweeting : " + display_time()
			sleep(time_dilatation[nb_dms])

			try:
				#dms = api.direct_messages(first_dm_id)
				dms = api.direct_messages()
			#	try:
			#		erasing_last_dm_batch(last_dm_id)
			#	except NameError:
			#		last_dm_id = first_dm_id
				dms = parsing_dm_list(dms)

			except tweepy.error.TweepError:pass
		except IndexError:pass

<|endoftext|>


<commit_before>"""
OAuth token endpoint support.
"""
from zope.interface import Attribute, Interface, implements

class ITokenRequest(Interface):
    client = Attribute(
        """
        The authenticated client making the request.

        @type: L{txoauth.clientcred.IClient}
        """)

class IAssertion(ITokenRequest):
    """
    A token request in the form of an assertion.
    """
    assertionType = Attribute(
        """
        The type of this assertion.

        @type: C{str}
        """)

    assertion = Attribute(
        """
        The value of this assertion.

        @type: C{str}
        """)

class Assertion(object):
    """
    A token request in the form of an assertion.
    """
    implements(IAssertion)

    def __init__(self, client, assertionType, assertion):
        self._client = client
        self._assertionType = assertionType
        self._assertion = assertion

    @property
    def client(self):
        return self._client

    @property
    def assertionType(self):
        return self._assertionType

    @property
    def assertion(self):
       return self._assertion
<commit_msg>Added EnforcedInvalidationException<commit_after>"""
OAuth token endpoint support.
"""
from zope.interface import Attribute, Interface, implements

class ITokenRequest(Interface):
    client = Attribute(
        """
        The authenticated client making the request.

        @type: L{txoauth.clientcred.IClient}
        """)

class IAssertion(ITokenRequest):
    """
    A token request in the form of an assertion.
    """
    assertionType = Attribute(
        """
        The type of this assertion.

        @type: C{str}
        """)

    assertion = Attribute(
        """
        The value of this assertion.

        @type: C{str}
        """)

class Assertion(object):
    """
    A token request in the form of an assertion.
    """
    implements(IAssertion)

    def __init__(self, client, assertionType, assertion):
        self._client = client
        self._assertionType = assertionType
        self._assertion = assertion

    @property
    def client(self):
        return self._client

    @property
    def assertionType(self):
        return self._assertionType

    @property
    def assertion(self):
       return self._assertion

class EnforcedInvalidationException(Exception):
    """
    Raised when attempting to check an assertion while not invalidating the
    assertion, if the L{txoauth.interfaces.IAssertionStore} does not allow
    that operation.

    The OAuth specification asserts that assertions should be invalidated.
    """
<|endoftext|>