<commit_before># Created By: Virgil Dupras
# Created On: 2008-04-20
# Copyright 2010 Hardcoded Software (http://www.hardcoded.net)

# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license

from __future__ import division

import copy
import pickle
from datetime import date
import sqlite3 as sqlite

from .. import testcase, io
from ..path import Path
from ..currency import Currency, RatesDB, CAD, EUR, PLN, USD

class TestCase(testcase.TestCase):
    def tearDown(self):
        Currency.set_rates_db(None)

class CurrencyTest(TestCase):
    def test_creation(self):
        """Different ways to create a currency."""
        self.assertEqual(Currency('CAD'), CAD)
        self.assertEqual(Currency(name='Canadian dollar'), CAD)

    def test_copy(self):
        """Currencies can be copied."""
        self.assertEqual(copy.copy(CAD), CAD)
        self.assertEqual(copy.deepcopy(CAD), CAD)

class EmptyMemoryDB(TestCase):
    def test_get_rate(self):
        """When there is no data available, use the fallback rate"""
        self.assertEqual(CAD.value_in(USD, date(2008, 4, 20)), 1 / USD.fallback_rate)

class DBWithDailyRate(TestCase):
    def setUp(self):
        USD.set_CAD_value(1/0.996115, date(2008, 4, 20))

    def test_get_rate(self):
        """Getting the rate exactly as set_rate happened returns the same rate"""
        self.assertAlmostEqual(CAD.value_in(USD, date(2008, 4, 20)), 0.996115)

    def test_get_rate_different_currency(self):
        """Use fallback rates when necessary"""
        self.assertEqual(CAD.value_in(EUR, date(2008, 4, 20)), 1 / EUR.fallback_rate)
        self.assertEqual(EUR.value_in(USD, date(2008, 4, 20)), EUR.fallback_rate * 0.996115)

    def test_get_rate_reverse(self):
        """It's possible to get the reverse value of a rate using the same data"""
        self.assertAlmostEqual(USD.value_in(CAD, date(2008, 4, 20)), 1 / 0.996115)

    def test_set_rate_twice(self):
        """When setting a rate for an index that already exists, the old rate is replaced by the new"""
        USD.set_CAD_value(1/42, date(2008, 4, 20))
        self.assertAlmostEqual(CAD.value_in(USD, date(2008, 4, 20)), 42)

    def test_set_rate_after_get(self):
        # When setting a rate after a get of the same rate, the rate cache is correctly updated.
        CAD.value_in(USD, date(2008, 4, 20)) # value will be cached
        USD.set_CAD_value(1/42, date(2008, 4, 20))
        self.assertAlmostEqual(CAD.value_in(USD, date(2008, 4, 20)), 42)

class DBWithTwoDailyRates(TestCase):
    def setUp(self):
        # Don't change the set order, it's important for the tests
        USD.set_CAD_value(1/0.997115, date(2008, 4, 25))
        USD.set_CAD_value(1/0.996115, date(2008, 4, 20))

    def test_date_range_range(self):
        """USD.rates_date_range() returns the USD's limits"""
        self.assertEqual(USD.rates_date_range(), (date(2008, 4, 20), date(2008, 4, 25)))

    def test_date_range_for_unfetched_currency(self):
        """If the curency is not in the DB, return None"""
        self.assertTrue(PLN.rates_date_range() is None)

    def test_seek_rate_middle(self):
        """A rate request with seek in the middle will return the lowest date"""
        self.assertEqual(USD.value_in(CAD, date(2008, 4, 24)), 1/0.996115)

    def test_seek_rate_after(self):
        """Make sure that the *nearest* lowest rate is returned. Because the 25th have been set
        before the 20th, an order by clause is required in the seek SQL to make this test pass"""
        self.assertEqual(USD.value_in(CAD, date(2008, 4, 26)), 1/0.997115)

    def test_seek_rate_before(self):
        """If there are no rate in the past, seek for a rate in the future"""
        self.assertEqual(USD.value_in(CAD, date(2008, 4, 19)), 1/0.996115)

class DBWithMutlipleCurrencies(TestCase):
    def setUp(self):
        USD.set_CAD_value(1/0.996115, date(2008, 4, 20))
        EUR.set_CAD_value(1/0.633141, date(2008, 4, 20))

    def test_get_rate(self):
        """Don't mix currency rates up"""
        self.assertAlmostEqual(CAD.value_in(USD, date(2008, 4, 20)), 0.996115)
        self.assertAlmostEqual(CAD.value_in(EUR, date(2008, 4, 20)), 0.633141)

    def test_get_rate_with_pivotal(self):
        """It's possible to get a rate by using 2 records"""
        # if 1 CAD = 0.996115 USD and 1 CAD = 0.633141 then 0.996115 USD = 0.633141 then 1 USD = 0.633141 / 0.996115 EUR
        self.assertAlmostEqual(USD.value_in(EUR, date(2008, 4, 20)), 0.633141 / 0.996115)

    def test_get_rate_doesnt_exist(self):
        """Don't crash when trying to do pivotal calculation with non-existing currencies"""
        self.assertEqual(USD.value_in(PLN, date(2008, 4, 20)), 1 / 0.996115 / PLN.fallback_rate)

class DBWithPath(TestCase):
    def setUp(self):
        self.dbpath = Path(self.tmpdir()) + 'foo.db'
        db = RatesDB(self.dbpath)
        db.set_CAD_value(date(2008, 4, 20), 'USD', 1/0.996115)

    def test_get_rate_new_instance(self):
        """Use another instance to call get_rate on"""
        db = RatesDB(self.dbpath)
        self.assertAlmostEqual(db.get_rate(date(2008, 4, 20), 'CAD', 'USD'), 0.996115)

class DBWithConnection(TestCase):
    def setUp(self):
        self.con = sqlite.connect(':memory:')
        self.db = RatesDB(self.con)

    def test_tables(self):
        """the supplied connection is used instead"""
        try:
            self.con.execute("select * from rates where 1=2")
        except sqlite.OperationalError: # new db
            self.fail()

class CorruptPath(TestCase):
    def setUp(self):
        self.dbpath = self.tmppath() + 'foo.db'
        fh = io.open(self.dbpath, 'w')
        fh.write('corrupted')
        fh.close()

    def test_open_db(self):
        db = RatesDB(self.dbpath) # no crash. deletes the old file and start a new db
        db.set_CAD_value(date(2008, 4, 20), 'USD', 42)
        db = RatesDB(self.dbpath)
        self.assertEqual(db.get_rate(date(2008, 4, 20), 'USD', 'CAD'), 42)

class DBProblemAfterConnection(TestCase):
    def setUp(self):
        class MockConnection(sqlite.Connection): # can't mock sqlite3.Connection's attribute, so we subclass it
            mocking = False
            def execute(self, *args, **kwargs):
                if self.mocking:
                    raise sqlite.OperationalError()
                else:
                    return sqlite.Connection.execute(self, *args, **kwargs)

        con = MockConnection(':memory:')
        self.db = RatesDB(con)
        con.mocking = True

    def test_date_range(self):
        self.db.date_range('USD') # no crash

    def test_get_rate(self):
        self.db.get_rate(date(2008, 4, 20), 'USD', 'CAD') # no crash

    def test_set_rate(self):
        self.db.set_CAD_value(date(2008, 4, 20), 'USD', 42) # no crash

<commit_msg>Nose-ified currency_test.<commit_after># Created By: Virgil Dupras
# Created On: 2008-04-20
# Copyright 2010 Hardcoded Software (http://www.hardcoded.net)

# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license

from __future__ import division

from datetime import date
import sqlite3 as sqlite

from nose.tools import eq_, assert_almost_equal, with_setup

from .. import io
from ..currency import Currency, RatesDB, CAD, EUR, PLN, USD
from ..testutil import with_tmpdir

def teardown():
    Currency.set_rates_db(None)

@with_setup(None, teardown)
def test_currency_creation():
    # Different ways to create a currency.
    eq_(Currency('CAD'), CAD)
    eq_(Currency(name='Canadian dollar'), CAD)

@with_setup(None, teardown)
def test_currency_copy():
    # Currencies can be copied.
    import copy
    eq_(copy.copy(CAD), CAD)
    eq_(copy.deepcopy(CAD), CAD)

@with_setup(None, teardown)
def test_get_rate_on_empty_db():
    # When there is no data available, use the fallback rate.
    eq_(CAD.value_in(USD, date(2008, 4, 20)), 1 / USD.fallback_rate)

@with_setup(None, teardown)
@with_tmpdir
def test_physical_rates_db_remember_rates(tmppath):
    # When a rates db uses a real file, rates are remembered
    dbpath = tmppath + 'foo.db'
    db = RatesDB(dbpath)
    db.set_CAD_value(date(2008, 4, 20), 'USD', 1/0.996115)
    db = RatesDB(dbpath)
    assert_almost_equal(db.get_rate(date(2008, 4, 20), 'CAD', 'USD'), 0.996115)

@with_setup(None, teardown)
def test_db_with_connection():
    # the supplied connection is used by the rates db.
    con = sqlite.connect(':memory:')
    db = RatesDB(con)
    try:
        con.execute("select * from rates where 1=2")
    except sqlite.OperationalError: # new db
        raise AssertionError()

@with_setup(None, teardown)
@with_tmpdir
def test_corrupt_db(tmppath):
    dbpath = tmppath + 'foo.db'
    fh = io.open(dbpath, 'w')
    fh.write('corrupted')
    fh.close()
    db = RatesDB(dbpath) # no crash. deletes the old file and start a new db
    db.set_CAD_value(date(2008, 4, 20), 'USD', 42)
    db = RatesDB(dbpath)
    eq_(db.get_rate(date(2008, 4, 20), 'USD', 'CAD'), 42)

#--- Daily rate
def setup_daily_rate():
    USD.set_CAD_value(1/0.996115, date(2008, 4, 20))

@with_setup(setup_daily_rate, teardown)
def test_get_rate_with_daily_rate():
    # Getting the rate exactly as set_rate happened returns the same rate.
    assert_almost_equal(CAD.value_in(USD, date(2008, 4, 20)), 0.996115)

@with_setup(setup_daily_rate, teardown)
def test_get_rate_different_currency():
    # Use fallback rates when necessary.
    eq_(CAD.value_in(EUR, date(2008, 4, 20)), 1 / EUR.fallback_rate)
    eq_(EUR.value_in(USD, date(2008, 4, 20)), EUR.fallback_rate * 0.996115)

@with_setup(setup_daily_rate, teardown)
def test_get_rate_reverse():
    # It's possible to get the reverse value of a rate using the same data.
    assert_almost_equal(USD.value_in(CAD, date(2008, 4, 20)), 1 / 0.996115)

@with_setup(setup_daily_rate, teardown)
def test_set_rate_twice():
    # When setting a rate for an index that already exists, the old rate is replaced by the new.
    USD.set_CAD_value(1/42, date(2008, 4, 20))
    assert_almost_equal(CAD.value_in(USD, date(2008, 4, 20)), 42)

@with_setup(setup_daily_rate, teardown)
def test_set_rate_after_get():
    # When setting a rate after a get of the same rate, the rate cache is correctly updated.
    CAD.value_in(USD, date(2008, 4, 20)) # value will be cached
    USD.set_CAD_value(1/42, date(2008, 4, 20))
    assert_almost_equal(CAD.value_in(USD, date(2008, 4, 20)), 42)

#--- Two daily rates
def setup_two_daily_rate():
    # Don't change the set order, it's important for the tests
    USD.set_CAD_value(1/0.997115, date(2008, 4, 25))
    USD.set_CAD_value(1/0.996115, date(2008, 4, 20))

@with_setup(setup_two_daily_rate, teardown)
def test_date_range_range():
    # USD.rates_date_range() returns the USD's limits.
    eq_(USD.rates_date_range(), (date(2008, 4, 20), date(2008, 4, 25)))

@with_setup(setup_two_daily_rate, teardown)
def test_date_range_for_unfetched_currency():
    # If the curency is not in the DB, return None.
    assert PLN.rates_date_range() is None

@with_setup(setup_two_daily_rate, teardown)
def test_seek_rate_middle():
    # A rate request with seek in the middle will return the lowest date.
    eq_(USD.value_in(CAD, date(2008, 4, 24)), 1/0.996115)

@with_setup(setup_two_daily_rate, teardown)
def test_seek_rate_after():
    # Make sure that the *nearest* lowest rate is returned. Because the 25th have been set
    # before the 20th, an order by clause is required in the seek SQL to make this test pass.
    eq_(USD.value_in(CAD, date(2008, 4, 26)), 1/0.997115)

@with_setup(setup_two_daily_rate, teardown)
def test_seek_rate_before():
    # If there are no rate in the past, seek for a rate in the future.
    eq_(USD.value_in(CAD, date(2008, 4, 19)), 1/0.996115)

#--- Rates of multiple currencies
def setup_rates_of_multiple_currencies():
    USD.set_CAD_value(1/0.996115, date(2008, 4, 20))
    EUR.set_CAD_value(1/0.633141, date(2008, 4, 20))

@with_setup(setup_rates_of_multiple_currencies, teardown)
def test_get_rate_multiple_currencies():
    # Don't mix currency rates up.
    assert_almost_equal(CAD.value_in(USD, date(2008, 4, 20)), 0.996115)
    assert_almost_equal(CAD.value_in(EUR, date(2008, 4, 20)), 0.633141)

@with_setup(setup_rates_of_multiple_currencies, teardown)
def test_get_rate_with_pivotal():
    # It's possible to get a rate by using 2 records.
    # if 1 CAD = 0.996115 USD and 1 CAD = 0.633141 then 0.996115 USD = 0.633141 then 1 USD = 0.633141 / 0.996115 EUR
    assert_almost_equal(USD.value_in(EUR, date(2008, 4, 20)), 0.633141 / 0.996115)

@with_setup(setup_rates_of_multiple_currencies, teardown)
def test_get_rate_doesnt_exist():
    # Don't crash when trying to do pivotal calculation with non-existing currencies.
    eq_(USD.value_in(PLN, date(2008, 4, 20)), 1 / 0.996115 / PLN.fallback_rate)

#--- Problems after connection
def get_problematic_db():
    class MockConnection(sqlite.Connection): # can't mock sqlite3.Connection's attribute, so we subclass it
        mocking = False
        def execute(self, *args, **kwargs):
            if self.mocking:
                raise sqlite.OperationalError()
            else:
                return sqlite.Connection.execute(self, *args, **kwargs)

    con = MockConnection(':memory:')
    db = RatesDB(con)
    con.mocking = True
    return db

@with_setup(None, teardown)
def test_date_range_with_problematic_db():
    db = get_problematic_db()
    db.date_range('USD') # no crash

@with_setup(None, teardown)
def test_get_rate_with_problematic_db():
    db = get_problematic_db()
    db.get_rate(date(2008, 4, 20), 'USD', 'CAD') # no crash

@with_setup(None, teardown)
def test_set_rate_with_problematic_db():
    db = get_problematic_db()
    db.set_CAD_value(date(2008, 4, 20), 'USD', 42) # no crash
<|endoftext|>


<commit_before>r"""
>>> s = fmtstr("Hey there!", 'red')
>>> s
red("Hey there!")
>>> s[4:7]
'\x1b[31mthe\x1b[39m'
"""
#TODO add a way to composite text without losing original formatting information
import itertools
import functools
import copy

from escseqparse import parse
from termformatconstants import FG_COLORS, BG_COLORS, STYLES
from termformatconstants import FG_NUMBER_TO_COLOR, BG_NUMBER_TO_COLOR
from termformatconstants import RESET_ALL, RESET_BG, RESET_FG
from termformatconstants import seq

xforms = {
    'fg' : lambda x, v: seq(v)+x+seq(RESET_FG),
    'bg' : lambda x, v: seq(v)+x+seq(RESET_BG),
    'bold' : lambda x: seq(STYLES['bold'])+x+seq(RESET_ALL),
    'underline' : lambda x: seq(STYLES['underline'])+x+seq(RESET_ALL),
    'blink' : lambda x: seq(STYLES['blink'])+x+seq(RESET_ALL),
    'invert' : lambda x: seq(STYLES['invert'])+x+seq(RESET_ALL),
}

class BaseFmtStr(object):
    """Formatting annotations on strings"""
    def __init__(self, string, atts=None):
        self.s = string
        self.atts = {k:v for k,v in atts.items()} if atts else {}

    def __len__(self):
        return len(self.s)

    def __str__(self):
        s = self.s
        for k, v in self.atts.items():
            if k not in xforms: continue
            if v is True:
                s = xforms[k](s)
            else:
                s = xforms[k](s, v)
        return s

    def __getitem__(self, index):
        return fmtstr(str(self)[index])

    def __repr__(self):
        def pp_att(att):
            if att == 'fg': return FG_NUMBER_TO_COLOR[self.atts[att]]
            elif att == 'bg': return 'on_' + BG_NUMBER_TO_COLOR[self.atts[att]]
            else: return att
        return (''.join(
                        pp_att(att)+'('
                        for att in sorted(self.atts)) +
               ('"%s"' % self.s) + ')'*len(self.atts))

class FmtStr(object):
    def __init__(self, *components):
        assert all(isinstance(x, BaseFmtStr) for x in components)
        self.fmtstrs = [x for x in components if len(x) > 0]

    @classmethod
    def from_str(cls, s):
        r"""
        >>> fmtstr("|"+on_blue(red("hey"))+"|")
        "|"+on_blue(red("hey"))+"|"
        >>> fmtstr('|\x1b[31m\x1b[44mhey\x1b[49m\x1b[39m|')
        "|"+on_blue(red("hey"))+"|"
        """
        tokens_and_strings = parse(s)
        bases = []
        cur_fmt = {}
        for x in tokens_and_strings:
            if isinstance(x, dict):
                cur_fmt.update(x)
            elif isinstance(x, basestring):
                atts = parse_args('', {k:v for k,v in cur_fmt.items() if v is not None})
                bases.append(BaseFmtStr(x, atts=atts))
            else:
                raise Exception("logic error")
        return FmtStr(*bases)

    def set_attributes(self, **attributes):
        for k, v in attributes.items():
            for fs in self.fmtstrs:
                fs.atts[k] = v

    def __str__(self):
        return ''.join(str(fs) for fs in self.fmtstrs)

    def __len__(self):
        return sum(len(fs) for fs in self.fmtstrs)

    def __repr__(self):
        return '+'.join(repr(fs) for fs in self.fmtstrs)

    def __add__(self, other):
        if isinstance(other, FmtStr):
            return FmtStr(*(copy.deepcopy(x) for x in (self.fmtstrs + other.fmtstrs)))
        elif isinstance(other, basestring):
            return FmtStr(*(copy.deepcopy(x) for x in (self.fmtstrs + [BaseFmtStr(other)])))
        else:
            raise TypeError('Can\'t add %r and %r' % (self, other))

    def __radd__(self, other):
        if isinstance(other, FmtStr):
            return FmtStr(*(copy.deepcopy(x) for x in (other.fmtstrs + self.fmtstrs)))
        elif isinstance(other, basestring):
            return FmtStr(*(copy.deepcopy(x) for x in ([BaseFmtStr(other)] + self.fmtstrs)))
        else:
            raise TypeError('Can\'t add those')

    @property
    def shared_atts(self):
        """Gets atts shared among all nonzero length component BaseFmtStrs"""
        #TODO cache this, could get ugly for large FmtStrs
        atts = {}
        first = self.fmtstrs[0]
        for att in first.atts:
            #TODO how to write this without the '???'?
            if all(fs.atts.get(att, '???') == first.atts[att] for fs in self.fmtstrs if len(fs) > 0):
                atts[att] = first.atts[att]
        return atts

    def __getattr__(self, att):
        # thanks to @aerenchyma/@jczetta
        def func_help(*args, **kwargs):
             result = getattr(self.s, att)(*args, **kwargs)
             if isinstance(result, basestring):
                 return fmtstr(result, **self.shared_atts)
             elif isinstance(result, list):
                 return [fmtstr(x, **self.shared_atts) for x in result]
             else:
                 return result
        return func_help

    @property
    def s(self):
        return "".join(fs.s for fs in self.fmtstrs)

    def __getitem__(self, index):
        index = normalize_slice(len(self), index)
        counter = 0
        output = ''
        for fs in self.fmtstrs:
            if index.start < counter + len(fs) and index.stop > counter:
                s_part = fs.s[max(0, index.start - counter):index.stop - counter]
                piece = str(BaseFmtStr(s_part, fs.atts))
                output += piece
            counter += len(fs)
        return output

    def __setitem__(self, index, value):
        index = normalize_slice(len(self), index)
        if isinstance(value, basestring):
            value = FmtStr(BaseFmtStr(value))
        elif not isinstance(value, FmtStr):
            raise ValueError('Should be str or FmtStr')
        counter = 0
        old_fmtstrs = self.fmtstrs[:]
        self.fmtstrs = []
        inserted = False
        for fs in old_fmtstrs:
            if index.start < counter + len(fs) and index.stop > counter:
                start = max(0, index.start - counter)
                end = index.stop - counter
                front = BaseFmtStr(fs.s[:start], fs.atts)
                # stuff
                new = value
                back = BaseFmtStr(fs.s[end:], fs.atts)
                if len(front) > 0:
                    self.fmtstrs.append(front)
                if len(new) > 0 and not inserted:
                    self.fmtstrs.extend(new.fmtstrs)
                    inserted = True
                if len(back) > 0:
                    self.fmtstrs.append(back)
            else:
                self.fmtstrs.append(fs)
            counter += len(fs)

def normalize_slice(length, index):
    is_int = False
    if isinstance(index, int):
        is_int = True
        index = slice(index, index+1)
    if index.start is None:
        index = slice(0, index.stop, index.step)
    if index.stop is None:
        index = slice(index.start, length, index.step)
    if index.start < -1:
        index = slice(length - index.start, index.stop, index.step)
    if index.stop < -1:
        index = slice(index.start, length - index.stop, index.step)
    if index.step is not None:
        raise NotImplementedError("You can't use steps with slicing yet")
    if is_int:
        if index.start < 0 or index.start > length:
            raise IndexError("index out of bounds")
    return index

def parse_args(args, kwargs):
    if 'style' in kwargs:
        args += (kwargs['style'],)
        del kwargs['style']
    for arg in args:
        if not isinstance(arg, basestring):
            raise ValueError("args must be strings:" + repr(args))
        if arg.lower() in FG_COLORS:
            if 'fg' in kwargs: raise ValueError("fg specified twice")
            kwargs['fg'] = FG_COLORS[arg]
        elif arg.lower().startswith('on_') and arg[3:].lower() in BG_COLORS:
            if 'bg' in kwargs: raise ValueError("fg specified twice")
            kwargs['bg'] = BG_COLORS[arg[3:]]
        elif arg.lower() in STYLES:
            kwargs[arg] = True
        else:
            raise ValueError("couldn't process arg: "+repr(arg))
    for k in kwargs:
        if k not in ['fg', 'bg'] + STYLES.keys():
            raise ValueError("Can't apply that transformation")
    if 'fg' in kwargs:
        if kwargs['fg'] in FG_COLORS:
            kwargs['fg'] = FG_COLORS[kwargs['fg']]
        if kwargs['fg'] not in FG_COLORS.values():
            raise ValueError("Bad fg value: %s", kwargs['fg'])
    if 'bg' in kwargs:
        if kwargs['bg'] in BG_COLORS:
            kwargs['bg'] = BG_COLORS[kwargs['bg']]
        if kwargs['bg'] not in BG_COLORS.values():
            raise ValueError("Bad bg value: %s", kwargs['bg'])
    return kwargs

# convenience functions
def fmtstr(string, *args, **kwargs):
    atts = parse_args(args, kwargs)
    if isinstance(string, FmtStr):
        string.set_attributes(**atts)
        return string
    elif isinstance(string, basestring):
        string = FmtStr.from_str(string)
        string.set_attributes(**atts)
        return string
    else:
        raise ValueError("Bad Args: %r %r %r" % (string, args, kwargs))

for att in itertools.chain(FG_COLORS, ('on_'+x for x in BG_COLORS), STYLES):
    locals()[att] = functools.partial(fmtstr, style=att)
plain = functools.partial(fmtstr)

if __name__ == '__main__':
    import doctest
    doctest.testmod()
    print blue('adf')
    print blue(on_red('ad'))
    print blue('asdf') + on_red('adsf')
    print (blue('asdf') + on_red('adsf'))[3:7]
    f = blue('hey there') + on_red(' Tom!')
    print f
    f[1:3] = 'ot'
    print repr(f)
    print f
    f = FmtStr.from_str(str(blue('tom')))
    print repr(f)

    print '=---'

    f = on_blue(red('stuff'))
    print repr(f)
    print repr(str(f))
    print f
    print (f + '!')[0:6] + '?'
<commit_msg>add eq<commit_after>r"""
>>> s = fmtstr("Hey there!", 'red')
>>> s
red("Hey there!")
>>> s[4:7]
'\x1b[31mthe\x1b[39m'
"""
#TODO add a way to composite text without losing original formatting information
import itertools
import functools
import copy

from escseqparse import parse
from termformatconstants import FG_COLORS, BG_COLORS, STYLES
from termformatconstants import FG_NUMBER_TO_COLOR, BG_NUMBER_TO_COLOR
from termformatconstants import RESET_ALL, RESET_BG, RESET_FG
from termformatconstants import seq

xforms = {
    'fg' : lambda x, v: seq(v)+x+seq(RESET_FG),
    'bg' : lambda x, v: seq(v)+x+seq(RESET_BG),
    'bold' : lambda x: seq(STYLES['bold'])+x+seq(RESET_ALL),
    'underline' : lambda x: seq(STYLES['underline'])+x+seq(RESET_ALL),
    'blink' : lambda x: seq(STYLES['blink'])+x+seq(RESET_ALL),
    'invert' : lambda x: seq(STYLES['invert'])+x+seq(RESET_ALL),
}

class BaseFmtStr(object):
    """Formatting annotations on strings"""
    def __init__(self, string, atts=None):
        self.s = string
        self.atts = {k:v for k,v in atts.items()} if atts else {}

    def __len__(self):
        return len(self.s)

    def __str__(self):
        s = self.s
        for k, v in self.atts.items():
            if k not in xforms: continue
            if v is True:
                s = xforms[k](s)
            else:
                s = xforms[k](s, v)
        return s

    def __getitem__(self, index):
        return fmtstr(str(self)[index])

    def __repr__(self):
        def pp_att(att):
            if att == 'fg': return FG_NUMBER_TO_COLOR[self.atts[att]]
            elif att == 'bg': return 'on_' + BG_NUMBER_TO_COLOR[self.atts[att]]
            else: return att
        return (''.join(
                        pp_att(att)+'('
                        for att in sorted(self.atts)) +
               ('"%s"' % self.s) + ')'*len(self.atts))

class FmtStr(object):
    def __init__(self, *components):
        assert all(isinstance(x, BaseFmtStr) for x in components)
        self.fmtstrs = [x for x in components if len(x) > 0]

    @classmethod
    def from_str(cls, s):
        r"""
        >>> fmtstr("|"+on_blue(red("hey"))+"|")
        "|"+on_blue(red("hey"))+"|"
        >>> fmtstr('|\x1b[31m\x1b[44mhey\x1b[49m\x1b[39m|')
        "|"+on_blue(red("hey"))+"|"
        """
        tokens_and_strings = parse(s)
        bases = []
        cur_fmt = {}
        for x in tokens_and_strings:
            if isinstance(x, dict):
                cur_fmt.update(x)
            elif isinstance(x, basestring):
                atts = parse_args('', {k:v for k,v in cur_fmt.items() if v is not None})
                bases.append(BaseFmtStr(x, atts=atts))
            else:
                raise Exception("logic error")
        return FmtStr(*bases)

    def set_attributes(self, **attributes):
        for k, v in attributes.items():
            for fs in self.fmtstrs:
                fs.atts[k] = v

    def __str__(self):
        return ''.join(str(fs) for fs in self.fmtstrs)

    def __len__(self):
        return sum(len(fs) for fs in self.fmtstrs)

    def __repr__(self):
        return '+'.join(repr(fs) for fs in self.fmtstrs)

    def __eq__(self, other):
        return str(self) == str(other)

    def __add__(self, other):
        if isinstance(other, FmtStr):
            return FmtStr(*(copy.deepcopy(x) for x in (self.fmtstrs + other.fmtstrs)))
        elif isinstance(other, basestring):
            return FmtStr(*(copy.deepcopy(x) for x in (self.fmtstrs + [BaseFmtStr(other)])))
        else:
            raise TypeError('Can\'t add %r and %r' % (self, other))

    def __radd__(self, other):
        if isinstance(other, FmtStr):
            return FmtStr(*(copy.deepcopy(x) for x in (other.fmtstrs + self.fmtstrs)))
        elif isinstance(other, basestring):
            return FmtStr(*(copy.deepcopy(x) for x in ([BaseFmtStr(other)] + self.fmtstrs)))
        else:
            raise TypeError('Can\'t add those')

    @property
    def shared_atts(self):
        """Gets atts shared among all nonzero length component BaseFmtStrs"""
        #TODO cache this, could get ugly for large FmtStrs
        atts = {}
        first = self.fmtstrs[0]
        for att in first.atts:
            #TODO how to write this without the '???'?
            if all(fs.atts.get(att, '???') == first.atts[att] for fs in self.fmtstrs if len(fs) > 0):
                atts[att] = first.atts[att]
        return atts

    def __getattr__(self, att):
        # thanks to @aerenchyma/@jczetta
        def func_help(*args, **kwargs):
             result = getattr(self.s, att)(*args, **kwargs)
             if isinstance(result, basestring):
                 return fmtstr(result, **self.shared_atts)
             elif isinstance(result, list):
                 return [fmtstr(x, **self.shared_atts) for x in result]
             else:
                 return result
        return func_help

    @property
    def s(self):
        return "".join(fs.s for fs in self.fmtstrs)

    def __getitem__(self, index):
        index = normalize_slice(len(self), index)
        counter = 0
        output = ''
        for fs in self.fmtstrs:
            if index.start < counter + len(fs) and index.stop > counter:
                s_part = fs.s[max(0, index.start - counter):index.stop - counter]
                piece = str(BaseFmtStr(s_part, fs.atts))
                output += piece
            counter += len(fs)
        return output

    def __setitem__(self, index, value):
        index = normalize_slice(len(self), index)
        if isinstance(value, basestring):
            value = FmtStr(BaseFmtStr(value))
        elif not isinstance(value, FmtStr):
            raise ValueError('Should be str or FmtStr')
        counter = 0
        old_fmtstrs = self.fmtstrs[:]
        self.fmtstrs = []
        inserted = False
        for fs in old_fmtstrs:
            if index.start < counter + len(fs) and index.stop > counter:
                start = max(0, index.start - counter)
                end = index.stop - counter
                front = BaseFmtStr(fs.s[:start], fs.atts)
                # stuff
                new = value
                back = BaseFmtStr(fs.s[end:], fs.atts)
                if len(front) > 0:
                    self.fmtstrs.append(front)
                if len(new) > 0 and not inserted:
                    self.fmtstrs.extend(new.fmtstrs)
                    inserted = True
                if len(back) > 0:
                    self.fmtstrs.append(back)
            else:
                self.fmtstrs.append(fs)
            counter += len(fs)

def normalize_slice(length, index):
    is_int = False
    if isinstance(index, int):
        is_int = True
        index = slice(index, index+1)
    if index.start is None:
        index = slice(0, index.stop, index.step)
    if index.stop is None:
        index = slice(index.start, length, index.step)
    if index.start < -1:
        index = slice(length - index.start, index.stop, index.step)
    if index.stop < -1:
        index = slice(index.start, length - index.stop, index.step)
    if index.step is not None:
        raise NotImplementedError("You can't use steps with slicing yet")
    if is_int:
        if index.start < 0 or index.start > length:
            raise IndexError("index out of bounds")
    return index

def parse_args(args, kwargs):
    if 'style' in kwargs:
        args += (kwargs['style'],)
        del kwargs['style']
    for arg in args:
        if not isinstance(arg, basestring):
            raise ValueError("args must be strings:" + repr(args))
        if arg.lower() in FG_COLORS:
            if 'fg' in kwargs: raise ValueError("fg specified twice")
            kwargs['fg'] = FG_COLORS[arg]
        elif arg.lower().startswith('on_') and arg[3:].lower() in BG_COLORS:
            if 'bg' in kwargs: raise ValueError("fg specified twice")
            kwargs['bg'] = BG_COLORS[arg[3:]]
        elif arg.lower() in STYLES:
            kwargs[arg] = True
        else:
            raise ValueError("couldn't process arg: "+repr(arg))
    for k in kwargs:
        if k not in ['fg', 'bg'] + STYLES.keys():
            raise ValueError("Can't apply that transformation")
    if 'fg' in kwargs:
        if kwargs['fg'] in FG_COLORS:
            kwargs['fg'] = FG_COLORS[kwargs['fg']]
        if kwargs['fg'] not in FG_COLORS.values():
            raise ValueError("Bad fg value: %s", kwargs['fg'])
    if 'bg' in kwargs:
        if kwargs['bg'] in BG_COLORS:
            kwargs['bg'] = BG_COLORS[kwargs['bg']]
        if kwargs['bg'] not in BG_COLORS.values():
            raise ValueError("Bad bg value: %s", kwargs['bg'])
    return kwargs

# convenience functions
def fmtstr(string, *args, **kwargs):
    atts = parse_args(args, kwargs)
    if isinstance(string, FmtStr):
        string.set_attributes(**atts)
        return string
    elif isinstance(string, basestring):
        string = FmtStr.from_str(string)
        string.set_attributes(**atts)
        return string
    else:
        raise ValueError("Bad Args: %r %r %r" % (string, args, kwargs))

for att in itertools.chain(FG_COLORS, ('on_'+x for x in BG_COLORS), STYLES):
    locals()[att] = functools.partial(fmtstr, style=att)
plain = functools.partial(fmtstr)

if __name__ == '__main__':
    import doctest
    doctest.testmod()
    print blue('adf')
    print blue(on_red('ad'))
    print blue('asdf') + on_red('adsf')
    print (blue('asdf') + on_red('adsf'))[3:7]
    f = blue('hey there') + on_red(' Tom!')
    print f
    f[1:3] = 'ot'
    print repr(f)
    print f
    f = FmtStr.from_str(str(blue('tom')))
    print repr(f)

    print '=---'

    f = on_blue(red('stuff'))
    print repr(f)
    print repr(str(f))
    print f
    print (f + '!')[0:6] + '?'
<|endoftext|>


<commit_before>import string
import sys

import googleapiclient.errors

import gam
from gam.var import *
from gam import controlflow
from gam import display
from gam import fileutils
from gam import gapi
from gam import utils
from gam.gapi import errors as gapi_errors
from gam.gapi import cloudresourcemanager as gapi_crm

THROW_REASONS = [gapi_errors.ErrorReason.FOUR_O_THREE]

def _gen_role_error(caa):
    sa_email = caa._http.credentials.signer_email
    role_error = f'Please grant service account {sa_email} the Access Context Manager Editor role to your GCP organization.'
    controlflow.system_error_exit(2, role_error)

def build():
    return gam.buildGAPIServiceObject('accesscontextmanager',
                                      act_as=None)

def get_access_policy(caa=None):
    if not caa:
        build()
    parent = gapi_crm.get_org_id()
    if not parent:
        _gen_role_error(caa)
    try:
        aps = gapi.get_all_pages(caa.accessPolicies(),
                                 'list',
                                 'accessPolicies',
                                 throw_reasons=THROW_REASONS,
                                 parent=parent,
                                 fields='accessPolicies(name,title)')
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)
    if not aps:
        controlflow.system_error_exit(2, 'You don\'t seem to have any access policies. That is odd.')
    elif len(aps) == 1:
        return aps[0]['name']
    for ap in aps:
        if ap.get('title') == 'Access policy created in Cloud Identity Console':
            return ap['name']
    controlflow.system_error_exit(2, ' Could not find a org level access policy. That is odd.')

def print_access_levels():
    caa = build()
    ap_name = get_access_policy(caa)
    try:
        levels = gapi.get_all_pages(caa.accessPolicies().accessLevels(),
                                    'list',
                                    'accessLevels',
                                    throw_reasons=THROW_REASONS,
                                    parent=ap_name,
                                    accessLevelFormat='CEL', fields='*')
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)
    for level in levels:
        display.print_json(level)
        print()

def build_os_constraints(constraints):
    consts_obj = []
    constraints = constraints.upper().split(',')
    valid_os_types = ['DESKTOP_MAC', 'DESKTOP_WINDOWS', 'DESKTOP_LINUX', 'DESKTOP_CHROMEOS', 'ANDROID', 'IOS']
    for constraint in constraints:
        new_const = {}
        new_const['osType'], new_const['minimumVersion'] = constraint.split(':')
        if new_const['osType'] == 'VERIFIED_DESKTOP_CHROME_OS':
            new_const['osType'] = 'DESKTOP_CHROME_OS'
            new_const['requireVerifiedChromeOs'] = True
        if new_const['osType'] not in valid_os_types:
            controlflow.system_error_exit(2, f'expected os type of {", ".join(valid_os_types)} got {new_const["osType"]}')
        consts_obj.append(new_const)
    return consts_obj

def build_device_policy(i, schemas):
    device_policy = {}
    while True:
        myarg = sys.argv[i].replace('_', '').lower()
        if myarg == 'requirescreenlock':
            device_policy['requireScreenLock'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'allowedencryptionstatuses':
            allowed_statuses = gapi.get_enum_values_minus_unspecified(schemas["DevicePolicy"]["properties"]["allowedEncryptionStatuses"]["items"]["enum"])
            device_policy['allowedEncryptionStatuses'] = sys.argv[i+1].upper().split(',')
            for status in device_policy['allowedEncryptionStatuses']:
                if status not in allowed_statuses:
                    controlflow.system_error_exit(2, f'expected encryption status of {", ".join(allowed_statuses)} got {status}')
            i += 2
        elif myarg == 'osconstraints':
            device_policy['osConstraints'] = build_os_constraints(sys.argv[i+1])
            i += 2
        elif myarg == 'alloweddevicemanagementlevels':
            allowed_levels = gapi.get_enum_values_minus_unspecified(schemas["DevicePolicy"]["properties"]["allowedDeviceManagementLevels"]["items"]["enum"])
            device_policy['allowedDeviceManagementLevels'] = sys.argv[i+1].upper().split(',')
            for level in device_policy['allowedDeviceManagementLevels']:
                if level == 'ADVANCED':
                    level == 'COMPLETE'
                if level not in allowed_levels:
                    controlflow.system_error_exit(2, f'expected device management level of {", ".join(allowed_levels)} got {level}')
            i += 2
        elif myarg == 'requireadminapproval':
            device_policy['requireAdminApproval'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'requirecorpowned':
            device_policy['requireCorpOwned'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'enddevicepolicy':
            i += 1
            break
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create/update caalevel')
    return i, device_policy

def build_condition(i, schemas):
    condition = {}
    while True:
        myarg = sys.argv[i].replace('_', '').lower()
        if myarg == 'ipsubnetworks':
            condition['ipSubnetworks'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'devicepolicy':
            i += 1
            i, condition['devicePolicy'] = build_device_policy(i, schemas)
        elif myarg == 'requiredaccesslevels':
            condition['requiredaccesslevels'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'negate':
            condition['negate'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'members':
            condition['members'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'regions':
            condition['regions'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'endcondition':
            i += 1
            break
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create/update caalevel')
    return i, condition

def build_basic_level(i, schemas):
    basic_level = {'conditions': []}
    valid_functions = gapi.get_enum_values_minus_unspecified(schemas['BasicLevel']['properties']['combiningFunction']['enum'])
    while i < len(sys.argv):
        myarg = sys.argv[i].replace('_', '').lower()
        if myarg == 'combiningfunction':
            combiningFunction = sys.argv[i+1].upper()
            if combiningFunction not in valid_functions:
                controlflow.system_error_exit(2, f'expected combining function of {",".join(valid_functions)} got {combiningFunction}')
            basic_level['combiningFunction'] = combiningFunction
            i += 2
        elif myarg == 'condition':
            i += 1
            i, condition = build_condition(i, schemas)
            basic_level['conditions'].append(condition)
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create/update caalevel')
    return i, basic_level

def create_access_level():
    caa = build()
    ap_name = get_access_policy(caa)
    title = sys.argv[3].replace(' ', '_')
    custom = {'expr': {'expression': sys.argv[4], 'title': 'expr'}}
    allowed_title_chars = string.ascii_letters + string.digits + '_'
    name = ''.join([c for c in title if c in allowed_title_chars])[:49]
    name = f'{ap_name}/accessLevels/{name}'
    body = {
            'name': name,
            'title': title,
           }
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'basic':
            schemas = caa._rootDesc['schemas']
            i += 1
            i, body['basic'] = build_basic_level(i, schemas)
        elif myarg == 'custom':
            body['custom'] = {'expr': {'expression': sys.argv[i+1], 'title': 'expr'}}
            i += 2
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create caalevel')
    print(f'Creating access level {name}...')
    try:
        gapi.call(caa.accessPolicies().accessLevels(),
                  'create',
                  throw_reasons=THROW_REASONS,
                  parent=ap_name,
                  body=body)
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)

def update_access_level():
    caa = build()
    name = sys.argv[3]
    if not name.startswith('accessPolicies/'):
        ap_name = get_access_policy(caa)
        name = f'{ap_name}/accessLevels/{name}'
    body = {}
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'basic':
            schemas = caa._rootDesc['schemas']
            i += 1
            i, body['basic'] = build_basic_level(i, schemas)
        elif myarg == 'custom':
            body['custom'] = {'expr': {'expression': sys.argv[i+1], 'title': 'expr'}}
            i += 2
        else:
            controlflow.invalid_argument_exit(myarg, 'gam update caalevel')
    updateMask = ','.join(body.keys())
    print(f'Updating access level {name}...')
    try:
        gapi.call(caa.accessPolicies().accessLevels(),
                  'patch',
                  throw_reasons=THROW_REASONS,
                  name=name,
                  updateMask=updateMask,
                  body=body)
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)

def delete_access_level():
    caa = build()
    name = sys.argv[3]
    if not name.startswith('accessPolicies/'):
        ap_name = get_access_policy(caa)
        name = f'{ap_name}/accessLevels/{name}'
    print(f'Deleting access level {name}...')
    try:
        gapi.call(caa.accessPolicies().accessLevels(),
                  'delete',
                  name=name)
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)
<commit_msg>Fix bug (#1502)<commit_after>import string
import sys

import googleapiclient.errors

import gam
from gam.var import *
from gam import controlflow
from gam import display
from gam import fileutils
from gam import gapi
from gam import utils
from gam.gapi import errors as gapi_errors
from gam.gapi import cloudresourcemanager as gapi_crm

THROW_REASONS = [gapi_errors.ErrorReason.FOUR_O_THREE]

def _gen_role_error(caa):
    sa_email = caa._http.credentials.signer_email
    role_error = f'Please grant service account {sa_email} the Access Context Manager Editor role to your GCP organization.'
    controlflow.system_error_exit(2, role_error)

def build():
    return gam.buildGAPIServiceObject('accesscontextmanager',
                                      act_as=None)

def get_access_policy(caa=None):
    if not caa:
        caa = build()
    parent = gapi_crm.get_org_id()
    if not parent:
        _gen_role_error(caa)
    try:
        aps = gapi.get_all_pages(caa.accessPolicies(),
                                 'list',
                                 'accessPolicies',
                                 throw_reasons=THROW_REASONS,
                                 parent=parent,
                                 fields='accessPolicies(name,title)')
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)
    if not aps:
        controlflow.system_error_exit(2, 'You don\'t seem to have any access policies. That is odd.')
    elif len(aps) == 1:
        return aps[0]['name']
    for ap in aps:
        if ap.get('title') == 'Access policy created in Cloud Identity Console':
            return ap['name']
    controlflow.system_error_exit(2, ' Could not find a org level access policy. That is odd.')

def print_access_levels():
    caa = build()
    ap_name = get_access_policy(caa)
    try:
        levels = gapi.get_all_pages(caa.accessPolicies().accessLevels(),
                                    'list',
                                    'accessLevels',
                                    throw_reasons=THROW_REASONS,
                                    parent=ap_name,
                                    accessLevelFormat='CEL', fields='*')
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)
    for level in levels:
        display.print_json(level)
        print()

def build_os_constraints(constraints):
    consts_obj = []
    constraints = constraints.upper().split(',')
    valid_os_types = ['DESKTOP_MAC', 'DESKTOP_WINDOWS', 'DESKTOP_LINUX', 'DESKTOP_CHROMEOS', 'ANDROID', 'IOS']
    for constraint in constraints:
        new_const = {}
        new_const['osType'], new_const['minimumVersion'] = constraint.split(':')
        if new_const['osType'] == 'VERIFIED_DESKTOP_CHROME_OS':
            new_const['osType'] = 'DESKTOP_CHROME_OS'
            new_const['requireVerifiedChromeOs'] = True
        if new_const['osType'] not in valid_os_types:
            controlflow.system_error_exit(2, f'expected os type of {", ".join(valid_os_types)} got {new_const["osType"]}')
        consts_obj.append(new_const)
    return consts_obj

def build_device_policy(i, schemas):
    device_policy = {}
    while True:
        myarg = sys.argv[i].replace('_', '').lower()
        if myarg == 'requirescreenlock':
            device_policy['requireScreenLock'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'allowedencryptionstatuses':
            allowed_statuses = gapi.get_enum_values_minus_unspecified(schemas["DevicePolicy"]["properties"]["allowedEncryptionStatuses"]["items"]["enum"])
            device_policy['allowedEncryptionStatuses'] = sys.argv[i+1].upper().split(',')
            for status in device_policy['allowedEncryptionStatuses']:
                if status not in allowed_statuses:
                    controlflow.system_error_exit(2, f'expected encryption status of {", ".join(allowed_statuses)} got {status}')
            i += 2
        elif myarg == 'osconstraints':
            device_policy['osConstraints'] = build_os_constraints(sys.argv[i+1])
            i += 2
        elif myarg == 'alloweddevicemanagementlevels':
            allowed_levels = gapi.get_enum_values_minus_unspecified(schemas["DevicePolicy"]["properties"]["allowedDeviceManagementLevels"]["items"]["enum"])
            device_policy['allowedDeviceManagementLevels'] = sys.argv[i+1].upper().split(',')
            for level in device_policy['allowedDeviceManagementLevels']:
                if level == 'ADVANCED':
                    level == 'COMPLETE'
                if level not in allowed_levels:
                    controlflow.system_error_exit(2, f'expected device management level of {", ".join(allowed_levels)} got {level}')
            i += 2
        elif myarg == 'requireadminapproval':
            device_policy['requireAdminApproval'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'requirecorpowned':
            device_policy['requireCorpOwned'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'enddevicepolicy':
            i += 1
            break
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create/update caalevel')
    return i, device_policy

def build_condition(i, schemas):
    condition = {}
    while True:
        myarg = sys.argv[i].replace('_', '').lower()
        if myarg == 'ipsubnetworks':
            condition['ipSubnetworks'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'devicepolicy':
            i += 1
            i, condition['devicePolicy'] = build_device_policy(i, schemas)
        elif myarg == 'requiredaccesslevels':
            condition['requiredaccesslevels'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'negate':
            condition['negate'] = gam.getBoolean(sys.argv[i+1], myarg)
            i += 2
        elif myarg == 'members':
            condition['members'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'regions':
            condition['regions'] = sys.argv[i+1].split(',')
            i += 2
        elif myarg == 'endcondition':
            i += 1
            break
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create/update caalevel')
    return i, condition

def build_basic_level(i, schemas):
    basic_level = {'conditions': []}
    valid_functions = gapi.get_enum_values_minus_unspecified(schemas['BasicLevel']['properties']['combiningFunction']['enum'])
    while i < len(sys.argv):
        myarg = sys.argv[i].replace('_', '').lower()
        if myarg == 'combiningfunction':
            combiningFunction = sys.argv[i+1].upper()
            if combiningFunction not in valid_functions:
                controlflow.system_error_exit(2, f'expected combining function of {",".join(valid_functions)} got {combiningFunction}')
            basic_level['combiningFunction'] = combiningFunction
            i += 2
        elif myarg == 'condition':
            i += 1
            i, condition = build_condition(i, schemas)
            basic_level['conditions'].append(condition)
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create/update caalevel')
    return i, basic_level

def create_access_level():
    caa = build()
    ap_name = get_access_policy(caa)
    title = sys.argv[3].replace(' ', '_')
    custom = {'expr': {'expression': sys.argv[4], 'title': 'expr'}}
    allowed_title_chars = string.ascii_letters + string.digits + '_'
    name = ''.join([c for c in title if c in allowed_title_chars])[:49]
    name = f'{ap_name}/accessLevels/{name}'
    body = {
            'name': name,
            'title': title,
           }
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'basic':
            schemas = caa._rootDesc['schemas']
            i += 1
            i, body['basic'] = build_basic_level(i, schemas)
        elif myarg == 'custom':
            body['custom'] = {'expr': {'expression': sys.argv[i+1], 'title': 'expr'}}
            i += 2
        else:
            controlflow.invalid_argument_exit(myarg, 'gam create caalevel')
    print(f'Creating access level {name}...')
    try:
        gapi.call(caa.accessPolicies().accessLevels(),
                  'create',
                  throw_reasons=THROW_REASONS,
                  parent=ap_name,
                  body=body)
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)

def update_access_level():
    caa = build()
    name = sys.argv[3]
    if not name.startswith('accessPolicies/'):
        ap_name = get_access_policy(caa)
        name = f'{ap_name}/accessLevels/{name}'
    body = {}
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'basic':
            schemas = caa._rootDesc['schemas']
            i += 1
            i, body['basic'] = build_basic_level(i, schemas)
        elif myarg == 'custom':
            body['custom'] = {'expr': {'expression': sys.argv[i+1], 'title': 'expr'}}
            i += 2
        else:
            controlflow.invalid_argument_exit(myarg, 'gam update caalevel')
    updateMask = ','.join(body.keys())
    print(f'Updating access level {name}...')
    try:
        gapi.call(caa.accessPolicies().accessLevels(),
                  'patch',
                  throw_reasons=THROW_REASONS,
                  name=name,
                  updateMask=updateMask,
                  body=body)
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)

def delete_access_level():
    caa = build()
    name = sys.argv[3]
    if not name.startswith('accessPolicies/'):
        ap_name = get_access_policy(caa)
        name = f'{ap_name}/accessLevels/{name}'
    print(f'Deleting access level {name}...')
    try:
        gapi.call(caa.accessPolicies().accessLevels(),
                  'delete',
                  name=name)
    except googleapiclient.errors.HttpError:
        _gen_role_error(caa)
<|endoftext|>


<commit_before>#!/usr/bin/python
import aws_instance
import subprocess
instance_id = 'i-318ca4c6'
image_type = "gluster"
new_instance_name = 'gluster0test.nmdev.us'
#create_instance_like(instance_id='i-318ca4c6', image_type="gluster", new_instance_name='gluster0test.nmdev.us')

user="root"
host="gluster01.nmdev.us"
aws_key='/var/jenkins_home/.ssh/aws.pem'
aws_key='/Users/csterling/.ssh/nmd/jenkins_ac.pem'
# ssh -p22 -i /var/jenkins_home/.ssh/aws.pem -o StrictHostKeyChecking=no {{ USERNAME }}@{{ HOST }}
ssh_cmd = ['ssh', '-p22', '-i', aws_key, '-o', 'StrictHostKeyChecking=no', "{user}@{host}".format(user=user,host=host)]
ssh_cmd = ssh_cmd + '"cat /etc/fstab"'.split(" ")
a = subprocess.check_output(ssh_cmd)

print ">>>>a<<<<"

# AWS devices /dev/sda and /dev/xvdf
# [csterling@gluster01 ~]$ cat /etc/fstab
# LABEL=centos_root   /        ext4      defaults         0 0
# devpts     /dev/pts  devpts  gid=5,mode=620   0 0
# tmpfs      /dev/shm  tmpfs   defaults         0 0
# proc       /proc     proc    defaults         0 0
# sysfs      /sys      sysfs   defaults         0 0
# /dev/xvdf /srv/sdb1 ext4 defaults 0 2

# # Get the fstab entry
# with open('/etc/fstab', 'r') as fp:
#   # Find the entry we want to operate on
#   lines = fp.readlines()
# for line in lines:
#   if line.startswith('/dev/xvdf'):
#     fstab_entry_line = line
#     fstab_entry = line.split(' ')
#     break

# # Write the fstab entry
# with open("/etc/fstab","r") as current_fstab:
#   with open("/tmp/fstab","wb") as tmp_fstab:
#     for line in current_fstab:
#       # Skip over the entry
#       if line != fstab_entry_line
#         tmp_fstab.write(line)
# # TODO - Move tmp fstab into place on old machine
<commit_msg>This should be the correct format for the SSH command now.<commit_after>#!/usr/bin/python
import aws_instance
import subprocess
instance_id = 'i-318ca4c6'
image_type = "gluster"
new_instance_name = 'gluster0test.nmdev.us'
new_instance_name = "gluster01.nmdev.us"
#create_instance_like(instance_id='i-318ca4c6', image_type="gluster", new_instance_name='gluster0test.nmdev.us')

user="root" if "nmdev.us" in new_instance_name else "ubuntu"
host=new_instance_name
aws_key='/var/jenkins_home/.ssh/aws.pem'
aws_key='/Users/csterling/.ssh/nmd/jenkins_ac.pem'
# ssh -p22 -i /var/jenkins_home/.ssh/aws.pem -o StrictHostKeyChecking=no {{ USERNAME }}@{{ HOST }}
ssh_cmd = ['ssh', '-p22', '-i', aws_key, '-o', 'StrictHostKeyChecking=no', "{user}@{host}".format(user=user,host=host)]
ssh_cmd = ssh_cmd + 'sudo cat /etc/fstab'.split(" ")
fstab_file_contents = subprocess.check_output(ssh_cmd)
for line in fstab_file_contents:
  if line.startswith('/dev/xvdf'):
    fstab_entry_line = line
    fstab_entry = line.split(' ')
    break
print fstab_entry

# AWS devices /dev/sda and /dev/xvdf
# [csterling@gluster01 ~]$ cat /etc/fstab
# LABEL=centos_root   /        ext4      defaults         0 0
# devpts     /dev/pts  devpts  gid=5,mode=620   0 0
# tmpfs      /dev/shm  tmpfs   defaults         0 0
# proc       /proc     proc    defaults         0 0
# sysfs      /sys      sysfs   defaults         0 0
# /dev/xvdf /srv/sdb1 ext4 defaults 0 2

# # Get the fstab entry
# with open('/etc/fstab', 'r') as fp:
#   # Find the entry we want to operate on
#   lines = fp.readlines()
# for line in lines:
#   if line.startswith('/dev/xvdf'):
#     fstab_entry_line = line
#     fstab_entry = line.split(' ')
#     break

# # Write the fstab entry
# with open("/etc/fstab","r") as current_fstab:
#   with open("/tmp/fstab","wb") as tmp_fstab:
#     for line in current_fstab:
#       # Skip over the entry
#       if line != fstab_entry_line
#         tmp_fstab.write(line)
# # TODO - Move tmp fstab into place on old machine
<|endoftext|>


<commit_before>"""
Rank all available models

@copyright: The Broad Institute of MIT and Harvard 2015
"""

import argparse
import os, csv
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

var_labels = { "OUT": "Outcome",
               "PCR": "PCR",
               "WEAK": "Weakness",
               "VOMIT": "Vomit",
               "AST_1": "AST",
               "Ca_1": "Ca",
               "AlkPhos_1": "ALK",
               "EDEMA": "Edema",
               "CONF": "Confussion",
               "Cl_1": "Cl",
               "TEMP": "Temperature",
               "RRATE": "Respiratory rate",
               "PBACK": "Back pain",
               "DIZZI": "Dizziness",
               "ALT_1": "ALT",
               "Cr_1": "CRE",
               "TCo2_1": "tCO2",
               "PRETROS": "Retrosternal pain",
               "DIARR": "Diarrhea",
               "HRATE": "Hearth rate",
               "Alb_1": "Alb",
               "BUN_1": "BUN",
               "TP_1": "TP",
               "PDIAST": "Diastolic pressure",
               "PABD": "Abdominal pain" }

parser = argparse.ArgumentParser()
parser.add_argument("-v", "--var_file", nargs=1, default=["./data/variables-master.txt"],
                    help="File containing list of variables to consider in the test")
args = parser.parse_args()

var_file = args.var_file[0]
# load_data(var_file)

depvar = None
numvar = []
catvar = []
with open(var_file, "rb") as vfile:
    for line in vfile.readlines():
        line = line.strip()
        if not line: continue
        parts = line.split()
        if not depvar: depvar = parts[0]
        else:
            if parts[1] == "category": catvar.append(parts[0])
            else: numvar.append(parts[0])
#
#         model_variables.append([0])

# dvar = model_variables[0]
# ivar = model_variables[1:]
# print dvar
# print ivar

src_file = "./data/sources.txt"
input_file = ""
with open(src_file, "rb") as sfile:
    for line in sfile.readlines():
        input_file = os.path.abspath(line.strip())
df = pd.read_csv(input_file, delimiter=",", na_values="\\N")

plt.clf()
fig = plt.figure()

# print df[dvar]
# color = dict(boxes='DarkGreen', whiskers='DarkOrange', medians='DarkBlue', caps='Gray')

range_file = "./data/ranges.txt"
range_variables = []
with open(range_file, "rb") as rfile:
    for line in rfile.readlines():
       line = line.strip()
       if not line: continue
       parts = line.strip().split()
       if 2 < len(parts):
           range_variables.append({"name":parts[0], "type":parts[1], "range":parts[2].split(",")})
ranges = (df["DIAG"] == 1) & (10 <= df["AGE"]) & (df["AGE"] <= 50) & ((df["OUT"] == 1) | (df["OUT"] == 0))

# sns.set(style="whitegrid")

for var in numvar:
    #df[ranges].boxplot(var, by=depvar)
    g = sns.factorplot("OUT", var, data=df[ranges], kind="box", ci=0, palette="coolwarm")
    g.despine(offset=10, trim=True)
    g.set_xticklabels(['Discharged', 'Died'])
    plt.xlabel('Outcome')
    plt.ylabel(var_labels[var])
    plt.savefig("./out/" + var + ".pdf")
    plt.clf()

for var in catvar:
	with mpl.rc_context({"lines.linewidth": 0.5}):
# 		g = sns.barplot(df[ranges][var], df[ranges]["OUT"], palette="coolwarm")
		g = sns.FacetGrid(df[ranges], aspect=1)
		g.map(sns.barplot, "OUT", var, palette="coolwarm");
		g.set_xticklabels(['Discharged', 'Died'])
		g.set(ylim=(0.0, 1.0))
        plt.xlabel('Outcome')
        plt.ylabel(var_labels[var])
#     df[ranges].hist(var, by=depvar)
#     df[ranges].plot(kind='area', var, by=depvar)
#
    	plt.savefig("./out/" + var + ".pdf")
    	plt.clf()
<commit_msg>some tweaks to the var plots<commit_after>"""
Generates plots for all variables

TODO: need to be generic for any type of dataset

@copyright: The Broad Institute of MIT and Harvard 2015
"""

import argparse
import os, csv
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

var_labels = { "OUT": "Outcome",
               "PCR": "PCR",
               "WEAK": "Weakness",
               "VOMIT": "Vomit",
               "AST_1": "AST",
               "Ca_1": "Ca",
               "AlkPhos_1": "ALK",
               "EDEMA": "Edema",
               "CONF": "Confussion",
               "Cl_1": "Cl",
               "TEMP": "Temperature",
               "RRATE": "Respiratory rate",
               "PBACK": "Back pain",
               "DIZZI": "Dizziness",
               "ALT_1": "ALT",
               "Cr_1": "CRE",
               "TCo2_1": "tCO2",
               "PRETROS": "Retrosternal pain",
               "DIARR": "Diarrhea",
               "HRATE": "Hearth rate",
               "Alb_1": "Alb",
               "BUN_1": "BUN",
               "TP_1": "TP",
               "PDIAST": "Diastolic pressure",
               "PABD": "Abdominal pain" }

var_pvalues = { "OUT": 0,
                "PCR": 0.001,
                "WEAK": 0.06,
                "VOMIT": 0.15,
                "AST_1": 0.0007,
                "Ca_1": 0.48,
                "AlkPhos_1": 0.008,
                "EDEMA": 0.3,
                "CONF": 0.56,
                "Cl_1": 0.24,
                "TEMP": 0.004,
                "RRATE": 0.41,
                "PBACK": 0.55,
                "DIZZI": 0.17,
                "ALT_1": 0.03,
                "Cr_1": 0.04,
                "TCo2_1": 0.009,
                "PRETROS": 0.57,
                "DIARR": 0.024,
                "HRATE": 0.07,
                "Alb_1": 0.16,
                "BUN_1": 0.02,
                "TP_1": 0.14,
                "PDIAST": 0.59,
                "PABD": 0.37 }

parser = argparse.ArgumentParser()
parser.add_argument("-v", "--var_file", nargs=1, default=["./data/variables-master.txt"],
                    help="File containing list of variables to consider in the test")
args = parser.parse_args()

var_file = args.var_file[0]
# load_data(var_file)

depvar = None
numvar = []
catvar = []
with open(var_file, "rb") as vfile:
    for line in vfile.readlines():
        line = line.strip()
        if not line: continue
        parts = line.split()
        if not depvar: depvar = parts[0]
        else:
            if parts[1] == "category": catvar.append(parts[0])
            else: numvar.append(parts[0])
#
#         model_variables.append([0])

# dvar = model_variables[0]
# ivar = model_variables[1:]
# print dvar
# print ivar

src_file = "./data/sources.txt"
input_file = ""
with open(src_file, "rb") as sfile:
    for line in sfile.readlines():
        input_file = os.path.abspath(line.strip())
df = pd.read_csv(input_file, delimiter=",", na_values="\\N")

plt.clf()
fig = plt.figure()

# print df[dvar]
# color = dict(boxes='DarkGreen', whiskers='DarkOrange', medians='DarkBlue', caps='Gray')

range_file = "./data/ranges.txt"
range_variables = []
with open(range_file, "rb") as rfile:
    for line in rfile.readlines():
       line = line.strip()
       if not line: continue
       parts = line.strip().split()
       if 2 < len(parts):
           range_variables.append({"name":parts[0], "type":parts[1], "range":parts[2].split(",")})
ranges = (df["DIAG"] == 1) & (10 <= df["AGE"]) & (df["AGE"] <= 50) & ((df["OUT"] == 1) | (df["OUT"] == 0))

# sns.set(style="whitegrid")

binary_palette = sns.color_palette([(122/255.0, 192/255.0,119/255.0),(153/255.0, 94/255.0, 78/255.0)])
# binary_palette = sns.color_palette("Set1", 2)
# binary_palette = "coolwarm"
# binary_palette = sns.xkcd_palette(["pale red", "denim blue"])

# sns.set_context(rc = {'lines.linewidth': 0.0})

sns.set_context("paper", rc={"lines.linewidth": 2})

# sns.axes_style({'axes.linewidth': 1})
for var in numvar:
    with mpl.rc_context({"lines.linewidth": 0.0, 'axes.linewidth': 0.0}):
        g = sns.factorplot("OUT", var, data=df[ranges], kind="box", ci=0, size=3, aspect=1, palette=binary_palette)
        g.despine(offset=10, trim=True)
        g.set_xticklabels(['Discharged', 'Died'])
        plt.title("P-value: " + str(var_pvalues[var]))
        plt.xlabel('Outcome')
        plt.ylabel(var_labels[var])
        plt.savefig("./out/" + var + ".pdf")
        plt.clf()

for var in catvar:
    with mpl.rc_context({"lines.linewidth": 0.5, 'patch.linewidth': 0.0}):
        g = sns.FacetGrid(df[ranges], aspect=1)
        g.map(sns.barplot, "OUT", var, palette=binary_palette)
        g.set_xticklabels(['Discharged', 'Died'])
        g.set(ylim=(0.0, 1.0))
        plt.title("P-value: " + str(var_pvalues[var]))
        plt.xlabel('Outcome')
        plt.ylabel(var_labels[var])
        plt.savefig("./out/" + var + ".pdf")
        plt.clf()
<|endoftext|>


<commit_before>from app import app
from flask.ext.appbuilder.security.models import is_menu_public, is_item_public
from flask.ext.appbuilder.models.datamodel import SQLAModel
from flask import g, request, url_for
from flask.ext.login import current_user

def app_template_filter(filter_name=''):
    """
        Use this decorator to expose views in your view classes.
    """
    def wrap(f):
        if not hasattr(f, '_filter'):
            f._filter = filter_name
        return f
    return wrap

class TemplateFilters(object):

    def __init__(self, app):
        for attr_name in dir(self):
            if hasattr(getattr(self, attr_name),'_filter'):
                attr = getattr(self, attr_name)
                app.jinja_env.filters[attr._filter] = attr

    @app_template_filter('link_order')
    def link_order_filter(self, column):
        new_args = request.view_args.copy()
        args = request.args.copy()
        if ('_oc_') in args:
            args['_oc_'] = column
            if args.get('_od_') == 'asc':
                args['_od_'] = 'desc'
            else:
                args['_od_'] = 'asc'
        else:
            args['_oc_'] = column
            args['_od_'] = 'asc'
        return url_for(request.endpoint,**dict(new_args.items() + args.to_dict().items()))

    @app_template_filter('link_page')
    def link_page_filter(self, page, generalview_name):

        new_args = request.view_args.copy()
        args = request.args.copy()
        args['page_' + generalview_name] = page
        return url_for(request.endpoint,**dict(new_args.items() + args.to_dict().items()))

    @app_template_filter('get_link_next')
    def get_link_next_filter(self, s):
        return request.args.get('next')

    # to improve
    @app_template_filter('set_link_filters')
    def set_link_filters_filter(self, path, filters, pk):
        lnkstr = path

        for _filter in filters:
            lnkstr = lnkstr + '&_flt_' + _filter + '=' + str(pk)
        return lnkstr

    @app_template_filter('get_link_order')
    def get_link_order_filter(self, s):
        if request.args.get('_oc_') == s:
            order_direction = request.args.get('_od_')
            if (order_direction):
                if (order_direction == 'asc'):
                    return 2
                else:
                    return 1
        else:
            return 0

    @app_template_filter('get_attr')
    def get_attr_filter(self, obj, item):
        return getattr(obj, item)

    @app_template_filter('is_menu_visible')
    def is_menu_visible(self, item):
        if current_user.is_authenticated():
            if is_menu_public(item) or g.user.has_menu_access(item.name):
                return True
            else:
                return False
        else:
            if is_menu_public(item.name):
                return True
            else:
                return False

    @app_template_filter('is_item_visible')
    def is_item_visible(self, permission, item):
        if current_user.is_authenticated():
            if g.user.has_permission_on_view(permission, item):
                return True
            else:
                return False
        else:
            if is_item_public(permission, item):
                return True
            else:
                return False
<commit_msg>Update filters.py<commit_after>from app import app
from flask.ext.appbuilder.security.models import is_menu_public, is_item_public
from flask.ext.appbuilder.models.datamodel import SQLAModel
from flask import g, request, url_for
from flask.ext.login import current_user

def app_template_filter(filter_name=''):
    """
        Use this decorator to expose views in your view classes.
    """
    def wrap(f):
        if not hasattr(f, '_filter'):
            f._filter = filter_name
        return f
    return wrap

class TemplateFilters(object):

    def __init__(self, app):
        for attr_name in dir(self):
            if hasattr(getattr(self, attr_name),'_filter'):
                attr = getattr(self, attr_name)
                app.jinja_env.filters[attr._filter] = attr

    @app_template_filter('link_order')
    """
    Arguments are passed like: _oc_<VIEW_NAME>=<COL_NAME>&_od_<VIEW_NAME>='asc'|'desc'
    """
    def link_order_filter(self, column, , generalview_name):
        new_args = request.view_args.copy()
        args = request.args.copy()
        if ('_oc_') in args:
            args['_oc_'] = column
            if args.get('_od_') == 'asc':
                args['_od_' + generalview_name] = 'desc'
            else:
                args['_od_' + generalview_name] = 'asc'
        else:
            args['_oc_' + generalview_name] = column
            args['_od_' + generalview_name] = 'asc'
        return url_for(request.endpoint,**dict(new_args.items() + args.to_dict().items()))

    @app_template_filter('link_page')
    """
    Arguments are passed like: page_<VIEW_NAME>=<PAGE_NUMBER>
    """
    def link_page_filter(self, page, generalview_name):

        new_args = request.view_args.copy()
        args = request.args.copy()
        args['page_' + generalview_name] = page
        return url_for(request.endpoint,**dict(new_args.items() + args.to_dict().items()))

    @app_template_filter('get_link_next')
    def get_link_next_filter(self, s):
        return request.args.get('next')

    # to improve
    @app_template_filter('set_link_filters')
    def set_link_filters_filter(self, path, filters, pk):
        lnkstr = path

        for _filter in filters:
            lnkstr = lnkstr + '&_flt_' + _filter + '=' + str(pk)
        return lnkstr

    @app_template_filter('get_link_order')
    def get_link_order_filter(self, s):
        if request.args.get('_oc_') == s:
            order_direction = request.args.get('_od_')
            if (order_direction):
                if (order_direction == 'asc'):
                    return 2
                else:
                    return 1
        else:
            return 0

    @app_template_filter('get_attr')
    def get_attr_filter(self, obj, item):
        return getattr(obj, item)

    @app_template_filter('is_menu_visible')
    def is_menu_visible(self, item):
        if current_user.is_authenticated():
            if is_menu_public(item) or g.user.has_menu_access(item.name):
                return True
            else:
                return False
        else:
            if is_menu_public(item.name):
                return True
            else:
                return False

    @app_template_filter('is_item_visible')
    def is_item_visible(self, permission, item):
        if current_user.is_authenticated():
            if g.user.has_permission_on_view(permission, item):
                return True
            else:
                return False
        else:
            if is_item_public(permission, item):
                return True
            else:
                return False
<|endoftext|>


<commit_before>"""Implement configuration file parsing."""

# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2018 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import re

from faucet import config_parser_util
from faucet.acl import ACL
from faucet.conf import test_config_condition, InvalidConfigError
from faucet.dp import DP
from faucet.meter import Meter
from faucet.port import Port
from faucet.router import Router
from faucet.vlan import VLAN
from faucet.watcher_conf import WatcherConf

V2_TOP_CONFS = (
    'acls',
    'dps',
    'meters',
    'routers',
    'vlans')

def dp_parser(config_file, logname):
    """Parse a config file into DP configuration objects with hashes of config include/files."""
    conf = config_parser_util.read_config(config_file, logname)
    config_hashes = None
    dps = None

    test_config_condition(conf is None, 'Config file is empty')
    test_config_condition(not isinstance(conf, dict), 'Config file does not have valid syntax')
    version = conf.pop('version', 2)
    test_config_condition(version != 2, 'Only config version 2 is supported')
    config_hashes, dps = _config_parser_v2(config_file, logname)
    test_config_condition(dps is None, 'no DPs are not defined')

    return config_hashes, dps

def _get_vlan_by_key(dp_id, vlan_key, vlans):
    test_config_condition(not isinstance(vlan_key, (str, int)), (
        'VLAN key must not be type %s' % type(vlan_key)))
    if vlan_key in vlans:
        return vlans[vlan_key]
    for vlan in vlans.values():
        if vlan_key == str(vlan.vid):
            return vlan
    # Create VLAN with VID, if not defined.
    return vlans.setdefault(vlan_key, VLAN(vlan_key, dp_id))

def _dp_parse_port(dp_id, port_key, port_conf, vlans):

    def _dp_parse_native_port_vlan():
        if port.native_vlan is not None:
            vlan = _get_vlan_by_key(dp_id, port.native_vlan, vlans)
            port.native_vlan = vlan

    def _dp_parse_tagged_port_vlans():
        if port.tagged_vlans:
            port_tagged_vlans = [
                _get_vlan_by_key(dp_id, vlan_key, vlans) for vlan_key in port.tagged_vlans]
            port.tagged_vlans = port_tagged_vlans

    port = Port(port_key, dp_id, port_conf)
    test_config_condition(str(port_key) not in (str(port.number), port.name), (
        'Port key %s match port name or port number' % port_key))
    _dp_parse_native_port_vlan()
    _dp_parse_tagged_port_vlans()
    return port

def _dp_add_ports(dp, dp_conf, dp_id, vlans):
    ports_conf = dp_conf.get('interfaces', {})
    port_ranges_conf = dp_conf.get('interface_ranges', {})
    # as users can config port VLAN by using VLAN name, we store vid in
    # Port instance instead of VLAN name for data consistency
    test_config_condition(not isinstance(ports_conf, dict), (
        'Invalid syntax in interface config'))
    test_config_condition(not isinstance(port_ranges_conf, dict), (
        'Invalid syntax in interface ranges config'))

    def _map_port_num_to_port(ports_conf):
        port_num_to_port_conf = {}
        for port_key, port_conf in ports_conf.items():
            test_config_condition(not isinstance(port_conf, dict), 'Invalid syntax in port config')
            port_num = port_conf.get('number', port_key)
            try:
                port_num_to_port_conf[port_num] = (port_key, port_conf)
            except TypeError:
                raise InvalidConfigError('Invalid syntax in port config')
        return port_num_to_port_conf

    def _parse_port_ranges(port_ranges_conf, port_num_to_port_conf):
        for port_range, port_conf in port_ranges_conf.items():
            # port range format: 1-6 OR 1-6,8-9 OR 1-3,5,7-9
            test_config_condition(not isinstance(port_conf, dict), 'Invalid syntax in port config')
            port_nums = set()
            if 'number' in port_conf:
                del port_conf['number']
            for range_ in re.findall(r'(\d+-\d+)', str(port_range)):
                start_num, end_num = [int(num) for num in range_.split('-')]
                test_config_condition(start_num >= end_num, (
                    'Incorrect port range (%d - %d)' % (start_num, end_num)))
                port_nums.update(range(start_num, end_num + 1))
                port_range = re.sub(range_, '', port_range)
            other_nums = [int(p) for p in re.findall(r'\d+', str(port_range))]
            port_nums.update(other_nums)
            test_config_condition(not port_nums, 'interface-ranges contain invalid config')
            for port_num in port_nums:
                if port_num in port_num_to_port_conf:
                    # port range config has lower priority than individual port config
                    for attr, value in port_conf.items():
                        port_num_to_port_conf[port_num][1].setdefault(attr, value)
                else:
                    port_num_to_port_conf[port_num] = (port_num, port_conf)

    port_num_to_port_conf = _map_port_num_to_port(ports_conf)
    _parse_port_ranges(port_ranges_conf, port_num_to_port_conf)

    for port_num, port_conf in port_num_to_port_conf.values():
        port = _dp_parse_port(dp_id, port_num, port_conf, vlans)
        dp.add_port(port)
    dp.reset_refs(vlans=vlans)

def _parse_dp(dp_key, dp_conf, acls_conf, meters_conf, routers_conf, vlans_conf):
    test_config_condition(not isinstance(dp_conf, dict), '')
    dp = DP(dp_key, dp_conf.get('dp_id', None), dp_conf)
    test_config_condition(dp.name != dp_key, (
        'DP key %s and DP name must match' % dp_key))
    dp_id = dp.dp_id
    vlans = {}
    vids = set()
    for vlan_key, vlan_conf in vlans_conf.items():
        vlan = VLAN(vlan_key, dp_id, vlan_conf)
        test_config_condition(str(vlan_key) not in (str(vlan.vid), vlan.name), (
            'VLAN %s key must match VLAN name or VLAN VID' % vlan_key))
        test_config_condition(vlan.vid in vids, (
            'VLAN VID %u multiply configured' % vlan.vid))
        vlans[vlan_key] = vlan
        vids.add(vlan.vid)
    for acl_key, acl_conf in acls_conf.items():
        acl = ACL(acl_key, dp_id, acl_conf)
        dp.add_acl(acl_key, acl)
    for router_key, router_conf in routers_conf.items():
        router = Router(router_key, dp_id, router_conf)
        dp.add_router(router_key, router)
    for meter_key, meter_conf in meters_conf.items():
        meter = Meter(meter_key, dp_id, meter_conf)
        dp.meters[meter_key] = meter
    _dp_add_ports(dp, dp_conf, dp_id, vlans)
    return dp

def _dp_parser_v2(acls_conf, dps_conf, meters_conf,
                  routers_conf, vlans_conf):

    dps = [_parse_dp(dp_key, dp_conf, acls_conf, meters_conf, routers_conf, vlans_conf)
           for dp_key, dp_conf in dps_conf.items()]
    for dp in dps:
        dp.finalize_config(dps)
    for dp in dps:
        dp.resolve_stack_topology(dps)

    router_ref_dps = collections.defaultdict(set)
    for dp in dps:
        for router in dp.routers.keys():
            router_ref_dps[router].add(dp)
    for router in routers_conf.keys():
        test_config_condition(not router_ref_dps[router], (
            'router %s configured but not used by any DP' % router))

    return dps

def _config_parser_v2(config_file, logname):
    config_path = config_parser_util.dp_config_path(config_file)
    top_confs = {}
    config_hashes = {}
    dps = None
    for top_conf in V2_TOP_CONFS:
        top_confs[top_conf] = {}

    if not config_parser_util.dp_include(
            config_hashes, config_path, logname, top_confs):
        raise InvalidConfigError('Error found while loading config file: %s' % config_path)
    elif not top_confs['dps']:
        raise InvalidConfigError('DPs not configured in file: %s' % config_path)
    else:
        dps = _dp_parser_v2(
            top_confs['acls'],
            top_confs['dps'],
            top_confs['meters'],
            top_confs['routers'],
            top_confs['vlans'])
    return (config_hashes, dps)

def get_config_for_api(valves):
    """Return config as dict for all DPs."""
    config = {}
    for i in V2_TOP_CONFS:
        config[i] = {}
    for valve in valves.values():
        valve_conf = valve.get_config_dict()
        for i in V2_TOP_CONFS:
            if i in valve_conf:
                config[i].update(valve_conf[i])
    return config

def watcher_parser(config_file, logname, prom_client):
    """Return Watcher instances from config."""
    conf = config_parser_util.read_config(config_file, logname)
    return _watcher_parser_v2(conf, logname, prom_client)

def _parse_dps_for_watchers(conf, logname):
    dps = {}
    if 'faucet_configs' in conf:
        for faucet_file in conf['faucet_configs']:
            _, dp_list = dp_parser(faucet_file, logname)
            if dp_list:
                for dp in dp_list:
                    dps[dp.name] = dp

    if 'faucet' in conf:
        faucet_conf = conf['faucet']
        acls = faucet_conf.get('acls', {})
        fct_dps = faucet_conf.get('dps', {})
        meters = faucet_conf.get('meters', {})
        routers = faucet_conf.get('routers', {})
        vlans = faucet_conf.get('vlans', {})
        for dp in _dp_parser_v2(acls, fct_dps, meters, routers, vlans):
            dps[dp.name] = dp

    if not dps:
        raise InvalidConfigError(
            'Gauge configured without any FAUCET configuration')
    return dps

def _watcher_parser_v2(conf, logname, prom_client):
    logger = config_parser_util.get_logger(logname)

    if conf is None:
        conf = {}
    dps = _parse_dps_for_watchers(conf, logname)
    dbs = conf.pop('dbs')

    result = []
    # pylint: disable=fixme
    for watcher_name, watcher_conf in conf['watchers'].items():
        if watcher_conf.get('all_dps', False):
            watcher_dps = dps.keys()
        else:
            watcher_dps = watcher_conf['dps']
        # Watcher config has a list of DPs, but actually a WatcherConf is
        # created for each DP.
        # TODO: refactor watcher_conf as a container.
        for dp_name in watcher_dps:
            if dp_name not in dps:
                logger.error('DP %s in Gauge but not configured in FAUCET', dp_name)
                continue
            dp = dps[dp_name]
            if 'dbs' in watcher_conf:
                watcher_dbs = watcher_conf['dbs']
            elif 'db' in watcher_conf:
                watcher_dbs = [watcher_conf['db']]
            else:
                raise InvalidConfigError('Watcher configured without DB')
            for db in watcher_dbs:
                watcher = WatcherConf(watcher_name, dp.dp_id, watcher_conf, prom_client)
                watcher.add_db(dbs[db])
                watcher.add_dp(dp)
                result.append(watcher)

    return result
<commit_msg>pylint, fix no-else-raise<commit_after>"""Implement configuration file parsing."""

# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2018 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import re

from faucet import config_parser_util
from faucet.acl import ACL
from faucet.conf import test_config_condition, InvalidConfigError
from faucet.dp import DP
from faucet.meter import Meter
from faucet.port import Port
from faucet.router import Router
from faucet.vlan import VLAN
from faucet.watcher_conf import WatcherConf

V2_TOP_CONFS = (
    'acls',
    'dps',
    'meters',
    'routers',
    'vlans')

def dp_parser(config_file, logname):
    """Parse a config file into DP configuration objects with hashes of config include/files."""
    conf = config_parser_util.read_config(config_file, logname)
    config_hashes = None
    dps = None

    test_config_condition(conf is None, 'Config file is empty')
    test_config_condition(not isinstance(conf, dict), 'Config file does not have valid syntax')
    version = conf.pop('version', 2)
    test_config_condition(version != 2, 'Only config version 2 is supported')
    config_hashes, dps = _config_parser_v2(config_file, logname)
    test_config_condition(dps is None, 'no DPs are not defined')

    return config_hashes, dps

def _get_vlan_by_key(dp_id, vlan_key, vlans):
    test_config_condition(not isinstance(vlan_key, (str, int)), (
        'VLAN key must not be type %s' % type(vlan_key)))
    if vlan_key in vlans:
        return vlans[vlan_key]
    for vlan in vlans.values():
        if vlan_key == str(vlan.vid):
            return vlan
    # Create VLAN with VID, if not defined.
    return vlans.setdefault(vlan_key, VLAN(vlan_key, dp_id))

def _dp_parse_port(dp_id, port_key, port_conf, vlans):

    def _dp_parse_native_port_vlan():
        if port.native_vlan is not None:
            vlan = _get_vlan_by_key(dp_id, port.native_vlan, vlans)
            port.native_vlan = vlan

    def _dp_parse_tagged_port_vlans():
        if port.tagged_vlans:
            port_tagged_vlans = [
                _get_vlan_by_key(dp_id, vlan_key, vlans) for vlan_key in port.tagged_vlans]
            port.tagged_vlans = port_tagged_vlans

    port = Port(port_key, dp_id, port_conf)
    test_config_condition(str(port_key) not in (str(port.number), port.name), (
        'Port key %s match port name or port number' % port_key))
    _dp_parse_native_port_vlan()
    _dp_parse_tagged_port_vlans()
    return port

def _dp_add_ports(dp, dp_conf, dp_id, vlans):
    ports_conf = dp_conf.get('interfaces', {})
    port_ranges_conf = dp_conf.get('interface_ranges', {})
    # as users can config port VLAN by using VLAN name, we store vid in
    # Port instance instead of VLAN name for data consistency
    test_config_condition(not isinstance(ports_conf, dict), (
        'Invalid syntax in interface config'))
    test_config_condition(not isinstance(port_ranges_conf, dict), (
        'Invalid syntax in interface ranges config'))

    def _map_port_num_to_port(ports_conf):
        port_num_to_port_conf = {}
        for port_key, port_conf in ports_conf.items():
            test_config_condition(not isinstance(port_conf, dict), 'Invalid syntax in port config')
            port_num = port_conf.get('number', port_key)
            try:
                port_num_to_port_conf[port_num] = (port_key, port_conf)
            except TypeError:
                raise InvalidConfigError('Invalid syntax in port config')
        return port_num_to_port_conf

    def _parse_port_ranges(port_ranges_conf, port_num_to_port_conf):
        for port_range, port_conf in port_ranges_conf.items():
            # port range format: 1-6 OR 1-6,8-9 OR 1-3,5,7-9
            test_config_condition(not isinstance(port_conf, dict), 'Invalid syntax in port config')
            port_nums = set()
            if 'number' in port_conf:
                del port_conf['number']
            for range_ in re.findall(r'(\d+-\d+)', str(port_range)):
                start_num, end_num = [int(num) for num in range_.split('-')]
                test_config_condition(start_num >= end_num, (
                    'Incorrect port range (%d - %d)' % (start_num, end_num)))
                port_nums.update(range(start_num, end_num + 1))
                port_range = re.sub(range_, '', port_range)
            other_nums = [int(p) for p in re.findall(r'\d+', str(port_range))]
            port_nums.update(other_nums)
            test_config_condition(not port_nums, 'interface-ranges contain invalid config')
            for port_num in port_nums:
                if port_num in port_num_to_port_conf:
                    # port range config has lower priority than individual port config
                    for attr, value in port_conf.items():
                        port_num_to_port_conf[port_num][1].setdefault(attr, value)
                else:
                    port_num_to_port_conf[port_num] = (port_num, port_conf)

    port_num_to_port_conf = _map_port_num_to_port(ports_conf)
    _parse_port_ranges(port_ranges_conf, port_num_to_port_conf)

    for port_num, port_conf in port_num_to_port_conf.values():
        port = _dp_parse_port(dp_id, port_num, port_conf, vlans)
        dp.add_port(port)
    dp.reset_refs(vlans=vlans)

def _parse_dp(dp_key, dp_conf, acls_conf, meters_conf, routers_conf, vlans_conf):
    test_config_condition(not isinstance(dp_conf, dict), '')
    dp = DP(dp_key, dp_conf.get('dp_id', None), dp_conf)
    test_config_condition(dp.name != dp_key, (
        'DP key %s and DP name must match' % dp_key))
    dp_id = dp.dp_id
    vlans = {}
    vids = set()
    for vlan_key, vlan_conf in vlans_conf.items():
        vlan = VLAN(vlan_key, dp_id, vlan_conf)
        test_config_condition(str(vlan_key) not in (str(vlan.vid), vlan.name), (
            'VLAN %s key must match VLAN name or VLAN VID' % vlan_key))
        test_config_condition(vlan.vid in vids, (
            'VLAN VID %u multiply configured' % vlan.vid))
        vlans[vlan_key] = vlan
        vids.add(vlan.vid)
    for acl_key, acl_conf in acls_conf.items():
        acl = ACL(acl_key, dp_id, acl_conf)
        dp.add_acl(acl_key, acl)
    for router_key, router_conf in routers_conf.items():
        router = Router(router_key, dp_id, router_conf)
        dp.add_router(router_key, router)
    for meter_key, meter_conf in meters_conf.items():
        meter = Meter(meter_key, dp_id, meter_conf)
        dp.meters[meter_key] = meter
    _dp_add_ports(dp, dp_conf, dp_id, vlans)
    return dp

def _dp_parser_v2(acls_conf, dps_conf, meters_conf,
                  routers_conf, vlans_conf):

    dps = [_parse_dp(dp_key, dp_conf, acls_conf, meters_conf, routers_conf, vlans_conf)
           for dp_key, dp_conf in dps_conf.items()]
    for dp in dps:
        dp.finalize_config(dps)
    for dp in dps:
        dp.resolve_stack_topology(dps)

    router_ref_dps = collections.defaultdict(set)
    for dp in dps:
        for router in dp.routers.keys():
            router_ref_dps[router].add(dp)
    for router in routers_conf.keys():
        test_config_condition(not router_ref_dps[router], (
            'router %s configured but not used by any DP' % router))

    return dps

def _config_parser_v2(config_file, logname):
    config_path = config_parser_util.dp_config_path(config_file)
    top_confs = {}
    config_hashes = {}
    dps = None
    for top_conf in V2_TOP_CONFS:
        top_confs[top_conf] = {}

    if not config_parser_util.dp_include(
            config_hashes, config_path, logname, top_confs):
        raise InvalidConfigError('Error found while loading config file: %s' % config_path)

    if not top_confs['dps']:
        raise InvalidConfigError('DPs not configured in file: %s' % config_path)

    dps = _dp_parser_v2(
        top_confs['acls'],
        top_confs['dps'],
        top_confs['meters'],
        top_confs['routers'],
        top_confs['vlans'])
    return (config_hashes, dps)

def get_config_for_api(valves):
    """Return config as dict for all DPs."""
    config = {}
    for i in V2_TOP_CONFS:
        config[i] = {}
    for valve in valves.values():
        valve_conf = valve.get_config_dict()
        for i in V2_TOP_CONFS:
            if i in valve_conf:
                config[i].update(valve_conf[i])
    return config

def watcher_parser(config_file, logname, prom_client):
    """Return Watcher instances from config."""
    conf = config_parser_util.read_config(config_file, logname)
    return _watcher_parser_v2(conf, logname, prom_client)

def _parse_dps_for_watchers(conf, logname):
    dps = {}
    if 'faucet_configs' in conf:
        for faucet_file in conf['faucet_configs']:
            _, dp_list = dp_parser(faucet_file, logname)
            if dp_list:
                for dp in dp_list:
                    dps[dp.name] = dp

    if 'faucet' in conf:
        faucet_conf = conf['faucet']
        acls = faucet_conf.get('acls', {})
        fct_dps = faucet_conf.get('dps', {})
        meters = faucet_conf.get('meters', {})
        routers = faucet_conf.get('routers', {})
        vlans = faucet_conf.get('vlans', {})
        for dp in _dp_parser_v2(acls, fct_dps, meters, routers, vlans):
            dps[dp.name] = dp

    if not dps:
        raise InvalidConfigError(
            'Gauge configured without any FAUCET configuration')
    return dps

def _watcher_parser_v2(conf, logname, prom_client):
    logger = config_parser_util.get_logger(logname)

    if conf is None:
        conf = {}
    dps = _parse_dps_for_watchers(conf, logname)
    dbs = conf.pop('dbs')

    result = []
    # pylint: disable=fixme
    for watcher_name, watcher_conf in conf['watchers'].items():
        if watcher_conf.get('all_dps', False):
            watcher_dps = dps.keys()
        else:
            watcher_dps = watcher_conf['dps']
        # Watcher config has a list of DPs, but actually a WatcherConf is
        # created for each DP.
        # TODO: refactor watcher_conf as a container.
        for dp_name in watcher_dps:
            if dp_name not in dps:
                logger.error('DP %s in Gauge but not configured in FAUCET', dp_name)
                continue
            dp = dps[dp_name]
            if 'dbs' in watcher_conf:
                watcher_dbs = watcher_conf['dbs']
            elif 'db' in watcher_conf:
                watcher_dbs = [watcher_conf['db']]
            else:
                raise InvalidConfigError('Watcher configured without DB')
            for db in watcher_dbs:
                watcher = WatcherConf(watcher_name, dp.dp_id, watcher_conf, prom_client)
                watcher.add_db(dbs[db])
                watcher.add_dp(dp)
                result.append(watcher)

    return result
<|endoftext|>