<commit_before>'''
@author: Loïc Grobol <loic.grobol@gmail.com>
Copyright © 2014, Loïc Grobol <loic.grobol@gmail.com>
Permission is granted to Do What The Fuck You Want To
with this document.

See the WTF Public License, Version 2 as published by Sam Hocevar
at http://www.wtfpl.net if you need more details.

A naïve syntax tree to help format function definitions for several engines.

Plans are for support of
  - Python
  - LaTeX maths
  - Geogebra
  - Pgf/TikZ
'''

import itertools as it
import numbers

class Term:
    def __add__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Sum((self, z))

    def __neg__(self):
        return Minus(self)

    def __sub__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return self + (-z)

    def __pow__(self, n):
        m = Scalar(n) if isinstance(n, numbers.Number) else m
        return Power(self, m)

    def __mul__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Scale(self, z)

    def __radd__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Sum((z, self))

    def __rsub__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return z + (-self)

    def __rmul__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Scale(z, self)

class Sum(Term):
    def __init__(self, summands):
        self.summands = summands

    def __str__(self):
        return '({})'.format('+'.join(('{}'.format(s) for s in self.summands)))

    def latex(self):
        return '({})'.format('+'.join(('{}'.format(s.latex()) for s in self.summands)))

    def geogebra(self):
        return '({})'.format('+'.join(('{}'.format(s.geogebra()) for s in self.summands)))

    def pgf(self):
        return '({})'.format('+'.join(('{}'.format(s.pgf()) for s in self.summands)))

    def value(self, *args, **kwargs):
        return sum(s.value(*args, **kwargs) for s in self.summands)

class Scale(Term):
    def __init__(self, scale, vector):
        self.scale = scale
        self.vector = vector

    def __str__(self):
        return '({scale}*{vector})'.format(**self.__dict__)

    def geogebra(self):
        return '({scale}*{vector})'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def latex(self):
        return '({scale}*{vector})'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def pgf(self):
        return '({scale}*{vector})'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def value(self, *args, **kwargs):
        return self.scale.value(*args, **kwargs)*self.vector.value(*args, **kwargs)

class Scalar(Term):
    def __init__(self, scalar):
        self.scalar = scalar

    def __str__(self):
        return str(self.scalar)

    def value(self, *args, **kwargs):
        return self.scalar

    geogebra = __str__
    pgf = __str__
    latex = __str__

class Integer(Scalar):
    pass

class Minus(Term):
    def __init__(self, val):
        self.val = val

    def __str__(self):
        return '(-{})'.format(self.val)

    def geogebra(self):
        return '(-{})'.format(self.val.geogebra())

    def latex(self):
        return '(-{})'.format(self.val.latex())

    def pgf(self):
        return '(-{})'.format(self.val.pgf())

    def value(self, *args, **kwargs):
        return -self.val.value(*args, **kwargs)

class Variable(Term):
    def __init__(self, varname='x'):
        self.varname = varname

    def __str__(self):
        return self.varname

    geogebra = __str__

    def pgf(self):
        return r'\{}'.format(self.varname)

    latex = __str__

    def value(self, *args, **kwargs):
        return kwargs[self.varname]

class Power(Term):
    def __init__(self, var, power):
        self.var = var
        self.power = power

    def __str__(self):
        return '({var}**{power})'.format(**self.__dict__)

    def geogebra(self):
        return '({var}^{power})'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def latex(self):
        return '({var}^{{{power}}})'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def pgf(self):
        return '({var}^{power})'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def value(self, *args, **kwargs):
        return self.var.value(*args, **kwargs)**self.power.value(*args, **kwargs)

class RealFun(Term):
    def __init__(self, a, b, image, variable):
        self.a = Scalar(a) if isinstance(a, numbers.Number) else a
        self.b = Scalar(b) if isinstance(b, numbers.Number) else b
        self.image = image
        self.variable = variable

    def __str__(self):
        return r'''def f({variable}):
    if {a}<={variable}<={b}:
        return {image}
    else:
        raise DomainError()'''.format(**self.__dict__)

    def geogebra(self):
        return 'Function[{a}, {b}, {image}]'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def pgf(self):
        return r'\draw[smooth,samples=100,domain={a}:{b}] plot({variable},{{{image}}});'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def latex(self):
        return '{variable}⟼{image}'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def value(self, *args, **kwargs):
        if not (a.value(*args, **kwargs) <= self.variable.value(*args, **kwargs) <=  a.value(*args, **kwargs)):
            raise DomainError()
        return self.image.value(*args, **kwargs)

class Indicator(Term):
    '''Indicator of [a,b[ (𝟙_{[a,b[}).'''
    def __init__(self, a, b, variable):
        self.a = Scalar(a) if isinstance(a, numbers.Number) else a
        self.b = Scalar(b) if isinstance(b, numbers.Number) else b
        self.variable = variable

    def __str__(self):
        return '(1 if {a} <= {variable} < {b} else 0)'.format(**self.__dict__)

    def geogebra(self):
        return '(If[x>={a} && x<{b},{variable},0])'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def latex(self):
        return r'𝟙_{{\left[{a}, {b}\right[}}({variable})'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def pgf(self):
        return 'and({variable}>={a},{variable}<{b})'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def value(self, *args, **kwargs):
        return 1 if a.value(*args, **kwargs) <= variable.value(*args, **kwargs) < b.value(*args, **kwargs) else 0

class DomainError(Exception):
    pass

def piecewise_polynomial(variable, coefs, bounds):
    bounds = sorted(list(bounds))
    img = Sum(Sum(c*(variable-a)**i for c,i in zip(piece_coefs,it.count()))*Indicator(a,b,variable) for a,b,piece_coefs in zip(bounds[:-1],bounds[1:],coefs))
    return RealFun(Scalar(bounds[0]), Scalar(bounds[-1]), img, variable)

def tree_fun(tree, *vars):
    '''Return a callable that returns the evaluation of `tree` afer the substitution of
       `vars`. That is `tree_fun(tree, ('x', 'y'))(2,5)` is `tree.value({'x': 2, 'y':5})`.'''
    def f(*vals):
        return tree.value({x: y for x,y in zip(vars, vals)})
    return f

if __name__=='__main__':
    x = Variable()
    import libinterpol
    c = list(libinterpol.cubic_coefs(((0,-2),(3,1),(2,7),(1,3),(5,-6))))
    f = piecewise_polynomial(x, c, (0,2,3,1,5))
    print(f.pgf())

<commit_msg>fixed quite a few typos in naive_tree<commit_after>'''
@author: Loïc Grobol <loic.grobol@gmail.com>
Copyright © 2014, Loïc Grobol <loic.grobol@gmail.com>
Permission is granted to Do What The Fuck You Want To
with this document.

See the WTF Public License, Version 2 as published by Sam Hocevar
at http://www.wtfpl.net if you need more details.

A naïve syntax tree to help format function definitions for several engines.

Plans are for support of
  - Python
  - LaTeX maths
  - Geogebra
  - Pgf/TikZ
'''

import itertools as it
import numbers

class Term:
    def __add__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Sum((self, z))

    def __neg__(self):
        return Minus(self)

    def __sub__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return self + (-z)

    def __pow__(self, n):
        m = Scalar(n) if isinstance(n, numbers.Number) else n
        return Power(self, m)

    def __mul__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Scale(self, z)

    def __radd__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Sum((z, self))

    def __rsub__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return z + (-self)

    def __rmul__(self, y):
        z = Scalar(y) if isinstance(y, numbers.Number) else y
        return Scale(z, self)

class Sum(Term):
    def __init__(self, summands):
        self.summands = summands

    def __str__(self):
        return '({})'.format('+'.join(('{}'.format(s) for s in self.summands)))

    def latex(self):
        return '({})'.format('+'.join(('{}'.format(s.latex()) for s in self.summands)))

    def geogebra(self):
        return '({})'.format('+'.join(('{}'.format(s.geogebra()) for s in self.summands)))

    def pgf(self):
        return '({})'.format('+'.join(('{}'.format(s.pgf()) for s in self.summands)))

    def value(self, *args, **kwargs):
        return sum(s.value(*args, **kwargs) for s in self.summands)

class Scale(Term):
    def __init__(self, scale, vector):
        self.scale = scale
        self.vector = vector

    def __str__(self):
        return '({scale}*{vector})'.format(**self.__dict__)

    def geogebra(self):
        return '({scale}*{vector})'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def latex(self):
        return '({scale}*{vector})'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def pgf(self):
        return '({scale}*{vector})'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def value(self, *args, **kwargs):
        return self.scale.value(*args, **kwargs)*self.vector.value(*args, **kwargs)

class Scalar(Term):
    def __init__(self, scalar):
        self.scalar = scalar

    def __str__(self):
        return str(self.scalar)

    def value(self, *args, **kwargs):
        return self.scalar

    geogebra = __str__
    pgf = __str__
    latex = __str__

class Integer(Scalar):
    pass

class Minus(Term):
    def __init__(self, val):
        self.val = val

    def __str__(self):
        return '(-{})'.format(self.val)

    def geogebra(self):
        return '(-{})'.format(self.val.geogebra())

    def latex(self):
        return '(-{})'.format(self.val.latex())

    def pgf(self):
        return '(-{})'.format(self.val.pgf())

    def value(self, *args, **kwargs):
        return -self.val.value(*args, **kwargs)

class Variable(Term):
    def __init__(self, varname='x'):
        self.varname = varname

    def __str__(self):
        return self.varname

    geogebra = __str__

    def pgf(self):
        return r'\{}'.format(self.varname)

    latex = __str__

    def value(self, *args, **kwargs):
        return kwargs[self.varname]

class Power(Term):
    def __init__(self, var, power):
        self.var = var
        self.power = power

    def __str__(self):
        return '({var}**{power})'.format(**self.__dict__)

    def geogebra(self):
        return '({var}^{power})'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def latex(self):
        return '({var}^{{{power}}})'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def pgf(self):
        return '({var}^{power})'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def value(self, *args, **kwargs):
        return self.var.value(*args, **kwargs)**self.power.value(*args, **kwargs)

class RealFun(Term):
    def __init__(self, a, b, image, variable):
        self.a = Scalar(a) if isinstance(a, numbers.Number) else a
        self.b = Scalar(b) if isinstance(b, numbers.Number) else b
        self.image = image
        self.variable = variable

    def __str__(self):
        return r'''def f({variable}):
    if {a}<={variable}<={b}:
        return {image}
    else:
        raise DomainError()'''.format(**self.__dict__)

    def geogebra(self):
        return 'Function[{a}, {b}, {image}]'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def pgf(self):
        return r'\draw[smooth,samples=100,domain={a}:{b}] plot({variable},{{{image}}});'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def latex(self):
        return '{variable}⟼{image}'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def value(self, *args, **kwargs):
        if not (self.a.value(*args, **kwargs) <= self.variable.value(*args, **kwargs) <=  self.b.value(*args, **kwargs)):
            raise DomainError()
        return self.image.value(*args, **kwargs)

class Indicator(Term):
    '''Indicator of [a,b[ (𝟙_{[a,b[}).'''
    def __init__(self, a, b, variable):
        self.a = Scalar(a) if isinstance(a, numbers.Number) else a
        self.b = Scalar(b) if isinstance(b, numbers.Number) else b
        self.variable = variable

    def __str__(self):
        return '(1 if {a} <= {variable} < {b} else 0)'.format(**self.__dict__)

    def geogebra(self):
        return '(If[x>={a} && x<{b},{variable},0])'.format(**{a: self.__dict__[a].geogebra() for a in self.__dict__})

    def latex(self):
        return r'𝟙_{{\left[{a}, {b}\right[}}({variable})'.format(**{a: self.__dict__[a].latex() for a in self.__dict__})

    def pgf(self):
        return 'and({variable}>={a},{variable}<{b})'.format(**{a: self.__dict__[a].pgf() for a in self.__dict__})

    def value(self, *args, **kwargs):
        return 1 if self.a.value(*args, **kwargs) <= self.variable.value(*args, **kwargs) < self.b.value(*args, **kwargs) else 0

class DomainError(Exception):
    pass

def piecewise_polynomial(variable, coefs, bounds):
    bounds = sorted(list(bounds))
    img = Sum(Sum(c*(variable-a)**i for c,i in zip(piece_coefs,it.count()))*Indicator(a,b,variable) for a,b,piece_coefs in zip(bounds[:-1],bounds[1:],coefs))
    return RealFun(Scalar(bounds[0]), Scalar(bounds[-1]), img, variable)

def tree_fun(tree, *vars):
    '''Return a callable that returns the evaluation of `tree` afer the substitution of
       `vars`. That is `tree_fun(tree, ('x', 'y'))(2,5)` is `tree.value({'x': 2, 'y':5})`.'''
    def f(*vals):
        return tree.value({x: y for x,y in zip(vars, vals)})
    return f

if __name__=='__main__':
    x = Variable()
    import libinterpol
    c = list(libinterpol.cubic_coefs(((0,-2),(3,1),(2,7),(1,3),(5,-6))))
    f = piecewise_polynomial(x, c, (0,2,3,1,5))
    print(f.pgf())

<|endoftext|>


<commit_before>"""
DB utilities and configuration.
"""

import os
import socket
import yaml

from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.automap import automap_base

class Config:
    def __init__(self, db_yaml, categories, db_config_dict=False):
        """
        Read the DB configuration file and the specified categories configuration from the database.
        """

        # Retrieve the database configuration.
        if os.path.exists(db_yaml):
            with open(db_yaml, 'r') as ymlfile:
                base_config = yaml.load(ymlfile)
        else:
            raise Exception('Configuration file "%s" does not exist.' % db_yaml)

        db_config = {}
        if 'database' in base_config:
            for item in base_config['database']:
                db_config[item] = base_config['database'][item]

            if 'db_table' not in db_config:
                db_config['db_table'] = 'configuration'

        if db_config_dict:
            self.db_config = db_config

        # Open the database.
        self.db_engine = create_engine(
            'mysql://%s:%s@%s:%s/%s' % (
                db_config['db_user'],
                db_config['db_password'],
                db_config['db_host'],
                str(db_config['db_port']),
                db_config['db_name']
                ),
            isolation_level="READ_COMMITTED",
            pool_pre_ping=True
            )

        self.db_connection = None
        self.db_map = automap_base()
        self.db_map.prepare(self.db_engine, reflect=True)

        # Create a unique instance ID from our FQDN and, if necessary, save it in the database
        self.csv2_host_id = sum(socket.getfqdn().encode())

        self.db_session = Session(self.db_engine)
        rows = self.db_session.query(self.db_map.classes[db_config['db_table']]).filter(
            (self.db_map.classes[db_config['db_table']].category == 'SQL') &
            (self.db_map.classes[db_config['db_table']].config_key == 'csv2_host_id')
            )

        if self.csv2_host_id != rows[0].config_value:
            try:
                table = Table(db_config['db_table']], MetaData(bind=self.db_engine), autoload=True)
                self.db_session.execute(table.update().where(
                    (self.db_map.classes[db_config['db_table']].category == 'SQL') &
                    (self.db_map.classes[db_config['db_table']].config_key == 'csv2_host_id')
                    ).values({'config_value': self.csv2_host_id}))

                self.db_session.commit()

            except Exception as msg:
                print("Error updating csv2_host_id in db_config: %s" % msg)

        # Retrieve the configuration for the specified category.
        if isinstance(categories, str):
            category_list = [ categories ]
        else:
            category_list = categories

        for category in category_list:
            rows = self.db_session.query(self.db_map.classes[db_config['db_table']]).filter(
                self.db_map.classes[db_config['db_table']].category == category
                )

            for row in rows:
                if row.config_type == 'bool':
                    self.__dict__[row.config_key] = row.config_value == '1' or row.config_value.lower() == 'yes' or row.config_value.lower() == 'true'
                elif row.config_type == 'float':
                    self.__dict__[row.config_key] = float(row.config_value)
                elif row.config_type == 'int':
                    self.__dict__[row.config_key] = int(row.config_value)
                elif row.config_type == 'null':
                    self.__dict__[row.config_key] = None
                else:
                    self.__dict__[row.config_key] = row.config_value

        # Close the session.
        self.db_session.close()
        self.db_session = None

#-------------------------------------------------------------------------------

    def db_close(self, commit=False):
        """
        Commit/rollback and close a session.
        """

        if self.db_session:
            if commit:
                self.db_session.commit()
            else:
                self.db_session.rollback()

            self.db_session.close()
            self.db_session = None

        if self.db_connection:
            self.db_connection.close()
            self.db_connection = None

#-------------------------------------------------------------------------------

    def db_open(self):
        """
        Open and return a database connection.
        """

        self.db_connection = self.db_engine.connect()
        self.db_session = Session(bind=self.db_engine)

#-------------------------------------------------------------------------------

    def db_session_execute(self, request, allow_no_rows=False):
        """
        Execute a DB request and return the response. Also, trap and return errors.
        """

        from sqlalchemy.engine.result import ResultProxy
        import sqlalchemy.exc

        try:
            result_proxy = self.db_session.execute(request)
            if result_proxy.rowcount == 0 and not allow_no_rows:
                return 1, 'the request did not match any rows'
            return 0, result_proxy.rowcount
        except sqlalchemy.exc.IntegrityError as ex:
            return 1, ex.orig
        except Exception as ex:
            return 1, ex

<commit_msg>bug fix<commit_after>"""
DB utilities and configuration.
"""

import os
import socket
import yaml

from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.automap import automap_base

class Config:
    def __init__(self, db_yaml, categories, db_config_dict=False):
        """
        Read the DB configuration file and the specified categories configuration from the database.
        """

        # Retrieve the database configuration.
        if os.path.exists(db_yaml):
            with open(db_yaml, 'r') as ymlfile:
                base_config = yaml.load(ymlfile)
        else:
            raise Exception('Configuration file "%s" does not exist.' % db_yaml)

        db_config = {}
        if 'database' in base_config:
            for item in base_config['database']:
                db_config[item] = base_config['database'][item]

            if 'db_table' not in db_config:
                db_config['db_table'] = 'configuration'

        if db_config_dict:
            self.db_config = db_config

        # Open the database.
        self.db_engine = create_engine(
            'mysql://%s:%s@%s:%s/%s' % (
                db_config['db_user'],
                db_config['db_password'],
                db_config['db_host'],
                str(db_config['db_port']),
                db_config['db_name']
                ),
            isolation_level="READ_COMMITTED",
            pool_pre_ping=True
            )

        self.db_connection = None
        self.db_map = automap_base()
        self.db_map.prepare(self.db_engine, reflect=True)

        # Create a unique instance ID from our FQDN and, if necessary, save it in the database
        self.csv2_host_id = sum(socket.getfqdn().encode())

        self.db_session = Session(self.db_engine)
        rows = self.db_session.query(self.db_map.classes[db_config['db_table']]).filter(
            (self.db_map.classes[db_config['db_table']].category == 'SQL') &
            (self.db_map.classes[db_config['db_table']].config_key == 'csv2_host_id')
            )

        if self.csv2_host_id != rows[0].config_value:
            try:
                table = Table(db_config['db_table'], MetaData(bind=self.db_engine), autoload=True)
                self.db_session.execute(table.update().where(
                    (self.db_map.classes[db_config['db_table']].category == 'SQL') &
                    (self.db_map.classes[db_config['db_table']].config_key == 'csv2_host_id')
                    ).values({'config_value': self.csv2_host_id}))

                self.db_session.commit()

            except Exception as msg:
                print("Error updating csv2_host_id in db_config: %s" % msg)

        # Retrieve the configuration for the specified category.
        if isinstance(categories, str):
            category_list = [ categories ]
        else:
            category_list = categories

        for category in category_list:
            rows = self.db_session.query(self.db_map.classes[db_config['db_table']]).filter(
                self.db_map.classes[db_config['db_table']].category == category
                )

            for row in rows:
                if row.config_type == 'bool':
                    self.__dict__[row.config_key] = row.config_value == '1' or row.config_value.lower() == 'yes' or row.config_value.lower() == 'true'
                elif row.config_type == 'float':
                    self.__dict__[row.config_key] = float(row.config_value)
                elif row.config_type == 'int':
                    self.__dict__[row.config_key] = int(row.config_value)
                elif row.config_type == 'null':
                    self.__dict__[row.config_key] = None
                else:
                    self.__dict__[row.config_key] = row.config_value

        # Close the session.
        self.db_session.close()
        self.db_session = None

#-------------------------------------------------------------------------------

    def db_close(self, commit=False):
        """
        Commit/rollback and close a session.
        """

        if self.db_session:
            if commit:
                self.db_session.commit()
            else:
                self.db_session.rollback()

            self.db_session.close()
            self.db_session = None

        if self.db_connection:
            self.db_connection.close()
            self.db_connection = None

#-------------------------------------------------------------------------------

    def db_open(self):
        """
        Open and return a database connection.
        """

        self.db_connection = self.db_engine.connect()
        self.db_session = Session(bind=self.db_engine)

#-------------------------------------------------------------------------------

    def db_session_execute(self, request, allow_no_rows=False):
        """
        Execute a DB request and return the response. Also, trap and return errors.
        """

        from sqlalchemy.engine.result import ResultProxy
        import sqlalchemy.exc

        try:
            result_proxy = self.db_session.execute(request)
            if result_proxy.rowcount == 0 and not allow_no_rows:
                return 1, 'the request did not match any rows'
            return 0, result_proxy.rowcount
        except sqlalchemy.exc.IntegrityError as ex:
            return 1, ex.orig
        except Exception as ex:
            return 1, ex

<|endoftext|>


<commit_before>from unittest import TestCase
from selenium import webdriver
from config import driver, platform, html_fixture_url

class TestObject(TestCase):
    driver = None

    @classmethod
    def setUpClass(cls):
        caps = {
            'takeScreenshot': False,
            'javascriptEnabled': True,
            'browserName': driver,
            'platform': platform
        }
        cls.driver = webdriver.Remote(
            command_executor="http://localhost:5556/wd/hub",
            desired_capabilities=caps)

    @classmethod
    def tearDownClass(cls):
        cls.driver.close()
        cls.driver = None

    def setUp(self):
        self.driver = self.__class__.driver
        self.driver.get(html_fixture_url)

    def tearDown(self):
        self.driver = None

    def _findBySelector(self, selector, parent=None):
        if parent is None:
            result = self.driver.find_element_by_css_selector(selector)
        else:
            result = parent.find_element_by_css_selector(selector)
        return result
<commit_msg>allow testobject to define their own fixture url<commit_after>from unittest import TestCase
from selenium import webdriver
from config import driver, platform, html_fixture_url

class TestObject(TestCase):
    driver = None
    fixture_url = html_fixture_url

    @classmethod
    def setUpClass(cls):
        caps = {
            'takeScreenshot': False,
            'javascriptEnabled': True,
            'browserName': driver,
            'platform': platform
        }
        cls.driver = webdriver.Remote(
            command_executor="http://localhost:5556/wd/hub",
            desired_capabilities=caps)

    @classmethod
    def tearDownClass(cls):
        cls.driver.close()
        cls.driver = None

    def setUp(self):
        self.driver = self.__class__.driver
        self.driver.get(self.fixture_url)

    def tearDown(self):
        self.driver = None

    def _findBySelector(self, selector, parent=None):
        if parent is None:
            result = self.driver.find_element_by_css_selector(selector)
        else:
            result = parent.find_element_by_css_selector(selector)
        return result
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2013, Inktank Storage, Inc.
# All rights reserved.
#
# This software and related documentation are provided under a license
# agreement containing restrictions on use and disclosure and are protected by
# intellectual property laws.  Except as expressly permitted in your license
# agreement or allowed by law, you may not use, copy, reproduce, translate,
# broadcast, modify, license, transmit, distribute, exhibit, perform, publish,
# or display any part, in any form, or by any means.  Reverse engineering,
# disassembly, or decompilation of this software, unless required by law for
# interoperability, is prohibited.
#
# The information contained herein is subject to change without notice and is
# not warranted to be error-free.  If you find any errors, please report them
# to us in writing.
#
# This software or hardware is developed for general use in a variety of
# information management applications.  It is not developed or intended for use
# in any inherently dangerous applications, including applications which may
# create a risk of personal injury.  If you use this software or hardware in
# dangerous applications, then you shall be responsible to take all appropriate
# fail-safe, backup, redundancy, and other measures to ensure its safe use.
# Inktank Storage, Inc.  and its affiliates disclaim any liability for any
# damages caused by use of this software or hardware in dangerous applications.

import logging
import os
import platform
import shutil
import subprocess
import sys
import tarfile
import tempfile

# =============================================================================
# Logging
# =============================================================================

logger = logging.getLogger('ice')

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)

COLORS = {
    'WARNING': YELLOW,
    'INFO': WHITE,
    'DEBUG': BLUE,
    'CRITICAL': RED,
    'ERROR': RED
}

RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"

BASE_COLOR_FORMAT = "%(color_levelname)s %(message)s"
VERBOSE_COLOR_FORMAT = "[%(name)s][$BOLD%(levelname)s] $RESET%(color_levelname)s %(message)s"

def color_message(message):
    message = message.replace("$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ)
    return message

class ColoredFormatter(logging.Formatter):
    """
    A very basic logging formatter that not only applies color to the levels of
    the ouput but will also truncate the level names so that they do not alter
    the visuals of logging when presented on the terminal.
    """

    def __init__(self, msg):
        logging.Formatter.__init__(self, msg)

    def format(self, record):
        levelname = record.levelname
        truncated_level = record.levelname[:6]
        if levelname in COLORS:
            levelname_color = COLOR_SEQ % (30 + COLORS[levelname]) + '-->' + RESET_SEQ
            record.color_levelname = levelname_color
        return logging.Formatter.format(self, record)

def color_format(verbose=False):
    """
    Main entry point to get a colored formatter, it will use the
    BASE_FORMAT by default.
    """
    if verbose:
        color_format = color_message(VERBOSE_COLOR_FORMAT)
    else:
        color_format = color_message(BASE_COLOR_FORMAT)
    return ColoredFormatter(color_format)

# =============================================================================
# Exceptions
# =============================================================================

class ICEError(Exception):
    """Base ICE Setup exception"""
    pass

class NonZeroExit(ICEError):
    """subprocess commands that exit with non-zero status"""
    pass

class UnsupportedPlatform(ICEError):
    pass

# =============================================================================
# Templates
# =============================================================================

ice_repo_template = """
[ice]
name=ice packages for $basearch
baseurl={repo_url}/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey={gpg_url}

[ice-noarch]
name=ice noarch packages
baseurl={repo_url}/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey={gpg_url}

[ice-source]
name=ice source packages
baseurl={repo_url}/SRPMS
enabled=0
gpgcheck=1
type=rpm-md
gpgkey={gpg_url}
"""

ice_list_template = """deb {repo_url} {codename} main\n"""

# =============================================================================
# Distributions
# =============================================================================

class CentOS(object):
    pkg_manager = Yum()

class Debian(object):
    pkg_manager = Apt()

# Normalize casing for easier mapping
centos = CentOS
debian = Debian

def get_distro():
    """
    Retrieve the class that matches the distribution of the current host. This
    function will call ``platform()`` and retrieve the distribution
    information, then return the appropriate class and slap a few attributes
    to that class defining the information it found from the host.

    For example, if the current host is an Ubuntu server, the ``Debian`` class
    would be returned (as it holds 1:1 parity with Ubuntu) and the following
    would be set::

        module.name = 'ubuntu'
        module.release = '12.04'
        module.codename = 'precise'

    """
    distro_name, release, codename = platform_information()

    if not codename or not _get_distro(distro_name):
        error_msg = 'platform is not supported: %s %s %s' % (
            distro_name,
            release,
            codename,
        )
        raise UnsupportedPlatform(error_msg)

    # TODO: make this part of the distro objects
    # no need for them to be inferred here
    module = _get_distro(distro_name)
    module.name = distro_name
    module.release = release
    module.codename = codename
    module.machine_type = platform.machine()

    # XXX: Do we need to know about the init?
    #module.init = _choose_init(distro_name, codename)

    return module

class Yum(object):

    @classmethod
    def create_repo_file(cls, gpg_url, repo_url, file_name=None, **kw):
        """set the contents of /etc/yum.repos.d/ice.repo"""
        etc_path = kw.pop('etc_path', '/etc/yum.repos.d')
        file_name = file_name or 'ice.repo'
        repo_file_path = os.path.join(etc_path, file_name)
        with open(repo_file_path, 'w') as repo_file:
            contents = ice_repo_template.format(
                gpg_url=gpg_url,
                repo_url=repo_url,
            )
            repo_file.write(contents)

    @classmethod
    def install(cls, package):
        cmd = [
            'yum',
            '-y',
            '-q',
            'install',
        ]
        cmd.append(package)
        run(cmd)

class Apt(object):

    @classmethod
    def create_repo_file(cls, repo_url, gpg_url, file_name=None, **kw):
        """add ceph deb repo to sources.list"""
        etc_path = kw.pop('etc_path', '/etc/apt/sources.list.d')
        file_name = file_name or 'ice.list'
        list_file_path = os.path.join(etc_path, file_name)
        with open(list_file_path, 'w') as list_file:
            list_file.write(ice_list_template.format(
                repo_url=repo_url, codename=kw.pop('codename'))
            )

    @classmethod
    def install(cls, package):
        cmd = [
            'sudo',
            'env',
            'DEBIAN_FRONTEND=noninteractive',
            'apt-get',
            '-q',
            'install',
            '--assume-yes',
        ]
        cmd.append(package)
        run(cmd)

# =============================================================================
# Subprocess
# =============================================================================

def run(cmd, **kw):
    stop_on_nonzero = kw.pop('stop_on_nonzero', True)

    process = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        close_fds=True,
        **kw
    )

    if process.stderr:
        while True:
            err = process.stderr.readline()
            if err == '' and process.poll() is not None:
                break
            if err != '':
                logger.warning(err)
                sys.stderr.flush()
    if process.stdout:
        while True:
            out = process.stdout.readline()
            if out == '' and process.poll() is not None:
                break
            if out != '':
                logger.debug(out)
                sys.stdout.flush()

    returncode = process.wait()
    if returncode != 0:
        error_msg = "command returned non-zero exit status: %s" % returncode
        if stop_on_nonzero:
            raise NonZeroExit(error_msg)
        else:
            logger.warning(error_msg)

def run_call(cmd, **kw):
    """
    a callable that will execute a subprocess without raising an exception if
    the exit status is non-zero.

    The ``logger`` argument is in the signature only for consistency in the
    API, it does nothing by default.
    """

    process = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw
    )
    stdout = [line.strip('\n') for line in process.stdout.readlines()]
    stderr = [line.strip('\n') for line in process.stderr.readlines()]
    return stdout, stderr, process.wait()

# =============================================================================
# System
# =============================================================================

def platform_information():
    """
    detect platform information, return a dictionary with the following keys:

    * distro_name: as in 'Ubuntu' or 'CentOS'
    * release:     like '5.0' or '13.04'
    * codename:    as in 'Wheezy' or 'Quantal'
    """
    distro, release, codename = platform.linux_distribution()

    # this could be an empty string in Debian
    if not codename and 'debian' in distro.lower():
        debian_codenames = {
            '8': 'jessie',
            '7': 'wheezy',
            '6': 'squeeze',
        }
        major_version = release.split('.')[0]
        codename = debian_codenames.get(major_version, '')
    return (
        _normalized_distro_name(distro.rstrip()),
        release.rstrip(),
        codename.rstrip(),
    )

def _get_distro(distro, fallback=None):
    if not distro:
        return

    distro = _normalized_distro_name(distro)
    distributions = {
        'debian': debian,
        'ubuntu': debian,
        'centos': centos,
        'redhat': centos,
        }

    return distributions.get(distro) or _get_distro(fallback)

def _normalized_distro_name(distro):
    """
    Normalizes the distribution name so it is easier to operate on well knowns
    rather than whatever small differences distributions decide to add to them

    Even though we do not support Suse or Scientific, because we are just
    normalizing we don't mind leaving them here.
    """
    distro = distro.lower()
    if distro.startswith(('redhat', 'red hat')):
        return 'redhat'
    elif distro.startswith(('scientific', 'scientific linux')):
        return 'scientific'
    elif distro.startswith(('suse', 'opensuse')):
        return 'suse'
    return distro

# =============================================================================
# File Utilities
# =============================================================================

def extract_file(file_path):
    """
    Decompress/Extract a tar file to a temporary location and return its full
    path so that it can be handled elsewhere.  If ``file_path`` is not a tar
    file and it is a directory holding decompressed files return ``file_path``,
    otherwise raise an error.

    Removal of the temporary directory files is responsibility of the caller.
    """
    if os.path.isdir(file_path):
        return file_path
    if tarfile.is_tarfile(file_path):
        tmp_dir = tempfile.mkdtemp()
        destination = os.path.join(tmp_dir, 'repo')
        tar = tarfile.open(file_path, 'r:gz')
        tar.extractall(destination)
        tar.close()
        return destination

def overwrite_dir(source, destination='/opt/ice-repo/'):
    """
    Copy all files from _source_ to a temporary location (if not in a temporary
    location already) and then overwrite the contents of its destination so
    that the contents are as up to date as possible
    """
    # TODO: Need to define the destination of respository files, for example:
    # /opt/ice-repo

    # remove destination to ensure we get a fresh copy
    try:
        shutil.rmtree(destination)
    except OSError:
        os.mkdir(destination)

    # now copy the contents
    shutil.copytree(source, destination)

    # finally remove the source
    shutil.rmtree(source)

def default_repo_location():
    """
    Calculates the default repository location of the repository files, for
    example if this script runs alongside the sources it would get the absolute
    path for the ``sources`` directory relative to this script.
    """
    # TODO: Needs to know about the location (and names) of directories
    # packaged alongside this script

    # XXX: bad naming here. Maybe `detect_repo_location`
    pass

# =============================================================================
# Prompts
# =============================================================================

def prompt(question, _raw_input=None):
    input_prompt = _raw_input or raw_input
    prefix = '%s-->%s ' % (COLOR_SEQ % (30 + COLORS['DEBUG']), RESET_SEQ)
    prompt_format = '{prefix} {question}'.format(prefix=prefix, question=question)
    response = input_prompt(prompt_format)
    try:
        return strtobool(response)
    except ValueError:
        logger.error('That response was invalid, please try again')
        return prompt(question, _raw_input=input_prompt)

def strtobool(val):
    """
    Convert a string representation of truth to true (1) or false (0).

    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are
    'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if 'val' is
    anything else.

    .. note:: lifted from distutils.utils.strtobool
    """
    try:
        val = val.lower()
    except AttributeError:
        val = str(val).lower()
    if val in ('y', 'yes', 't', 'true', 'on', '1', '', None):
        return 1
    elif val in ('n', 'no', 'f', 'false', 'off', '0'):
        return 0
    else:
        raise ValueError("invalid input value: %r" % (val,))

# =============================================================================
# Actions
# =============================================================================

def configure(tar_file):
    """
    Decompress a tar file for the current host so that it can serve as a repo
    server and we can then install Calamari and ceph-deploy.
    """
    distro = get_distro()
    decompressed_repo = extract_file(tar_file)
    overwrite_dir(decompressed_repo)

    # TODO: Allow custom destinations
    distro.pkg_manager.create_repo_file(
        gpg_url='/opt/ice/repo/release.asc',
        repo_url='/opt/ice/repo',
        codename=distro.codename,
    )

def install(package):
    """
    Perform a package installation (e.g. Calamari or ceph-deploy) in the
    current host, abstracted away from the underlying package manager.
    """

# =============================================================================
# Main
# =============================================================================

def main(argv=None):
    argv = argv or sys.argv
    # Console Logger
    terminal_log = logging.StreamHandler()
    terminal_log.setFormatter(color_format(verbose='-v' in argv))
    logger.addHandler(terminal_log)
    logger.setLevel(logging.DEBUG)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>create structure for installing packages<commit_after>#!/usr/bin/env python

# Copyright 2013, Inktank Storage, Inc.
# All rights reserved.
#
# This software and related documentation are provided under a license
# agreement containing restrictions on use and disclosure and are protected by
# intellectual property laws.  Except as expressly permitted in your license
# agreement or allowed by law, you may not use, copy, reproduce, translate,
# broadcast, modify, license, transmit, distribute, exhibit, perform, publish,
# or display any part, in any form, or by any means.  Reverse engineering,
# disassembly, or decompilation of this software, unless required by law for
# interoperability, is prohibited.
#
# The information contained herein is subject to change without notice and is
# not warranted to be error-free.  If you find any errors, please report them
# to us in writing.
#
# This software or hardware is developed for general use in a variety of
# information management applications.  It is not developed or intended for use
# in any inherently dangerous applications, including applications which may
# create a risk of personal injury.  If you use this software or hardware in
# dangerous applications, then you shall be responsible to take all appropriate
# fail-safe, backup, redundancy, and other measures to ensure its safe use.
# Inktank Storage, Inc.  and its affiliates disclaim any liability for any
# damages caused by use of this software or hardware in dangerous applications.

import logging
import os
import platform
import shutil
import subprocess
import sys
import tarfile
import tempfile

help_heather = """

8888888      .d8888b.      8888888888
  888       d88P  Y88b     888
  888       888    888     888
  888       888            8888888
  888       888            888
  888       888    888     888
  888   d8b Y88b  d88P d8b 888        d8b
8888888 Y8P  "Y8888P"  Y8P 8888888888 Y8P

      Inktank Ceph Enterprise Setup
"""

# =============================================================================
# Logging
# =============================================================================

logger = logging.getLogger('ice')

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)

COLORS = {
    'WARNING': YELLOW,
    'INFO': WHITE,
    'DEBUG': BLUE,
    'CRITICAL': RED,
    'ERROR': RED
}

RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"

BASE_COLOR_FORMAT = "%(color_levelname)s %(message)s"
VERBOSE_COLOR_FORMAT = "[%(name)s][$BOLD%(levelname)s] $RESET%(color_levelname)s %(message)s"

def color_message(message):
    message = message.replace("$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ)
    return message

class ColoredFormatter(logging.Formatter):
    """
    A very basic logging formatter that not only applies color to the levels of
    the ouput but will also truncate the level names so that they do not alter
    the visuals of logging when presented on the terminal.
    """

    def __init__(self, msg):
        logging.Formatter.__init__(self, msg)

    def format(self, record):
        levelname = record.levelname
        truncated_level = record.levelname[:6]
        if levelname in COLORS:
            levelname_color = COLOR_SEQ % (30 + COLORS[levelname]) + '-->' + RESET_SEQ
            record.color_levelname = levelname_color
        return logging.Formatter.format(self, record)

def color_format(verbose=False):
    """
    Main entry point to get a colored formatter, it will use the
    BASE_FORMAT by default.
    """
    if verbose:
        color_format = color_message(VERBOSE_COLOR_FORMAT)
    else:
        color_format = color_message(BASE_COLOR_FORMAT)
    return ColoredFormatter(color_format)

# =============================================================================
# Exceptions
# =============================================================================

class ICEError(Exception):
    """Base ICE Setup exception"""
    pass

class NonZeroExit(ICEError):
    """subprocess commands that exit with non-zero status"""
    pass

class UnsupportedPlatform(ICEError):
    pass

# =============================================================================
# Templates
# =============================================================================

ice_repo_template = """
[ice]
name=ice packages for $basearch
baseurl={repo_url}/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey={gpg_url}

[ice-noarch]
name=ice noarch packages
baseurl={repo_url}/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey={gpg_url}

[ice-source]
name=ice source packages
baseurl={repo_url}/SRPMS
enabled=0
gpgcheck=1
type=rpm-md
gpgkey={gpg_url}
"""

ice_list_template = """deb {repo_url} {codename} main\n"""

# =============================================================================
# Distributions
# =============================================================================

class CentOS(object):
    pkg_manager = Yum()

class Debian(object):
    pkg_manager = Apt()

# Normalize casing for easier mapping
centos = CentOS
debian = Debian

def get_distro():
    """
    Retrieve the class that matches the distribution of the current host. This
    function will call ``platform()`` and retrieve the distribution
    information, then return the appropriate class and slap a few attributes
    to that class defining the information it found from the host.

    For example, if the current host is an Ubuntu server, the ``Debian`` class
    would be returned (as it holds 1:1 parity with Ubuntu) and the following
    would be set::

        module.name = 'ubuntu'
        module.release = '12.04'
        module.codename = 'precise'

    """
    distro_name, release, codename = platform_information()

    if not codename or not _get_distro(distro_name):
        error_msg = 'platform is not supported: %s %s %s' % (
            distro_name,
            release,
            codename,
        )
        raise UnsupportedPlatform(error_msg)

    # TODO: make this part of the distro objects
    # no need for them to be inferred here
    module = _get_distro(distro_name)
    module.name = distro_name
    module.release = release
    module.codename = codename
    module.machine_type = platform.machine()

    # XXX: Do we need to know about the init?
    #module.init = _choose_init(distro_name, codename)

    return module

# XXX These probably do not need to be full of classmethods but can be
# instantiated when the distro detection happens

class Yum(object):

    @classmethod
    def create_repo_file(cls, gpg_url, repo_url, file_name=None, **kw):
        """set the contents of /etc/yum.repos.d/ice.repo"""
        etc_path = kw.pop('etc_path', '/etc/yum.repos.d')
        file_name = file_name or 'ice.repo'
        repo_file_path = os.path.join(etc_path, file_name)
        with open(repo_file_path, 'w') as repo_file:
            contents = ice_repo_template.format(
                gpg_url=gpg_url,
                repo_url=repo_url,
            )
            repo_file.write(contents)

    @classmethod
    def install(cls, package):
        cmd = [
            'yum',
            '-y',
            '-q',
            'install',
        ]
        cmd.append(package)
        run(cmd)

class Apt(object):

    @classmethod
    def create_repo_file(cls, repo_url, gpg_url, file_name=None, **kw):
        """add ceph deb repo to sources.list"""
        etc_path = kw.pop('etc_path', '/etc/apt/sources.list.d')
        file_name = file_name or 'ice.list'
        list_file_path = os.path.join(etc_path, file_name)
        with open(list_file_path, 'w') as list_file:
            list_file.write(ice_list_template.format(
                repo_url=repo_url, codename=kw.pop('codename'))
            )

    @classmethod
    def install(cls, package):
        cmd = [
            'sudo',
            'env',
            'DEBIAN_FRONTEND=noninteractive',
            'apt-get',
            '-q',
            'install',
            '--assume-yes',
        ]
        cmd.append(package)
        run(cmd)

# =============================================================================
# Subprocess
# =============================================================================

def run(cmd, **kw):
    stop_on_nonzero = kw.pop('stop_on_nonzero', True)

    process = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        close_fds=True,
        **kw
    )

    if process.stderr:
        while True:
            err = process.stderr.readline()
            if err == '' and process.poll() is not None:
                break
            if err != '':
                logger.warning(err)
                sys.stderr.flush()
    if process.stdout:
        while True:
            out = process.stdout.readline()
            if out == '' and process.poll() is not None:
                break
            if out != '':
                logger.debug(out)
                sys.stdout.flush()

    returncode = process.wait()
    if returncode != 0:
        error_msg = "command returned non-zero exit status: %s" % returncode
        if stop_on_nonzero:
            raise NonZeroExit(error_msg)
        else:
            logger.warning(error_msg)

def run_call(cmd, **kw):
    """
    a callable that will execute a subprocess without raising an exception if
    the exit status is non-zero.

    The ``logger`` argument is in the signature only for consistency in the
    API, it does nothing by default.
    """

    process = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw
    )
    stdout = [line.strip('\n') for line in process.stdout.readlines()]
    stderr = [line.strip('\n') for line in process.stderr.readlines()]
    return stdout, stderr, process.wait()

# =============================================================================
# System
# =============================================================================

def platform_information():
    """
    detect platform information, return a dictionary with the following keys:

    * distro_name: as in 'Ubuntu' or 'CentOS'
    * release:     like '5.0' or '13.04'
    * codename:    as in 'Wheezy' or 'Quantal'
    """
    distro, release, codename = platform.linux_distribution()

    # this could be an empty string in Debian
    if not codename and 'debian' in distro.lower():
        debian_codenames = {
            '8': 'jessie',
            '7': 'wheezy',
            '6': 'squeeze',
        }
        major_version = release.split('.')[0]
        codename = debian_codenames.get(major_version, '')
    return (
        _normalized_distro_name(distro.rstrip()),
        release.rstrip(),
        codename.rstrip(),
    )

def _get_distro(distro, fallback=None):
    if not distro:
        return

    distro = _normalized_distro_name(distro)
    distributions = {
        'debian': debian,
        'ubuntu': debian,
        'centos': centos,
        'redhat': centos,
        }

    return distributions.get(distro) or _get_distro(fallback)

def _normalized_distro_name(distro):
    """
    Normalizes the distribution name so it is easier to operate on well knowns
    rather than whatever small differences distributions decide to add to them

    Even though we do not support Suse or Scientific, because we are just
    normalizing we don't mind leaving them here.
    """
    distro = distro.lower()
    if distro.startswith(('redhat', 'red hat')):
        return 'redhat'
    elif distro.startswith(('scientific', 'scientific linux')):
        return 'scientific'
    elif distro.startswith(('suse', 'opensuse')):
        return 'suse'
    return distro

# =============================================================================
# File Utilities
# =============================================================================

def extract_file(file_path):
    """
    Decompress/Extract a tar file to a temporary location and return its full
    path so that it can be handled elsewhere.  If ``file_path`` is not a tar
    file and it is a directory holding decompressed files return ``file_path``,
    otherwise raise an error.

    Removal of the temporary directory files is responsibility of the caller.
    """
    if os.path.isdir(file_path):
        return file_path
    if tarfile.is_tarfile(file_path):
        tmp_dir = tempfile.mkdtemp()
        destination = os.path.join(tmp_dir, 'repo')
        tar = tarfile.open(file_path, 'r:gz')
        tar.extractall(destination)
        tar.close()
        return destination

def overwrite_dir(source, destination='/opt/ice-repo/'):
    """
    Copy all files from _source_ to a temporary location (if not in a temporary
    location already) and then overwrite the contents of its destination so
    that the contents are as up to date as possible
    """
    # TODO: Need to define the destination of respository files, for example:
    # /opt/ice-repo

    # remove destination to ensure we get a fresh copy
    try:
        shutil.rmtree(destination)
    except OSError:
        os.mkdir(destination)

    # now copy the contents
    shutil.copytree(source, destination)

    # finally remove the source
    shutil.rmtree(source)

def default_repo_location():
    """
    Calculates the default repository location of the repository files, for
    example if this script runs alongside the sources it would get the absolute
    path for the ``sources`` directory relative to this script.
    """
    # TODO: Needs to know about the location (and names) of directories
    # packaged alongside this script

    # XXX: bad naming here. Maybe `detect_repo_location`
    pass

# =============================================================================
# Prompts
# =============================================================================

def prompt(question, _raw_input=None):
    input_prompt = _raw_input or raw_input
    prefix = '%s-->%s ' % (COLOR_SEQ % (30 + COLORS['DEBUG']), RESET_SEQ)
    prompt_format = '{prefix} {question}'.format(prefix=prefix, question=question)
    response = input_prompt(prompt_format)
    try:
        return strtobool(response)
    except ValueError:
        logger.error('That response was invalid, please try again')
        return prompt(question, _raw_input=input_prompt)

def strtobool(val):
    """
    Convert a string representation of truth to true (1) or false (0).

    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are
    'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if 'val' is
    anything else.

    .. note:: lifted from distutils.utils.strtobool
    """
    try:
        val = val.lower()
    except AttributeError:
        val = str(val).lower()
    if val in ('y', 'yes', 't', 'true', 'on', '1', '', None):
        return 1
    elif val in ('n', 'no', 'f', 'false', 'off', '0'):
        return 0
    else:
        raise ValueError("invalid input value: %r" % (val,))

# =============================================================================
# Actions
# =============================================================================

def configure(tar_file):
    """
    Decompress a tar file for the current host so that it can serve as a repo
    server and we can then install Calamari and ceph-deploy.
    """
    distro = get_distro()
    decompressed_repo = extract_file(tar_file)
    overwrite_dir(decompressed_repo)

    # TODO: Allow custom destinations
    distro.pkg_manager.create_repo_file(
        gpg_url='/opt/ice/repo/release.asc',
        repo_url='/opt/ice/repo',
        codename=distro.codename,
    )

def install(package):
    """
    Perform a package installation (e.g. Calamari or ceph-deploy) in the
    current host, abstracted away from the underlying package manager.
    """
    distro = get_distro()
    distro.pkg_manager.install(package)

# =============================================================================
# Main
# =============================================================================

def main(argv=None):
    # TODO:
    # * add the user prompts for first time runs
    # * implement hybrid behavior via commands and/or prompts
    #   ice_setup.py install calamari
    argv = argv or sys.argv
    # Console Logger
    terminal_log = logging.StreamHandler()
    terminal_log.setFormatter(color_format(verbose='-v' in argv))
    logger.addHandler(terminal_log)
    logger.setLevel(logging.DEBUG)

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import sys

from optparse import OptionParser, make_option as Option
from pprint import pprint

from anyjson import deserialize

from celery import __version__

class Command(object):
    help = ""
    args = ""
    option_list = (
        Option("--conf", dest="conf",
            help="Celery config module name (default: celeryconfig)"),
        Option("--loader", dest="loader",
            help="Celery loaders module name (default: default)"),
    )

    def create_parser(self, prog_name, subcommand):
        return OptionParser(prog=prog_name,
                            usage=self.usage(subcommand),
                            version=self.get_version(),
                            option_list=self.option_list)

    def run_from_argv(self, argv):
        parser = self.create_parser(argv[0], argv[1])
        options, args = parser.parse_args(argv[2:])
        if options.loader:
            os.environ["CELERY_LOADER"] = options.loader
        if options.conf:
            os.environ["CELERY_CONFIG_MODULE"] = options.conf
        self(*args, **options.__dict__)

    def __call__(self, *args, **kwargs):
        out = self.run(*args, **kwargs)
        if out:
            sys.stderr.write("%s\n" % out)

    def run(self, *args, **kwargs):
        raise NotImplementedError()

    def get_version(self):
        return __version__

    def usage(self, subcommand):
        return "%%prog %s [options] %s" % (subcommand, self.args)

class apply(Command):
    option_list = Command.option_list + (
            Option("--args", "-a", dest="args"),
            Option("--kwargs", "-k", dest="kwargs"),
            Option("--eta", dest="eta"),
            Option("--countdown", dest="countdown", type="int"),
            Option("--expires", dest="expires"),
            Option("--serializer", dest="serializer", default="json"),
            Option("--queue", dest="queue"),
            Option("--exchange", dest="exchange"),
            Option("--routing-key", dest="routing_key"),
    )

    def run(self, name, *_, **kw):
        from celery.execute import send_task

        # Positional args.
        args = kw.get("args") or ()
        if isinstance(args, basestring):
            args = deserialize(args)

        # Keyword args.
        kwargs = kw.get("kwargs") or {}
        if isinstance(kwargs, basestring):
            kwargs = deserialize(kwargs)

        # Expires can be int.
        expires = kw.get("expires") or None
        try:
            expires = int(expires)
        except (TypeError, ValueError):
            pass

        res = send_task(name, args=args, kwargs=kwargs,
                        countdown=kw.get("countdown"),
                        serializer=kw.get("serializer"),
                        queue=kw.get("queue"),
                        exchange=kw.get("exchange"),
                        routing_key=kw.get("routing_key"),
                        eta=kw.get("eta"),
                        expires=expires)

        print(res.task_id)

class inspect(Command):
    choices = ("active", "scheduled", "reserved",
               "stats", "revoked", "registered_tasks",
               "enable_events", "disable_events")
    option_list = Command.option_list + (
                Option("--timeout", "-t", type="float", dest="timeout",
                    default=1.0,
                    help="Timeout in seconds (float) waiting for reply"),
                Option("--destination", "-d", dest="destination",
                    help="Comma separated list of destination node names."))

    def usage(self, subcommand):
        return "%%prog %s [options] %s [%s]" % (
                subcommand, self.args, "|".join(self.choices))

    def run(self, *args, **kwargs):
        if not args:
            return "Missing inspect command. See --help"
        command = args[0]
        if command not in self.choices:
            return "Unknown inspect command: %s" % command
        from celery.task.control import inspect

        destination = kwargs.get("destination")
        timeout = kwargs.get("timeout", 2.0)
        if destination and isinstance(destination, basestring):
            destination = map(str.strip, destination.split(","))

        i = inspect(destination=destination, timeout=timeout)
        pprint(getattr(i, command)())

commands = {"apply": apply,
            "inspect": inspect}

def main():
    subcommand = sys.argv[1]
    commands[subcommand]().run_from_argv(sys.argv)

if __name__ == "__main__":
    main()
<commit_msg>Some polishing work on celeryctl<commit_after>import os
import sys

from optparse import OptionParser, make_option as Option
from pprint import pprint

from anyjson import deserialize

from celery import __version__

commands = {}

def command(fun):
    commands[fun.__name__] = fun
    return fun

class Command(object):
    help = ""
    args = ""
    option_list = (
        Option("--conf", dest="conf",
            help="Celery config module name (default: celeryconfig)"),
        Option("--loader", dest="loader",
            help="Celery loaders module name (default: default)"),
    )

    def create_parser(self, prog_name, command):
        return OptionParser(prog=prog_name,
                            usage=self.usage(command),
                            version=self.get_version(),
                            option_list=self.option_list)

    def run_from_argv(self, argv):
        self.prog_name = os.path.basename(argv[0])
        self.command = argv[1]
        self.arglist = argv[2:]
        self.parser = self.create_parser(self.prog_name, self.command)
        options, args = self.parser.parse_args(self.arglist)
        if options.loader:
            os.environ["CELERY_LOADER"] = options.loader
        if options.conf:
            os.environ["CELERY_CONFIG_MODULE"] = options.conf
        self(*args, **options.__dict__)

    def __call__(self, *args, **kwargs):
        out = self.run(*args, **kwargs)
        if out:
            sys.stderr.write("%s\n" % out)

    def run(self, *args, **kwargs):
        raise NotImplementedError()

    def get_version(self):
        return __version__

    def usage(self, command):
        return "%%prog %s [options] %s" % (command, self.args)

class apply(Command):
    args = "<task_name>"
    option_list = Command.option_list + (
            Option("--args", "-a", dest="args"),
            Option("--kwargs", "-k", dest="kwargs"),
            Option("--eta", dest="eta"),
            Option("--countdown", dest="countdown", type="int"),
            Option("--expires", dest="expires"),
            Option("--serializer", dest="serializer", default="json"),
            Option("--queue", dest="queue"),
            Option("--exchange", dest="exchange"),
            Option("--routing-key", dest="routing_key"),
    )

    def run(self, name, *_, **kw):
        from celery.execute import send_task

        # Positional args.
        args = kw.get("args") or ()
        if isinstance(args, basestring):
            args = deserialize(args)

        # Keyword args.
        kwargs = kw.get("kwargs") or {}
        if isinstance(kwargs, basestring):
            kwargs = deserialize(kwargs)

        # Expires can be int.
        expires = kw.get("expires") or None
        try:
            expires = int(expires)
        except (TypeError, ValueError):
            pass

        res = send_task(name, args=args, kwargs=kwargs,
                        countdown=kw.get("countdown"),
                        serializer=kw.get("serializer"),
                        queue=kw.get("queue"),
                        exchange=kw.get("exchange"),
                        routing_key=kw.get("routing_key"),
                        eta=kw.get("eta"),
                        expires=expires)

        print(res.task_id)
apply = command(apply)

class inspect(Command):
    choices = ("active", "scheduled", "reserved",
               "stats", "revoked", "registered_tasks",
               "enable_events", "disable_events")
    option_list = Command.option_list + (
                Option("--timeout", "-t", type="float", dest="timeout",
                    default=1.0,
                    help="Timeout in seconds (float) waiting for reply"),
                Option("--destination", "-d", dest="destination",
                    help="Comma separated list of destination node names."))

    def usage(self, command):
        return "%%prog %s [options] %s [%s]" % (
                command, self.args, "|".join(self.choices))

    def run(self, *args, **kwargs):
        if not args:
            return "Missing inspect command. See --help"
        command = args[0]
        if command not in self.choices:
            return "Unknown inspect command: %s" % command
        from celery.task.control import inspect

        destination = kwargs.get("destination")
        timeout = kwargs.get("timeout", 2.0)
        if destination and isinstance(destination, basestring):
            destination = map(str.strip, destination.split(","))

        i = inspect(destination=destination, timeout=timeout)
        pprint(getattr(i, command)())
inspect = command(inspect)

class help(Command):

    def usage(self, command):
        return "%%prog <command> [options] %s" % (self.args, )

    def run(self, *args, **kwargs):
        self.parser.print_help()
        usage = ["",
                "Type '%s <command> --help' for help on a "
                "specific command." % (self.prog_name, ),
                "",
                "Available commands:"]
        for command in list(sorted(commands.keys())):
            usage.append("    %s" % command)
        print("\n".join(usage))
help = command(help)

class celeryctl(object):
    commands = commands

    def execute(self, command, argv=None):
        if argv is None:
            argv = sys.arg
        argv = list(argv)
        try:
            cls = self.commands[command]
        except KeyError:
            cls = self.commands["help"]
            argv.insert(1, "help")
        cls = self.commands.get(command) or self.commands["help"]
        try:
            cls().run_from_argv(argv)
        except TypeError:
            return self.execute("help", argv)

    def execute_from_commandline(self, argv=None):
        if argv is None:
            argv = sys.argv
        try:
            command = argv[1]
        except IndexError:
            command = "help"
        return self.execute(command, argv)

def main():
    celeryctl().execute_from_commandline()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import unittest, time, sys
sys.path.extend(['.','..','py'])
import h2o, h2o_cmd, h2o_kmeans, h2o_hosts, h2o_browse as h2b, h2o_import2 as h2i

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global localhost
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(1, java_heap_GB=14)
        else:
            h2o_hosts.build_cloud_with_hosts(1)

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_KMeans_twit(self):
        csvFilename = "Twitter2DB.txt"
        print "\nStarting", csvFilename

        # h2b.browseTheCloud()
        parseResult = h2i.import_parse(bucket='smalldata', path=csvFilename, hex_key=csvFilename + ".hex", schema='put')

        # loop, to see if we get same centers
        # should check the means?
        # FIX! have to fix these to right answers
        expected = [
                # expected centers are from R. rest is just from h2o
                ([310527.2, 13433.89], 11340, None),
                ([5647967.1, 40487.76], 550, None),
                ([21765291.7, 93129.26], 14,  None),
            ]
        # all are multipliers of expected tuple value
        allowedDelta = (0.01, 0.01, 0.01)
        for trial in range(2):
            kwargs = {
                'k': 3,
                'max_iter': 50,
                'epsilon': 1e-4,
                'normalize': 0,
                'cols': '0,1',
                'initialization': 'Furthest',
                # 'initialization': 'PlusPlus',
                'destination_key': 'kmeans_dest_key',
                # reuse the same seed, to get deterministic results (otherwise sometimes fails
                'seed': 265211114317615310
            }

            kmeans = h2o_cmd.runKMeansOnly(parseResult=parseResult, timeoutSecs=5, **kwargs)
            (centers, tupleResultList) = h2o_kmeans.bigCheckResults(self, kmeans, csvFilename, parseResult, 'd', **kwargs)

            if 1==0:
                h2b.browseJsonHistoryAsUrlLastMatch("KMeansScore")
                h2b.browseJsonHistoryAsUrlLastMatch("KMeansApply")
                h2b.browseJsonHistoryAsUrlLastMatch("KMeans")
                # Comment sleep out to get a clean grep.
                # time.sleep(3600)

            h2o_kmeans.compareResultsToExpected(self, tupleResultList, expected, allowedDelta, trial=trial)

if __name__ == '__main__':
    h2o.unit_main()
<commit_msg>adjust this tests center compares<commit_after>import unittest, time, sys
sys.path.extend(['.','..','py'])
import h2o, h2o_cmd, h2o_kmeans, h2o_hosts, h2o_browse as h2b, h2o_import2 as h2i

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global localhost
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(1, java_heap_GB=14)
        else:
            h2o_hosts.build_cloud_with_hosts(1)

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_KMeans_twit(self):
        csvFilename = "Twitter2DB.txt"
        print "\nStarting", csvFilename

        # h2b.browseTheCloud()
        parseResult = h2i.import_parse(bucket='smalldata', path=csvFilename, hex_key=csvFilename + ".hex", schema='put')

        # loop, to see if we get same centers
        # should check the means?
        # both of these centers match what different R/Scikit packages get
        expected1 = [
                # expected centers are from R. rest is just from h2o
                ([310527.2, 13433.89], 11340, None),
                ([5647967.1, 40487.76], 550, None),
                ([21765291.7, 93129.26], 14,  None),
            ]

        # this is what we get with Furthest
        expected2 = [
                ([351104.74065255735, 15421.749823633158], 11340, 5021682274541967.0) ,
                ([7292636.589090909, 7575.630909090909], 550, 6373072701775582.0) ,
                ([34406781.071428575, 244878.0], 14, 123310713697348.92) ,
            ]

        # all are multipliers of expected tuple value
        allowedDelta = (0.0001, 0.0001, 0.0001)
        for trial in range(2):
            kwargs = {
                'k': 3,
                'max_iter': 50,
                'epsilon': 1e-4,
                'normalize': 0,
                'cols': '0,1',
                'initialization': 'Furthest',
                # 'initialization': 'PlusPlus',
                'destination_key': 'kmeans_dest_key',
                # reuse the same seed, to get deterministic results (otherwise sometimes fails
                'seed': 265211114317615310
            }

            kmeans = h2o_cmd.runKMeansOnly(parseResult=parseResult, timeoutSecs=5, **kwargs)
            (centers, tupleResultList) = h2o_kmeans.bigCheckResults(self, kmeans, csvFilename, parseResult, 'd', **kwargs)

            if 1==0:
                h2b.browseJsonHistoryAsUrlLastMatch("KMeansScore")
                h2b.browseJsonHistoryAsUrlLastMatch("KMeansApply")
                h2b.browseJsonHistoryAsUrlLastMatch("KMeans")
                # Comment sleep out to get a clean grep.
                # time.sleep(3600)

            h2o_kmeans.compareResultsToExpected(self, tupleResultList, expected2, allowedDelta, trial=trial)

if __name__ == '__main__':
    h2o.unit_main()
<|endoftext|>


<commit_before>
import discord
from discord.ext import commands, tasks

import asyncio
import datetime
import functools
import html
import io
import logging
import re
import sys
import time
import textwrap
import traceback
import urllib

import aiohttp
from bs4 import BeautifulSoup
import dateutil.parser
import dateutil.tz
import feedparser
import pytz

from utilities import checks

errors_logger = logging.getLogger("errors")

def setup(bot):
	bot.add_cog(RSS(bot))

class RSS(commands.Cog):

	def __init__(self, bot):
		self.bot = bot

		# Generate tzinfos
		self.tzinfos = {}
		for timezone_abbreviation in ("EDT", "EST"):
			matching_timezones = list(filter(lambda t: datetime.datetime.now(pytz.timezone(t)).strftime("%Z") == timezone_abbreviation, pytz.common_timezones))
			matching_utc_offsets = set(datetime.datetime.now(pytz.timezone(t)).strftime("%z") for t in matching_timezones)
			if len(matching_utc_offsets) == 1:
				self.tzinfos[timezone_abbreviation] = dateutil.tz.gettz(matching_timezones[0])

		self.new_feed = asyncio.Event()
		self.check_feeds.start().set_name("RSS")

	def cog_unload(self):
		self.check_feeds.cancel()

	async def inititalize_database(self):
		await self.bot.connect_to_database()
		await self.bot.db.execute("CREATE SCHEMA IF NOT EXISTS rss")
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS rss.feeds (
				channel_id		BIGINT,
				feed			TEXT,
				last_checked	TIMESTAMPTZ,
				ttl				INT,
				PRIMARY KEY		(channel_id, feed)
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS rss.entries (
				entry			TEXT,
				feed			TEXT,
				PRIMARY KEY		(entry, feed)
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS rss.errors (
				timestamp		TIMESTAMPTZ PRIMARY KEY DEFAULT NOW(),
				feed			TEXT,
				type			TEXT,
				message			TEXT
			)
			"""
		)

	@commands.group(aliases = ["feed"], invoke_without_command = True, case_insensitive = True)
	@checks.not_forbidden()
	async def rss(self, ctx):
		'''RSS'''
		await ctx.send_help(ctx.command)

	@rss.command()
	@commands.check_any(checks.is_permitted(), checks.is_guild_owner())
	async def add(self, ctx, url: str):
		'''Add a feed to a channel'''
		following = await ctx.bot.db.fetchval(
			"""
			SELECT EXISTS (
				SELECT FROM rss.feeds
				WHERE channel_id = $1 AND feed = $2
			)
			""",
			ctx.channel.id, url
		)
		if following:
			return await ctx.embed_reply(":no_entry: This text channel is already following that feed")
		async with ctx.bot.aiohttp_session.get(url) as resp:
			feed_text = await resp.text()
		# TODO: Handle issues getting URL
		partial = functools.partial(feedparser.parse, io.BytesIO(feed_text.encode("UTF-8")),
																	response_headers = {"Content-Location": url})
		feed_info = await self.bot.loop.run_in_executor(None, partial)
		# Still necessary to run in executor?
		# TODO: Handle if feed already being followed elsewhere
		ttl = None
		if "ttl" in feed_info.feed:
			ttl = int(feed_info.feed.ttl)
		for entry in feed_info.entries:
			await ctx.bot.db.execute(
				"""
				INSERT INTO rss.entries (entry, feed)
				VALUES ($1, $2)
				ON CONFLICT (entry, feed) DO NOTHING
				""",
				entry.id, url
			)
		await ctx.bot.db.execute(
			"""
			INSERT INTO rss.feeds (channel_id, feed, last_checked, ttl)
			VALUES ($1, $2, NOW(), $3)
			""",
			ctx.channel.id, url, ttl
		)
		await ctx.embed_reply(f"The feed, {url}, has been added to this channel")
		self.new_feed.set()

	@rss.command(aliases = ["delete"])
	@commands.check_any(checks.is_permitted(), checks.is_guild_owner())
	async def remove(self, ctx, url: str):
		'''Remove a feed from a channel'''
		deleted = await ctx.bot.db.fetchval(
			"""
			DELETE FROM rss.feeds
			WHERE channel_id = $1 AND feed = $2
			RETURNING *
			""",
			ctx.channel.id, url
		)
		if not deleted:
			return await ctx.embed_reply(":no_entry: This channel isn't following that feed")
		await ctx.embed_reply(f"The feed, {url}, has been removed from this channel")

	@rss.command(aliases = ["feed"])
	@checks.not_forbidden()
	async def feeds(self, ctx):
		'''Show feeds being followed in this channel'''
		records = await ctx.bot.db.fetch("SELECT feed FROM rss.feeds WHERE channel_id = $1", ctx.channel.id)
		await ctx.embed_reply('\n'.join(record["feed"] for record in records),
								title = "RSS feeds being followed in this channel")

	# R/PT60S
	@tasks.loop(seconds = 60)
	async def check_feeds(self):
		records = await self.bot.db.fetch(
			"""
			SELECT DISTINCT ON (feed) feed, last_checked, ttl
			FROM rss.feeds
			ORDER BY feed, last_checked
			"""
		)
		if not records:
			self.new_feed.clear()
			await self.new_feed.wait()
		for record in records:
			feed = record["feed"]
			if record["ttl"] and datetime.datetime.now(datetime.timezone.utc) < record["last_checked"] + datetime.timedelta(minutes = record["ttl"]):
				continue
			try:
				async with self.bot.aiohttp_session.get(feed) as resp:
					feed_text = await resp.text()
				feed_info = await self.bot.loop.run_in_executor(None, functools.partial(feedparser.parse, io.BytesIO(feed_text.encode("UTF-8")), response_headers = {"Content-Location": feed}))
				# Still necessary to run in executor?
				ttl = None
				if "ttl" in feed_info.feed:
					ttl = int(feed_info.feed.ttl)
				await self.bot.db.execute(
					"""
					UPDATE rss.feeds
					SET last_checked = NOW(),
						ttl = $1
					WHERE feed = $2
					""",
					ttl, feed
				)
				for entry in feed_info.entries:
					if "id" not in entry:
						continue
					inserted = await self.bot.db.fetchrow(
						"""
						INSERT INTO rss.entries (entry, feed)
						VALUES ($1, $2)
						ON CONFLICT DO NOTHING
						RETURNING *
						""",
						entry.id, feed
					)
					if not inserted:
						continue
					# Get timestamp
					## if "published_parsed" in entry:
					##  timestamp = datetime.datetime.fromtimestamp(time.mktime(entry.published_parsed))
					### inaccurate
					if "published" in entry and entry.published:
						timestamp = dateutil.parser.parse(entry.published, tzinfos = self.tzinfos)
					elif "updated" in entry:  # and entry.updated necessary?; check updated first?
						timestamp = dateutil.parser.parse(entry.updated, tzinfos = self.tzinfos)
					else:
						timestamp = discord.Embed.Empty
					# Get and set description, title, url + set timestamp
					if not (description := entry.get("summary")) and "content" in entry:
						description = entry["content"][0].get("value")
					if description:
						description = BeautifulSoup(description, "lxml").get_text(separator = '\n')
						description = re.sub(r"\n\s*\n", '\n', description)
						if len(description) > self.bot.EMBED_DESCRIPTION_CHARACTER_LIMIT:
							space_index = description.rfind(' ', 0, self.bot.EDCL - 3)
							# EDCL: Embed Description Character Limit
							description = description[:space_index] + "..."
					title = textwrap.shorten(entry.get("title"), width = self.bot.ETiCL, placeholder = "...")
					# ETiCL: Embed Title Character Limit
					embed = discord.Embed(title = html.unescape(title),
											url = entry.link,
											description = description,
											timestamp = timestamp,
											color = self.bot.rss_color)
					# Get and set thumbnail url
					thumbnail_url = (
						(media_thumbnail := entry.get("media_thumbnail")) and media_thumbnail[0].get("url") or
						(
							(media_content := entry.get("media_content")) and
							(media_image := discord.utils.find(lambda c: "image" in c.get("medium", ""), media_content)) and
							media_image.get("url")
						) or
						(
							(links := entry.get("links")) and
							(image_link := discord.utils.find(lambda l: "image" in l.get("type", ""), links)) and
							image_link.get("href")
						 ) or
						(
							(content := entry.get("content")) and (content_value := content[0].get("value")) and
							(content_img := getattr(BeautifulSoup(content_value, "lxml"), "img")) and
							content_img.get("src")
						) or
						(
							(media_content := entry.get("media_content")) and
							(media_content := discord.utils.find(lambda c: "url" in c, media_content)) and
							media_content["url"]
						) or
						(
							(description := entry.get("description")) and
							(description_img := getattr(BeautifulSoup(description, "lxml"), "img")) and
							description_img.get("src")
						)
					)
					if thumbnail_url:
						if not urllib.parse.urlparse(thumbnail_url).netloc:
							thumbnail_url = feed_info.feed.link + thumbnail_url
						embed.set_thumbnail(url = thumbnail_url)
					# Get and set footer icon url
					footer_icon_url = (
						feed_info.feed.get("icon") or feed_info.feed.get("logo") or
						(feed_image := feed_info.feed.get("image")) and feed_image.get("href") or
						(parsed_image := BeautifulSoup(feed_text, "lxml").image) and next(iter(parsed_image.attrs.values()), None) or
						discord.Embed.Empty
					)
					embed.set_footer(text = feed_info.feed.title, icon_url = footer_icon_url)
					# Send embed(s)
					channel_records = await self.bot.db.fetch("SELECT channel_id FROM rss.feeds WHERE feed = $1", feed)
					for record in channel_records:
						if text_channel := self.bot.get_channel(record["channel_id"]):
							try:
								await text_channel.send(embed = embed)
							except discord.Forbidden:
								pass
							except discord.HTTPException as e:
								if e.status == 400 and e.code == 50035:
									if "In embed.url: Not a well formed URL." in e.text:
										embed.url = discord.Embed.Empty
									if "In embed.thumbnail.url: Not a well formed URL." in e.text:
										embed.set_thumbnail(url = "")
									if "In embed.footer.icon_url: Not a well formed URL." in e.text:
										embed.set_footer(text = feed_info.feed.title)
									await text_channel.send(embed = embed)
								else:
									raise
						# TODO: Remove text channel data if now non-existent
			except (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError,
					aiohttp.TooManyRedirects, asyncio.TimeoutError,
					UnicodeDecodeError) as e:
				await self.bot.db.execute(
					"""
					INSERT INTO rss.errors (feed, type, message)
					VALUES ($1, $2, $3)
					""",
					feed, type(e).__name__, str(e)
				)
				# Print error?
				await asyncio.sleep(10)
				# TODO: Add variable for sleep time
				# TODO: Remove persistently erroring feed or exponentially backoff?
			except discord.DiscordServerError as e:
				self.bot.print(f"RSS Task Discord Server Error: {e}")
				await asyncio.sleep(60)
			except Exception as e:
				print("Exception in RSS Task", file = sys.stderr)
				traceback.print_exception(type(e), e, e.__traceback__, file = sys.stderr)
				errors_logger.error("Uncaught RSS Task exception\n", exc_info = (type(e), e, e.__traceback__))
				print(f" (feed: {feed})")
				await asyncio.sleep(60)

	@check_feeds.before_loop
	async def before_check_feeds(self):
		await self.inititalize_database()
		await self.bot.wait_until_ready()

	@check_feeds.after_loop
	async def after_check_feeds(self):
		self.bot.print("RSS task cancelled")

<commit_msg>[Discord] Handle unsupported schemes for RSS embed footer icon URLs<commit_after>
import discord
from discord.ext import commands, tasks

import asyncio
import datetime
import functools
import html
import io
import logging
import re
import sys
import time
import textwrap
import traceback
import urllib

import aiohttp
from bs4 import BeautifulSoup
import dateutil.parser
import dateutil.tz
import feedparser
import pytz

from utilities import checks

errors_logger = logging.getLogger("errors")

def setup(bot):
	bot.add_cog(RSS(bot))

class RSS(commands.Cog):

	def __init__(self, bot):
		self.bot = bot

		# Generate tzinfos
		self.tzinfos = {}
		for timezone_abbreviation in ("EDT", "EST"):
			matching_timezones = list(filter(lambda t: datetime.datetime.now(pytz.timezone(t)).strftime("%Z") == timezone_abbreviation, pytz.common_timezones))
			matching_utc_offsets = set(datetime.datetime.now(pytz.timezone(t)).strftime("%z") for t in matching_timezones)
			if len(matching_utc_offsets) == 1:
				self.tzinfos[timezone_abbreviation] = dateutil.tz.gettz(matching_timezones[0])

		self.new_feed = asyncio.Event()
		self.check_feeds.start().set_name("RSS")

	def cog_unload(self):
		self.check_feeds.cancel()

	async def inititalize_database(self):
		await self.bot.connect_to_database()
		await self.bot.db.execute("CREATE SCHEMA IF NOT EXISTS rss")
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS rss.feeds (
				channel_id		BIGINT,
				feed			TEXT,
				last_checked	TIMESTAMPTZ,
				ttl				INT,
				PRIMARY KEY		(channel_id, feed)
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS rss.entries (
				entry			TEXT,
				feed			TEXT,
				PRIMARY KEY		(entry, feed)
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS rss.errors (
				timestamp		TIMESTAMPTZ PRIMARY KEY DEFAULT NOW(),
				feed			TEXT,
				type			TEXT,
				message			TEXT
			)
			"""
		)

	@commands.group(aliases = ["feed"], invoke_without_command = True, case_insensitive = True)
	@checks.not_forbidden()
	async def rss(self, ctx):
		'''RSS'''
		await ctx.send_help(ctx.command)

	@rss.command()
	@commands.check_any(checks.is_permitted(), checks.is_guild_owner())
	async def add(self, ctx, url: str):
		'''Add a feed to a channel'''
		following = await ctx.bot.db.fetchval(
			"""
			SELECT EXISTS (
				SELECT FROM rss.feeds
				WHERE channel_id = $1 AND feed = $2
			)
			""",
			ctx.channel.id, url
		)
		if following:
			return await ctx.embed_reply(":no_entry: This text channel is already following that feed")
		async with ctx.bot.aiohttp_session.get(url) as resp:
			feed_text = await resp.text()
		# TODO: Handle issues getting URL
		partial = functools.partial(feedparser.parse, io.BytesIO(feed_text.encode("UTF-8")),
																	response_headers = {"Content-Location": url})
		feed_info = await self.bot.loop.run_in_executor(None, partial)
		# Still necessary to run in executor?
		# TODO: Handle if feed already being followed elsewhere
		ttl = None
		if "ttl" in feed_info.feed:
			ttl = int(feed_info.feed.ttl)
		for entry in feed_info.entries:
			await ctx.bot.db.execute(
				"""
				INSERT INTO rss.entries (entry, feed)
				VALUES ($1, $2)
				ON CONFLICT (entry, feed) DO NOTHING
				""",
				entry.id, url
			)
		await ctx.bot.db.execute(
			"""
			INSERT INTO rss.feeds (channel_id, feed, last_checked, ttl)
			VALUES ($1, $2, NOW(), $3)
			""",
			ctx.channel.id, url, ttl
		)
		await ctx.embed_reply(f"The feed, {url}, has been added to this channel")
		self.new_feed.set()

	@rss.command(aliases = ["delete"])
	@commands.check_any(checks.is_permitted(), checks.is_guild_owner())
	async def remove(self, ctx, url: str):
		'''Remove a feed from a channel'''
		deleted = await ctx.bot.db.fetchval(
			"""
			DELETE FROM rss.feeds
			WHERE channel_id = $1 AND feed = $2
			RETURNING *
			""",
			ctx.channel.id, url
		)
		if not deleted:
			return await ctx.embed_reply(":no_entry: This channel isn't following that feed")
		await ctx.embed_reply(f"The feed, {url}, has been removed from this channel")

	@rss.command(aliases = ["feed"])
	@checks.not_forbidden()
	async def feeds(self, ctx):
		'''Show feeds being followed in this channel'''
		records = await ctx.bot.db.fetch("SELECT feed FROM rss.feeds WHERE channel_id = $1", ctx.channel.id)
		await ctx.embed_reply('\n'.join(record["feed"] for record in records),
								title = "RSS feeds being followed in this channel")

	# R/PT60S
	@tasks.loop(seconds = 60)
	async def check_feeds(self):
		records = await self.bot.db.fetch(
			"""
			SELECT DISTINCT ON (feed) feed, last_checked, ttl
			FROM rss.feeds
			ORDER BY feed, last_checked
			"""
		)
		if not records:
			self.new_feed.clear()
			await self.new_feed.wait()
		for record in records:
			feed = record["feed"]
			if record["ttl"] and datetime.datetime.now(datetime.timezone.utc) < record["last_checked"] + datetime.timedelta(minutes = record["ttl"]):
				continue
			try:
				async with self.bot.aiohttp_session.get(feed) as resp:
					feed_text = await resp.text()
				feed_info = await self.bot.loop.run_in_executor(None, functools.partial(feedparser.parse, io.BytesIO(feed_text.encode("UTF-8")), response_headers = {"Content-Location": feed}))
				# Still necessary to run in executor?
				ttl = None
				if "ttl" in feed_info.feed:
					ttl = int(feed_info.feed.ttl)
				await self.bot.db.execute(
					"""
					UPDATE rss.feeds
					SET last_checked = NOW(),
						ttl = $1
					WHERE feed = $2
					""",
					ttl, feed
				)
				for entry in feed_info.entries:
					if "id" not in entry:
						continue
					inserted = await self.bot.db.fetchrow(
						"""
						INSERT INTO rss.entries (entry, feed)
						VALUES ($1, $2)
						ON CONFLICT DO NOTHING
						RETURNING *
						""",
						entry.id, feed
					)
					if not inserted:
						continue
					# Get timestamp
					## if "published_parsed" in entry:
					##  timestamp = datetime.datetime.fromtimestamp(time.mktime(entry.published_parsed))
					### inaccurate
					if "published" in entry and entry.published:
						timestamp = dateutil.parser.parse(entry.published, tzinfos = self.tzinfos)
					elif "updated" in entry:  # and entry.updated necessary?; check updated first?
						timestamp = dateutil.parser.parse(entry.updated, tzinfos = self.tzinfos)
					else:
						timestamp = discord.Embed.Empty
					# Get and set description, title, url + set timestamp
					if not (description := entry.get("summary")) and "content" in entry:
						description = entry["content"][0].get("value")
					if description:
						description = BeautifulSoup(description, "lxml").get_text(separator = '\n')
						description = re.sub(r"\n\s*\n", '\n', description)
						if len(description) > self.bot.EMBED_DESCRIPTION_CHARACTER_LIMIT:
							space_index = description.rfind(' ', 0, self.bot.EDCL - 3)
							# EDCL: Embed Description Character Limit
							description = description[:space_index] + "..."
					title = textwrap.shorten(entry.get("title"), width = self.bot.ETiCL, placeholder = "...")
					# ETiCL: Embed Title Character Limit
					embed = discord.Embed(title = html.unescape(title),
											url = entry.link,
											description = description,
											timestamp = timestamp,
											color = self.bot.rss_color)
					# Get and set thumbnail url
					thumbnail_url = (
						(media_thumbnail := entry.get("media_thumbnail")) and media_thumbnail[0].get("url") or
						(
							(media_content := entry.get("media_content")) and
							(media_image := discord.utils.find(lambda c: "image" in c.get("medium", ""), media_content)) and
							media_image.get("url")
						) or
						(
							(links := entry.get("links")) and
							(image_link := discord.utils.find(lambda l: "image" in l.get("type", ""), links)) and
							image_link.get("href")
						 ) or
						(
							(content := entry.get("content")) and (content_value := content[0].get("value")) and
							(content_img := getattr(BeautifulSoup(content_value, "lxml"), "img")) and
							content_img.get("src")
						) or
						(
							(media_content := entry.get("media_content")) and
							(media_content := discord.utils.find(lambda c: "url" in c, media_content)) and
							media_content["url"]
						) or
						(
							(description := entry.get("description")) and
							(description_img := getattr(BeautifulSoup(description, "lxml"), "img")) and
							description_img.get("src")
						)
					)
					if thumbnail_url:
						if not urllib.parse.urlparse(thumbnail_url).netloc:
							thumbnail_url = feed_info.feed.link + thumbnail_url
						embed.set_thumbnail(url = thumbnail_url)
					# Get and set footer icon url
					footer_icon_url = (
						feed_info.feed.get("icon") or feed_info.feed.get("logo") or
						(feed_image := feed_info.feed.get("image")) and feed_image.get("href") or
						(parsed_image := BeautifulSoup(feed_text, "lxml").image) and next(iter(parsed_image.attrs.values()), None) or
						discord.Embed.Empty
					)
					embed.set_footer(text = feed_info.feed.title, icon_url = footer_icon_url)
					# Send embed(s)
					channel_records = await self.bot.db.fetch("SELECT channel_id FROM rss.feeds WHERE feed = $1", feed)
					for record in channel_records:
						if text_channel := self.bot.get_channel(record["channel_id"]):
							try:
								await text_channel.send(embed = embed)
							except discord.Forbidden:
								pass
							except discord.HTTPException as e:
								if e.status == 400 and e.code == 50035:
									if "In embed.url: Not a well formed URL." in e.text:
										embed.url = discord.Embed.Empty
									if "In embed.thumbnail.url: Not a well formed URL." in e.text:
										embed.set_thumbnail(url = "")
									if ("In embed.footer.icon_url: Not a well formed URL." in e.text or
										("In embed.footer.icon_url: Scheme" in e.text and
											"is not supported. Scheme must be one of ('http', 'https')." in e.text)):
										embed.set_footer(text = feed_info.feed.title)
									await text_channel.send(embed = embed)
								else:
									raise
						# TODO: Remove text channel data if now non-existent
			except (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError,
					aiohttp.TooManyRedirects, asyncio.TimeoutError,
					UnicodeDecodeError) as e:
				await self.bot.db.execute(
					"""
					INSERT INTO rss.errors (feed, type, message)
					VALUES ($1, $2, $3)
					""",
					feed, type(e).__name__, str(e)
				)
				# Print error?
				await asyncio.sleep(10)
				# TODO: Add variable for sleep time
				# TODO: Remove persistently erroring feed or exponentially backoff?
			except discord.DiscordServerError as e:
				self.bot.print(f"RSS Task Discord Server Error: {e}")
				await asyncio.sleep(60)
			except Exception as e:
				print("Exception in RSS Task", file = sys.stderr)
				traceback.print_exception(type(e), e, e.__traceback__, file = sys.stderr)
				errors_logger.error("Uncaught RSS Task exception\n", exc_info = (type(e), e, e.__traceback__))
				print(f" (feed: {feed})")
				await asyncio.sleep(60)

	@check_feeds.before_loop
	async def before_check_feeds(self):
		await self.inititalize_database()
		await self.bot.wait_until_ready()

	@check_feeds.after_loop
	async def after_check_feeds(self):
		self.bot.print("RSS task cancelled")

<|endoftext|>