<commit_before>from __future__ import unicode_literals
import six
import pytz
from copy import copy
from math import ceil
from .utils import comma_join

# TODO
# - and/or between Q objects
# - check that field names are valid
# - operators for arrays: length, has, empty

class Operator(object):
    """
    Base class for filtering operators.
    """

    def to_sql(self, model_cls, field_name, value):
        """
        Subclasses should implement this method. It returns an SQL string
        that applies this operator on the given field and value.
        """
        raise NotImplementedError   # pragma: no cover

class SimpleOperator(Operator):
    """
    A simple binary operator such as a=b, a<b, a>b etc.
    """

    def __init__(self, sql_operator):
        self._sql_operator = sql_operator

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        value = field.to_db_string(field.to_python(value, pytz.utc))
        return ' '.join([field_name, self._sql_operator, value])

class InOperator(Operator):
    """
    An operator that implements IN.
    Accepts 3 different types of values:
    - a list or tuple of simple values
    - a string (used verbatim as the contents of the parenthesis)
    - a queryset (subquery)
    """

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        if isinstance(value, QuerySet):
            value = value.as_sql()
        elif isinstance(value, six.string_types):
            pass
        else:
            value = comma_join([field.to_db_string(field.to_python(v, pytz.utc)) for v in value])
        return '%s IN (%s)' % (field_name, value)

class LikeOperator(Operator):
    """
    A LIKE operator that matches the field to a given pattern. Can be
    case sensitive or insensitive.
    """

    def __init__(self, pattern, case_sensitive=True):
        self._pattern = pattern
        self._case_sensitive = case_sensitive

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        value = field.to_db_string(field.to_python(value, pytz.utc), quote=False)
        value = value.replace('\\', '\\\\').replace('%', '\\\\%').replace('_', '\\\\_')
        pattern = self._pattern.format(value)
        if self._case_sensitive:
            return '%s LIKE \'%s\'' % (field_name, pattern)
        else:
            return 'lowerUTF8(%s) LIKE lowerUTF8(\'%s\')' % (field_name, pattern)

class IExactOperator(Operator):
    """
    An operator for case insensitive string comparison.
    """

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        value = field.to_db_string(field.to_python(value, pytz.utc))
        return 'lowerUTF8(%s) = lowerUTF8(%s)' % (field_name, value)

class NotOperator(Operator):
    """
    A wrapper around another operator, which negates it.
    """

    def __init__(self, base_operator):
        self._base_operator = base_operator

    def to_sql(self, model_cls, field_name, value):
        # Negate the base operator
        return 'NOT (%s)' % self._base_operator.to_sql(model_cls, field_name, value)

# Define the set of builtin operators

_operators = {}

def register_operator(name, sql):
    _operators[name] = sql

register_operator('eq',          SimpleOperator('='))
register_operator('ne',          SimpleOperator('!='))
register_operator('gt',          SimpleOperator('>'))
register_operator('gte',         SimpleOperator('>='))
register_operator('lt',          SimpleOperator('<'))
register_operator('lte',         SimpleOperator('<='))
register_operator('in',          InOperator())
register_operator('not_in',      NotOperator(InOperator()))
register_operator('contains',    LikeOperator('%{}%'))
register_operator('startswith',  LikeOperator('{}%'))
register_operator('endswith',    LikeOperator('%{}'))
register_operator('icontains',   LikeOperator('%{}%', False))
register_operator('istartswith', LikeOperator('{}%', False))
register_operator('iendswith',   LikeOperator('%{}', False))
register_operator('iexact',      IExactOperator())

class FOV(object):
    """
    An object for storing Field + Operator + Value.
    """

    def __init__(self, field_name, operator, value):
        self._field_name = field_name
        self._operator = _operators[operator]
        self._value = value

    def to_sql(self, model_cls):
        return self._operator.to_sql(model_cls, self._field_name, self._value)

class Q(object):

    def __init__(self, **kwargs):
        self._fovs = [self._build_fov(k, v) for k, v in six.iteritems(kwargs)]
        self._negate = False

    def _build_fov(self, key, value):
        if '__' in key:
            field_name, operator = key.rsplit('__', 1)
        else:
            field_name, operator = key, 'eq'
        return FOV(field_name, operator, value)

    def to_sql(self, model_cls):
        if not self._fovs:
            return '1'
        sql = ' AND '.join(fov.to_sql(model_cls) for fov in self._fovs)
        if self._negate:
            sql = 'NOT (%s)' % sql
        return sql

    def __invert__(self):
        q = copy(self)
        q._negate = True
        return q

@six.python_2_unicode_compatible
class QuerySet(object):
    """
    A queryset is an object that represents a database query using a specific `Model`.
    It is lazy, meaning that it does not hit the database until you iterate over its
    matching rows (model instances).
    """

    def __init__(self, model_cls, database):
        """
        Initializer. It is possible to create a queryset like this, but the standard
        way is to use `MyModel.objects_in(database)`.
        """
        self._model_cls = model_cls
        self._database = database
        self._order_by = []
        self._q = []
        self._fields = []
        self._limits = None
        self._distinct = False

    def __iter__(self):
        """
        Iterates over the model instances matching this queryset
        """
        return self._database.select(self.as_sql(), self._model_cls)

    def __bool__(self):
        """
        Returns true if this queryset matches any rows.
        """
        return bool(self.count())

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def __str__(self):
        return self.as_sql()

    def __getitem__(self, s):
        if isinstance(s, six.integer_types):
            # Single index
            assert s >= 0, 'negative indexes are not supported'
            qs = copy(self)
            qs._limits = (s, 1)
            return six.next(iter(qs))
        else:
            # Slice
            assert s.step in (None, 1), 'step is not supported in slices'
            start = s.start or 0
            stop = s.stop or 2**63 - 1
            assert start >= 0 and stop >= 0, 'negative indexes are not supported'
            assert start <= stop, 'start of slice cannot be smaller than its end'
            qs = copy(self)
            qs._limits = (start, stop - start)
            return qs

    def as_sql(self):
        """
        Returns the whole query as a SQL string.
        """
        distinct = 'DISTINCT ' if self._distinct else ''
        fields = '*'
        if self._fields:
            fields = comma_join('`%s`' % field for field in self._fields)
        ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
        limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
        params = (distinct, fields, self._model_cls.table_name(),
                  self.conditions_as_sql(), ordering, limit)
        return u'SELECT %s%s\nFROM `%s`\nWHERE %s%s%s' % params

    def order_by_as_sql(self):
        """
        Returns the contents of the query's `ORDER BY` clause as a string.
        """
        return comma_join([
            '%s DESC' % field[1:] if field[0] == '-' else field
            for field in self._order_by
        ])

    def conditions_as_sql(self):
        """
        Returns the contents of the query's `WHERE` clause as a string.
        """
        if self._q:
            return u' AND '.join([q.to_sql(self._model_cls) for q in self._q])
        else:
            return u'1'

    def count(self):
        """
        Returns the number of matching model instances.
        """
        if self._distinct:
            # Use a subquery, since a simple count won't be accurate
            sql = u'SELECT count() FROM (%s)' % self.as_sql()
            raw = self._database.raw(sql)
            return int(raw) if raw else 0
        return self._database.count(self._model_cls, self.conditions_as_sql())

    def order_by(self, *field_names):
        """
        Returns a copy of this queryset with the ordering changed.
        """
        qs = copy(self)
        qs._order_by = field_names
        return qs

    def only(self, *field_names):
        """
        Returns a copy of this queryset limited to the specified field names.
        Useful when there are large fields that are not needed,
        or for creating a subquery to use with an IN operator.
        """
        qs = copy(self)
        qs._fields = field_names
        return qs

    def filter(self, **kwargs):
        """
        Returns a copy of this queryset that includes only rows matching the conditions.
        """
        qs = copy(self)
        qs._q = list(self._q) + [Q(**kwargs)]
        return qs

    def exclude(self, **kwargs):
        """
        Returns a copy of this queryset that excludes all rows matching the conditions.
        """
        qs = copy(self)
        qs._q = list(self._q) + [~Q(**kwargs)]
        return qs

    def paginate(self, page_num=1, page_size=100):
        """
        Returns a single page of model instances that match the queryset.
        Note that `order_by` should be used first, to ensure a correct
        partitioning of records into pages.

        - `page_num`: the page number (1-based), or -1 to get the last page.
        - `page_size`: number of records to return per page.

        The result is a namedtuple containing `objects` (list), `number_of_objects`,
        `pages_total`, `number` (of the current page), and `page_size`.
        """
        from .database import Page
        count = self.count()
        pages_total = int(ceil(count / float(page_size)))
        if page_num == -1:
            page_num = pages_total
        elif page_num < 1:
            raise ValueError('Invalid page number: %d' % page_num)
        offset = (page_num - 1) * page_size
        return Page(
            objects=list(self[offset : offset + page_size]),
            number_of_objects=count,
            pages_total=pages_total,
            number=page_num,
            page_size=page_size
        )

    def distinct(self):
        """
        Adds a DISTINCT clause to the query, meaning that any duplicate rows
        in the results will be omitted.
        """
        qs = copy(self)
        qs._distinct = True
        return qs

    def aggregate(self, *args, **kwargs):
        """
        Returns an `AggregateQuerySet` over this query, with `args` serving as
        grouping fields and `kwargs` serving as calculated fields. At least one
        calculated field is required. For example:
        ```
            Event.objects_in(database).filter(date__gt='2017-08-01').aggregate('event_type', count='count()')
        ```
        is equivalent to:
        ```
            SELECT event_type, count() AS count FROM event
            WHERE data > '2017-08-01'
            GROUP BY event_type
        ```
        """
        return AggregateQuerySet(self, args, kwargs)

class AggregateQuerySet(QuerySet):
    """
    A queryset used for aggregation.
    """

    def __init__(self, base_qs, grouping_fields, calculated_fields):
        """
        Initializer. Normally you should not call this but rather use `QuerySet.aggregate()`.

        The grouping fields should be a list/tuple of field names from the model. For example:
        ```
            ('event_type', 'event_subtype')
        ```
        The calculated fields should be a mapping from name to a ClickHouse aggregation function. For example:
        ```
            {'weekday': 'toDayOfWeek(event_date)', 'number_of_events': 'count()'}
        ```
        At least one calculated field is required.
        """
        super(AggregateQuerySet, self).__init__(base_qs._model_cls, base_qs._database)
        assert calculated_fields, 'No calculated fields specified for aggregation'
        self._fields = grouping_fields
        self._grouping_fields = grouping_fields
        self._calculated_fields = calculated_fields
        self._order_by = list(base_qs._order_by)
        self._q = list(base_qs._q)
        self._limits = base_qs._limits
        self._distinct = base_qs._distinct

    def group_by(self, *args):
        """
        This method lets you specify the grouping fields explicitly. The `args` must
        be names of grouping fields or calculated fields that this queryset was
        created with.
        """
        for name in args:
            assert name in self._fields or name in self._calculated_fields, \
                   'Cannot group by `%s` since it is not included in the query' % name
        qs = copy(self)
        qs._grouping_fields = args
        return qs

    def only(self, *field_names):
        """
        This method is not supported on `AggregateQuerySet`.
        """
        raise NotImplementedError('Cannot use "only" with AggregateQuerySet')

    def aggregate(self, *args, **kwargs):
        """
        This method is not supported on `AggregateQuerySet`.
        """
        raise NotImplementedError('Cannot re-aggregate an AggregateQuerySet')

    def as_sql(self):
        """
        Returns the whole query as a SQL string.
        """
        distinct = 'DISTINCT ' if self._distinct else ''
        grouping = comma_join('`%s`' % field for field in self._grouping_fields)
        fields = comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
        params = dict(
            distinct=distinct,
            grouping=grouping or "''",
            fields=fields,
            table=self._model_cls.table_name(),
            conds=self.conditions_as_sql()
        )
        sql = u'SELECT %(distinct)s%(fields)s\nFROM `%(table)s`\nWHERE %(conds)s\nGROUP BY %(grouping)s' % params
        if self._order_by:
            sql += '\nORDER BY ' + self.order_by_as_sql()
        if self._limits:
            sql += '\nLIMIT %d, %d' % self._limits
        return sql

    def __iter__(self):
        return self._database.select(self.as_sql()) # using an ad-hoc model

    def count(self):
        """
        Returns the number of rows after aggregation.
        """
        sql = u'SELECT count() FROM (%s)' % self.as_sql()
        raw = self._database.raw(sql)
        return int(raw) if raw else 0
<commit_msg>Add OR and AND operations for Q objects<commit_after>from __future__ import unicode_literals
import six
import pytz
from copy import copy
from math import ceil
from .utils import comma_join

# TODO
# - and/or between Q objects
# - check that field names are valid
# - operators for arrays: length, has, empty

class Operator(object):
    """
    Base class for filtering operators.
    """

    def to_sql(self, model_cls, field_name, value):
        """
        Subclasses should implement this method. It returns an SQL string
        that applies this operator on the given field and value.
        """
        raise NotImplementedError   # pragma: no cover

class SimpleOperator(Operator):
    """
    A simple binary operator such as a=b, a<b, a>b etc.
    """

    def __init__(self, sql_operator):
        self._sql_operator = sql_operator

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        value = field.to_db_string(field.to_python(value, pytz.utc))
        return ' '.join([field_name, self._sql_operator, value])

class InOperator(Operator):
    """
    An operator that implements IN.
    Accepts 3 different types of values:
    - a list or tuple of simple values
    - a string (used verbatim as the contents of the parenthesis)
    - a queryset (subquery)
    """

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        if isinstance(value, QuerySet):
            value = value.as_sql()
        elif isinstance(value, six.string_types):
            pass
        else:
            value = comma_join([field.to_db_string(field.to_python(v, pytz.utc)) for v in value])
        return '%s IN (%s)' % (field_name, value)

class LikeOperator(Operator):
    """
    A LIKE operator that matches the field to a given pattern. Can be
    case sensitive or insensitive.
    """

    def __init__(self, pattern, case_sensitive=True):
        self._pattern = pattern
        self._case_sensitive = case_sensitive

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        value = field.to_db_string(field.to_python(value, pytz.utc), quote=False)
        value = value.replace('\\', '\\\\').replace('%', '\\\\%').replace('_', '\\\\_')
        pattern = self._pattern.format(value)
        if self._case_sensitive:
            return '%s LIKE \'%s\'' % (field_name, pattern)
        else:
            return 'lowerUTF8(%s) LIKE lowerUTF8(\'%s\')' % (field_name, pattern)

class IExactOperator(Operator):
    """
    An operator for case insensitive string comparison.
    """

    def to_sql(self, model_cls, field_name, value):
        field = getattr(model_cls, field_name)
        value = field.to_db_string(field.to_python(value, pytz.utc))
        return 'lowerUTF8(%s) = lowerUTF8(%s)' % (field_name, value)

class NotOperator(Operator):
    """
    A wrapper around another operator, which negates it.
    """

    def __init__(self, base_operator):
        self._base_operator = base_operator

    def to_sql(self, model_cls, field_name, value):
        # Negate the base operator
        return 'NOT (%s)' % self._base_operator.to_sql(model_cls, field_name, value)

# Define the set of builtin operators

_operators = {}

def register_operator(name, sql):
    _operators[name] = sql

register_operator('eq',          SimpleOperator('='))
register_operator('ne',          SimpleOperator('!='))
register_operator('gt',          SimpleOperator('>'))
register_operator('gte',         SimpleOperator('>='))
register_operator('lt',          SimpleOperator('<'))
register_operator('lte',         SimpleOperator('<='))
register_operator('in',          InOperator())
register_operator('not_in',      NotOperator(InOperator()))
register_operator('contains',    LikeOperator('%{}%'))
register_operator('startswith',  LikeOperator('{}%'))
register_operator('endswith',    LikeOperator('%{}'))
register_operator('icontains',   LikeOperator('%{}%', False))
register_operator('istartswith', LikeOperator('{}%', False))
register_operator('iendswith',   LikeOperator('%{}', False))
register_operator('iexact',      IExactOperator())

class FOV(object):
    """
    An object for storing Field + Operator + Value.
    """

    def __init__(self, field_name, operator, value):
        self._field_name = field_name
        self._operator = _operators[operator]
        self._value = value

    def to_sql(self, model_cls):
        return self._operator.to_sql(model_cls, self._field_name, self._value)

class Q(object):

    AND_MODE = 'AND'
    OR_MODE = 'OR'

    def __init__(self, **filter_fields):
        self._fovs = [self._build_fov(k, v) for k, v in six.iteritems(filter_fields)]
        self._l_child = None
        self._r_child = None
        self._negate = False
        self._mode = self.AND_MODE

    @classmethod
    def _construct_from(cls, l_child, r_child, mode):
        q = Q()
        q._l_child = l_child
        q._r_child = r_child
        q._mode = mode
        return q

    def _build_fov(self, key, value):
        if '__' in key:
            field_name, operator = key.rsplit('__', 1)
        else:
            field_name, operator = key, 'eq'
        return FOV(field_name, operator, value)

    def to_sql(self, model_cls):
        if self._fovs:
            sql = ' {} '.format(self._mode).join(fov.to_sql(model_cls) for fov in self._fovs)
        else:
            if self._l_child and self._r_child:
                sql = '({}) {} ({})'.format(self._l_child.to_sql(model_cls), self._mode, self._r_child.to_sql(model_cls))
            else:
                return '1'
        if self._negate:
            sql = 'NOT (%s)' % sql
        return sql

    def __or__(self, other):
        return Q._construct_from(self, other, self.OR_MODE)

    def __and__(self, other):
        return Q._construct_from(self, other, self.AND_MODE)

    def __invert__(self):
        q = copy(self)
        q._negate = True
        return q

@six.python_2_unicode_compatible
class QuerySet(object):
    """
    A queryset is an object that represents a database query using a specific `Model`.
    It is lazy, meaning that it does not hit the database until you iterate over its
    matching rows (model instances).
    """

    def __init__(self, model_cls, database):
        """
        Initializer. It is possible to create a queryset like this, but the standard
        way is to use `MyModel.objects_in(database)`.
        """
        self._model_cls = model_cls
        self._database = database
        self._order_by = []
        self._q = []
        self._fields = []
        self._limits = None
        self._distinct = False

    def __iter__(self):
        """
        Iterates over the model instances matching this queryset
        """
        return self._database.select(self.as_sql(), self._model_cls)

    def __bool__(self):
        """
        Returns true if this queryset matches any rows.
        """
        return bool(self.count())

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def __str__(self):
        return self.as_sql()

    def __getitem__(self, s):
        if isinstance(s, six.integer_types):
            # Single index
            assert s >= 0, 'negative indexes are not supported'
            qs = copy(self)
            qs._limits = (s, 1)
            return six.next(iter(qs))
        else:
            # Slice
            assert s.step in (None, 1), 'step is not supported in slices'
            start = s.start or 0
            stop = s.stop or 2**63 - 1
            assert start >= 0 and stop >= 0, 'negative indexes are not supported'
            assert start <= stop, 'start of slice cannot be smaller than its end'
            qs = copy(self)
            qs._limits = (start, stop - start)
            return qs

    def as_sql(self):
        """
        Returns the whole query as a SQL string.
        """
        distinct = 'DISTINCT ' if self._distinct else ''
        fields = '*'
        if self._fields:
            fields = comma_join('`%s`' % field for field in self._fields)
        ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
        limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
        params = (distinct, fields, self._model_cls.table_name(),
                  self.conditions_as_sql(), ordering, limit)
        return u'SELECT %s%s\nFROM `%s`\nWHERE %s%s%s' % params

    def order_by_as_sql(self):
        """
        Returns the contents of the query's `ORDER BY` clause as a string.
        """
        return comma_join([
            '%s DESC' % field[1:] if field[0] == '-' else field
            for field in self._order_by
        ])

    def conditions_as_sql(self):
        """
        Returns the contents of the query's `WHERE` clause as a string.
        """
        if self._q:
            return u' AND '.join([q.to_sql(self._model_cls) for q in self._q])
        else:
            return u'1'

    def count(self):
        """
        Returns the number of matching model instances.
        """
        if self._distinct:
            # Use a subquery, since a simple count won't be accurate
            sql = u'SELECT count() FROM (%s)' % self.as_sql()
            raw = self._database.raw(sql)
            return int(raw) if raw else 0
        return self._database.count(self._model_cls, self.conditions_as_sql())

    def order_by(self, *field_names):
        """
        Returns a copy of this queryset with the ordering changed.
        """
        qs = copy(self)
        qs._order_by = field_names
        return qs

    def only(self, *field_names):
        """
        Returns a copy of this queryset limited to the specified field names.
        Useful when there are large fields that are not needed,
        or for creating a subquery to use with an IN operator.
        """
        qs = copy(self)
        qs._fields = field_names
        return qs

    def filter(self, *q, **filter_fields):
        """
        Returns a copy of this queryset that includes only rows matching the conditions.
        Add q object to query if it specified.
        """
        qs = copy(self)
        if q:
            qs._q = list(self._q) + list(q)
        else:
            qs._q = list(self._q) + [Q(**filter_fields)]
        return qs

    def exclude(self, **filter_fields):
        """
        Returns a copy of this queryset that excludes all rows matching the conditions.
        """
        qs = copy(self)
        qs._q = list(self._q) + [~Q(**filter_fields)]
        return qs

    def paginate(self, page_num=1, page_size=100):
        """
        Returns a single page of model instances that match the queryset.
        Note that `order_by` should be used first, to ensure a correct
        partitioning of records into pages.

        - `page_num`: the page number (1-based), or -1 to get the last page.
        - `page_size`: number of records to return per page.

        The result is a namedtuple containing `objects` (list), `number_of_objects`,
        `pages_total`, `number` (of the current page), and `page_size`.
        """
        from .database import Page
        count = self.count()
        pages_total = int(ceil(count / float(page_size)))
        if page_num == -1:
            page_num = pages_total
        elif page_num < 1:
            raise ValueError('Invalid page number: %d' % page_num)
        offset = (page_num - 1) * page_size
        return Page(
            objects=list(self[offset : offset + page_size]),
            number_of_objects=count,
            pages_total=pages_total,
            number=page_num,
            page_size=page_size
        )

    def distinct(self):
        """
        Adds a DISTINCT clause to the query, meaning that any duplicate rows
        in the results will be omitted.
        """
        qs = copy(self)
        qs._distinct = True
        return qs

    def aggregate(self, *args, **kwargs):
        """
        Returns an `AggregateQuerySet` over this query, with `args` serving as
        grouping fields and `kwargs` serving as calculated fields. At least one
        calculated field is required. For example:
        ```
            Event.objects_in(database).filter(date__gt='2017-08-01').aggregate('event_type', count='count()')
        ```
        is equivalent to:
        ```
            SELECT event_type, count() AS count FROM event
            WHERE data > '2017-08-01'
            GROUP BY event_type
        ```
        """
        return AggregateQuerySet(self, args, kwargs)

class AggregateQuerySet(QuerySet):
    """
    A queryset used for aggregation.
    """

    def __init__(self, base_qs, grouping_fields, calculated_fields):
        """
        Initializer. Normally you should not call this but rather use `QuerySet.aggregate()`.

        The grouping fields should be a list/tuple of field names from the model. For example:
        ```
            ('event_type', 'event_subtype')
        ```
        The calculated fields should be a mapping from name to a ClickHouse aggregation function. For example:
        ```
            {'weekday': 'toDayOfWeek(event_date)', 'number_of_events': 'count()'}
        ```
        At least one calculated field is required.
        """
        super(AggregateQuerySet, self).__init__(base_qs._model_cls, base_qs._database)
        assert calculated_fields, 'No calculated fields specified for aggregation'
        self._fields = grouping_fields
        self._grouping_fields = grouping_fields
        self._calculated_fields = calculated_fields
        self._order_by = list(base_qs._order_by)
        self._q = list(base_qs._q)
        self._limits = base_qs._limits
        self._distinct = base_qs._distinct

    def group_by(self, *args):
        """
        This method lets you specify the grouping fields explicitly. The `args` must
        be names of grouping fields or calculated fields that this queryset was
        created with.
        """
        for name in args:
            assert name in self._fields or name in self._calculated_fields, \
                   'Cannot group by `%s` since it is not included in the query' % name
        qs = copy(self)
        qs._grouping_fields = args
        return qs

    def only(self, *field_names):
        """
        This method is not supported on `AggregateQuerySet`.
        """
        raise NotImplementedError('Cannot use "only" with AggregateQuerySet')

    def aggregate(self, *args, **kwargs):
        """
        This method is not supported on `AggregateQuerySet`.
        """
        raise NotImplementedError('Cannot re-aggregate an AggregateQuerySet')

    def as_sql(self):
        """
        Returns the whole query as a SQL string.
        """
        distinct = 'DISTINCT ' if self._distinct else ''
        grouping = comma_join('`%s`' % field for field in self._grouping_fields)
        fields = comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
        params = dict(
            distinct=distinct,
            grouping=grouping or "''",
            fields=fields,
            table=self._model_cls.table_name(),
            conds=self.conditions_as_sql()
        )
        sql = u'SELECT %(distinct)s%(fields)s\nFROM `%(table)s`\nWHERE %(conds)s\nGROUP BY %(grouping)s' % params
        if self._order_by:
            sql += '\nORDER BY ' + self.order_by_as_sql()
        if self._limits:
            sql += '\nLIMIT %d, %d' % self._limits
        return sql

    def __iter__(self):
        return self._database.select(self.as_sql()) # using an ad-hoc model

    def count(self):
        """
        Returns the number of rows after aggregation.
        """
        sql = u'SELECT count() FROM (%s)' % self.as_sql()
        raw = self._database.raw(sql)
        return int(raw) if raw else 0
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2012-2014 CoNWeT Lab., Universidad Politécnica de Madrid

# This file is part of Wirecloud.

# Wirecloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# Wirecloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with Wirecloud.  If not, see <http://www.gnu.org/licenses/>.

__version_info__ = (2, 3, 1)
__version__ = '.'.join(map(str, __version_info__))

DEFAULT_FIWARE_PORTALS = (
    {'name': 'Cloud', 'url': 'http://cloud.lab.fi-ware.org', "logout_path": "/logout"},
    {'name': 'Store', 'url': 'https://store.lab.fi-ware.org', "logout_path": "/logout"},
    {'name': 'Mashup', 'url': 'https://mashup.lab.fi-ware.org', "logout_path": "/logout", "active": True},
    {'name': 'Account', 'url': 'https://account.lab.fi-ware.org', "logout_path": "/users/sign_out"},
    {'name': 'Help&Info', 'url': 'https://lab.fi-ware.org'},
)

DEFAULT_FIWARE_HOME = 'https://lab.fi-ware.org';
<commit_msg>Bump fiware version<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2012-2014 CoNWeT Lab., Universidad Politécnica de Madrid

# This file is part of Wirecloud.

# Wirecloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# Wirecloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with Wirecloud.  If not, see <http://www.gnu.org/licenses/>.

__version_info__ = (3, 2)
__version__ = '.'.join(map(str, __version_info__))

DEFAULT_FIWARE_PORTALS = (
    {'name': 'Cloud', 'url': 'http://cloud.lab.fi-ware.org', "logout_path": "/logout"},
    {'name': 'Store', 'url': 'https://store.lab.fi-ware.org', "logout_path": "/logout"},
    {'name': 'Mashup', 'url': 'https://mashup.lab.fi-ware.org', "logout_path": "/logout", "active": True},
    {'name': 'Account', 'url': 'https://account.lab.fi-ware.org', "logout_path": "/users/sign_out"},
    {'name': 'Help&Info', 'url': 'https://lab.fi-ware.org'},
)

DEFAULT_FIWARE_HOME = 'https://lab.fi-ware.org';
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- python -*-
#
# Git Version: @git@

from __future__ import print_function
from util       import capture, which, config_logger

import os, re, sys, subprocess, time, socket, json, argparse, platform

logger = config_logger()

def syshost():
  hostA = platform.node().split('.')
  idx = 1
  if (len(hostA) < 2):
    idx = 0
  return hostA[idx]

class CmdLineOptions(object):
  def __init__(self):
    pass

  def execute(self):
    parser = argparse.ArgumentParser()
    parser.add_argument("--start",   dest='startTime', action="store", type=float, default="0.0", help="start time")
    parser.add_argument("--end",     dest='endTime',   action="store", type=float, default="0.0", help="end time")
    parser.add_argument("--fn",      dest='resultFn',  action="store", default = "/dev/null",     help="resultFn")
    parser.add_argument("--ntasks",  dest='ntasks',    action="store", default = "-1",            help="number of mpi tasks")
    parser.add_argument("--syshost", dest='syshost',   action="store", default = syshost(),       help="system host name")
    parser.add_argument("--run_uuid",dest='run_uuid',  action="store", default = None,            help="run uuid")
    parser.add_argument("exec_prog", nargs='+',        help="user program")

    args = parser.parse_args()

    return args

keyPat = re.compile(r'.*<(.*)>.*')

class ExtractXALT(object):

  def __init__(self, cmd):
    self.__fieldT = self.__build_fieldT(cmd)

  def xaltRecordT(self):
    return self.__fieldT

  def __build_fieldT(self, cmd):
    outStr  = capture(["objdump", "-s", "-j", ".xalt", cmd])

    fieldT = {}
    if (not outStr.find("Contents of section .xalt:") != -1):
      return fieldT

    outputA = outStr.split('\n')
    outputA.pop(0)
    outputA.pop(0)
    outputA.pop(0)
    outputA.pop(0)

    sA = []
    for line in outputA:
      split = line.split()
      if (len(split) > 0):
        sA.append(split[-1])
    s = "".join(sA)

    xaltA   = re.split('%%', s)

    fieldT = {}

    N = len(xaltA)
    idx = -1
    while (True):
      idx  = idx + 1
      line = xaltA[idx]
      if (line.find("XALT_Link_Info_End") != -1):
        break
      m = keyPat.search(line)
      if (m):
        key   = m.group(1)
        idx   = idx + 1
        value = xaltA[idx].replace("_%_%_"," ")
        fieldT[key] = value

    return fieldT

class UserEnvT(object):
  def __init__(self, args, userExec):

    ltime                 = time.time()
    userT                 = {}
    userT['cwd']          = os.getcwd()
    userT['syshost']      = args.syshost
    userT['run_uuid']     = args.run_uuid
    userT['num_threads']  = int(os.environ.get("OMP_NUM_THREADS","0"))
    userT['user']         = os.environ.get("USER","unknown")
    userT['num_tasks']    = args.ntasks
    userT['start_date']   = time.strftime("%c",time.localtime(args.startTime))
    userT['start_time']   = args.startTime
    userT['currentEpoch'] = ltime
    userT['end_time']     = args.endTime
    userT['run_time']     = max(0, args.endTime - args.startTime)
    userT['exec_path']    = userExec.execName()
    userT['exec_type']    = userExec.execType()
    userT['exec_epoch']   = userExec.execEpoch()
    userT['execModify']   = userExec.execModify()

    self.__userT = userT

  def userT(self):
    return self.__userT

class UserExec(object):

  def __init__(self, exec_progA):

    ignoreT = {
      'env'              : True,
      'time'             : True,
    }
    cmd = None
    for prog in exec_progA:
      bare = os.path.basename(prog)
      if (not (bare in ignoreT)):
        cmd = prog
        break

    self.__execType = None
    self.__execName = which(cmd)
    ldd = None
    if (self.__execName):
      outStr = capture(["file", self.__execName])
      if (outStr.find("script") > 0 or outStr.find("text") > 0):
        self.__execType = "script"
      else:
        self.__execType = "binary"
        ldd             = capture(["ldd", self.__execName])

      info = os.stat(self.__execName)
      self.__modify = info.st_mtime
      self.__libA   = self.__parseLDD(ldd)
      self.__hash   = self.__computeHash(self.__execName)

  def execName(self):
    return self.__execName

  def execType(self):
    return self.__execType

  def execEpoch(self):
    return self.__modify

  def execModify(self):
    return time.strftime("%c",time.localtime(self.__modify))

  def libA(self):
    return self.__libA

  def hash(self):
    return self.__hash

  def __computeHash(self, cmd):
    fieldA = capture(["sha1sum", cmd]).split()
    return fieldA[0]

  def __parseLDD(self,ldd):
    if (not ldd or ldd.find("not a dynamic executable") > 0):
      return []

    lineA = ldd.split('\n')

    libA = []
    d    = {}
    for line in lineA:
      fieldA = line.split()
      N      = len(fieldA)

      if (N < 1):
        break
      elif (N == 4):
        lib = fieldA[2]
      else:
        lib = fieldA[0]
      lib = os.path.realpath(lib)
      d[lib] = True

    libA = d.keys()
    libA = sorted(libA)

    libB = []
    for lib in libA:
      hash_line = capture(['sha1sum', lib])
      if (hash_line.find("No such file or directory") != -1):
        v = "unknown"
      else:
        v = hash_line.split()[0]

      libB.append([lib, v])

    return libB

class EnvT(object):
  def __init__(self):
    self.__envT = self.__reportUserEnv()
  def envT(self):
    return self.__envT
  def __reportUserEnv(self):
    ignoreKeyA = [
      re.compile(r'^LESS_TERMCAP_.*$'),
      re.compile(r'^LS_COLORS$'),
      re.compile(r'^SSH_.*$'),
      re.compile(r'^XDG_.*$'),
      re.compile(r'^PS1$'),
    ]

    envT = {}
    for k in os.environ:
      keep = True
      for pat in ignoreKeyA:
        m = pat.search(k)
        if (m):
          keep = False
          break
      if (keep):
        envT[k] = os.environ[k]
    return envT

def main():

  try:
    # parse command line options:
    args = CmdLineOptions().execute()

    userExec = UserExec(args.exec_prog)
    if (not userExec.execName()):
      return

    userT    = UserEnvT(args, userExec).userT()

    submitT              = {}
    submitT['userT']     = userT
    submitT['xaltLinkT'] = ExtractXALT(userExec.execName()).xaltRecordT()
    submitT['libA']      = userExec.libA()
    submitT['envT']      = EnvT().envT()
    submitT['hash_id']   = userExec.hash()

    dirname,fn = os.path.split(os.path.abspath(args.resultFn))
    tmpFn      = os.path.join(dirname, "." + fn)

    if (not os.path.isdir(dirname)):
      os.mkdir(dirname);

    s = json.dumps(submitT, sort_keys=True, indent=2, separators=(',',': '))

    f = open(tmpFn,'w')
    f.write(s)
    f.close()
    os.rename(tmpFn, args.resultFn)
  except:
    logger.exception("XALT_EXCEPTION:xalt_run_submission.py")

if ( __name__ == '__main__'): main()
<commit_msg>better handling of scripts vs. binaries<commit_after>#!/usr/bin/env python
# -*- python -*-
#
# Git Version: @git@

from __future__ import print_function
from util       import capture, which, config_logger

import os, re, sys, subprocess, time, socket, json, argparse, platform

logger = config_logger()

def syshost():
  hostA = platform.node().split('.')
  idx = 1
  if (len(hostA) < 2):
    idx = 0
  return hostA[idx]

class CmdLineOptions(object):
  def __init__(self):
    pass

  def execute(self):
    parser = argparse.ArgumentParser()
    parser.add_argument("--start",   dest='startTime', action="store", type=float, default="0.0", help="start time")
    parser.add_argument("--end",     dest='endTime',   action="store", type=float, default="0.0", help="end time")
    parser.add_argument("--fn",      dest='resultFn',  action="store", default = "/dev/null",     help="resultFn")
    parser.add_argument("--ntasks",  dest='ntasks',    action="store", default = "-1",            help="number of mpi tasks")
    parser.add_argument("--syshost", dest='syshost',   action="store", default = syshost(),       help="system host name")
    parser.add_argument("--run_uuid",dest='run_uuid',  action="store", default = None,            help="run uuid")
    parser.add_argument("exec_prog", nargs='+',        help="user program")

    args = parser.parse_args()

    return args

keyPat = re.compile(r'.*<(.*)>.*')

class ExtractXALT(object):

  def __init__(self, cmd):
    self.__fieldT = self.__build_fieldT(cmd)

  def xaltRecordT(self):
    return self.__fieldT

  def __build_fieldT(self, cmd):
    outStr  = capture(["objdump", "-s", "-j", ".xalt", cmd])

    fieldT = {}
    if (not outStr.find("Contents of section .xalt:") != -1):
      return fieldT

    outputA = outStr.split('\n')
    outputA.pop(0)
    outputA.pop(0)
    outputA.pop(0)
    outputA.pop(0)

    sA = []
    for line in outputA:
      split = line.split()
      if (len(split) > 0):
        sA.append(split[-1])
    s = "".join(sA)

    xaltA   = re.split('%%', s)

    fieldT = {}

    N = len(xaltA)
    idx = -1
    while (True):
      idx  = idx + 1
      line = xaltA[idx]
      if (line.find("XALT_Link_Info_End") != -1):
        break
      m = keyPat.search(line)
      if (m):
        key   = m.group(1)
        idx   = idx + 1
        value = xaltA[idx].replace("_%_%_"," ")
        fieldT[key] = value

    return fieldT

class UserEnvT(object):
  def __init__(self, args, userExec):

    ltime                 = time.time()
    userT                 = {}
    userT['cwd']          = os.getcwd()
    userT['syshost']      = args.syshost
    userT['run_uuid']     = args.run_uuid
    userT['num_threads']  = int(os.environ.get("OMP_NUM_THREADS","0"))
    userT['user']         = os.environ.get("USER","unknown")
    userT['num_tasks']    = args.ntasks
    userT['start_date']   = time.strftime("%c",time.localtime(args.startTime))
    userT['start_time']   = args.startTime
    userT['currentEpoch'] = ltime
    userT['end_time']     = args.endTime
    userT['run_time']     = max(0, args.endTime - args.startTime)
    userT['exec_path']    = userExec.execName()
    userT['exec_type']    = userExec.execType()
    userT['exec_epoch']   = userExec.execEpoch()
    userT['execModify']   = userExec.execModify()

    self.__userT = userT

  def userT(self):
    return self.__userT

class UserExec(object):

  def __init__(self, exec_progA):

    ignoreT = {
      'env'              : True,
      'time'             : True,
    }
    cmd = None
    for prog in exec_progA:
      bare = os.path.basename(prog)
      if (not (bare in ignoreT)):
        cmd = prog
        break

    self.__execType = None
    self.__execName = which(cmd)
    self.__libA     = []
    if (self.__execName):
      outStr = capture(["file", self.__execName])
      if (outStr.find("script") > 0 or outStr.find("text") > 0):
        self.__execType = "script"
      else:
        self.__execType = "binary"
        ldd             = capture(["ldd", self.__execName])
        self.__libA     = self.__parseLDD(ldd)

      info = os.stat(self.__execName)
      self.__modify = info.st_mtime
      self.__hash   = self.__computeHash(self.__execName)

  def execName(self):
    return self.__execName

  def execType(self):
    return self.__execType

  def execEpoch(self):
    return self.__modify

  def execModify(self):
    return time.strftime("%c",time.localtime(self.__modify))

  def libA(self):
    return self.__libA

  def hash(self):
    return self.__hash

  def __computeHash(self, cmd):
    fieldA = capture(["sha1sum", cmd]).split()
    return fieldA[0]

  def __parseLDD(self,ldd):
    if (not ldd or ldd.find("not a dynamic executable") > 0):
      return []

    lineA = ldd.split('\n')

    libA = []
    d    = {}
    for line in lineA:
      fieldA = line.split()
      N      = len(fieldA)

      if (N < 1):
        break
      elif (N == 4):
        lib = fieldA[2]
      else:
        lib = fieldA[0]
      lib = os.path.realpath(lib)
      d[lib] = True

    libA = d.keys()
    libA = sorted(libA)

    libB = []
    for lib in libA:
      hash_line = capture(['sha1sum', lib])
      if (hash_line.find("No such file or directory") != -1):
        v = "unknown"
      else:
        v = hash_line.split()[0]

      libB.append([lib, v])

    return libB

class EnvT(object):
  def __init__(self):
    self.__envT = self.__reportUserEnv()
  def envT(self):
    return self.__envT
  def __reportUserEnv(self):
    ignoreKeyA = [
      re.compile(r'^LESS_TERMCAP_.*$'),
      re.compile(r'^LS_COLORS$'),
      re.compile(r'^SSH_.*$'),
      re.compile(r'^XDG_.*$'),
      re.compile(r'^PS1$'),
    ]

    envT = {}
    for k in os.environ:
      keep = True
      for pat in ignoreKeyA:
        m = pat.search(k)
        if (m):
          keep = False
          break
      if (keep):
        envT[k] = os.environ[k]
    return envT

def main():

  try:
    # parse command line options:
    args = CmdLineOptions().execute()

    userExec = UserExec(args.exec_prog)
    if (not userExec.execName()):
      return

    userT    = UserEnvT(args, userExec).userT()

    submitT              = {}
    submitT['userT']     = userT
    submitT['xaltLinkT'] = ExtractXALT(userExec.execName()).xaltRecordT()
    submitT['libA']      = userExec.libA()
    submitT['envT']      = EnvT().envT()
    submitT['hash_id']   = userExec.hash()

    dirname,fn = os.path.split(os.path.abspath(args.resultFn))
    tmpFn      = os.path.join(dirname, "." + fn)

    if (not os.path.isdir(dirname)):
      os.mkdir(dirname);

    s = json.dumps(submitT, sort_keys=True, indent=2, separators=(',',': '))

    f = open(tmpFn,'w')
    f.write(s)
    f.close()
    os.rename(tmpFn, args.resultFn)
  except:
    logger.exception("XALT_EXCEPTION:xalt_run_submission.py")

if ( __name__ == '__main__'): main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import hexchat
import re

__module_name__ = "DeadKeyFix"
__module_version__ = "2.1"
__module_description__ = "Fixes the Us-International deadkey issue"

prev = ''

def keypress_cb(word, word_eol, userdata):
	global prev
	specialChars = {
		'65104': {
			'a': u'à',
			'o': u'ò',
			'e': u'è',
			'i': u'ì',
			'u': u'ù',

			'A': u'À',
			'O': u'Ò',
			'E': u'È',
			'I': u'Ì',
			'U': u'Ù'
		},
		'65105': {
			'a': u'á',
			'o': u'ó',
			'e': u'é',
			'i': u'í',
			'u': u'ú',
			'y': u'ý',
			'c': u'ç',

			'A': u'Á',
			'O': u'Ó',
			'E': u'É',
			'I': u'Í',
			'U': u'Ú',
			'Y': u'Ý',
			'C': u'Ç'
		},
		'65106': {
			'a': u'â',
			'o': u'ô',
			'e': u'ê',
			'i': u'î',
			'u': u'û',

			'A': u'Â',
			'O': u'Ô',
			'E': u'Ê',
			'I': u'Î',
			'U': u'Û'
		},
		'65107': {
			'a': u'ã',
			'o': u'õ',
			'n': u'ñ',

			'A': u'Ã',
			'O': u'Õ',
			'N': u'Ñ'
		},
		'65111': {
			'a': u'ä',
			'o': u'ö',
			'e': u'ë',
			'i': u'ï',
			'u': u'ü',
			'y': u'ÿ',

			'A': u'Ä',
			'O': u'Ö',
			'E': u'Ë',
			'I': u'Ï',
			'U': u'Ü',
			'Y': u'Ÿ'
		}
	}
	accents = {
		'65104': '`',
		'65105': "'",
		'65106': '^',
		'65107': '~',
		'65111': '"'
	}

	charset =  hexchat.get_info('charset')
	#When there is no current charset derived from server or channel it is set to IRC
	#IRC is not a recognized encoding type so default to utf-8 in that case.
	if(charset == "IRC"):
		charset = "utf-8"

	textraw = hexchat.get_info('inputbox')
	text = unicode(textraw, charset)
	loc = hexchat.get_prefs("state_cursor")

	if prev in accents and word[2] in specialChars[prev]:
		#insert an accented character
		text = insert(specialChars[prev][word[2]],text,loc)
	elif prev in accents and word[2] == ' ':
		#insert a clean accent ( input was accent followed by space )
		text = insert(accents[prev],text,loc)
	elif prev in accents and word[0] in accents:
		#Insert two accents ( input was accent followed by accent )
		text = insert(accents[prev] + accents[word[0]],text,loc)
		loc+=1
	elif prev in accents and int(word[3]) != 0:
		#insert an accent and a character (character and accent do not combine)
		text = insert(accents[prev] + word[2],text,loc)
		loc+=1
	elif word[0] in accents:
		#store an accent input
		prev = word[0]
		return
	else:
		#regular character input
		if int(word[3]) != 0:
			prev = ''
		return
	prev = ''

	settex = u"settext " + text
	hexchat.command( settex.encode(charset) )

	hexchat.command('setcursor {}'.format(loc+1))

	return hexchat.EAT_HEXCHAT

def unload_cb(userdata):
	print(__module_name__, 'version', __module_version__, 'unloaded.')

def insert(char,text,loc):
	return u"{}{}{}".format(text[:loc] , char , text[loc:])

hexchat.hook_print('Key Press', keypress_cb)
hexchat.hook_unload(unload_cb)

print(__module_name__, 'version', __module_version__, 'loaded.')
<commit_msg>Version bump<commit_after># -*- coding: utf-8 -*-

import hexchat
import re

__module_name__ = "DeadKeyFix"
__module_version__ = "2.2"
__module_description__ = "Fixes the Us-International deadkey issue"

prev = ''

def keypress_cb(word, word_eol, userdata):
	global prev
	specialChars = {
		'65104': {
			'a': u'à',
			'o': u'ò',
			'e': u'è',
			'i': u'ì',
			'u': u'ù',

			'A': u'À',
			'O': u'Ò',
			'E': u'È',
			'I': u'Ì',
			'U': u'Ù'
		},
		'65105': {
			'a': u'á',
			'o': u'ó',
			'e': u'é',
			'i': u'í',
			'u': u'ú',
			'y': u'ý',
			'c': u'ç',

			'A': u'Á',
			'O': u'Ó',
			'E': u'É',
			'I': u'Í',
			'U': u'Ú',
			'Y': u'Ý',
			'C': u'Ç'
		},
		'65106': {
			'a': u'â',
			'o': u'ô',
			'e': u'ê',
			'i': u'î',
			'u': u'û',

			'A': u'Â',
			'O': u'Ô',
			'E': u'Ê',
			'I': u'Î',
			'U': u'Û'
		},
		'65107': {
			'a': u'ã',
			'o': u'õ',
			'n': u'ñ',

			'A': u'Ã',
			'O': u'Õ',
			'N': u'Ñ'
		},
		'65111': {
			'a': u'ä',
			'o': u'ö',
			'e': u'ë',
			'i': u'ï',
			'u': u'ü',
			'y': u'ÿ',

			'A': u'Ä',
			'O': u'Ö',
			'E': u'Ë',
			'I': u'Ï',
			'U': u'Ü',
			'Y': u'Ÿ'
		}
	}
	accents = {
		'65104': '`',
		'65105': "'",
		'65106': '^',
		'65107': '~',
		'65111': '"'
	}

	charset =  hexchat.get_info('charset')
	#When there is no current charset derived from server or channel it is set to IRC
	#IRC is not a recognized encoding type so default to utf-8 in that case.
	if(charset == "IRC"):
		charset = "utf-8"

	textraw = hexchat.get_info('inputbox')
	text = unicode(textraw, charset)
	loc = hexchat.get_prefs("state_cursor")

	if prev in accents and word[2] in specialChars[prev]:
		#insert an accented character
		text = insert(specialChars[prev][word[2]],text,loc)
	elif prev in accents and word[2] == ' ':
		#insert a clean accent ( input was accent followed by space )
		text = insert(accents[prev],text,loc)
	elif prev in accents and word[0] in accents:
		#Insert two accents ( input was accent followed by accent )
		text = insert(accents[prev] + accents[word[0]],text,loc)
		loc+=1
	elif prev in accents and int(word[3]) != 0:
		#insert an accent and a character (character and accent do not combine)
		text = insert(accents[prev] + word[2],text,loc)
		loc+=1
	elif word[0] in accents:
		#store an accent input
		prev = word[0]
		return
	else:
		#regular character input
		if int(word[3]) != 0:
			prev = ''
		return
	prev = ''

	settex = u"settext " + text
	hexchat.command( settex.encode(charset) )

	hexchat.command('setcursor {}'.format(loc+1))

	return hexchat.EAT_HEXCHAT

def unload_cb(userdata):
	print(__module_name__, 'version', __module_version__, 'unloaded.')

def insert(char,text,loc):
	return u"{}{}{}".format(text[:loc] , char , text[loc:])

hexchat.hook_print('Key Press', keypress_cb)
hexchat.hook_unload(unload_cb)

print(__module_name__, 'version', __module_version__, 'loaded.')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys

####################################################################################
#Master Script that calls subscripts to be deployed to new Linux systems
####################################################################################

def main(*args,**kwargs) :

    #System variables
    scriptname = sys.argv[0]
    workingdir = '/usr/tmp/systemprep'
    readyfile = '/var/run/system-is-ready'
    scriptstart = '+' * 80
    scriptend = '-' * 80

    print scriptstart
    print 'Entering script -- ', scriptname
    print 'Printing parameters...'

    print 'kwargs = ', kwargs
    for key,value in kwargs.items() :
        print ( str(key) + ' = ' + str(value) )

#    raw_input("\n\nPress the enter key to exit.")

if __name__ == "__main__" :
    kwargs = dict( x.split('=', 1) for x in sys.argv[1:] )
    main(**kwargs)

<commit_msg>-Removed stylistic commenting -Removed unused variables -Removed variables for formatting, this is an old approach and creates unnecessary variables which muddies the code.<commit_after>#!/usr/bin/env python
import os
import sys
import platform
import tempfile

def main(**kwargs):
    """
Master Script that calls subscripts to be deployed to new Linux systems
    """

    scriptname = sys.argv[0]
    #Logic to define variables based on OS
    if 'Windows' in platform.system():
        #workingdir = 'C:\WINDOWS\TEMP\SYSTEMPREP'
        workingdir = 'C:\WINDOWS\TEMP'
        readyfile = 'C:\WINDOWS\TEMP\system-is-ready'
    elif 'Linux' in platform.system():
        #workingdir = '/usr/tmp/systemprep'
        workingdir = '/usr/tmp/'
        readyfile = '/var/run/system-is-ready'
    else:
        raise SystemError('platform.system() is not recognized?')

    print('+' * 80)
    print('Entering script -- ' + scriptname)
    if not os.path.exists(readyfile):
        raise SystemError(readyfile + ' does not exist! System is not ready?')

    if os.path.exists(workingdir):
        print('Creating working directory.')
        workingdir = tempfile.mkdtemp(prefix='systemprep-', dir=workingdir)
        print(workingdir)
    else:
        raise SystemError('Defined workingdir does not exist!')

    print('Printing parameters...')
    print('kwargs = ' + str(kwargs))

    for key, value in kwargs.items():
        print(str(key) + ' = ' + str(value))

    #TODO: Add Cleanup function and replace cleanup time. This does not cleanup if SystemError is raised after creation.
    #todo Cleanup should be a function that is called during a SystemError so files are never left behind unless needed.

    print('+-' * 40)
    print('Cleanup Time...')
    print('Removing temporary data...')
    os.removedirs(workingdir)
    print('-' * 80)
#    raw_input("\n\nPress the enter key to exit.")

if __name__ == "__main__":
    kwargs = dict(x.split('=', 1) for x in sys.argv[1:])
    main(**kwargs)<|endoftext|>


<commit_before>import os, re, sys
import fileinput
from mergeex import mergeex
import simplejson as json
from glob import glob

filters = ()
for js in glob("../plugins/*.json"):
    try:
        f = open(js, 'r')
        js = json.load(f, "utf-8")
        filters += js['match'],
        f.close()
    except Exception,e:
        print sys.stderr, js
        raise e

filters = mergeex(map(lambda x: re.sub(r'/', '\\/', x), filters))

print r'pref("extensions.dta.filters.deffilter-ac.label", "AntiContainer");'
print r'pref("extensions.dta.filters.deffilter-ac.test", "/' +  filters + '/i");'
print r'pref("extensions.dta.filters.deffilter-ac.active", true);'
print r'pref("extensions.dta.filters.deffilter-ac.type", 1);'<commit_msg>json/simplejson for json-to-filters<commit_after>import os, re, sys
import fileinput
from mergeex import mergeex
try:
    import simplejson as json
except:
    import json
from glob import glob

filters = ()
for js in glob("../plugins/*.json"):
    try:
        f = open(js, 'r')
        js = json.load(f, "utf-8")
        filters += js['match'],
        f.close()
    except Exception,e:
        print sys.stderr, js
        raise e

filters = mergeex(map(lambda x: re.sub(r'/', '\\/', x), filters))

print r'pref("extensions.dta.filters.deffilter-ac.label", "AntiContainer");'
print r'pref("extensions.dta.filters.deffilter-ac.test", "/' +  filters + '/i");'
print r'pref("extensions.dta.filters.deffilter-ac.active", true);'
print r'pref("extensions.dta.filters.deffilter-ac.type", 1);'<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
@brief      test log(time=67s)
"""

import sys
import os
import unittest

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder, is_travis_or_appveyor
from pyquickhelper.ipythonhelper import execute_notebook_list, execute_notebook_list_finalize_ut
from pyquickhelper.ipythonhelper import install_python_kernel_for_unittest
import src.teachpyx

class TestRunNotebooksPythonProtobuf(unittest.TestCase):

    def test_run_notebook_python_protobuf(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        kernel_name = None if is_travis_or_appveyor() else install_python_kernel_for_unittest(
            "python3_module_template")

        temp = get_temp_folder(__file__, "temp_run_notebooks_python_protobuf")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "python"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "protobuf" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, kernel_name=kernel_name)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=src.teachpyx)

if __name__ == "__main__":
    unittest.main()
<commit_msg>disable unit test on travis<commit_after># -*- coding: utf-8 -*-
"""
@brief      test log(time=67s)
"""

import sys
import os
import unittest

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder, is_travis_or_appveyor, skipif_circleci, skipif_travis
from pyquickhelper.ipythonhelper import execute_notebook_list, execute_notebook_list_finalize_ut
from pyquickhelper.ipythonhelper import install_python_kernel_for_unittest
import src.teachpyx

class TestRunNotebooksPythonProtobuf(unittest.TestCase):

    @skipif_circleci("'Permission denied: 'bin/protoc'")
    @skipif_travis("'Permission denied: 'bin/protoc'")
    def test_run_notebook_python_protobuf(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        kernel_name = None if is_travis_or_appveyor() else install_python_kernel_for_unittest(
            "python3_module_template")

        temp = get_temp_folder(__file__, "temp_run_notebooks_python_protobuf")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "python"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "protobuf" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, kernel_name=kernel_name)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=src.teachpyx)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>"""
Simple example of GLM fitting in fMRI
======================================

Full step-by-step example of fitting a GLM to experimental data and visualizing
the results. This is done on two runs of one subject of the FIAC dataset.
For details on the data, please see:

Dehaene-Lambertz G, Dehaene S, Anton JL, Campagne A, Ciuciu P, Dehaene
G, Denghien I, Jobert A, LeBihan D, Sigman M, Pallier C, Poline
JB. Functional segregation of cortical language areas by sentence
repetition. Hum Brain Mapp. 2006: 27:360--371.
http://www.pubmedcentral.nih.gov/articlerender.fcgi?artid=2653076#R11

More specifically:

1. A sequence of fMRI volumes are loaded
2. A design matrix describing all the effects related to the data is computed
3. a mask of the useful brain volume is computed
4. A GLM is applied to the dataset (effect/covariance,
   then contrast estimation)

Author: Bertrand Thirion, 2015
"""
from os import mkdir, path, getcwd

import numpy as np
import pandas as pd

from nilearn import plotting
from nilearn.image import mean_img
import nibabel as nib

from nistats.first_level_model import FirstLevelModel
from nistats import datasets

# # write directory
# write_dir = path.join(getcwd(), 'results')
# if not path.exists(write_dir):
#     mkdir(write_dir)

#########################################################################
# Prepare data and analysis parameters
# ----------------------------------
data = datasets.fetch_fiac_first_level()
fmri_img = [data['func1'], data['func2']]
mean_img_ = mean_img(fmri_img[0])
design_files = [data['design_matrix1'], data['design_matrix2']]
design_matrices = [pd.DataFrame(np.load(df)['X']) for df in design_files]

#########################################################################
# GLM estimation
# ----------------------------------
# GLM specification
fmri_glm = FirstLevelModel(mask=data['mask'], minimize_memory=True)

#########################################################################
# GLM fitting
fmri_glm = fmri_glm.fit(fmri_img, design_matrices=design_matrices)

#########################################################################
# compute fixed effects of the two runs and compute related images
n_columns = design_matrices[0].shape[1]

def pad_vector(contrast_, n_columns):
    return np.hstack((contrast_, np.zeros(n_columns - len(contrast_))))

contrasts = {'SStSSp_minus_DStDSp': pad_vector([1, 0, 0, -1], n_columns),
             'DStDSp_minus_SStSSp': pad_vector([-1, 0, 0, 1], n_columns),
             'DSt_minus_SSt': pad_vector([-1, -1, 1, 1], n_columns),
             'DSp_minus_SSp': pad_vector([-1, 1, -1, 1], n_columns),
             'DSt_minus_SSt_for_DSp': pad_vector([0, -1, 0, 1], n_columns),
             'DSp_minus_SSp_for_DSt': pad_vector([0, 0, -1, 1], n_columns),
             'Deactivation': pad_vector([-1, -1, -1, -1, 4], n_columns),
             'Effects_of_interest': np.eye(n_columns)[:5]}

# print('Computing contrasts...')
for index, (contrast_id, contrast_val) in enumerate(contrasts.items()):
    # print('  Contrast % 2i out of %i: %s' % (
        # index + 1, len(contrasts), contrast_id))
    # z_image_path = path.join(write_dir, '%s_z_map.nii' % contrast_id)
    z_map = fmri_glm.compute_contrast(
        contrast_val, output_type='z_score')
    # nib.save(z_map, z_image_path)

    # make a snapshot of the contrast activation
    if contrast_id == 'Effects_of_interest':
        display = plotting.plot_stat_map(
            z_map, bg_img=mean_img_, threshold=2.5, title=contrast_id)
        # display.savefig(path.join(write_dir, '%s_z_map.png' % contrast_id))

# print('All the  results were witten in %s' % write_dir)
plotting.show()
<commit_msg>another underline fix<commit_after>"""
Simple example of GLM fitting in fMRI
======================================

Full step-by-step example of fitting a GLM to experimental data and visualizing
the results. This is done on two runs of one subject of the FIAC dataset.
For details on the data, please see:

Dehaene-Lambertz G, Dehaene S, Anton JL, Campagne A, Ciuciu P, Dehaene
G, Denghien I, Jobert A, LeBihan D, Sigman M, Pallier C, Poline
JB. Functional segregation of cortical language areas by sentence
repetition. Hum Brain Mapp. 2006: 27:360--371.
http://www.pubmedcentral.nih.gov/articlerender.fcgi?artid=2653076#R11

More specifically:

1. A sequence of fMRI volumes are loaded
2. A design matrix describing all the effects related to the data is computed
3. a mask of the useful brain volume is computed
4. A GLM is applied to the dataset (effect/covariance,
   then contrast estimation)

Author: Bertrand Thirion, 2015
"""
from os import mkdir, path, getcwd

import numpy as np
import pandas as pd

from nilearn import plotting
from nilearn.image import mean_img
import nibabel as nib

from nistats.first_level_model import FirstLevelModel
from nistats import datasets

# # write directory
# write_dir = path.join(getcwd(), 'results')
# if not path.exists(write_dir):
#     mkdir(write_dir)

#########################################################################
# Prepare data and analysis parameters
# --------------------------------------
data = datasets.fetch_fiac_first_level()
fmri_img = [data['func1'], data['func2']]
mean_img_ = mean_img(fmri_img[0])
design_files = [data['design_matrix1'], data['design_matrix2']]
design_matrices = [pd.DataFrame(np.load(df)['X']) for df in design_files]

#########################################################################
# GLM estimation
# ----------------------------------
# GLM specification
fmri_glm = FirstLevelModel(mask=data['mask'], minimize_memory=True)

#########################################################################
# GLM fitting
fmri_glm = fmri_glm.fit(fmri_img, design_matrices=design_matrices)

#########################################################################
# compute fixed effects of the two runs and compute related images
n_columns = design_matrices[0].shape[1]

def pad_vector(contrast_, n_columns):
    return np.hstack((contrast_, np.zeros(n_columns - len(contrast_))))

contrasts = {'SStSSp_minus_DStDSp': pad_vector([1, 0, 0, -1], n_columns),
             'DStDSp_minus_SStSSp': pad_vector([-1, 0, 0, 1], n_columns),
             'DSt_minus_SSt': pad_vector([-1, -1, 1, 1], n_columns),
             'DSp_minus_SSp': pad_vector([-1, 1, -1, 1], n_columns),
             'DSt_minus_SSt_for_DSp': pad_vector([0, -1, 0, 1], n_columns),
             'DSp_minus_SSp_for_DSt': pad_vector([0, 0, -1, 1], n_columns),
             'Deactivation': pad_vector([-1, -1, -1, -1, 4], n_columns),
             'Effects_of_interest': np.eye(n_columns)[:5]}

# print('Computing contrasts...')
for index, (contrast_id, contrast_val) in enumerate(contrasts.items()):
    # print('  Contrast % 2i out of %i: %s' % (
        # index + 1, len(contrasts), contrast_id))
    # z_image_path = path.join(write_dir, '%s_z_map.nii' % contrast_id)
    z_map = fmri_glm.compute_contrast(
        contrast_val, output_type='z_score')
    # nib.save(z_map, z_image_path)

    # make a snapshot of the contrast activation
    if contrast_id == 'Effects_of_interest':
        display = plotting.plot_stat_map(
            z_map, bg_img=mean_img_, threshold=2.5, title=contrast_id)
        # display.savefig(path.join(write_dir, '%s_z_map.png' % contrast_id))

# print('All the  results were witten in %s' % write_dir)
plotting.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = ['sina_download', 'sina_download_by_vid', 'sina_download_by_vkey']

from ..common import *

from hashlib import md5
from random import randint
from time import time

def get_k(vid, rand):
    t = str(int('{0:b}'.format(int(time()))[:-6], 2))
    return md5((vid + 'Z6prk18aWxP278cVAH' + t + rand).encode('utf-8')).hexdigest()[:16] + t

def video_info_xml(vid):
    rand = "0.{0}{1}".format(randint(10000, 10000000), randint(10000, 10000000))
    url = 'http://v.iask.com/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, get_k(vid, rand))
    xml = get_content(url, headers=fake_headers, decoded=True)
    return xml

def video_info(xml):
    urls = re.findall(r'<url>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</url>', xml)
    name = match1(xml, r'<vname>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vname>')
    vstr = match1(xml, r'<vstr>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vstr>')
    return urls, name, vstr

def sina_download_by_vid(vid, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vid.
    http://video.sina.com.cn/
    """

    xml = video_info_xml(vid)
    sina_download_by_urls(xml, title, output_dir, merge, info_only)

def sina_download_by_xml(xml, title, output_dir, merge, info_only):
    urls, name, vstr = video_info(xml)
    title = title or name
    assert title
    size = 0
    for url in urls:
        _, _, temp = url_info(url)
        size += temp

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls(urls, title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download_by_vkey(vkey, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vkey.
    http://video.sina.com/
    """

    url = 'http://video.sina.com/v/flvideo/%s_0.flv' % vkey
    type, ext, size = url_info(url)

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls([url], title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download(url, output_dir='.', merge=True, info_only=False):
    """Downloads Sina videos by URL.
    """

    vid = match1(url, r'vid=(\d+)')
    if vid is None:
        video_page = get_content(url)
        vid = hd_vid = match1(video_page, r'hd_vid\s*:\s*\'([^\']+)\'')
        if hd_vid == '0':
            vids = match1(video_page, r'[^\w]vid\s*:\s*\'([^\']+)\'').split('|')
            vid = vids[-1]

    if vid:
        title = match1(video_page, r'title\s*:\s*\'([^\']+)\'')
        sina_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
    else:
        vkey = match1(video_page, r'vkey\s*:\s*"([^"]+)"')
        title = match1(video_page, r'title\s*:\s*"([^"]+)"')
        sina_download_by_vkey(vkey, title=title, output_dir=output_dir, merge=merge, info_only=info_only)

site_info = "Sina.com"
download = sina_download
download_playlist = playlist_not_supported('sina')
<commit_msg>repair mistake in sina.py:line 33 sina_download_by_urls ->sina_download_by_xml<commit_after>#!/usr/bin/env python

__all__ = ['sina_download', 'sina_download_by_vid', 'sina_download_by_vkey']

from ..common import *

from hashlib import md5
from random import randint
from time import time

def get_k(vid, rand):
    t = str(int('{0:b}'.format(int(time()))[:-6], 2))
    return md5((vid + 'Z6prk18aWxP278cVAH' + t + rand).encode('utf-8')).hexdigest()[:16] + t

def video_info_xml(vid):
    rand = "0.{0}{1}".format(randint(10000, 10000000), randint(10000, 10000000))
    url = 'http://v.iask.com/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, get_k(vid, rand))
    xml = get_content(url, headers=fake_headers, decoded=True)
    return xml

def video_info(xml):
    urls = re.findall(r'<url>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</url>', xml)
    name = match1(xml, r'<vname>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vname>')
    vstr = match1(xml, r'<vstr>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vstr>')
    return urls, name, vstr

def sina_download_by_vid(vid, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vid.
    http://video.sina.com.cn/
    """

    xml = video_info_xml(vid)
    sina_download_by_xml(xml, title, output_dir, merge, info_only)

def sina_download_by_xml(xml, title, output_dir, merge, info_only):
    urls, name, vstr = video_info(xml)
    title = title or name
    assert title
    size = 0
    for url in urls:
        _, _, temp = url_info(url)
        size += temp

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls(urls, title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download_by_vkey(vkey, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vkey.
    http://video.sina.com/
    """

    url = 'http://video.sina.com/v/flvideo/%s_0.flv' % vkey
    type, ext, size = url_info(url)

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls([url], title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download(url, output_dir='.', merge=True, info_only=False):
    """Downloads Sina videos by URL.
    """

    vid = match1(url, r'vid=(\d+)')
    if vid is None:
        video_page = get_content(url)
        vid = hd_vid = match1(video_page, r'hd_vid\s*:\s*\'([^\']+)\'')
        if hd_vid == '0':
            vids = match1(video_page, r'[^\w]vid\s*:\s*\'([^\']+)\'').split('|')
            vid = vids[-1]

    if vid:
        title = match1(video_page, r'title\s*:\s*\'([^\']+)\'')
        sina_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
    else:
        vkey = match1(video_page, r'vkey\s*:\s*"([^"]+)"')
        title = match1(video_page, r'title\s*:\s*"([^"]+)"')
        sina_download_by_vkey(vkey, title=title, output_dir=output_dir, merge=merge, info_only=info_only)

site_info = "Sina.com"
download = sina_download
download_playlist = playlist_not_supported('sina')
<|endoftext|>


<commit_before># Copyright 2010-2018 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Integer programming examples that show how to use the APIs."""

from ortools.linear_solver import pywraplp
from ortools.init import init

def Announce(solver, api_type):
    print('---- Integer programming example with ' + solver + ' (' + api_type +
          ') -----')

def RunIntegerExampleNaturalLanguageAPI(optimization_problem_type):
    """Example of simple integer program with natural language API."""

    solver = pywraplp.Solver.CreateSolver(optimization_problem_type)
    if not solver:
        return

    Announce(optimization_problem_type, 'natural language API')

    infinity = solver.infinity()
    # x1 and x2 are integer non-negative variables.
    x1 = solver.IntVar(0.0, infinity, 'x1')
    x2 = solver.IntVar(0.0, infinity, 'x2')

    solver.Minimize(x1 + 2 * x2)
    solver.Add(3 * x1 + 2 * x2 >= 17)

    SolveAndPrint(solver, [x1, x2])

def RunIntegerExampleCppStyleAPI(optimization_problem_type):
    """Example of simple integer program with the C++ style API."""
    solver = pywraplp.Solver.CreateSolver(optimization_problem_type)
    if not solver:
        return

    Announce(optimization_problem_type, 'C++ style API')

    infinity = solver.infinity()
    # x1 and x2 are integer non-negative variables.
    x1 = solver.IntVar(0.0, infinity, 'x1')
    x2 = solver.IntVar(0.0, infinity, 'x2')

    # Minimize x1 + 2 * x2.
    objective = solver.Objective()
    objective.SetCoefficient(x1, 1)
    objective.SetCoefficient(x2, 2)

    # 2 * x2 + 3 * x1 >= 17.
    ct = solver.Constraint(17, infinity)
    ct.SetCoefficient(x1, 3)
    ct.SetCoefficient(x2, 2)

    SolveAndPrint(solver, [x1, x2])

def SolveAndPrint(solver, variable_list):
    """Solve the problem and print the solution."""
    print('Number of variables = %d' % solver.NumVariables())
    print('Number of constraints = %d' % solver.NumConstraints())

    result_status = solver.Solve()

    # The problem has an optimal solution.
    assert result_status == pywraplp.Solver.OPTIMAL

    # The solution looks legit (when using solvers others than
    # GLOP_LINEAR_PROGRAMMING, verifying the solution is highly recommended!).
    assert solver.VerifySolution(1e-7, True)

    print('Problem solved in %f milliseconds' % solver.wall_time())

    # The objective value of the solution.
    print('Optimal objective value = %f' % solver.Objective().Value())

    # The value of each variable in the solution.
    for variable in variable_list:
        print('%s = %f' % (variable.name(), variable.solution_value()))

    print('Advanced usage:')
    print('Problem solved in %d branch-and-bound nodes' % solver.nodes())

def RunAllIntegerExampleNaturalLanguageAPI():
    RunIntegerExampleNaturalLanguageAPI('GLPK')
    RunIntegerExampleNaturalLanguageAPI('CBC')
    RunIntegerExampleNaturalLanguageAPI('SCIP')
    RunIntegerExampleNaturalLanguageAPI('SAT')
    RunIntegerExampleNaturalLanguageAPI('Gurobi')

def RunAllIntegerExampleCppStyleAPI():
    RunIntegerExampleCppStyleAPI('GLPK')
    RunIntegerExampleCppStyleAPI('CBC')
    RunIntegerExampleCppStyleAPI('SCIP')
    RunIntegerExampleCppStyleAPI('SAT')
    RunIntegerExampleCppStyleAPI('Gurobi')

def main():
    RunAllIntegerExampleNaturalLanguageAPI()
    RunAllIntegerExampleCppStyleAPI()

if __name__ == '__main__':
    init.CppBridge.InitLogging('integer_programming.py')
    cpp_flags = init.CppFlags()
    cpp_flags.logtostderr = True
    cpp_flags.log_prefix = False
    init.CppBridge.SetFlags(cpp_flags)
    main()
<commit_msg>python: Fix integer_programming.py<commit_after># Copyright 2010-2018 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Integer programming examples that show how to use the APIs."""

from ortools.linear_solver import pywraplp
from ortools.init import pywrapinit

def Announce(solver, api_type):
    print('---- Integer programming example with ' + solver + ' (' + api_type +
          ') -----')

def RunIntegerExampleNaturalLanguageAPI(optimization_problem_type):
    """Example of simple integer program with natural language API."""

    solver = pywraplp.Solver.CreateSolver(optimization_problem_type)
    if not solver:
        return

    Announce(optimization_problem_type, 'natural language API')

    infinity = solver.infinity()
    # x1 and x2 are integer non-negative variables.
    x1 = solver.IntVar(0.0, infinity, 'x1')
    x2 = solver.IntVar(0.0, infinity, 'x2')

    solver.Minimize(x1 + 2 * x2)
    solver.Add(3 * x1 + 2 * x2 >= 17)

    SolveAndPrint(solver, [x1, x2])

def RunIntegerExampleCppStyleAPI(optimization_problem_type):
    """Example of simple integer program with the C++ style API."""
    solver = pywraplp.Solver.CreateSolver(optimization_problem_type)
    if not solver:
        return

    Announce(optimization_problem_type, 'C++ style API')

    infinity = solver.infinity()
    # x1 and x2 are integer non-negative variables.
    x1 = solver.IntVar(0.0, infinity, 'x1')
    x2 = solver.IntVar(0.0, infinity, 'x2')

    # Minimize x1 + 2 * x2.
    objective = solver.Objective()
    objective.SetCoefficient(x1, 1)
    objective.SetCoefficient(x2, 2)

    # 2 * x2 + 3 * x1 >= 17.
    ct = solver.Constraint(17, infinity)
    ct.SetCoefficient(x1, 3)
    ct.SetCoefficient(x2, 2)

    SolveAndPrint(solver, [x1, x2])

def SolveAndPrint(solver, variable_list):
    """Solve the problem and print the solution."""
    print('Number of variables = %d' % solver.NumVariables())
    print('Number of constraints = %d' % solver.NumConstraints())

    result_status = solver.Solve()

    # The problem has an optimal solution.
    assert result_status == pywraplp.Solver.OPTIMAL

    # The solution looks legit (when using solvers others than
    # GLOP_LINEAR_PROGRAMMING, verifying the solution is highly recommended!).
    assert solver.VerifySolution(1e-7, True)

    print('Problem solved in %f milliseconds' % solver.wall_time())

    # The objective value of the solution.
    print('Optimal objective value = %f' % solver.Objective().Value())

    # The value of each variable in the solution.
    for variable in variable_list:
        print('%s = %f' % (variable.name(), variable.solution_value()))

    print('Advanced usage:')
    print('Problem solved in %d branch-and-bound nodes' % solver.nodes())

def RunAllIntegerExampleNaturalLanguageAPI():
    RunIntegerExampleNaturalLanguageAPI('GLPK')
    RunIntegerExampleNaturalLanguageAPI('CBC')
    RunIntegerExampleNaturalLanguageAPI('SCIP')
    RunIntegerExampleNaturalLanguageAPI('SAT')
    RunIntegerExampleNaturalLanguageAPI('Gurobi')

def RunAllIntegerExampleCppStyleAPI():
    RunIntegerExampleCppStyleAPI('GLPK')
    RunIntegerExampleCppStyleAPI('CBC')
    RunIntegerExampleCppStyleAPI('SCIP')
    RunIntegerExampleCppStyleAPI('SAT')
    RunIntegerExampleCppStyleAPI('Gurobi')

def main():
    RunAllIntegerExampleNaturalLanguageAPI()
    RunAllIntegerExampleCppStyleAPI()

if __name__ == '__main__':
    pywrapinit.CppBridge.InitLogging('integer_programming.py')
    cpp_flags = pywrapinit.CppFlags()
    cpp_flags.logtostderr = True
    cpp_flags.log_prefix = False
    pywrapinit.CppBridge.SetFlags(cpp_flags)
    main()
<|endoftext|>


<commit_before>import ggame
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame, CircleAsset, PolygonAsset, TextAsset

SCREEN_WIDTH = 700
SCREEN_HEIGHT = 700

# Barrel
class Barrel(Sprite):
    brown = Color(0x996633, 1.0)
    Black = Color(1,0)
    yellow = Color(0xfdd835, 1.0)
    noline = LineStyle(1, Black)
    asset = CircleAsset(25, noline, brown)

    def __init__(self, position):
        super().__init__(Barrel.asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0

# THE WALLS
class wall(Sprite):
    Red = Color(0xF44366, 1.0)
    noline = LineStyle(0, Red)

    def __init__(self, asset, position):
        super().__init__(asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0

class ladder(Sprite):
    Blue = Color(0x558b24, 1.0)
    noline = LineStyle(0, Blue)

    def __init__(self, asset, position):
        super().__init__(asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0

class player(Sprite):
    purple = Color(0x9575cd, 1.0)
    white = Color(0xfafafa, 1.0)
    line= LineStyle(1, white)
    asset = RectangleAsset(25, 25, line, purple)

    def __init__(self, position):
        super().__init__(player.asset, position)
        self.vx = 0
        self.vy = 5
        self.vr = 0
        self.a = self.collidingWithSprites(wall)
        self.b = self.collidingWithSprites(ladder)
        self.fxcenter = self.fycenter = 0.25
        self.YourDad = True
        self.YourUncle = False
        self.You = False
        self.YourAunt = False
        Kong.listenKeyEvent("keydown", "right arrow", self.MoveRIGHT)
        Kong.listenKeyEvent("keyup", "right arrow", self.MoveOff)
        Kong.listenKeyEvent("keydown", "left arrow", self.MoveLEFT)
        Kong.listenKeyEvent("keyup", "left arrow", self.MoveOff)
        Kong.listenKeyEvent("keydown", "up arrow", self.JumpOn)
        Kong.listenKeyEvent("keyup", "up arrow", self.JumpOff)
        Kong.listenKeyEvent("keydown", "right arrow", self.falling)
        Kong.listenKeyEvent("keydown", "left arrow", self.falling)

    def step(self):
        self.vy = self.vy + 1.25
        self.y += self.vy
        self.a = self.collidingWithSprites(wall)
        self.b = self.collidingWithSprites(ladder)
        if len(self.b) != 0:
            self.You = True
        if len(self.b) == 0:
            self.You == False
        if len(self.a) != 0:
            self.y -= self.vy
            self.vy = 0
            self.YourDad = False
        else:
            self.YourDad = True
        if self.YourDad == True:
            self.YourUncle = True
        else:
            self.YourUncle = False
        self.x += self.vx
        self.a = self.collidingWithSprites(wall)
        if len(self.a) != 0:
            self.x -= self.vx
            self.vx = 0
            self.YourDad = False
        else:
            self.YourDad = True

    def falling(self, event):
        if self.YourDad == True:
            self.vy = self.vy + 1

    def MoveRIGHT(self, event):
        if len(self.a) == 0:
            self.YourDad = True
        if self.YourDad == False:
            self.vx = 5
        else:
            self.vx = 5
            self.vy = 5

    def MoveLEFT(self, event):
        if len(self.a) == 0:
            self.YourDad = True
        if self.YourDad == False:
            self.vx = -5
        else:
            self.vx = -5
            self.vy = 5

    def MoveOff(self, event):
        self.vx = 0
        self.vy = 5

    def JumpOn(self, event):
        if self.You == True:
            self.vy = -5
        if len(self.a) == 0:
            self.YourDad = True
        if self.YourUncle == False:
            if self.YourDad == True:
                self.vy = -18
        else:
            self.vy = 5

    def JumpOff(self, event):
        self. vy = self.vy + 1

class trophy(Sprite):
    yellow = Color(0xffca28, 1.0)
    white = Color(0xfafafa, 1.0)
    liner = LineStyle(1, white)

    def __init__(self, asset, position):
        super().__init__(asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0

# THE WORLD
class Kong(App):

    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        yellow = Color(0xffeb3b, 1.0)
        white = Color(0xfafafa, 1.0)
        liner = LineStyle(1, white)
        line = LineStyle(1, black)
        self.play = False
        self.ComicSans = True
        bg_asset=RectangleAsset(width, height, line, black)
        bg=Sprite(bg_asset, (0, 0))
        text = TextAsset("Press ENTER To Start", style='40pt Comic Sans MS', fill= Color(0xffeb3b, 1), width=700)
        self.prompt = Sprite(text,(50, 250))
        Kong.listenKeyEvent("keydown", "enter", self.playing)

    def playing(self, event):
        self.prompt.destroy()
        self.play = True
        if self.play == True:
            player((50, 640))
            black=Color(1, 0)
            Black=Color(0, 1)
            Red = Color(0xF44366, 1.0)
            noline=LineStyle(1000, Black)
            oline= LineStyle(0, Red)
            white = Color(0xfafafa, 1.0)
            yellow = Color(0xffca28, 1.0)
            liner = LineStyle(1, white)
            Blue = Color(0x558b24, 1.0)
            wall(RectangleAsset(700, 30, oline, Red), (0, 670))
            wall(RectangleAsset(550, 30, oline, Red), (0, 500))
            wall(RectangleAsset(550, 30, oline, Red), (150, 330))
            wall(RectangleAsset(600, 30, oline, Red), (0, 160))
            wall(RectangleAsset(100, 30, oline, Red), (100, 50))
            ladder(RectangleAsset(10, 170, oline, Blue), (550, 500))
            ladder(RectangleAsset(10, 170, oline, Blue), (300, 500))
            ladder(RectangleAsset(10, 170, oline, Blue), (400, 330))
            ladder(RectangleAsset(10, 170, oline, Blue), (150, 330))
            ladder(RectangleAsset(10, 100, oline, Blue), (250, 160))
            ladder(RectangleAsset(10, 170, oline, Blue), (330, 160))
            ladder(RectangleAsset(10, 170, oline, Blue), (600, 160))
            ladder(RectangleAsset(10, 110, oline, Blue), (150, 50))
            trophy(RectangleAsset(25, 25, liner, yellow), (100, 25))

    def step(self):
        for ship in self.getSpritesbyClass(player):
            ship.step()

myapp = Kong(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
<commit_msg>Updated from Brython Server: 12/16/2016 1:40:29 PM<commit_after>import ggame
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame, CircleAsset, PolygonAsset, TextAsset

SCREEN_WIDTH = 700
SCREEN_HEIGHT = 700

# Barrel
class Barrel(Sprite):
    brown = Color(0x996633, 1.0)
    Black = Color(1,0)
    yellow = Color(0xfdd835, 1.0)
    noline = LineStyle(1, Black)
    asset = CircleAsset(25, noline, brown)

    def __init__(self, position):
        super().__init__(Barrel.asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0
# THE WALLS
class wall(Sprite):
    Red = Color(0xF44366, 1.0)
    noline = LineStyle(0, Red)

    def __init__(self, asset, position):
        super().__init__(asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0
# LADDER
class ladder(Sprite):
    Blue = Color(0x558b24, 1.0)
    noline = LineStyle(0, Blue)

    def __init__(self, asset, position):
        super().__init__(asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0
# PLAYER
class player(Sprite):
    purple = Color(0x9575cd, 1.0)
    white = Color(0xfafafa, 1.0)
    line= LineStyle(1, white)
    asset = RectangleAsset(25, 25, line, purple)

    def __init__(self, position):
        super().__init__(player.asset, position)
        self.vx = 0
        self.vy = 5
        self.vr = 0
        self.a = self.collidingWithSprites(wall)
        self.b = self.collidingWithSprites(ladder)
        self.fxcenter = self.fycenter = 0.25
        self.YourDad = True
        self.YourUncle = False
        self.You = False
        self.YourAunt = False
        Kong.listenKeyEvent("keydown", "right arrow", self.MoveRIGHT)
        Kong.listenKeyEvent("keyup", "right arrow", self.MoveOff)
        Kong.listenKeyEvent("keydown", "left arrow", self.MoveLEFT)
        Kong.listenKeyEvent("keyup", "left arrow", self.MoveOff)
        Kong.listenKeyEvent("keydown", "up arrow", self.JumpOn)
        Kong.listenKeyEvent("keyup", "up arrow", self.JumpOff)
        Kong.listenKeyEvent("keydown", "right arrow", self.falling)
        Kong.listenKeyEvent("keydown", "left arrow", self.falling)

    def step(self):
        self.a = self.collidingWithSprites(wall)
        self.b = self.collidingWithSprites(ladder)
        if len(self.b) != 0:
            self.You = True
        if len(self.b) == 0:
            self.vy = self.vy + 1.25
            self.You == False
        self.y += self.vy
        if len(self.a) != 0:
            self.y -= self.vy
            self.vy = 1
            self.YourDad = False
        else:
            self.YourDad = True
        if self.YourDad == True:
            self.YourUncle = True
        else:
            self.YourUncle = False
        self.x += self.vx
        self.a = self.collidingWithSprites(wall)
        if len(self.a) != 0:
            self.x -= self.vx
            self.vx = 0
            self.YourDad = False
        else:
            self.YourDad = True

    def falling(self, event):
        if self.YourDad == True:
            self.vy = self.vy + 1

    def MoveRIGHT(self, event):
        if len(self.a) == 0:
            self.YourDad = True
        if self.YourDad == False:
            self.vx = 5
        else:
            self.vx = 5
            self.vy = 5

    def MoveLEFT(self, event):
        if len(self.a) == 0:
            self.YourDad = True
        if self.YourDad == False:
            self.vx = -5
        else:
            self.vx = -5
            self.vy = 5

    def MoveOff(self, event):
        self.vx = 0
        self.vy = 5

    def JumpOn(self, event):
        if self.You == True:
            self.vy = -5
        if len(self.a) == 0:
            self.YourDad = True
        if self.YourUncle == False:
            if self.YourDad == True:
                self.vy = -18
        else:
            self.vy = 5

    def JumpOff(self, event):
        self. vy = self.vy + 1
#TROPY
class trophy(Sprite):
    yellow = Color(0xffca28, 1.0)
    white = Color(0xfafafa, 1.0)
    liner = LineStyle(1, white)

    def __init__(self, asset, position):
        super().__init__(asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.fxcenter = self.fycenter = 0
# THE WORLD
class Kong(App):

    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        yellow = Color(0xffeb3b, 1.0)
        white = Color(0xfafafa, 1.0)
        liner = LineStyle(1, white)
        line = LineStyle(1, black)
        self.play = False
        self.ComicSans = True
        bg_asset=RectangleAsset(width, height, line, black)
        bg=Sprite(bg_asset, (0, 0))
        text = TextAsset("Press ENTER To Start", style='40pt Comic Sans MS', fill= Color(0xffeb3b, 1), width=700)
        self.prompt = Sprite(text,(50, 250))
        Kong.listenKeyEvent("keydown", "enter", self.playing)

    def playing(self, event):
        self.prompt.destroy()
        self.play = True
        if self.play == True:
            player((50, 640))
            black=Color(1, 0)
            Black=Color(0, 1)
            Red = Color(0xF44366, 1.0)
            noline=LineStyle(1000, Black)
            oline= LineStyle(0, Red)
            white = Color(0xfafafa, 1.0)
            yellow = Color(0xffca28, 1.0)
            liner = LineStyle(1, white)
            Blue = Color(0x558b24, 1.0)
            wall(RectangleAsset(700, 30, oline, Red), (0, 670))
            wall(RectangleAsset(550, 30, oline, Red), (0, 500))
            wall(RectangleAsset(550, 30, oline, Red), (150, 330))
            wall(RectangleAsset(600, 30, oline, Red), (0, 160))
            wall(RectangleAsset(100, 30, oline, Red), (100, 50))
            ladder(RectangleAsset(10, 170, oline, Blue), (550, 500))
            ladder(RectangleAsset(10, 170, oline, Blue), (300, 500))
            ladder(RectangleAsset(10, 170, oline, Blue), (400, 330))
            ladder(RectangleAsset(10, 170, oline, Blue), (150, 330))
            ladder(RectangleAsset(10, 100, oline, Blue), (250, 160))
            ladder(RectangleAsset(10, 170, oline, Blue), (330, 160))
            ladder(RectangleAsset(10, 170, oline, Blue), (600, 160))
            ladder(RectangleAsset(10, 110, oline, Blue), (150, 50))
            trophy(RectangleAsset(25, 25, liner, yellow), (100, 25))

    def step(self):
        for ship in self.getSpritesbyClass(player):
            ship.step()
# I DON'T KNOW WHAT THIS IS BUT I NEED IT
myapp = Kong(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
<|endoftext|>