<commit_before># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2015 YUNOHOST.ORG

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""
import re
import os
import logging
from collections import OrderedDict

import apt
from apt_pkg import version_compare

from moulinette import m18n

logger = logging.getLogger('yunohost.utils.packages')

# Exceptions -----------------------------------------------------------------

class PackageException(Exception):

    """Base exception related to a package

    Represent an exception related to the package named `pkgname`. If no
    `message` is provided, it will first try to use the translation key
    `message_key` if defined by the derived class. Otherwise, a standard
    message will be used.

    """
    message_key = 'package_unexpected_error'

    def __init__(self, pkgname, message=None):
        super(PackageException, self).__init__(
            message or m18n.n(self.message_key, pkgname=pkgname))
        self.pkgname = pkgname

class UnknownPackage(PackageException):

    """The package is not found in the cache."""
    message_key = 'package_unknown'

class UninstalledPackage(PackageException):

    """The package is not installed."""
    message_key = 'package_not_installed'

class InvalidSpecifier(ValueError):

    """An invalid specifier was found."""

# Version specifier ----------------------------------------------------------
# The packaging package has been a nice inspiration for the following classes.
# See: https://github.com/pypa/packaging

class Specifier(object):

    """Unique package version specifier

    Restrict a package version according to the `spec`. It must be a string
    containing a relation from the list below followed by a version number
    value. The relations allowed are, as defined by the Debian Policy Manual:

      - `<<` for strictly lower
      - `<=` for lower or equal
      - `=` for exactly equal
      - `>=` for greater or equal
      - `>>` for strictly greater

    """
    _regex_str = (
        r"""
        (?P<relation>(<<|<=|=|>=|>>))
        \s*
        (?P<version>[^,;\s)]*)
        """
    )
    _regex = re.compile(
        r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE)

    _relations = {
        "<<": "lower_than",
        "<=": "lower_or_equal_than",
        "=": "equal",
        ">=": "greater_or_equal_than",
        ">>": "greater_than",
    }

    def __init__(self, spec):
        if isinstance(spec, basestring):
            match = self._regex.search(spec)
            if not match:
                raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec))

            self._spec = (
                match.group("relation").strip(),
                match.group("version").strip(),
            )
        elif isinstance(spec, self.__class__):
            self._spec = spec._spec
        else:
            return NotImplemented

    def __repr__(self):
        return "<Specifier({0!r})>".format(str(self))

    def __str__(self):
        return "{0}{1}".format(*self._spec)

    def __hash__(self):
        return hash(self._spec)

    def __eq__(self, other):
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        return self._spec == other._spec

    def __ne__(self, other):
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        return self._spec != other._spec

    def __and__(self, other):
        return self.intersection(other)

    def __or__(self, other):
        return self.union(other)

    def _get_relation(self, op):
        return getattr(self, "_compare_{0}".format(self._relations[op]))

    def _compare_lower_than(self, version, spec):
        return version_compare(version, spec) < 0

    def _compare_lower_or_equal_than(self, version, spec):
        return version_compare(version, spec) <= 0

    def _compare_equal(self, version, spec):
        return version_compare(version, spec) == 0

    def _compare_greater_or_equal_than(self, version, spec):
        return version_compare(version, spec) >= 0

    def _compare_greater_than(self, version, spec):
        return version_compare(version, spec) > 0

    @property
    def relation(self):
        return self._spec[0]

    @property
    def version(self):
        return self._spec[1]

    def __contains__(self, item):
        return self.contains(item)

    def intersection(self, other):
        """Make the intersection of two specifiers

        Return a new `SpecifierSet` with version specifier(s) common to the
        specifier and the other.

        Example:
            >>> Specifier('>= 2.2') & '>> 2.2.1' == '>> 2.2.1'
            >>> Specifier('>= 2.2') & '<< 2.3' == '>= 2.2, << 2.3'

        """
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        # store spec parts for easy access
        rel1, v1 = self.relation, self.version
        rel2, v2 = other.relation, other.version
        result = []

        if other == self:
            result = [other]
        elif rel1 == '=':
            result = [self] if v1 in other else None
        elif rel2 == '=':
            result = [other] if v2 in self else None
        elif v1 == v2:
            result = [other if rel1[1] == '=' else self]
        elif v2 in self or v1 in other:
            is_self_greater = version_compare(v1, v2) > 0
            if rel1[0] == rel2[0]:
                if rel1[0] == '>':
                    result = [self if is_self_greater else other]
                else:
                    result = [other if is_self_greater else self]
            else:
                result = [self, other]
        return SpecifierSet(result if result is not None else '')

    def union(self, other):
        """Make the union of two version specifiers

        Return a new `SpecifierSet` with version specifiers from the
        specifier and the other.

        Example:
            >>> Specifier('>= 2.2') | '<< 2.3' == '>= 2.2, << 2.3'

        """
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        return SpecifierSet([self, other])

    def contains(self, item):
        """Check if the specifier contains an other

        Return whether the item is contained in the version specifier.

        Example:
            >>> '2.2.1' in Specifier('<< 2.3')
            >>> '2.4' not in Specifier('<< 2.3')

        """
        return self._get_relation(self.relation)(item, self.version)

class SpecifierSet(object):

    """A set of package version specifiers

    Combine several Specifier separated by a comma. It allows to restrict
    more precisely a package version. Each package version specifier must be
    meet. Note than an empty set of specifiers will always be meet.

    """

    def __init__(self, specifiers):
        if isinstance(specifiers, basestring):
            specifiers = [s.strip() for s in specifiers.split(",")
                          if s.strip()]

        parsed = set()
        for specifier in specifiers:
            parsed.add(Specifier(specifier))

        self._specs = frozenset(parsed)

    def __repr__(self):
        return "<SpecifierSet({0!r})>".format(str(self))

    def __str__(self):
        return ",".join(sorted(str(s) for s in self._specs))

    def __hash__(self):
        return hash(self._specs)

    def __and__(self, other):
        return self.intersection(other)

    def __or__(self, other):
        return self.union(other)

    def __eq__(self, other):
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif isinstance(other, Specifier):
            other = SpecifierSet(str(other))
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        return self._specs == other._specs

    def __ne__(self, other):
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif isinstance(other, Specifier):
            other = SpecifierSet(str(other))
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        return self._specs != other._specs

    def __len__(self):
        return len(self._specs)

    def __iter__(self):
        return iter(self._specs)

    def __contains__(self, item):
        return self.contains(item)

    def intersection(self, other):
        """Make the intersection of two specifiers sets

        Return a new `SpecifierSet` with version specifier(s) common to the
        set and the other.

        Example:
            >>> SpecifierSet('>= 2.2') & '>> 2.2.1' == '>> 2.2.1'
            >>> SpecifierSet('>= 2.2, << 2.4') & '<< 2.3' == '>= 2.2, << 2.3'
            >>> SpecifierSet('>= 2.2, << 2.3') & '>= 2.4' == ''

        """
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        specifiers = set(self._specs | other._specs)
        intersection = [specifiers.pop()] if specifiers else []

        for specifier in specifiers:
            parsed = set()
            for spec in intersection:
                inter = spec & specifier
                if not inter:
                    parsed.clear()
                    break
                # TODO: validate with other specs in parsed
                parsed.update(inter._specs)
            intersection = parsed
            if not intersection:
                break
        return SpecifierSet(intersection)

    def union(self, other):
        """Make the union of two specifiers sets

        Return a new `SpecifierSet` with version specifiers from the set
        and the other.

        Example:
            >>> SpecifierSet('>= 2.2') | '<< 2.3' == '>= 2.2, << 2.3'

        """
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        specifiers = SpecifierSet([])
        specifiers._specs = frozenset(self._specs | other._specs)
        return specifiers

    def contains(self, item):
        """Check if the set contains a version specifier

        Return whether the item is contained in all version specifiers.

        Example:
            >>> '2.2.1' in SpecifierSet('>= 2.2, << 2.3')
            >>> '2.4' not in SpecifierSet('>= 2.2, << 2.3')

        """
        return all(
            s.contains(item)
            for s in self._specs
        )

# Packages and cache helpers -------------------------------------------------

def get_installed_version(*pkgnames, **kwargs):
    """Get the installed version of package(s)

    Retrieve one or more packages named `pkgnames` and return their installed
    version as a dict or as a string if only one is requested and `as_dict` is
    `False`. If `strict` is `True`, an exception will be raised if a package
    is unknown or not installed.

    """
    versions = OrderedDict()
    cache = apt.Cache()

    # Retrieve options
    as_dict = kwargs.get('as_dict', False)
    strict = kwargs.get('strict', False)
    with_repo = kwargs.get('with_repo', False)

    for pkgname in pkgnames:
        try:
            pkg = cache[pkgname]
        except KeyError:
            if strict:
                raise UnknownPackage(pkgname)
            logger.warning(m18n.n('package_unknown', pkgname=pkgname))
            continue

        try:
            version = pkg.installed.version
        except AttributeError:
            if strict:
                raise UninstalledPackage(pkgname)
            version = None

        try:
            # stable, testing, unstable
            repo = pkg.installed.origins[0].component
        except AttributeError:
            if strict:
                raise UninstalledPackage(pkgname)
            repo = ""

        if with_repo:
            versions[pkgname] = {
                "version": version,
                # when we don't have component it's because it's from a local
                # install or from an image (like in vagrant)
                "repo": repo if repo else "local",
            }
        else:
            versions[pkgname] = version

    if len(pkgnames) == 1 and not as_dict:
        return versions[pkgnames[0]]
    return versions

def meets_version_specifier(pkgname, specifier):
    """Check if a package installed version meets specifier"""
    spec = SpecifierSet(specifier)
    return get_installed_version(pkgname) in spec

# YunoHost related methods ---------------------------------------------------

def ynh_packages_version(*args, **kwargs):
    # from cli the received arguments are:
    # (Namespace(_callbacks=deque([]), _tid='_global', _to_return={}), []) {}
    # they don't seem to serve any purpose
    """Return the version of each YunoHost package"""
    return get_installed_version(
        'yunohost', 'yunohost-admin', 'moulinette', 'ssowat',
        with_repo=True
    )

def dpkg_is_broken():
    # If dpkg is broken, /var/lib/dpkg/updates
    # will contains files like 0001, 0002, ...
    # ref: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174
    return any(re.match("^[0-9]+$", f)
               for f in os.listdir("/var/lib/dpkg/updates/"))
<commit_msg>Who knows, maybe this folder doesn't exist in some context<commit_after># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2015 YUNOHOST.ORG

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""
import re
import os
import logging
from collections import OrderedDict

import apt
from apt_pkg import version_compare

from moulinette import m18n

logger = logging.getLogger('yunohost.utils.packages')

# Exceptions -----------------------------------------------------------------

class PackageException(Exception):

    """Base exception related to a package

    Represent an exception related to the package named `pkgname`. If no
    `message` is provided, it will first try to use the translation key
    `message_key` if defined by the derived class. Otherwise, a standard
    message will be used.

    """
    message_key = 'package_unexpected_error'

    def __init__(self, pkgname, message=None):
        super(PackageException, self).__init__(
            message or m18n.n(self.message_key, pkgname=pkgname))
        self.pkgname = pkgname

class UnknownPackage(PackageException):

    """The package is not found in the cache."""
    message_key = 'package_unknown'

class UninstalledPackage(PackageException):

    """The package is not installed."""
    message_key = 'package_not_installed'

class InvalidSpecifier(ValueError):

    """An invalid specifier was found."""

# Version specifier ----------------------------------------------------------
# The packaging package has been a nice inspiration for the following classes.
# See: https://github.com/pypa/packaging

class Specifier(object):

    """Unique package version specifier

    Restrict a package version according to the `spec`. It must be a string
    containing a relation from the list below followed by a version number
    value. The relations allowed are, as defined by the Debian Policy Manual:

      - `<<` for strictly lower
      - `<=` for lower or equal
      - `=` for exactly equal
      - `>=` for greater or equal
      - `>>` for strictly greater

    """
    _regex_str = (
        r"""
        (?P<relation>(<<|<=|=|>=|>>))
        \s*
        (?P<version>[^,;\s)]*)
        """
    )
    _regex = re.compile(
        r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE)

    _relations = {
        "<<": "lower_than",
        "<=": "lower_or_equal_than",
        "=": "equal",
        ">=": "greater_or_equal_than",
        ">>": "greater_than",
    }

    def __init__(self, spec):
        if isinstance(spec, basestring):
            match = self._regex.search(spec)
            if not match:
                raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec))

            self._spec = (
                match.group("relation").strip(),
                match.group("version").strip(),
            )
        elif isinstance(spec, self.__class__):
            self._spec = spec._spec
        else:
            return NotImplemented

    def __repr__(self):
        return "<Specifier({0!r})>".format(str(self))

    def __str__(self):
        return "{0}{1}".format(*self._spec)

    def __hash__(self):
        return hash(self._spec)

    def __eq__(self, other):
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        return self._spec == other._spec

    def __ne__(self, other):
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        return self._spec != other._spec

    def __and__(self, other):
        return self.intersection(other)

    def __or__(self, other):
        return self.union(other)

    def _get_relation(self, op):
        return getattr(self, "_compare_{0}".format(self._relations[op]))

    def _compare_lower_than(self, version, spec):
        return version_compare(version, spec) < 0

    def _compare_lower_or_equal_than(self, version, spec):
        return version_compare(version, spec) <= 0

    def _compare_equal(self, version, spec):
        return version_compare(version, spec) == 0

    def _compare_greater_or_equal_than(self, version, spec):
        return version_compare(version, spec) >= 0

    def _compare_greater_than(self, version, spec):
        return version_compare(version, spec) > 0

    @property
    def relation(self):
        return self._spec[0]

    @property
    def version(self):
        return self._spec[1]

    def __contains__(self, item):
        return self.contains(item)

    def intersection(self, other):
        """Make the intersection of two specifiers

        Return a new `SpecifierSet` with version specifier(s) common to the
        specifier and the other.

        Example:
            >>> Specifier('>= 2.2') & '>> 2.2.1' == '>> 2.2.1'
            >>> Specifier('>= 2.2') & '<< 2.3' == '>= 2.2, << 2.3'

        """
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        # store spec parts for easy access
        rel1, v1 = self.relation, self.version
        rel2, v2 = other.relation, other.version
        result = []

        if other == self:
            result = [other]
        elif rel1 == '=':
            result = [self] if v1 in other else None
        elif rel2 == '=':
            result = [other] if v2 in self else None
        elif v1 == v2:
            result = [other if rel1[1] == '=' else self]
        elif v2 in self or v1 in other:
            is_self_greater = version_compare(v1, v2) > 0
            if rel1[0] == rel2[0]:
                if rel1[0] == '>':
                    result = [self if is_self_greater else other]
                else:
                    result = [other if is_self_greater else self]
            else:
                result = [self, other]
        return SpecifierSet(result if result is not None else '')

    def union(self, other):
        """Make the union of two version specifiers

        Return a new `SpecifierSet` with version specifiers from the
        specifier and the other.

        Example:
            >>> Specifier('>= 2.2') | '<< 2.3' == '>= 2.2, << 2.3'

        """
        if isinstance(other, basestring):
            try:
                other = self.__class__(other)
            except InvalidSpecifier:
                return NotImplemented
        elif not isinstance(other, self.__class__):
            return NotImplemented

        return SpecifierSet([self, other])

    def contains(self, item):
        """Check if the specifier contains an other

        Return whether the item is contained in the version specifier.

        Example:
            >>> '2.2.1' in Specifier('<< 2.3')
            >>> '2.4' not in Specifier('<< 2.3')

        """
        return self._get_relation(self.relation)(item, self.version)

class SpecifierSet(object):

    """A set of package version specifiers

    Combine several Specifier separated by a comma. It allows to restrict
    more precisely a package version. Each package version specifier must be
    meet. Note than an empty set of specifiers will always be meet.

    """

    def __init__(self, specifiers):
        if isinstance(specifiers, basestring):
            specifiers = [s.strip() for s in specifiers.split(",")
                          if s.strip()]

        parsed = set()
        for specifier in specifiers:
            parsed.add(Specifier(specifier))

        self._specs = frozenset(parsed)

    def __repr__(self):
        return "<SpecifierSet({0!r})>".format(str(self))

    def __str__(self):
        return ",".join(sorted(str(s) for s in self._specs))

    def __hash__(self):
        return hash(self._specs)

    def __and__(self, other):
        return self.intersection(other)

    def __or__(self, other):
        return self.union(other)

    def __eq__(self, other):
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif isinstance(other, Specifier):
            other = SpecifierSet(str(other))
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        return self._specs == other._specs

    def __ne__(self, other):
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif isinstance(other, Specifier):
            other = SpecifierSet(str(other))
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        return self._specs != other._specs

    def __len__(self):
        return len(self._specs)

    def __iter__(self):
        return iter(self._specs)

    def __contains__(self, item):
        return self.contains(item)

    def intersection(self, other):
        """Make the intersection of two specifiers sets

        Return a new `SpecifierSet` with version specifier(s) common to the
        set and the other.

        Example:
            >>> SpecifierSet('>= 2.2') & '>> 2.2.1' == '>> 2.2.1'
            >>> SpecifierSet('>= 2.2, << 2.4') & '<< 2.3' == '>= 2.2, << 2.3'
            >>> SpecifierSet('>= 2.2, << 2.3') & '>= 2.4' == ''

        """
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        specifiers = set(self._specs | other._specs)
        intersection = [specifiers.pop()] if specifiers else []

        for specifier in specifiers:
            parsed = set()
            for spec in intersection:
                inter = spec & specifier
                if not inter:
                    parsed.clear()
                    break
                # TODO: validate with other specs in parsed
                parsed.update(inter._specs)
            intersection = parsed
            if not intersection:
                break
        return SpecifierSet(intersection)

    def union(self, other):
        """Make the union of two specifiers sets

        Return a new `SpecifierSet` with version specifiers from the set
        and the other.

        Example:
            >>> SpecifierSet('>= 2.2') | '<< 2.3' == '>= 2.2, << 2.3'

        """
        if isinstance(other, basestring):
            other = SpecifierSet(other)
        elif not isinstance(other, SpecifierSet):
            return NotImplemented

        specifiers = SpecifierSet([])
        specifiers._specs = frozenset(self._specs | other._specs)
        return specifiers

    def contains(self, item):
        """Check if the set contains a version specifier

        Return whether the item is contained in all version specifiers.

        Example:
            >>> '2.2.1' in SpecifierSet('>= 2.2, << 2.3')
            >>> '2.4' not in SpecifierSet('>= 2.2, << 2.3')

        """
        return all(
            s.contains(item)
            for s in self._specs
        )

# Packages and cache helpers -------------------------------------------------

def get_installed_version(*pkgnames, **kwargs):
    """Get the installed version of package(s)

    Retrieve one or more packages named `pkgnames` and return their installed
    version as a dict or as a string if only one is requested and `as_dict` is
    `False`. If `strict` is `True`, an exception will be raised if a package
    is unknown or not installed.

    """
    versions = OrderedDict()
    cache = apt.Cache()

    # Retrieve options
    as_dict = kwargs.get('as_dict', False)
    strict = kwargs.get('strict', False)
    with_repo = kwargs.get('with_repo', False)

    for pkgname in pkgnames:
        try:
            pkg = cache[pkgname]
        except KeyError:
            if strict:
                raise UnknownPackage(pkgname)
            logger.warning(m18n.n('package_unknown', pkgname=pkgname))
            continue

        try:
            version = pkg.installed.version
        except AttributeError:
            if strict:
                raise UninstalledPackage(pkgname)
            version = None

        try:
            # stable, testing, unstable
            repo = pkg.installed.origins[0].component
        except AttributeError:
            if strict:
                raise UninstalledPackage(pkgname)
            repo = ""

        if with_repo:
            versions[pkgname] = {
                "version": version,
                # when we don't have component it's because it's from a local
                # install or from an image (like in vagrant)
                "repo": repo if repo else "local",
            }
        else:
            versions[pkgname] = version

    if len(pkgnames) == 1 and not as_dict:
        return versions[pkgnames[0]]
    return versions

def meets_version_specifier(pkgname, specifier):
    """Check if a package installed version meets specifier"""
    spec = SpecifierSet(specifier)
    return get_installed_version(pkgname) in spec

# YunoHost related methods ---------------------------------------------------

def ynh_packages_version(*args, **kwargs):
    # from cli the received arguments are:
    # (Namespace(_callbacks=deque([]), _tid='_global', _to_return={}), []) {}
    # they don't seem to serve any purpose
    """Return the version of each YunoHost package"""
    return get_installed_version(
        'yunohost', 'yunohost-admin', 'moulinette', 'ssowat',
        with_repo=True
    )

def dpkg_is_broken():
    # If dpkg is broken, /var/lib/dpkg/updates
    # will contains files like 0001, 0002, ...
    # ref: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174
    if not os.path.isdir("/var/lib/dpkg/updates/"):
        return False
    return any(re.match("^[0-9]+$", f)
               for f in os.listdir("/var/lib/dpkg/updates/"))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from distutils import version

__version__ = '0.2.0'
version_info = version.StrictVersion(__version__).version
<commit_msg>Bump version<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from distutils import version

__version__ = '0.3.0'
version_info = version.StrictVersion(__version__).version
<|endoftext|>


<commit_before>import logging
from typing import Any, Dict, List

from overrides import overrides

from keras import backend as K
from keras.layers import merge, Dropout, TimeDistributed

from ...common.checks import ConfigurationError
from ...data.dataset import TextDataset
from ...layers.wrappers import EncoderWrapper
from ...training.models import DeepQaModel
from ...training.pretraining.text_pretrainer import TextPretrainer
from ..with_memory.memory_network import MemoryNetworkSolver

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

class AttentionPretrainer(TextPretrainer):
    """
    This is a generic Pretrainer for the attention mechanism of a MemoryNetworkSolver.

    The data that we take as input here is a train file, just like you would pass to a
    MemoryNetworkSolver, and a background file that has _labeled_ attention (this is different from
    what you typically pass to a MemoryNetworkSolver).  See LabeledBackgroundInstance and
    TextDataset.read_labeled_background_from_file() for more information on the expected input
    here.

    The label we get from a LabeledBackgroundInstance is the expected attention over the
    background sentences.  We use that signal to pretrain the attention component of the memory
    network.

    Because it seems very difficult to get this to train correctly with hard attention, we always
    do pre-training with soft attention, whatever you set for knowledge selector during actual
    training.  We'll set it back to what it was when we're done with pre-training.
    """
    # While it's not great, we need access to a few of the internals of the trainer, so we'll
    # disable protected access checks.
    def __init__(self, trainer, params: Dict[str, Any]):
        if not isinstance(trainer, MemoryNetworkSolver):
            raise ConfigurationError("The AttentionPretrainer needs a subclass of MemoryNetworkSolver")
        super(AttentionPretrainer, self).__init__(trainer, params)
        # NOTE: the default here needs to match the default in the KnowledgeSelector classes.
        self._old_hard_attention_setting = self.trainer.knowledge_selector_params.get('hard_selection', False)
        self.trainer.knowledge_selector_params['hard_selection'] = False
        self.name = 'AttentionPretrainer'

    @overrides
    def on_finished(self):
        self.trainer.knowledge_selector_params['hard_selection'] = self._old_hard_attention_setting
        for layer in self.trainer.knowledge_selector_layers.values():
            layer.hard_selection = self._old_hard_attention_setting

    @overrides
    def _instance_type(self):
        # pylint: disable=protected-access
        return self.trainer._instance_type()

    @overrides
    def _load_dataset_from_files(self, files: List[str]):
        """
        This method requires two input files, one with training examples, and one with labeled
        background corresponding to the training examples.

        Note that we're calling TextDataset.read_labeled_background_from_file() here, not
        TextDataset.read_background_from_file(), because we want our Instances to have labeled
        attention for pretraining, not labeled answers.
        """
        dataset = super(AttentionPretrainer, self)._load_dataset_from_files(files)
        return TextDataset.read_labeled_background_from_file(dataset, files[1])

    @overrides
    def _build_model(self):
        """
        This model basically just pulls out the first half of the memory network model, up until
        the first attention layer.

        Because the trainer we're pretraining might have some funny input shapes, we don't use
        trainer._get_question_shape() directly; instead we re-create it for the case where we don't
        have TimeDistributed input.
        """
        # What follows is a lightly-edited version of the code from
        # MemoryNetworkSolver._build_model().
        # pylint: disable=protected-access
        sentence_shape = (self.trainer.max_sentence_length,)
        background_shape = (self.trainer.max_knowledge_length, self.trainer.max_sentence_length)

        sentence_input_layer, sentence_embedding = self.trainer._get_embedded_sentence_input(
                input_shape=sentence_shape, name_prefix="sentence")
        background_input_layer, background_embedding = self.trainer._get_embedded_sentence_input(
                input_shape=background_shape, name_prefix="background")

        sentence_encoder = self.trainer._get_sentence_encoder()
        while isinstance(sentence_encoder, TimeDistributed):
            sentence_encoder = sentence_encoder.layer

        background_encoder = EncoderWrapper(sentence_encoder, name='background_encoder')
        encoded_sentence = sentence_encoder(sentence_embedding)  # (samples, word_dim)
        encoded_background = background_encoder(background_embedding)  # (samples, background_len, word_dim)

        merge_mode = lambda layer_outs: K.concatenate([K.expand_dims(layer_outs[0], dim=1),
                                                       K.expand_dims(layer_outs[1], dim=1),
                                                       layer_outs[2]],
                                                      axis=1)
        def merge_mask(mask_outs):
            return K.concatenate([K.expand_dims(K.zeros_like(mask_outs[2][:, 0]), dim=1),
                                  K.expand_dims(K.zeros_like(mask_outs[2][:, 0]), dim=1),
                                  mask_outs[2]],
                                 axis=1)

        # We don't generate a memory representation here so we just pass another encoded_sentence.
        merged_encoded_rep = merge([encoded_sentence, encoded_sentence, encoded_background],
                                   mode=merge_mode,
                                   output_shape=(self.trainer.max_knowledge_length + 2,
                                                 self.trainer.max_sentence_length),
                                   output_mask=merge_mask,
                                   name='concat_sentence_with_background_%d' % 0)

        regularized_merged_rep = Dropout(0.2)(merged_encoded_rep)
        knowledge_selector = self.trainer._get_knowledge_selector(0)
        while isinstance(knowledge_selector, TimeDistributed):
            knowledge_selector = knowledge_selector.layer
        attention_weights = knowledge_selector(regularized_merged_rep)

        input_layers = [sentence_input_layer, background_input_layer]
        return DeepQaModel(input=input_layers, output=attention_weights)
<commit_msg>Fixed a crazy bug (not sure how this even ran previously)<commit_after>import logging
from typing import Any, Dict, List

from overrides import overrides

from keras import backend as K
from keras.layers import merge, Dropout, TimeDistributed

from ...common.checks import ConfigurationError
from ...data.dataset import TextDataset
from ...layers.wrappers import EncoderWrapper
from ...training.models import DeepQaModel
from ...training.pretraining.text_pretrainer import TextPretrainer
from ..with_memory.memory_network import MemoryNetworkSolver

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

class AttentionPretrainer(TextPretrainer):
    """
    This is a generic Pretrainer for the attention mechanism of a MemoryNetworkSolver.

    The data that we take as input here is a train file, just like you would pass to a
    MemoryNetworkSolver, and a background file that has _labeled_ attention (this is different from
    what you typically pass to a MemoryNetworkSolver).  See LabeledBackgroundInstance and
    TextDataset.read_labeled_background_from_file() for more information on the expected input
    here.

    The label we get from a LabeledBackgroundInstance is the expected attention over the
    background sentences.  We use that signal to pretrain the attention component of the memory
    network.

    Because it seems very difficult to get this to train correctly with hard attention, we always
    do pre-training with soft attention, whatever you set for knowledge selector during actual
    training.  We'll set it back to what it was when we're done with pre-training.
    """
    # While it's not great, we need access to a few of the internals of the trainer, so we'll
    # disable protected access checks.
    def __init__(self, trainer, params: Dict[str, Any]):
        if not isinstance(trainer, MemoryNetworkSolver):
            raise ConfigurationError("The AttentionPretrainer needs a subclass of MemoryNetworkSolver")
        super(AttentionPretrainer, self).__init__(trainer, params)
        # NOTE: the default here needs to match the default in the KnowledgeSelector classes.
        self._old_hard_attention_setting = self.trainer.knowledge_selector_params.get('hard_selection', False)
        self.trainer.knowledge_selector_params['hard_selection'] = False
        self.name = 'AttentionPretrainer'

    @overrides
    def on_finished(self):
        self.trainer.knowledge_selector_params['hard_selection'] = self._old_hard_attention_setting
        for layer in self.trainer.knowledge_selector_layers.values():
            layer.hard_selection = self._old_hard_attention_setting

    @overrides
    def _instance_type(self):
        # pylint: disable=protected-access
        return self.trainer._instance_type()

    @overrides
    def _load_dataset_from_files(self, files: List[str]):
        """
        This method requires two input files, one with training examples, and one with labeled
        background corresponding to the training examples.

        Note that we're calling TextDataset.read_labeled_background_from_file() here, not
        TextDataset.read_background_from_file(), because we want our Instances to have labeled
        attention for pretraining, not labeled answers.
        """
        dataset = super(AttentionPretrainer, self)._load_dataset_from_files(files)
        return TextDataset.read_labeled_background_from_file(dataset, files[1])

    @overrides
    def _build_model(self):
        """
        This model basically just pulls out the first half of the memory network model, up until
        the first attention layer.

        Because the trainer we're pretraining might have some funny input shapes, we don't use
        trainer._get_question_shape() directly; instead we re-create it for the case where we don't
        have TimeDistributed input.
        """
        # What follows is a lightly-edited version of the code from
        # MemoryNetworkSolver._build_model().
        # pylint: disable=protected-access
        sentence_shape = (self.trainer.max_sentence_length,)
        background_shape = (self.trainer.max_knowledge_length, self.trainer.max_sentence_length)

        sentence_input_layer, sentence_embedding = self.trainer._get_embedded_sentence_input(
                input_shape=sentence_shape, name_prefix="sentence")
        background_input_layer, background_embedding = self.trainer._get_embedded_sentence_input(
                input_shape=background_shape, name_prefix="background")

        sentence_encoder = self.trainer._get_sentence_encoder()
        while isinstance(sentence_encoder, TimeDistributed):
            sentence_encoder = sentence_encoder.layer

        background_encoder = EncoderWrapper(sentence_encoder, name='background_encoder')
        encoded_sentence = sentence_encoder(sentence_embedding)  # (samples, word_dim)
        encoded_background = background_encoder(background_embedding)  # (samples, background_len, word_dim)

        merge_mode = lambda layer_outs: K.concatenate([K.expand_dims(layer_outs[0], dim=1),
                                                       K.expand_dims(layer_outs[1], dim=1),
                                                       layer_outs[2]],
                                                      axis=1)
        def merge_mask(mask_outs):
            return K.concatenate([K.expand_dims(K.zeros_like(mask_outs[2][:, 0]), dim=1),
                                  K.expand_dims(K.zeros_like(mask_outs[2][:, 0]), dim=1),
                                  mask_outs[2]],
                                 axis=1)

        # We don't generate a memory representation here so we just pass another encoded_sentence.
        merged_encoded_rep = merge([encoded_sentence, encoded_sentence, encoded_background],
                                   mode=merge_mode,
                                   output_shape=(self.trainer.max_knowledge_length + 2,
                                                 self.trainer.embedding_size),
                                   output_mask=merge_mask,
                                   name='concat_sentence_with_background_%d' % 0)

        regularized_merged_rep = Dropout(0.2)(merged_encoded_rep)
        knowledge_selector = self.trainer._get_knowledge_selector(0)
        while isinstance(knowledge_selector, TimeDistributed):
            knowledge_selector = knowledge_selector.layer
        attention_weights = knowledge_selector(regularized_merged_rep)

        input_layers = [sentence_input_layer, background_input_layer]
        return DeepQaModel(input=input_layers, output=attention_weights)
<|endoftext|>


<commit_before>#!/usr/bin/python3 python3

# This is a heavily modified version of the jupyterhub cull-idle script, retrieved October 31st 2016 from
# https://github.com/jupyterhub/jupyterhub/blob/master/examples/cull-idle/cull_idle_servers.py
# Due to environment variable bugs we have changed the mechanism by which we attain the jupyterhub
# API key.
# Author: Eli <eli@zagaran.com>

import datetime
import json
import os
import sys
import boto3
import logging

sys.path.insert(1, '/etc/jupyterhub')
from models import Server

from dateutil.parser import parse as parse_date

from botocore.exceptions import ClientError, WaiterError
from concurrent.futures import ThreadPoolExecutor
from tornado.gen import coroutine, sleep
from tornado.log import app_log
from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPError
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.options import define, options, parse_command_line

with open("/etc/jupyterhub/server_config.json", "r") as f:
    SERVER_PARAMS = json.load(f) # load local server parameters

app_log.setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)

thread_pool = ThreadPoolExecutor(10)

@coroutine
def retry(function, *args, **kwargs):
    """ Retries a function up to max_retries, waiting `timeout` seconds between tries.
        This function is designed to retry both boto3 and fabric calls.  In the
        case of boto3, it is necessary because sometimes aws calls return too
        early and a resource needed by the next call is not yet available. """
    max_retries = kwargs.pop("max_retries", 20)
    timeout = kwargs.pop("timeout", 0.25)
    for attempt in range(max_retries):
        try:
            ret = yield thread_pool.submit(function, *args, **kwargs)
            return ret
        except (ClientError, WaiterError) as e:
            app_log.warn("encountered %s, waiting for %s seconds before retrying..." % (type(e), timeout) )
            yield sleep(timeout)
    else:
        print("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
        raise e

@coroutine
def manually_kill_server(user_name):
    # Get our AWS server db's instance for the user
    try:
        server = Server.get_server(user_name)
        app_log.debug("Checking server for %s manually..." % user_name)
    except Server.DoesNotExist:
        # it is not necessarily the case that a server will exist, we return early if that is the case.
        app_log.warn("There is no matching, allocated server for user %s" % user_name)
        return
    # get server instance information
    resource = yield retry(boto3.resource, "ec2", region_name=SERVER_PARAMS["REGION"])
    instance = yield retry(resource.Instance, server.server_id)
    # instance object is lazy, run this to get full info...
    yield retry(instance.load)

    #stop server if state is running (possible states are stopped, stopping, pending, shutting-down, terminated, and running)
    if instance.state["Name"] == "running":
        retry(instance.stop)
        app_log.info("manually killed server for user %s" % user_name)
    else:
        app_log.debug("server state for user %s is %s, no action taken" % (user_name, instance.state["Name"]))

@coroutine
def cull_idle(url, api_token, timeout):
    #last valid activity timestame
    cull_limit = datetime.datetime.utcnow() - datetime.timedelta(seconds=timeout)

    #get user list
    hub_api_authorization_header = { 'Authorization': 'token %s' % api_token}
    users_request = HTTPRequest(url=url + '/users', headers=hub_api_authorization_header )

    #run request tornado-asynchronously, extract user list (contains more information)
    resp = yield AsyncHTTPClient().fetch(users_request)
    all_users = json.loads(resp.body.decode('utf8', 'replace'))

    #build a bunch of (asynchronous) HTTP request futures...
    stop_notebook_futures = []
    servers_to_check = []
    dont_cull_these = set()
    for user in all_users:

        #extract last activity time, determine cullability of the server.
        last_activity = parse_date(user['last_activity'])
        should_cull = last_activity < cull_limit
        user_name = user['name']
        app_log.debug("checking %s, last activity: %s, server: %s" % (user_name, last_activity, user['server']) )

        if not should_cull:
            dont_cull_these.add(user_name)

        #server should be culled:
        if user['server'] and should_cull:
            app_log.info("Culling %s (inactive since %s)", user_name, last_activity)
            stop_user_request = HTTPRequest(url=url + '/users/%s/server' % user_name,
                                            method='DELETE',
                                            headers=hub_api_authorization_header )
            stop_notebook_futures.append( (user_name, AsyncHTTPClient().fetch(stop_user_request)) )

        #Server status is None, which means actual status needs to be checked.
        if not user['server'] and should_cull:
            servers_to_check.append(user_name)

        #server should not be culled, just a log statement
        if user['server'] and not should_cull:
            app_log.info("Not culling %s (active since %s)", user['name'], last_activity)

    # Cull notebooks using normal API.
    for (user_name, cull_request) in stop_notebook_futures:
        try:
            yield cull_request #this line actually runs the api call to kill a server
        except HTTPError:
            #Due to a bug in Jupyterhub
            app_log.error("Something went wrong culling %s, will be manually killing it.", user_name)
            servers_to_check.append( user_name )
            continue
        app_log.info("Finished culling %s", user_name)

    for user_name in servers_to_check:
        if user_name not in dont_cull_these:
            yield manually_kill_server(user_name)

if __name__ == '__main__':
    define('url', default=os.environ.get('JUPYTERHUB_API_URL'), help="The JupyterHub API URL")
    define('timeout', default=SERVER_PARAMS["JUPYTER_NOTEBOOK_TIMEOUT"], help="The idle timeout (in seconds)")
    define('cull_every', default=300, help="The interval (in seconds) for checking for idle servers to cull")

    parse_command_line()
    if not options.cull_every:
        options.cull_every = options.timeout // 2

    # we were having significant issues with environment variables, we will just read from a file
    # api_token = os.environ['JUPYTERHUB_API_TOKEN']
    with open("/etc/jupyterhub/api_token.txt", 'r') as f:
        # for culler script to work without modification we need to set this value in the machine environment.
        api_token = f.read().strip()

    loop = IOLoop.current()
    cull = lambda: cull_idle(options.url, api_token, options.timeout)
    # run once before scheduling periodic call
    loop.run_sync(cull)
    # schedule periodic cull
    pc = PeriodicCallback(cull, 1e3 * options.cull_every)
    pc.start()
    try:
        loop.start()
    except KeyboardInterrupt:
        pass
<commit_msg>use timezone None when compare server activities in cull_idle_servers.py<commit_after>#!/usr/bin/python3 python3

# This is a heavily modified version of the jupyterhub cull-idle script, retrieved October 31st 2016 from
# https://github.com/jupyterhub/jupyterhub/blob/master/examples/cull-idle/cull_idle_servers.py
# Due to environment variable bugs we have changed the mechanism by which we attain the jupyterhub
# API key.
# Author: Eli <eli@zagaran.com>

import datetime
import json
import os
import sys
import boto3
import logging

sys.path.insert(1, '/etc/jupyterhub')
from models import Server

from dateutil.parser import parse as parse_date

from botocore.exceptions import ClientError, WaiterError
from concurrent.futures import ThreadPoolExecutor
from tornado.gen import coroutine, sleep
from tornado.log import app_log
from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPError
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.options import define, options, parse_command_line

with open("/etc/jupyterhub/server_config.json", "r") as f:
    SERVER_PARAMS = json.load(f) # load local server parameters

app_log.setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)

thread_pool = ThreadPoolExecutor(10)

@coroutine
def retry(function, *args, **kwargs):
    """ Retries a function up to max_retries, waiting `timeout` seconds between tries.
        This function is designed to retry both boto3 and fabric calls.  In the
        case of boto3, it is necessary because sometimes aws calls return too
        early and a resource needed by the next call is not yet available. """
    max_retries = kwargs.pop("max_retries", 20)
    timeout = kwargs.pop("timeout", 0.25)
    for attempt in range(max_retries):
        try:
            ret = yield thread_pool.submit(function, *args, **kwargs)
            return ret
        except (ClientError, WaiterError) as e:
            app_log.warn("encountered %s, waiting for %s seconds before retrying..." % (type(e), timeout) )
            yield sleep(timeout)
    else:
        print("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
        raise e

@coroutine
def manually_kill_server(user_name):
    # Get our AWS server db's instance for the user
    try:
        server = Server.get_server(user_name)
        app_log.debug("Checking server for %s manually..." % user_name)
    except Server.DoesNotExist:
        # it is not necessarily the case that a server will exist, we return early if that is the case.
        app_log.warn("There is no matching, allocated server for user %s" % user_name)
        return
    # get server instance information
    resource = yield retry(boto3.resource, "ec2", region_name=SERVER_PARAMS["REGION"])
    instance = yield retry(resource.Instance, server.server_id)
    # instance object is lazy, run this to get full info...
    yield retry(instance.load)

    #stop server if state is running (possible states are stopped, stopping, pending, shutting-down, terminated, and running)
    if instance.state["Name"] == "running":
        retry(instance.stop)
        app_log.info("manually killed server for user %s" % user_name)
    else:
        app_log.debug("server state for user %s is %s, no action taken" % (user_name, instance.state["Name"]))

@coroutine
def cull_idle(url, api_token, timeout):
    #last valid activity timestame
    cull_limit = datetime.datetime.utcnow() - datetime.timedelta(seconds=timeout)

    #get user list
    hub_api_authorization_header = { 'Authorization': 'token %s' % api_token}
    users_request = HTTPRequest(url=url + '/users', headers=hub_api_authorization_header )

    #run request tornado-asynchronously, extract user list (contains more information)
    resp = yield AsyncHTTPClient().fetch(users_request)
    all_users = json.loads(resp.body.decode('utf8', 'replace'))

    #build a bunch of (asynchronous) HTTP request futures...
    stop_notebook_futures = []
    servers_to_check = []
    dont_cull_these = set()
    for user in all_users:

        #extract last activity time, determine cullability of the server.
        last_activity = parse_date(user['last_activity'])
        #should_cull = last_activity < cull_limit
        should_cull = last_activity.replace(tzinfo=None)  < cull_limit.replace(tzinfo=None)
        user_name = user['name']
        app_log.debug("checking %s, last activity: %s, server: %s" % (user_name, last_activity, user['server']) )

        if not should_cull:
            dont_cull_these.add(user_name)

        #server should be culled:
        if user['server'] and should_cull:
            app_log.info("Culling %s (inactive since %s)", user_name, last_activity)
            stop_user_request = HTTPRequest(url=url + '/users/%s/server' % user_name,
                                            method='DELETE',
                                            headers=hub_api_authorization_header )
            stop_notebook_futures.append( (user_name, AsyncHTTPClient().fetch(stop_user_request)) )

        #Server status is None, which means actual status needs to be checked.
        if not user['server'] and should_cull:
            servers_to_check.append(user_name)

        #server should not be culled, just a log statement
        if user['server'] and not should_cull:
            app_log.info("Not culling %s (active since %s)", user['name'], last_activity)

    # Cull notebooks using normal API.
    for (user_name, cull_request) in stop_notebook_futures:
        try:
            yield cull_request #this line actually runs the api call to kill a server
        except HTTPError:
            #Due to a bug in Jupyterhub
            app_log.error("Something went wrong culling %s, will be manually killing it.", user_name)
            servers_to_check.append( user_name )
            continue
        app_log.info("Finished culling %s", user_name)

    for user_name in servers_to_check:
        if user_name not in dont_cull_these:
            yield manually_kill_server(user_name)

if __name__ == '__main__':
    define('url', default=os.environ.get('JUPYTERHUB_API_URL'), help="The JupyterHub API URL")
    define('timeout', default=SERVER_PARAMS["JUPYTER_NOTEBOOK_TIMEOUT"], help="The idle timeout (in seconds)")
    define('cull_every', default=300, help="The interval (in seconds) for checking for idle servers to cull")

    parse_command_line()
    if not options.cull_every:
        options.cull_every = options.timeout // 2

    # we were having significant issues with environment variables, we will just read from a file
    # api_token = os.environ['JUPYTERHUB_API_TOKEN']
    with open("/etc/jupyterhub/api_token.txt", 'r') as f:
        # for culler script to work without modification we need to set this value in the machine environment.
        api_token = f.read().strip()

    loop = IOLoop.current()
    cull = lambda: cull_idle(options.url, api_token, options.timeout)
    # run once before scheduling periodic call
    loop.run_sync(cull)
    # schedule periodic cull
    pc = PeriodicCallback(cull, 1e3 * options.cull_every)
    pc.start()
    try:
        loop.start()
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>#script to generate all keys
import os

gsonPath = "/usr/local/google/users/arkajit/eclipse/workspaces/keymaster/" + \
           "Keyczar-Java/third_party/gson/gson.jar"
binPath = "/usr/local/google/users/arkajit/eclipse/workspaces/keymaster/" + \
           "Keyczar-Java/bin/"
paths = binPath + ":" + gsonPath
cmd = "java -cp " + paths + " com.google.keyczar.KeyczarTool"

keyFiles = [("../testdata/aes/", "crypt", None),
            ("../testdata/rsa/", "crypt", "rsa"),
            ("../testdata/hmac/", "sign", None),
            ("../testdata/dsa/", "sign", "dsa"),
            ("../testdata/rsa-sign/", "sign", "rsa")]

pubKeyFiles = [("../testdata/dsa/", "../testdata/dsa.public/"),
               ("../testdata/rsa-sign/", "../testdata/rsa-sign.public/")]

def cleanUp(directory):
  for file in os.listdir(directory):
    filePath = os.path.join(directory, file)
    if not os.path.isdir(filePath): os.remove(filePath)

def createFlags(loc, name=None, dest=None, purpose=None, status=None,
                version=None, asymmetric=None):
  flags = " "
  if name is not None: flags += "--name="+name+" "
  if loc is not None: flags += "--location="+loc+" "
  if dest is not None: flags += "--destination="+dest+" "
  if purpose is not None: flags += "--purpose="+purpose+" "
  if status is not None: flags += "--status="+status+" "
  if version is not None: flags += "--version="+version+" "
  if asymmetric is not None: flags += "--asymmetric="+asymmetric+" "
  return flags[:-1]

def create(loc, purpose, name=None, asymmetric=None):
  args = createFlags(name=name, loc=loc, purpose=purpose, asymmetric=asymmetric)
  os.chdir(binPath)
  os.system(cmd + " create" + args)

def addKey(loc, status="active"):
  args = createFlags(loc=loc, status=status)
  os.chdir(binPath)
  os.system(cmd + " addkey" + args)

def pubKey(loc, dest):
  args = createFlags(loc=loc, dest=dest)
  os.chdir(binPath)
  os.system(cmd + " pubkey" + args)

def useKey(loc, dest, data="This is a test"):
  args = createFlags(loc=loc, dest=dest)
  os.chdir(binPath)
  os.system(cmd + ' usekey "' + data + '"' + args)

print useKey("../testdata/aes/", "../testdata/aes/output")

##generate private key sets
#print "Generating private key sets..."
#for (loc, purpose, asymmetric) in keyFiles:
#  cleanUp(loc)
#  create(name="test", loc=loc, purpose=purpose, asymmetric=asymmetric)
#  addKey(loc, "primary")
#  addKey(loc, "primary")
#
##export public key sets
#print "Exporting public key sets..."
#for (loc, dest) in pubKeyFiles:
#  cleanUp(dest)
#  pubKey(loc, dest)
#
#print "Done!"
  <commit_msg>Maybe fixed path issue<commit_after>#script to generate all keys
import os

cwd = os.getcwd()

gsonPath = cwd + "/../third_party/gson/gson.jar"
log4jPath = cwd + "/../third_party/log4j/log4j-1.2.15.jar"
binPath = cwd + "/../bin/"
paths = binPath + ":" + gsonPath + ":" + log4jPath
cmd = "java -cp " + paths + " com.google.keyczar.KeyczarTool"

keyFiles = [("../testdata/aes/", "crypt", None),
            ("../testdata/rsa/", "crypt", "rsa"),
            ("../testdata/hmac/", "sign", None),
            ("../testdata/dsa/", "sign", "dsa"),
            ("../testdata/rsa-sign/", "sign", "rsa")]

pubKeyFiles = [("../testdata/dsa/", "../testdata/dsa.public/"),
               ("../testdata/rsa-sign/", "../testdata/rsa-sign.public/")]

def cleanUp(directory):
  for file in os.listdir(directory):
    filePath = os.path.join(directory, file)
    if not os.path.isdir(filePath): os.remove(filePath)

def createFlags(loc, name=None, dest=None, purpose=None, status=None,
                version=None, asymmetric=None):
  flags = " "
  if name is not None: flags += "--name="+name+" "
  if loc is not None: flags += "--location="+loc+" "
  if dest is not None: flags += "--destination="+dest+" "
  if purpose is not None: flags += "--purpose="+purpose+" "
  if status is not None: flags += "--status="+status+" "
  if version is not None: flags += "--version="+version+" "
  if asymmetric is not None: flags += "--asymmetric="+asymmetric+" "
  return flags[:-1]

def create(loc, purpose, name=None, asymmetric=None):
  args = createFlags(name=name, loc=loc, purpose=purpose, asymmetric=asymmetric)
  os.chdir(binPath)
  os.system(cmd + " create" + args)

def addKey(loc, status="active"):
  args = createFlags(loc=loc, status=status)
  os.chdir(binPath)
  os.system(cmd + " addkey" + args)

def pubKey(loc, dest):
  args = createFlags(loc=loc, dest=dest)
  os.chdir(binPath)
  os.system(cmd + " pubkey" + args)

def useKey(loc, dest, data="This is a test"):
  args = createFlags(loc=loc, dest=dest)
  os.chdir(binPath)
  os.system(cmd + ' usekey "' + data + '"' + args)

print useKey("../testdata/aes/", "../testdata/aes/output")

##generate private key sets
#print "Generating private key sets..."
#for (loc, purpose, asymmetric) in keyFiles:
#  cleanUp(loc)
#  create(name="test", loc=loc, purpose=purpose, asymmetric=asymmetric)
#  addKey(loc, "primary")
#  addKey(loc, "primary")
#
##export public key sets
#print "Exporting public key sets..."
#for (loc, dest) in pubKeyFiles:
#  cleanUp(dest)
#  pubKey(loc, dest)
#
#print "Done!"
  <|endoftext|>


<commit_before>#!/usr/bin/env python

from mininet.cli import CLI
from mininet.link import Link
from mininet.net import Mininet
from mininet.node import RemoteController
from mininet.term import makeTerm

if '__main__' == __name__:
    net = Mininet(controller=RemoteController)

    c0 = net.addController('c0', port=6633)

    s1 = net.addSwitch('s1')

    h1 = net.addHost('h1')
    h2 = net.addHost('h2', mac='00:00:00:00:00:22')
    h3 = net.addHost('h3', mac='00:00:00:00:00:23')
    h4 = net.addHost('h4', mac='00:00:00:00:00:24')

    Link(s1, h1)
    Link(s1, h1)
    Link(s1, h2)
    Link(s1, h3)
    Link(s1, h4)

    net.build()
    c0.start()
    s1.start([c0])

    net.startTerms()

    CLI(net)

    net.stop()
<commit_msg>ko: link_aggregation: Fix a import Link error problem<commit_after>#!/usr/bin/env python

from mininet.cli import CLI
from mininet.net import Mininet
from mininet.node import RemoteController
from mininet.term import makeTerm

if '__main__' == __name__:
    net = Mininet(controller=RemoteController)

    c0 = net.addController('c0', port=6633)

    s1 = net.addSwitch('s1')

    h1 = net.addHost('h1')
    h2 = net.addHost('h2', mac='00:00:00:00:00:22')
    h3 = net.addHost('h3', mac='00:00:00:00:00:23')
    h4 = net.addHost('h4', mac='00:00:00:00:00:24')

    net.addLink(s1, h1)
    net.addLink(s1, h1)
    net.addLink(s1, h2)
    net.addLink(s1, h3)
    net.addLink(s1, h4)

    net.build()
    c0.start()
    s1.start([c0])

    net.startTerms()

    CLI(net)

    net.stop()
<|endoftext|>


<commit_before>import logging
import requests
import waffle

from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from django.utils import timezone

from goals.sequence import get_next_useractions_in_sequence
from utils.user_utils import to_utc

logger = logging.getLogger(__name__)

FB_MESSENGER_URL = 'https://brad.ngrok.io/send/{user_id}'

# -----------------------------------------------------------------------------
#
# NOTICE:
#
# This script will likely send a message to a user every time it is run, so
# don't schedule it to run too quickly.
#
# -----------------------------------------------------------------------------

class Command(BaseCommand):
    help = 'Sends messages to our FB messenger users.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--user',
            action='store',
            dest='user',
            default=None,
            help=("Restrict this command to the given User. "
                  "Accepts a username, email, or id")
        )

    def _get_users(self, options):
        User = get_user_model()

        # Check to see if we're just running this for a specific user
        if options.get('user', False):
            user = options['user']
            try:
                if user.isnumeric():
                    criteria = Q(id=user)
                else:
                    criteria = (Q(username=user) | Q(email=user))
                # Return an iterable, but use .get() to raise an exception
                # if there's not matching user.
                user = User.objects.get(criteria)
                return [user]
            except User.DoesNotExist:
                raise CommandError("Could not find user: {0}".format(user))

        # Find all the users that no email, no GCM devices and only 16-digit usernames.
        return User.objects.filter(
            email='',
            username__regex=r'\d{16}',
            gcmdevice__isnull=True
        ).distinct()

    def send(self, user_id, message):
        payload = {'message': message}
        requests.post(FB_MESSENGER_URL.format(user_id=user_id), payload)
        self.stdout.write("Sent to {}: {}".format(user_id, message))

    def handle(self, *args, **options):
        # This switch allows us to completely disable creation of notifications
        if not waffle.switch_is_active('goals-fb-messenger'):
            return None

        now = timezone.now()

        # XXX: Assume we're all in central time, don't send at night.
        if now.hour < 12 or now.hour > 23:
            console.log("not running during evening/early morning")
            return None

        users = self._get_users(options)
        self.stdout.write("Found {} users".format(len(users)))

        for user in users:
            # Only support Sequenced Goals, Actions
            for ua in get_next_useractions_in_sequence(user):
                # Retrive the `next_reminder`, which will be in the
                # user's timezone, and may have been created weeks or
                # months ago. This is slightly different from other
                # UserActions, because we don't re-generate
                # these on the fly
                deliver_on = to_utc(ua.next_reminder)

                if deliver_on < now:
                    text = "{} {}".format(
                        ua.get_notification_text(),
                        ua.action.description
                    )
                    self.send(user.username, text)
                    break  # only send 1 message.
<commit_msg>includ useraction's id in messages to FB messenger users<commit_after>import logging
import requests
import waffle

from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from django.utils import timezone

from goals.sequence import get_next_useractions_in_sequence
from utils.user_utils import to_utc

logger = logging.getLogger(__name__)

FB_MESSENGER_URL = 'https://brad.ngrok.io/send/{user_id}'

# -----------------------------------------------------------------------------
#
# NOTICE:
#
# This script will likely send a message to a user every time it is run, so
# don't schedule it to run too quickly.
#
# -----------------------------------------------------------------------------

class Command(BaseCommand):
    help = 'Sends messages to our FB messenger users.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--user',
            action='store',
            dest='user',
            default=None,
            help=("Restrict this command to the given User. "
                  "Accepts a username, email, or id")
        )

    def _get_users(self, options):
        User = get_user_model()

        # Check to see if we're just running this for a specific user
        if options.get('user', False):
            user = options['user']
            try:
                if user.isnumeric():
                    criteria = Q(id=user)
                else:
                    criteria = (Q(username=user) | Q(email=user))
                # Return an iterable, but use .get() to raise an exception
                # if there's not matching user.
                user = User.objects.get(criteria)
                return [user]
            except User.DoesNotExist:
                raise CommandError("Could not find user: {0}".format(user))

        # Find all the users that no email, no GCM devices and only 16-digit usernames.
        return User.objects.filter(
            email='',
            username__regex=r'\d{16}',
            gcmdevice__isnull=True
        ).distinct()

    def send(self, user_id, message, useraction_id):
        payload = {'message': message, 'id': useraction_id}
        requests.post(FB_MESSENGER_URL.format(user_id=user_id), payload)
        self.stdout.write("Sent to {}: {}".format(user_id, message))

    def handle(self, *args, **options):
        # This switch allows us to completely disable creation of notifications
        if not waffle.switch_is_active('goals-fb-messenger'):
            return None

        now = timezone.now()

        # XXX: Assume we're all in central time, don't send at night.
        if now.hour < 12 or now.hour > 23:
            self.stderr.write("not running during evening/early morning")
            return None

        users = self._get_users(options)
        self.stdout.write("Found {} users".format(len(users)))

        for user in users:
            # Only support Sequenced Goals, Actions
            for ua in get_next_useractions_in_sequence(user):
                # Retrive the `next_reminder`, which will be in the
                # user's timezone, and may have been created weeks or
                # months ago. This is slightly different from other
                # UserActions, because we don't re-generate
                # these on the fly
                deliver_on = to_utc(ua.next_reminder)

                if deliver_on < now:
                    text = "{} {}".format(
                        ua.get_notification_text(),
                        ua.action.description
                    )
                    self.send(user.username, text, ua.id)
                    break  # only send 1 message.
<|endoftext|>


<commit_before>import kronos

from django.core.management.base import BaseCommand, CommandError

class Command(BaseCommand):
    args = '<task>'
    help = 'Run the given task'

    def handle(self, task_name, **options):
        kronos.load()

        for task in kronos.registry:
            if task.name == task_name:
                if task.function:
                    return task.function()
                else:
                    raise CommandError('This is a django command. You have '
                        'to run it via python manage.py {0}'
                        .format(task_name))
        raise CommandError('Task \'%s\' not found' % task_name)
<commit_msg>Fixed argument handling in 'runtask' command<commit_after>import kronos

from django.core.management.base import BaseCommand, CommandError

class Command(BaseCommand):
    args = '<task>'
    help = 'Run the given task'

    def add_arguments(self, parser):
        parser.add_argument('task', nargs='?', type=str)

    def handle(self, **options):
        kronos.load()

        task_name = options.get('task')

        for task in kronos.registry:
            if task.name == task_name:
                if task.function:
                    return task.function()
                else:
                    raise CommandError('This is a django command. You have '
                        'to run it via python manage.py {0}'
                        .format(task_name))
        raise CommandError('Task \'%s\' not found' % task_name)
<|endoftext|>


<commit_before># Copyright 2021 Binovo IT Human Project SL
# Copyright 2021 Landoo Sistemas de Informacion SL
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

{
    "name": "TicketBAI - API",
    "version": "14.0.2.1.0",
    "category": "Accounting & Finance",
    "website": "https://github.com/OCA/l10n-spain",
    "author": "Binovo," "Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "application": False,
    "installable": True,
    "auto_install": False,
    "development_status": "Beta",
    "maintainers": ["ao-landoo"],
    "depends": ["base", "base_setup"],
    "external_dependencies": {
        "python": [
            "cryptography",
            "pyOpenSSL",
            "qrcode",
            "xmlsig",
            "xmltodict",
        ],
    },
    "data": [
        "security/ir.model.access.csv",
        "data/tax_agency_data.xml",
        "data/ticketbai_invoice.xml",
        "views/l10n_es_ticketbai_api_views.xml",
        "views/res_company_views.xml",
        "views/res_config_settings_views.xml",
        "views/ticketbai_certificate_views.xml",
        "views/ticketbai_installation_views.xml",
    ],
    "demo": ["demo/res_partner_demo.xml"],
}
<commit_msg>l10n_es_ticketbai_api 14.0.2.1.1<commit_after># Copyright 2021 Binovo IT Human Project SL
# Copyright 2021 Landoo Sistemas de Informacion SL
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

{
    "name": "TicketBAI - API",
    "version": "14.0.2.1.1",
    "category": "Accounting & Finance",
    "website": "https://github.com/OCA/l10n-spain",
    "author": "Binovo," "Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "application": False,
    "installable": True,
    "auto_install": False,
    "development_status": "Beta",
    "maintainers": ["ao-landoo"],
    "depends": ["base", "base_setup"],
    "external_dependencies": {
        "python": [
            "cryptography",
            "pyOpenSSL",
            "qrcode",
            "xmlsig",
            "xmltodict",
        ],
    },
    "data": [
        "security/ir.model.access.csv",
        "data/tax_agency_data.xml",
        "data/ticketbai_invoice.xml",
        "views/l10n_es_ticketbai_api_views.xml",
        "views/res_company_views.xml",
        "views/res_config_settings_views.xml",
        "views/ticketbai_certificate_views.xml",
        "views/ticketbai_installation_views.xml",
    ],
    "demo": ["demo/res_partner_demo.xml"],
}
<|endoftext|>