<commit_before># Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Tools for compiling and importing Python protos on the fly."""

from collections.abc import Mapping
import importlib.util
import logging
import os
from pathlib import Path
import subprocess
import shlex
import tempfile
from types import ModuleType
from typing import (Dict, Generic, Iterable, Iterator, List, NamedTuple, Set,
                    Tuple, TypeVar, Union)

# Temporarily set the root logger level to critical while importing yapf.
# This silences INFO level messages from
# environment/cipd/packages/python/lib/python3.9/lib2to3/driver.py
# when it writes Grammar3.*.pickle and PatternGrammar3.*.pickle files.
_original_level = 0
for handler in logging.getLogger().handlers:
    if type(handler) == logging.StreamHandler:  # pylint: disable=unidiomatic-typecheck
        if handler.level > _original_level:
            _original_level = handler.level
        handler.level = logging.CRITICAL

try:
    from yapf.yapflib import yapf_api  # type: ignore[import] # pylint: disable=wrong-import-position
except ImportError:
    yapf_api = None

# Restore the original stderr/out log handler level.
for handler in logging.getLogger().handlers:
    # Must use type() check here since isinstance returns True for FileHandlers
    # and StreamHandler: isinstance(logging.FileHandler, logging.StreamHandler)
    if type(handler) == logging.StreamHandler:  # pylint: disable=unidiomatic-typecheck
        handler.level = _original_level
del _original_level

_LOG = logging.getLogger(__name__)

PathOrStr = Union[Path, str]

def compile_protos(
    output_dir: PathOrStr,
    proto_files: Iterable[PathOrStr],
    includes: Iterable[PathOrStr] = ()) -> None:
    """Compiles proto files for Python by invoking the protobuf compiler.

    Proto files not covered by one of the provided include paths will have their
    directory added as an include path.
    """
    proto_paths: List[Path] = [Path(f).resolve() for f in proto_files]
    include_paths: Set[Path] = set(Path(d).resolve() for d in includes)

    for path in proto_paths:
        if not any(include in path.parents for include in include_paths):
            include_paths.add(path.parent)

    cmd: Tuple[PathOrStr, ...] = (
        'protoc',
        '--experimental_allow_proto3_optional',
        '--python_out',
        os.path.abspath(output_dir),
        *(f'-I{d}' for d in include_paths),
        *proto_paths,
    )

    _LOG.debug('%s', ' '.join(shlex.quote(str(c)) for c in cmd))
    process = subprocess.run(cmd, capture_output=True)

    if process.returncode:
        _LOG.error('protoc invocation failed!\n%s\n%s',
                   ' '.join(shlex.quote(str(c)) for c in cmd),
                   process.stderr.decode())
        process.check_returncode()

def _import_module(name: str, path: str) -> ModuleType:
    spec = importlib.util.spec_from_file_location(name, path)
    assert spec is not None
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)  # type: ignore[union-attr]
    return module

def import_modules(directory: PathOrStr) -> Iterator:
    """Imports modules in a directory and yields them."""
    parent = os.path.dirname(directory)

    for dirpath, _, files in os.walk(directory):
        path_parts = os.path.relpath(dirpath, parent).split(os.sep)

        for file in files:
            name, ext = os.path.splitext(file)

            if ext == '.py':
                yield _import_module(f'{".".join(path_parts)}.{name}',
                                     os.path.join(dirpath, file))

def compile_and_import(proto_files: Iterable[PathOrStr],
                       includes: Iterable[PathOrStr] = (),
                       output_dir: PathOrStr = None) -> Iterator:
    """Compiles protos and imports their modules; yields the proto modules.

    Args:
      proto_files: paths to .proto files to compile
      includes: include paths to use for .proto compilation
      output_dir: where to place the generated modules; a temporary directory is
          used if omitted

    Yields:
      the generated protobuf Python modules
    """

    if output_dir:
        compile_protos(output_dir, proto_files, includes)
        yield from import_modules(output_dir)
    else:
        with tempfile.TemporaryDirectory(prefix='compiled_protos_') as tempdir:
            compile_protos(tempdir, proto_files, includes)
            yield from import_modules(tempdir)

def compile_and_import_file(proto_file: PathOrStr,
                            includes: Iterable[PathOrStr] = (),
                            output_dir: PathOrStr = None):
    """Compiles and imports the module for a single .proto file."""
    return next(iter(compile_and_import([proto_file], includes, output_dir)))

def compile_and_import_strings(contents: Iterable[str],
                               includes: Iterable[PathOrStr] = (),
                               output_dir: PathOrStr = None) -> Iterator:
    """Compiles protos in one or more strings."""

    if isinstance(contents, str):
        contents = [contents]

    with tempfile.TemporaryDirectory(prefix='proto_sources_') as path:
        protos = []

        for proto in contents:
            # Use a hash of the proto so the same contents map to the same file
            # name. The protobuf package complains if it seems the same contents
            # in files with different names.
            protos.append(Path(path, f'protobuf_{hash(proto):x}.proto'))
            protos[-1].write_text(proto)

        yield from compile_and_import(protos, includes, output_dir)

T = TypeVar('T')

class _NestedPackage(Generic[T]):
    """Facilitates navigating protobuf packages as attributes."""
    def __init__(self, package: str):
        self._packages: Dict[str, _NestedPackage[T]] = {}
        self._items: List[T] = []
        self._package = package

    def _add_package(self, subpackage: str, package: '_NestedPackage') -> None:
        self._packages[subpackage] = package

    def _add_item(self, item) -> None:
        if item not in self._items:  # Don't store the same item multiple times.
            self._items.append(item)

    def __getattr__(self, attr: str):
        """Look up subpackages or package members."""
        if attr in self._packages:
            return self._packages[attr]

        for item in self._items:
            if hasattr(item, attr):
                return getattr(item, attr)

        raise AttributeError(
            f'Proto package "{self._package}" does not contain "{attr}"')

    def __getitem__(self, subpackage: str) -> '_NestedPackage[T]':
        """Support accessing nested packages by name."""
        result = self

        for package in subpackage.split('.'):
            result = result._packages[package]

        return result

    def __dir__(self) -> List[str]:
        """List subpackages and members of modules as attributes."""
        attributes = list(self._packages)

        for item in self._items:
            for attr, value in vars(item).items():
                # Exclude private variables and modules from dir().
                if not attr.startswith('_') and not isinstance(
                        value, ModuleType):
                    attributes.append(attr)

        return attributes

    def __iter__(self) -> Iterator['_NestedPackage[T]']:
        """Iterate over nested packages."""
        return iter(self._packages.values())

    def __repr__(self) -> str:
        msg = [f'ProtoPackage({self._package!r}']

        public_members = [
            i for i in vars(self)
            if i not in self._packages and not i.startswith('_')
        ]
        if public_members:
            msg.append(f'members={str(public_members)}')

        if self._packages:
            msg.append(f'subpackages={str(list(self._packages))}')

        return ', '.join(msg) + ')'

    def __str__(self) -> str:
        return self._package

class Packages(NamedTuple):
    """Items in a protobuf package structure; returned from as_package."""
    items_by_package: Dict[str, List]
    packages: _NestedPackage

def as_packages(items: Iterable[Tuple[str, T]],
                packages: Packages = None) -> Packages:
    """Places items in a proto-style package structure navigable by attributes.

    Args:
      items: (package, item) tuples to insert into the package structure
      packages: if provided, update this Packages instead of creating a new one
    """
    if packages is None:
        packages = Packages({}, _NestedPackage(''))

    for package, item in items:
        packages.items_by_package.setdefault(package, []).append(item)

        entry = packages.packages
        subpackages = package.split('.')

        # pylint: disable=protected-access
        for i, subpackage in enumerate(subpackages, 1):
            if subpackage not in entry._packages:
                entry._add_package(subpackage,
                                   _NestedPackage('.'.join(subpackages[:i])))

            entry = entry._packages[subpackage]

        entry._add_item(item)
        # pylint: enable=protected-access

    return packages

PathOrModule = Union[str, Path, ModuleType]

class Library:
    """A collection of protocol buffer modules sorted by package.

    In Python, each .proto file is compiled into a Python module. The Library
    class makes it simple to navigate a collection of Python modules
    corresponding to .proto files, without relying on the location of these
    compiled modules.

    Proto messages and other types can be directly accessed by their protocol
    buffer package name. For example, the foo.bar.Baz message can be accessed
    in a Library called `protos` as:

      protos.packages.foo.bar.Baz

    A Library also provides the modules_by_package dictionary, for looking up
    the list of modules in a particular package, and the modules() generator
    for iterating over all modules.
    """
    @classmethod
    def from_paths(cls, protos: Iterable[PathOrModule]) -> 'Library':
        """Creates a Library from paths to proto files or proto modules."""
        paths: List[PathOrStr] = []
        modules: List[ModuleType] = []

        for proto in protos:
            if isinstance(proto, (Path, str)):
                paths.append(proto)
            else:
                modules.append(proto)

        if paths:
            modules += compile_and_import(paths)
        return Library(modules)

    @classmethod
    def from_strings(cls,
                     contents: Iterable[str],
                     includes: Iterable[PathOrStr] = (),
                     output_dir: PathOrStr = None) -> 'Library':
        """Creates a proto library from protos in the provided strings."""
        return cls(compile_and_import_strings(contents, includes, output_dir))

    def __init__(self, modules: Iterable[ModuleType]):
        """Constructs a Library from an iterable of modules.

        A Library can be constructed with modules dynamically compiled by
        compile_and_import. For example:

            protos = Library(compile_and_import(list_of_proto_files))
        """
        self.modules_by_package, self.packages = as_packages(
            (m.DESCRIPTOR.package, m)  # type: ignore[attr-defined]
            for m in modules)

    def modules(self) -> Iterable:
        """Iterates over all protobuf modules in this library."""
        for module_list in self.modules_by_package.values():
            yield from module_list

    def messages(self) -> Iterable:
        """Iterates over all protobuf messages in this library."""
        for module in self.modules():
            yield from _nested_messages(
                module, module.DESCRIPTOR.message_types_by_name)

def _nested_messages(scope, message_names: Iterable[str]) -> Iterator:
    for name in message_names:
        msg = getattr(scope, name)
        yield msg
        yield from _nested_messages(msg, msg.DESCRIPTOR.nested_types_by_name)

def _repr_char(char: int) -> str:
    r"""Returns an ASCII char or the \x code for non-printable values."""
    if ord(' ') <= char <= ord('~'):
        return r"\'" if chr(char) == "'" else chr(char)

    return f'\\x{char:02X}'

def bytes_repr(value: bytes) -> str:
    """Prints bytes as mixed ASCII only if at least half are printable."""
    ascii_char_count = sum(ord(' ') <= c <= ord('~') for c in value)
    if ascii_char_count >= len(value) / 2:
        contents = ''.join(_repr_char(c) for c in value)
    else:
        contents = ''.join(f'\\x{c:02X}' for c in value)

    return f"b'{contents}'"

def _field_repr(field, value) -> str:
    if field.type == field.TYPE_ENUM:
        try:
            enum = field.enum_type.values_by_number[value]
            return f'{field.enum_type.full_name}.{enum.name}'
        except KeyError:
            return repr(value)

    if field.type == field.TYPE_MESSAGE:
        return proto_repr(value)

    if field.type == field.TYPE_BYTES:
        return bytes_repr(value)

    return repr(value)

def _proto_repr(message) -> Iterator[str]:
    for field in message.DESCRIPTOR.fields:
        value = getattr(message, field.name)

        # Skip fields that are not present.
        try:
            if not message.HasField(field.name):
                continue
        except ValueError:
            # Skip default-valued fields that don't support HasField.
            if (field.label != field.LABEL_REPEATED
                    and value == field.default_value):
                continue

        if field.label == field.LABEL_REPEATED:
            if not value:
                continue

            if isinstance(value, Mapping):
                key_desc, value_desc = field.message_type.fields
                values = ', '.join(
                    f'{_field_repr(key_desc, k)}: {_field_repr(value_desc, v)}'
                    for k, v in value.items())
                yield f'{field.name}={{{values}}}'
            else:
                values = ', '.join(_field_repr(field, v) for v in value)
                yield f'{field.name}=[{values}]'
        else:
            yield f'{field.name}={_field_repr(field, value)}'

def proto_repr(message, *, wrap: bool = True) -> str:
    """Creates a repr-like string for a protobuf.

    In an interactive console that imports proto objects into the namespace, the
    output of proto_repr() can be used as Python source to create a proto
    object.

    Args:
      message: The protobuf message to format
      wrap: If true and YAPF is available, the output is wrapped according to
          PEP8 using YAPF.
    """
    raw = f'{message.DESCRIPTOR.full_name}({", ".join(_proto_repr(message))})'

    if wrap and yapf_api is not None:
        return yapf_api.FormatCode(raw, style_config='PEP8')[0].rstrip()

    return raw
<commit_msg>pw_protobuf_compiler: Support mypy --no-implicit-optional<commit_after># Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Tools for compiling and importing Python protos on the fly."""

from collections.abc import Mapping
import importlib.util
import logging
import os
from pathlib import Path
import subprocess
import shlex
import tempfile
from types import ModuleType
from typing import (Dict, Generic, Iterable, Iterator, List, NamedTuple,
                    Optional, Set, Tuple, TypeVar, Union)

# Temporarily set the root logger level to critical while importing yapf.
# This silences INFO level messages from
# environment/cipd/packages/python/lib/python3.9/lib2to3/driver.py
# when it writes Grammar3.*.pickle and PatternGrammar3.*.pickle files.
_original_level = 0
for handler in logging.getLogger().handlers:
    if type(handler) == logging.StreamHandler:  # pylint: disable=unidiomatic-typecheck
        if handler.level > _original_level:
            _original_level = handler.level
        handler.level = logging.CRITICAL

try:
    from yapf.yapflib import yapf_api  # type: ignore[import] # pylint: disable=wrong-import-position
except ImportError:
    yapf_api = None

# Restore the original stderr/out log handler level.
for handler in logging.getLogger().handlers:
    # Must use type() check here since isinstance returns True for FileHandlers
    # and StreamHandler: isinstance(logging.FileHandler, logging.StreamHandler)
    if type(handler) == logging.StreamHandler:  # pylint: disable=unidiomatic-typecheck
        handler.level = _original_level
del _original_level

_LOG = logging.getLogger(__name__)

PathOrStr = Union[Path, str]

def compile_protos(
    output_dir: PathOrStr,
    proto_files: Iterable[PathOrStr],
    includes: Iterable[PathOrStr] = ()) -> None:
    """Compiles proto files for Python by invoking the protobuf compiler.

    Proto files not covered by one of the provided include paths will have their
    directory added as an include path.
    """
    proto_paths: List[Path] = [Path(f).resolve() for f in proto_files]
    include_paths: Set[Path] = set(Path(d).resolve() for d in includes)

    for path in proto_paths:
        if not any(include in path.parents for include in include_paths):
            include_paths.add(path.parent)

    cmd: Tuple[PathOrStr, ...] = (
        'protoc',
        '--experimental_allow_proto3_optional',
        '--python_out',
        os.path.abspath(output_dir),
        *(f'-I{d}' for d in include_paths),
        *proto_paths,
    )

    _LOG.debug('%s', ' '.join(shlex.quote(str(c)) for c in cmd))
    process = subprocess.run(cmd, capture_output=True)

    if process.returncode:
        _LOG.error('protoc invocation failed!\n%s\n%s',
                   ' '.join(shlex.quote(str(c)) for c in cmd),
                   process.stderr.decode())
        process.check_returncode()

def _import_module(name: str, path: str) -> ModuleType:
    spec = importlib.util.spec_from_file_location(name, path)
    assert spec is not None
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)  # type: ignore[union-attr]
    return module

def import_modules(directory: PathOrStr) -> Iterator:
    """Imports modules in a directory and yields them."""
    parent = os.path.dirname(directory)

    for dirpath, _, files in os.walk(directory):
        path_parts = os.path.relpath(dirpath, parent).split(os.sep)

        for file in files:
            name, ext = os.path.splitext(file)

            if ext == '.py':
                yield _import_module(f'{".".join(path_parts)}.{name}',
                                     os.path.join(dirpath, file))

def compile_and_import(proto_files: Iterable[PathOrStr],
                       includes: Iterable[PathOrStr] = (),
                       output_dir: Optional[PathOrStr] = None) -> Iterator:
    """Compiles protos and imports their modules; yields the proto modules.

    Args:
      proto_files: paths to .proto files to compile
      includes: include paths to use for .proto compilation
      output_dir: where to place the generated modules; a temporary directory is
          used if omitted

    Yields:
      the generated protobuf Python modules
    """

    if output_dir:
        compile_protos(output_dir, proto_files, includes)
        yield from import_modules(output_dir)
    else:
        with tempfile.TemporaryDirectory(prefix='compiled_protos_') as tempdir:
            compile_protos(tempdir, proto_files, includes)
            yield from import_modules(tempdir)

def compile_and_import_file(proto_file: PathOrStr,
                            includes: Iterable[PathOrStr] = (),
                            output_dir: Optional[PathOrStr] = None):
    """Compiles and imports the module for a single .proto file."""
    return next(iter(compile_and_import([proto_file], includes, output_dir)))

def compile_and_import_strings(
        contents: Iterable[str],
        includes: Iterable[PathOrStr] = (),
        output_dir: Optional[PathOrStr] = None) -> Iterator:
    """Compiles protos in one or more strings."""

    if isinstance(contents, str):
        contents = [contents]

    with tempfile.TemporaryDirectory(prefix='proto_sources_') as path:
        protos = []

        for proto in contents:
            # Use a hash of the proto so the same contents map to the same file
            # name. The protobuf package complains if it seems the same contents
            # in files with different names.
            protos.append(Path(path, f'protobuf_{hash(proto):x}.proto'))
            protos[-1].write_text(proto)

        yield from compile_and_import(protos, includes, output_dir)

T = TypeVar('T')

class _NestedPackage(Generic[T]):
    """Facilitates navigating protobuf packages as attributes."""
    def __init__(self, package: str):
        self._packages: Dict[str, _NestedPackage[T]] = {}
        self._items: List[T] = []
        self._package = package

    def _add_package(self, subpackage: str, package: '_NestedPackage') -> None:
        self._packages[subpackage] = package

    def _add_item(self, item) -> None:
        if item not in self._items:  # Don't store the same item multiple times.
            self._items.append(item)

    def __getattr__(self, attr: str):
        """Look up subpackages or package members."""
        if attr in self._packages:
            return self._packages[attr]

        for item in self._items:
            if hasattr(item, attr):
                return getattr(item, attr)

        raise AttributeError(
            f'Proto package "{self._package}" does not contain "{attr}"')

    def __getitem__(self, subpackage: str) -> '_NestedPackage[T]':
        """Support accessing nested packages by name."""
        result = self

        for package in subpackage.split('.'):
            result = result._packages[package]

        return result

    def __dir__(self) -> List[str]:
        """List subpackages and members of modules as attributes."""
        attributes = list(self._packages)

        for item in self._items:
            for attr, value in vars(item).items():
                # Exclude private variables and modules from dir().
                if not attr.startswith('_') and not isinstance(
                        value, ModuleType):
                    attributes.append(attr)

        return attributes

    def __iter__(self) -> Iterator['_NestedPackage[T]']:
        """Iterate over nested packages."""
        return iter(self._packages.values())

    def __repr__(self) -> str:
        msg = [f'ProtoPackage({self._package!r}']

        public_members = [
            i for i in vars(self)
            if i not in self._packages and not i.startswith('_')
        ]
        if public_members:
            msg.append(f'members={str(public_members)}')

        if self._packages:
            msg.append(f'subpackages={str(list(self._packages))}')

        return ', '.join(msg) + ')'

    def __str__(self) -> str:
        return self._package

class Packages(NamedTuple):
    """Items in a protobuf package structure; returned from as_package."""
    items_by_package: Dict[str, List]
    packages: _NestedPackage

def as_packages(items: Iterable[Tuple[str, T]],
                packages: Optional[Packages] = None) -> Packages:
    """Places items in a proto-style package structure navigable by attributes.

    Args:
      items: (package, item) tuples to insert into the package structure
      packages: if provided, update this Packages instead of creating a new one
    """
    if packages is None:
        packages = Packages({}, _NestedPackage(''))

    for package, item in items:
        packages.items_by_package.setdefault(package, []).append(item)

        entry = packages.packages
        subpackages = package.split('.')

        # pylint: disable=protected-access
        for i, subpackage in enumerate(subpackages, 1):
            if subpackage not in entry._packages:
                entry._add_package(subpackage,
                                   _NestedPackage('.'.join(subpackages[:i])))

            entry = entry._packages[subpackage]

        entry._add_item(item)
        # pylint: enable=protected-access

    return packages

PathOrModule = Union[str, Path, ModuleType]

class Library:
    """A collection of protocol buffer modules sorted by package.

    In Python, each .proto file is compiled into a Python module. The Library
    class makes it simple to navigate a collection of Python modules
    corresponding to .proto files, without relying on the location of these
    compiled modules.

    Proto messages and other types can be directly accessed by their protocol
    buffer package name. For example, the foo.bar.Baz message can be accessed
    in a Library called `protos` as:

      protos.packages.foo.bar.Baz

    A Library also provides the modules_by_package dictionary, for looking up
    the list of modules in a particular package, and the modules() generator
    for iterating over all modules.
    """
    @classmethod
    def from_paths(cls, protos: Iterable[PathOrModule]) -> 'Library':
        """Creates a Library from paths to proto files or proto modules."""
        paths: List[PathOrStr] = []
        modules: List[ModuleType] = []

        for proto in protos:
            if isinstance(proto, (Path, str)):
                paths.append(proto)
            else:
                modules.append(proto)

        if paths:
            modules += compile_and_import(paths)
        return Library(modules)

    @classmethod
    def from_strings(cls,
                     contents: Iterable[str],
                     includes: Iterable[PathOrStr] = (),
                     output_dir: Optional[PathOrStr] = None) -> 'Library':
        """Creates a proto library from protos in the provided strings."""
        return cls(compile_and_import_strings(contents, includes, output_dir))

    def __init__(self, modules: Iterable[ModuleType]):
        """Constructs a Library from an iterable of modules.

        A Library can be constructed with modules dynamically compiled by
        compile_and_import. For example:

            protos = Library(compile_and_import(list_of_proto_files))
        """
        self.modules_by_package, self.packages = as_packages(
            (m.DESCRIPTOR.package, m)  # type: ignore[attr-defined]
            for m in modules)

    def modules(self) -> Iterable:
        """Iterates over all protobuf modules in this library."""
        for module_list in self.modules_by_package.values():
            yield from module_list

    def messages(self) -> Iterable:
        """Iterates over all protobuf messages in this library."""
        for module in self.modules():
            yield from _nested_messages(
                module, module.DESCRIPTOR.message_types_by_name)

def _nested_messages(scope, message_names: Iterable[str]) -> Iterator:
    for name in message_names:
        msg = getattr(scope, name)
        yield msg
        yield from _nested_messages(msg, msg.DESCRIPTOR.nested_types_by_name)

def _repr_char(char: int) -> str:
    r"""Returns an ASCII char or the \x code for non-printable values."""
    if ord(' ') <= char <= ord('~'):
        return r"\'" if chr(char) == "'" else chr(char)

    return f'\\x{char:02X}'

def bytes_repr(value: bytes) -> str:
    """Prints bytes as mixed ASCII only if at least half are printable."""
    ascii_char_count = sum(ord(' ') <= c <= ord('~') for c in value)
    if ascii_char_count >= len(value) / 2:
        contents = ''.join(_repr_char(c) for c in value)
    else:
        contents = ''.join(f'\\x{c:02X}' for c in value)

    return f"b'{contents}'"

def _field_repr(field, value) -> str:
    if field.type == field.TYPE_ENUM:
        try:
            enum = field.enum_type.values_by_number[value]
            return f'{field.enum_type.full_name}.{enum.name}'
        except KeyError:
            return repr(value)

    if field.type == field.TYPE_MESSAGE:
        return proto_repr(value)

    if field.type == field.TYPE_BYTES:
        return bytes_repr(value)

    return repr(value)

def _proto_repr(message) -> Iterator[str]:
    for field in message.DESCRIPTOR.fields:
        value = getattr(message, field.name)

        # Skip fields that are not present.
        try:
            if not message.HasField(field.name):
                continue
        except ValueError:
            # Skip default-valued fields that don't support HasField.
            if (field.label != field.LABEL_REPEATED
                    and value == field.default_value):
                continue

        if field.label == field.LABEL_REPEATED:
            if not value:
                continue

            if isinstance(value, Mapping):
                key_desc, value_desc = field.message_type.fields
                values = ', '.join(
                    f'{_field_repr(key_desc, k)}: {_field_repr(value_desc, v)}'
                    for k, v in value.items())
                yield f'{field.name}={{{values}}}'
            else:
                values = ', '.join(_field_repr(field, v) for v in value)
                yield f'{field.name}=[{values}]'
        else:
            yield f'{field.name}={_field_repr(field, value)}'

def proto_repr(message, *, wrap: bool = True) -> str:
    """Creates a repr-like string for a protobuf.

    In an interactive console that imports proto objects into the namespace, the
    output of proto_repr() can be used as Python source to create a proto
    object.

    Args:
      message: The protobuf message to format
      wrap: If true and YAPF is available, the output is wrapped according to
          PEP8 using YAPF.
    """
    raw = f'{message.DESCRIPTOR.full_name}({", ".join(_proto_repr(message))})'

    if wrap and yapf_api is not None:
        return yapf_api.FormatCode(raw, style_config='PEP8')[0].rstrip()

    return raw
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
RDS Functions
"""

import sys
from datetime import datetime, timedelta
from exclusions import excluded_instances
import boto3
import botocore

def get_session(access_key_id, secret_access_key):
    " Establishes a session with AWS "
    return boto3.session.Session(
        aws_access_key_id=access_key_id,
        aws_secret_access_key=secret_access_key
    )

def get_rds_client(session):
    " Returns a RDS boto client "
    return session.client('rds')

def get_cloudwatch_client(session):
    " Returns a CloudWatch boto client "
    return session.client('cloudwatch')

def get_ec2_client(session):
    " Returns a EC2 boto client "
    return session.client('ec2')

def get_vpc_ids(client):
    " Returns a list of VPC IDs in the account "
    vpc_ids = []
    vpcs = client.describe_vpcs()['Vpcs']
    for vpc in vpcs:
        vpc_ids.append(vpc['VpcId'])
    return vpc_ids

def get_isolated_sgs(client):
    " Returns a dict of rds-isolate SG IDs for each VPC in account. "
    vpc_ids = get_vpc_ids(client)
    isolated_sgs = {}
    for vpc in vpc_ids:
        sec_groups = client.describe_security_groups(
            Filters=[
                {
                    "Name": "vpc-id",
                    "Values": [vpc]
                },
                {
                    "Name": "group-name",
                    "Values": ["rds-isolate"]
                }
            ]
        )['SecurityGroups']
        try:
            isolated_sgs[vpc] = sec_groups[0]['GroupId']
        except IndexError:
            print("No rds-isolate group found for VPC: {}".format(vpc))
    return isolated_sgs

def get_connections_statistics(client, rds_instances):
    " Returns a dict of all instances and their avg DB conns over all datapoints "
    rds_stats = {}
    for rds_instance in rds_instances:
        stats = client.get_metric_statistics(
            Namespace="AWS/RDS",
            MetricName="DatabaseConnections",
            Statistics=['Average'],
            Period=57600,
            StartTime=(datetime.today() - timedelta(days=3)),
            EndTime=datetime.today(),
            Dimensions=[
                {
                    'Name': 'DBInstanceIdentifier',
                    'Value': rds_instance['DBInstanceIdentifier']
                }
            ]
        )['Datapoints']
        datapoints = []
        for stat in stats:
            datapoints.append(stat['Average'])
        if len(datapoints) > 0:
            dp_conns = sum(datapoints)/float(len(datapoints))
            rds_stats[rds_instance['DBInstanceIdentifier']] = dp_conns
        else:
            print("Instance: %s has no datapoints." % rds_instance['DBInstanceIdentifier'])

    return rds_stats

def get_rds_instances(client, vpc_id=None):
    " Gets all RDS instances, per VPC, if specified. "

    rds_instances = []
    resp = client.describe_db_instances()
    while 'Marker' in resp:
        rds_instances.extend(resp['DBInstances'])
        resp = client.describe_db_instances(Marker=resp['Marker'])
    rds_instances.extend(resp['DBInstances'])
    if not vpc_id:
        return rds_instances
    else:
        return [r for r in rds_instances if r['DBSubnetGroup']['VpcId'] == vpc_id]

def set_no_multiaz(client, rds_instance):
    " Takes a rds instance obj and turns off MultiAZ "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            MultiAZ=False,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting no-multiaz on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_security_group(client, rds_instance, sg_id):
    " Sets the rds_instance Security Group to sg_id "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            VpcSecurityGroupIds=[sg_id]
        )
    except botocore.exceptions.ClientError:
        print("Error setting SG on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_instance_size(client, rds_instance, size=None):
    " Sets instance to the smallest available size "
    if not size:
        available_sizes = client.describe_orderable_db_instance_options(
            Engine=rds_instance['Engine']
        )['OrderableDBInstanceOptions']
        size = available_sizes[0]['DBInstanceClass']
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            DBInstanceClass=size,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting size on instance %s" % rds_instance['DBInstanceIdentifier'])

def get_isolated_instances(client, sg_id, vpc_id=None):
    """ Gets all RDS instances that are using the sg_id """
    rds_instances = get_rds_instances(client, vpc_id)

    isolated_instances = []
    for instance in rds_instances:
        sgs = instance['VpcSecurityGroups']
        for sg in sgs:
            if sg['VpcSecurityGroupId'] == sg_id:
                isolated_instances.append(instance)

    return isolated_instances

    def take_snapshot(client, rds_instance):
        """ Takes a snapshot of an RDS instance """
        #TODO
        pass

    def cfn_destroy(client, cfn_stack):
        """ Destroys a Cloudformation stack """
        #TODO
        pass

def main():
    " Main execution "
    debug = False
    dry_run = False
    session = get_session('', '')
    ec2 = get_ec2_client(session)
    rds = get_rds_client(session)
    cloudwatch = get_cloudwatch_client(session)
def destroy_cfn_stack(client, cfn_stack):
    """ Destroys a Cloudformation stack """
    #TODO
    pass

    isolated_sgs = get_isolated_sgs(ec2)
    all_rds_instances = get_rds_instances(rds)
    if debug:
        print("DEBUG: Number of RDS instances found: %s" % len(all_rds_instances))
    all_rds_stats = get_connections_statistics(cloudwatch, all_rds_instances)
    if debug:
        print("DEBUG: Isolated SGs {}".format(isolated_sgs))
        print("DEBUG: All RDS Instances: ")
        for instance in all_rds_instances:
            print(instance['DBInstanceIdentifier'])
    abandoned_instances = []
    if len(excluded_instances) > 0:
        print("\nThe following instances meet low connections criteria, but have been excluded.")
    for key in all_rds_stats:
        if all_rds_stats[key] == 0 and key not in excluded_instances:
            abandoned_instances.append(key)
        elif all_rds_stats[key] == 0 and key in excluded_instances:
            print(key)
        if debug:
            print("DEBUG: Instance: %s. Connections: %s" % (key, all_rds_stats[key]))
    if len(abandoned_instances) > 0:
        print("\nThe following instances appear to be abandoned. Please investigate.")
        for instance in abandoned_instances:
            print(instance)
    else:
        print("\nNo instances appear to be abandoned.")
        sys.exit(0)
    print("\nTaking action on the following instances: ")
    for rds_instance in all_rds_instances:
        if rds_instance['DBInstanceIdentifier'] in abandoned_instances:
            if dry_run:
                print("DRYRUN: %s would have been isolated and downsized."
                      % rds_instance['DBInstanceIdentifier'])
            else:
                print("Isolating and downsizing instance: %s"
                      % rds_instance['DBInstanceIdentifier'])
                set_security_group(rds,
                                   rds_instance,
                                   isolated_sgs[rds_instance['DBSubnetGroup']['VpcId']])

                set_instance_size(rds,
                                  rds_instance,
                                  'db.t2.micro')

                set_no_multiaz(rds, rds_instance)

main()
<commit_msg>More sensible naming for get_isolated_instances<commit_after>#!/usr/bin/env python3
"""
RDS Functions
"""

import sys
from datetime import datetime, timedelta
from exclusions import excluded_instances
import boto3
import botocore

def get_session(access_key_id, secret_access_key):
    " Establishes a session with AWS "
    return boto3.session.Session(
        aws_access_key_id=access_key_id,
        aws_secret_access_key=secret_access_key
    )

def get_rds_client(session):
    " Returns a RDS boto client "
    return session.client('rds')

def get_cloudwatch_client(session):
    " Returns a CloudWatch boto client "
    return session.client('cloudwatch')

def get_ec2_client(session):
    " Returns a EC2 boto client "
    return session.client('ec2')

def get_vpc_ids(client):
    " Returns a list of VPC IDs in the account "
    vpc_ids = []
    vpcs = client.describe_vpcs()['Vpcs']
    for vpc in vpcs:
        vpc_ids.append(vpc['VpcId'])
    return vpc_ids

def get_isolated_sgs(client):
    " Returns a dict of rds-isolate SG IDs for each VPC in account. "
    vpc_ids = get_vpc_ids(client)
    isolated_sgs = {}
    for vpc in vpc_ids:
        sec_groups = client.describe_security_groups(
            Filters=[
                {
                    "Name": "vpc-id",
                    "Values": [vpc]
                },
                {
                    "Name": "group-name",
                    "Values": ["rds-isolate"]
                }
            ]
        )['SecurityGroups']
        try:
            isolated_sgs[vpc] = sec_groups[0]['GroupId']
        except IndexError:
            print("No rds-isolate group found for VPC: {}".format(vpc))
    return isolated_sgs

def get_connections_statistics(client, rds_instances):
    " Returns a dict of all instances and their avg DB conns over all datapoints "
    rds_stats = {}
    for rds_instance in rds_instances:
        stats = client.get_metric_statistics(
            Namespace="AWS/RDS",
            MetricName="DatabaseConnections",
            Statistics=['Average'],
            Period=57600,
            StartTime=(datetime.today() - timedelta(days=3)),
            EndTime=datetime.today(),
            Dimensions=[
                {
                    'Name': 'DBInstanceIdentifier',
                    'Value': rds_instance['DBInstanceIdentifier']
                }
            ]
        )['Datapoints']
        datapoints = []
        for stat in stats:
            datapoints.append(stat['Average'])
        if len(datapoints) > 0:
            dp_conns = sum(datapoints)/float(len(datapoints))
            rds_stats[rds_instance['DBInstanceIdentifier']] = dp_conns
        else:
            print("Instance: %s has no datapoints." % rds_instance['DBInstanceIdentifier'])

    return rds_stats

def get_rds_instances(client, vpc_id=None):
    " Gets all RDS instances, per VPC, if specified. "

    rds_instances = []
    resp = client.describe_db_instances()
    while 'Marker' in resp:
        rds_instances.extend(resp['DBInstances'])
        resp = client.describe_db_instances(Marker=resp['Marker'])
    rds_instances.extend(resp['DBInstances'])
    if not vpc_id:
        return rds_instances
    else:
        return [r for r in rds_instances if r['DBSubnetGroup']['VpcId'] == vpc_id]

def set_no_multiaz(client, rds_instance):
    " Takes a rds instance obj and turns off MultiAZ "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            MultiAZ=False,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting no-multiaz on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_security_group(client, rds_instance, sg_id):
    " Sets the rds_instance Security Group to sg_id "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            VpcSecurityGroupIds=[sg_id]
        )
    except botocore.exceptions.ClientError:
        print("Error setting SG on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_instance_size(client, rds_instance, size=None):
    " Sets instance to the smallest available size "
    if not size:
        available_sizes = client.describe_orderable_db_instance_options(
            Engine=rds_instance['Engine']
        )['OrderableDBInstanceOptions']
        size = available_sizes[0]['DBInstanceClass']
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            DBInstanceClass=size,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting size on instance %s" % rds_instance['DBInstanceIdentifier'])

def get_instances_with_sg(client, sg_id, vpc_id=None):
    """ Gets all RDS instances that are using the sg_id """
    rds_instances = get_rds_instances(client, vpc_id)

    instances_with_sg = []
    for instance in rds_instances:
        sgs = instance['VpcSecurityGroups']
        for sg in sgs:
            if sg['VpcSecurityGroupId'] == sg_id:
                instances_with_sg.append(instance)

    return instances_with_sg

    def take_snapshot(client, rds_instance):
        """ Takes a snapshot of an RDS instance """
        #TODO
        pass

    def cfn_destroy(client, cfn_stack):
        """ Destroys a Cloudformation stack """
        #TODO
        pass

def main():
    " Main execution "
    debug = False
    dry_run = False
    session = get_session('', '')
    ec2 = get_ec2_client(session)
    rds = get_rds_client(session)
    cloudwatch = get_cloudwatch_client(session)
def destroy_cfn_stack(client, cfn_stack):
    """ Destroys a Cloudformation stack """
    #TODO
    pass

    isolated_sgs = get_isolated_sgs(ec2)
    all_rds_instances = get_rds_instances(rds)
    if debug:
        print("DEBUG: Number of RDS instances found: %s" % len(all_rds_instances))
    all_rds_stats = get_connections_statistics(cloudwatch, all_rds_instances)
    if debug:
        print("DEBUG: Isolated SGs {}".format(isolated_sgs))
        print("DEBUG: All RDS Instances: ")
        for instance in all_rds_instances:
            print(instance['DBInstanceIdentifier'])
    abandoned_instances = []
    if len(excluded_instances) > 0:
        print("\nThe following instances meet low connections criteria, but have been excluded.")
    for key in all_rds_stats:
        if all_rds_stats[key] == 0 and key not in excluded_instances:
            abandoned_instances.append(key)
        elif all_rds_stats[key] == 0 and key in excluded_instances:
            print(key)
        if debug:
            print("DEBUG: Instance: %s. Connections: %s" % (key, all_rds_stats[key]))
    if len(abandoned_instances) > 0:
        print("\nThe following instances appear to be abandoned. Please investigate.")
        for instance in abandoned_instances:
            print(instance)
    else:
        print("\nNo instances appear to be abandoned.")
        sys.exit(0)
    print("\nTaking action on the following instances: ")
    for rds_instance in all_rds_instances:
        if rds_instance['DBInstanceIdentifier'] in abandoned_instances:
            if dry_run:
                print("DRYRUN: %s would have been isolated and downsized."
                      % rds_instance['DBInstanceIdentifier'])
            else:
                print("Isolating and downsizing instance: %s"
                      % rds_instance['DBInstanceIdentifier'])
                set_security_group(rds,
                                   rds_instance,
                                   isolated_sgs[rds_instance['DBSubnetGroup']['VpcId']])

                set_instance_size(rds,
                                  rds_instance,
                                  'db.t2.micro')

                set_no_multiaz(rds, rds_instance)

main()
<|endoftext|>


<commit_before>from regolith.dates import get_dates
from regolith.helpers.basehelper import SoutHelperBase
from regolith.fsclient import _id_key
from regolith.tools import (
    all_docs_from_collection,
    get_pi_id,
    get_person_contact
)
from gooey import GooeyParser

TARGET_COLL = "presentations"
HELPER_TARGET = "l_abstract"

def subparser(subpi):
    int_kwargs = {}
    if isinstance(subpi, GooeyParser):
        int_kwargs['widget'] = 'IntegerField'
        int_kwargs['gooey_options'] = {'min': 2000, 'max': 2100}

    subpi.add_argument(
        "-a",
        "--author",
        help="Filter abstracts or this author ID (e.g., sbillinge)."
             )
    subpi.add_argument(
        "-y",
        "--year",
        help='Get presentations since this year'
    )
    subpi.add_argument(
        "-l",
        "--loc_inst",
        help='location of presentation, either a fragment of an institution, '
             'country, city, state, or university. If an institution is entered,'
             'the search will be for seminars or colloquiums, otherwise the '
             'search will be for all other meetings')
    subpi.add_argument(
        "-t",
        "--title",
        help='fragment of the title of the abstract or talk to use to '
             'filter presentations')
    return subpi

class AbstractListerHelper(SoutHelperBase):
    """Helper for finding and listing abstracts from the presentations.yml file
    """
    # btype must be the same as helper target in helper.py
    btype = HELPER_TARGET
    needed_dbs = [f'{TARGET_COLL}', 'people', 'contacts']

    def construct_global_ctx(self):
        """Constructs the global context"""
        super().construct_global_ctx()
        gtx = self.gtx
        rc = self.rc
        if "groups" in self.needed_dbs:
            rc.pi_id = get_pi_id(rc)
        rc.coll = f"{TARGET_COLL}"
        try:
            if not rc.database:
                rc.database = rc.databases[0]
        except BaseException:
            pass
        colls = [
            sorted(
                all_docs_from_collection(rc.client, collname), key=_id_key
            )
            for collname in self.needed_dbs
        ]
        for db, coll in zip(self.needed_dbs, colls):
            gtx[db] = coll
        gtx["all_docs_from_collection"] = all_docs_from_collection
        gtx["float"] = float
        gtx["str"] = str
        gtx["zip"] = zip

    def sout(self):
        rc = self.rc

        presentations = self.gtx["presentations"]
        SEMINAR_TYPES = ['seminar', 'colloquium']
        filtered_title, filtered_authors, filtered_years, filtered_inst, filtered_loc = ([] for i in range(5))

        if (not rc.author) and (not rc.year) and (not rc.loc_inst) and (not rc.title):
            print("-------------------------------------------")
            print("please rerun specifying at least one filter")
            print("-------------------------------------------")
            return

        if rc.title:
            presentations = [presentation for presentation in presentations
                              if rc.title.casefold() in presentation.get('title').casefold()]
        if rc.author:
            presentations = [presentation for presentation in presentations
                                if rc.author in presentation.get('authors')]
        if rc.year:
            presentations = [presentation for presentation in presentations
                              if get_dates(presentation).get("date",
                                                             get_dates(presentation).get("end_date",
                                                             get_dates(presentation).get("begin_date"))).year == int(rc.year)]
        if rc.loc_inst:
            filtered_loc = [presentation for presentation in presentations
                             if presentation.get('type') in SEMINAR_TYPES and
                             rc.loc_inst.casefold() in presentation.get('institution').casefold()]
            filtered_loc.append([presentation for presentation in presentations
                            if rc.loc_inst.casefold() in presentation.get('location', 'institution').casefold()
                            and rc.loc_inst.casefold() not in presentation.get('institution').casefold()])
            presentations = list(set(filtered_loc))

        # filtered_presentations_by_args = [filtered_inst, filtered_years, filtered_title,
        #                                   filtered_authors, filtered_loc]
        # nonempty_filtered_presentations_by_args = [filtered_presentations
        #                                            for filtered_presentations in filtered_presentations_by_args
        #                                            if filtered_presentations]
        # filtered_presentations = [talk for presentations in nonempty_filtered_presentations_by_args
        #                           for talk in presentations
        #                           if all(talk in presentations
        #                                  for presentations in nonempty_filtered_presentations_by_args)]
        flat_filtered_presentations = list({talk['_id']: talk for talk in presentations}.values())

        for presentation in flat_filtered_presentations:
            print("---------------------------------------")
            print(f"Title: {presentation.get('title')}\n")
            author_list = [author
                           if not get_person_contact(author, self.gtx["people"], self.gtx["contacts"])
                           else get_person_contact(author, self.gtx["people"], self.gtx["contacts"]).get('name')
                           for author in presentation.get('authors')]
            print(", ".join(author_list))
            print(f"\nAbstract: {presentation.get('abstract')}")
        return
<commit_msg>revert edits in abstract lister so helper tests pass<commit_after>from regolith.dates import get_dates
from regolith.helpers.basehelper import SoutHelperBase
from regolith.fsclient import _id_key
from regolith.tools import (
    all_docs_from_collection,
    get_pi_id,
    get_person_contact
)
from gooey import GooeyParser

TARGET_COLL = "presentations"
HELPER_TARGET = "l_abstract"

def subparser(subpi):
    int_kwargs = {}
    if isinstance(subpi, GooeyParser):
        int_kwargs['widget'] = 'IntegerField'
        int_kwargs['gooey_options'] = {'min': 2000, 'max': 2100}

    subpi.add_argument(
        "-a",
        "--author",
        help="Filter abstracts or this author ID (e.g., sbillinge)."
             )
    subpi.add_argument(
        "-y",
        "--year",
        help='Get presentations since this year'
    )
    subpi.add_argument(
        "-l",
        "--loc_inst",
        help='location of presentation, either a fragment of an institution, '
             'country, city, state, or university. If an institution is entered,'
             'the search will be for seminars or colloquiums, otherwise the '
             'search will be for all other meetings')
    subpi.add_argument(
        "-t",
        "--title",
        help='fragment of the title of the abstract or talk to use to '
             'filter presentations')
    return subpi

class AbstractListerHelper(SoutHelperBase):
    """Helper for finding and listing abstracts from the presentations.yml file
    """
    # btype must be the same as helper target in helper.py
    btype = HELPER_TARGET
    needed_dbs = [f'{TARGET_COLL}', 'people', 'contacts']

    def construct_global_ctx(self):
        """Constructs the global context"""
        super().construct_global_ctx()
        gtx = self.gtx
        rc = self.rc
        if "groups" in self.needed_dbs:
            rc.pi_id = get_pi_id(rc)
        rc.coll = f"{TARGET_COLL}"
        try:
            if not rc.database:
                rc.database = rc.databases[0]
        except BaseException:
            pass
        colls = [
            sorted(
                all_docs_from_collection(rc.client, collname), key=_id_key
            )
            for collname in self.needed_dbs
        ]
        for db, coll in zip(self.needed_dbs, colls):
            gtx[db] = coll
        gtx["all_docs_from_collection"] = all_docs_from_collection
        gtx["float"] = float
        gtx["str"] = str
        gtx["zip"] = zip

    def sout(self):
        rc = self.rc

        presentations = self.gtx["presentations"]
        SEMINAR_TYPES = ['seminar', 'colloquium']
        filtered_title, filtered_authors, filtered_years, filtered_inst, filtered_loc = ([] for i in range(5))

        if (not rc.author) and (not rc.year) and (not rc.loc_inst) and (not rc.title):
            print("-------------------------------------------")
            print("please rerun specifying at least one filter")
            print("-------------------------------------------")
            return

        if rc.title:
            filtered_title = [presentation for presentation in presentations
                              if rc.title.casefold() in presentation.get('title').casefold()]
        if rc.author:
            filtered_authors = [presentation for presentation in presentations
                                if rc.author in presentation.get('authors')]
        if rc.year:
            filtered_years = [presentation for presentation in presentations
                              if get_dates(presentation).get("date",
                                                             get_dates(presentation).get("end_date",
                                                             get_dates(presentation).get("begin_date"))).year == int(rc.year)]
        if rc.loc_inst:
            filtered_inst = [presentation for presentation in presentations
                             if presentation.get('type') in SEMINAR_TYPES and
                             rc.loc_inst.casefold() in presentation.get('institution').casefold()]
            filtered_loc = [presentation for presentation in presentations
                            if rc.loc_inst.casefold() in presentation.get('location', 'institution').casefold()
                            and rc.loc_inst.casefold() not in presentation.get('institution').casefold()]

        filtered_presentations_by_args = [filtered_inst, filtered_years, filtered_title,
                                          filtered_authors, filtered_loc]
        nonempty_filtered_presentations_by_args = [filtered_presentations
                                                   for filtered_presentations in filtered_presentations_by_args
                                                   if filtered_presentations]
        filtered_presentations = [talk for presentations in nonempty_filtered_presentations_by_args
                                  for talk in presentations
                                  if all(talk in presentations
                                         for presentations in nonempty_filtered_presentations_by_args)]
        flat_filtered_presentations = list({talk['_id']: talk for talk in filtered_presentations}.values())

        for presentation in flat_filtered_presentations:
            print("---------------------------------------")
            print(f"Title: {presentation.get('title')}\n")
            author_list = [author
                           if not get_person_contact(author, self.gtx["people"], self.gtx["contacts"])
                           else get_person_contact(author, self.gtx["people"], self.gtx["contacts"]).get('name')
                           for author in presentation.get('authors')]
            print(", ".join(author_list))
            print(f"\nAbstract: {presentation.get('abstract')}")
        return
<|endoftext|>


<commit_before># Copyright 2017-2018 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import os
import subprocess

__version__ = "0.5.0.rc1"

__requires__ = [
    # (<required module>, <distutils package req>)
    ("pip", "pip"),
    ("yaml", "PyYAML"),
    ("setuptools", "setuptools"),
    ("six", "six"),
    ("tabview", "tabview"),
    ("twine", "twine"),
    ("werkzeug", "Werkzeug"),
    ("whoosh", "Whoosh"),
]

__pkgdir__ = os.path.dirname(os.path.dirname(__file__))

def _try_init_git_attrs():
    try:
        _init_git_commit()
    except (OSError, subprocess.CalledProcessError):
        pass
    else:
        try:
            _init_git_status()
        except (OSError, subprocess.CalledProcessError):
            pass

def _init_git_commit():
    commit = _git_cmd("git -C \"%(repo)s\" log -1 --oneline | cut -d' ' -f1")
    globals()["__git_commit__"] = commit

def _init_git_status():
    raw = _git_cmd("git -C \"%(repo)s\" status -s")
    globals()["__git_status__"] = raw.split("\n") if raw else []

def _git_cmd(cmd, **kw):
    repo = os.path.dirname(__file__)
    cmd = cmd % dict(repo=repo, **kw)
    null = open(os.devnull, "w")
    out = subprocess.check_output(cmd, stderr=null, shell=True)
    return out.decode("utf-8").strip()

def version():
    git_commit = globals().get("__git_commit__")
    if git_commit:
        git_status = globals().get("__git_status__", [])
        workspace_changed_marker = "*" if git_status else ""
        return "%s (dev %s%s)" % (__version__, git_commit,
                                 workspace_changed_marker)
    else:
        return __version__

_try_init_git_attrs()
<commit_msg>0.5.0.rc2<commit_after># Copyright 2017-2018 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import os
import subprocess

__version__ = "0.5.0.rc2"

__requires__ = [
    # (<required module>, <distutils package req>)
    ("pip", "pip"),
    ("yaml", "PyYAML"),
    ("setuptools", "setuptools"),
    ("six", "six"),
    ("tabview", "tabview"),
    ("twine", "twine"),
    ("werkzeug", "Werkzeug"),
    ("whoosh", "Whoosh"),
]

__pkgdir__ = os.path.dirname(os.path.dirname(__file__))

def _try_init_git_attrs():
    try:
        _init_git_commit()
    except (OSError, subprocess.CalledProcessError):
        pass
    else:
        try:
            _init_git_status()
        except (OSError, subprocess.CalledProcessError):
            pass

def _init_git_commit():
    commit = _git_cmd("git -C \"%(repo)s\" log -1 --oneline | cut -d' ' -f1")
    globals()["__git_commit__"] = commit

def _init_git_status():
    raw = _git_cmd("git -C \"%(repo)s\" status -s")
    globals()["__git_status__"] = raw.split("\n") if raw else []

def _git_cmd(cmd, **kw):
    repo = os.path.dirname(__file__)
    cmd = cmd % dict(repo=repo, **kw)
    null = open(os.devnull, "w")
    out = subprocess.check_output(cmd, stderr=null, shell=True)
    return out.decode("utf-8").strip()

def version():
    git_commit = globals().get("__git_commit__")
    if git_commit:
        git_status = globals().get("__git_status__", [])
        workspace_changed_marker = "*" if git_status else ""
        return "%s (dev %s%s)" % (__version__, git_commit,
                                 workspace_changed_marker)
    else:
        return __version__

_try_init_git_attrs()
<|endoftext|>


<commit_before>from enum import Enum
from typing import Union

_VALUES = {}

class EnumSpecDescription(Enum):
    def __new__(cls, description, *spec):
        obj = object.__new__(cls)
        obj._value_ = len(_VALUES) + 1
        _VALUES[obj._value_] = obj
        obj.__description = description
        obj.__spec = spec
        return obj

    def __repr__(self):  # pragma: no cover
        return '<{}.{}: {} {!r}>'.format(self.__class__.__name__, self.name,
                                         self.value, self.__description)

    def __lt__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value < other.value

    def __le__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value <= other.value

    def __gt__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value > other.value

    def __ge__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value >= other.value

    @property
    def spec(self):
        return self.__spec

    @property
    def description(self):
        return self.__description

class ModeTask(EnumSpecDescription):
    TASK = 'name, uuid, routing key, state and retries', False, False, False
    SENT = 'TASK + args/kwargs being sent', True, False, False
    RESULT = 'TASK + result only', False, True, False
    ERROR = 'TASK + error only', False, False, True
    OUTCOME = 'RESULT + ERROR', False, True, True
    SUCCESS = 'RESULT + args/kwargs', None, True, False
    FAILURE = 'ERROR + args/kwargs', None, False, True
    DONE = 'SUCCESS + FAILURE', None, True, True
    THROUGH = 'SENT + SUCCESS', True, True, False
    CUTOFF = 'SENT + FAILURE', True, False, True
    ALL = 'SENT + DONE', True, True, True

class ModeWorker(EnumSpecDescription):
    WORKER = 'name, status and pid', False
    STATS = 'WORKER + version, load and heartbeats', True

def find_mode(constant: Union[None, int, ModeTask, ModeWorker]) \
        -> Union[None, ModeTask, ModeWorker]:
    if isinstance(constant, (type(None), ModeTask, ModeWorker)):
        return constant
    if isinstance(constant, int) and constant in _VALUES:
        return _VALUES[constant]
    raise UserWarning('Invalid mode constant.')
<commit_msg>organize display modes<commit_after>from enum import Enum
from typing import Union

_VALUES = {}

class EnumSpecDescription(Enum):
    def __new__(cls, description, *spec):
        obj = object.__new__(cls)
        obj._value_ = len(_VALUES) + 1
        _VALUES[obj._value_] = obj
        obj.__description = description
        obj.__spec = spec
        return obj

    def __repr__(self):  # pragma: no cover
        return '<{}.{}: {} {!r}>'.format(self.__class__.__name__, self.name,
                                         self.value, self.__description)

    def __lt__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value < other.value

    def __le__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value <= other.value

    def __gt__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value > other.value

    def __ge__(self, other):  # pragma: no cover
        if issubclass(self.__class__, EnumSpecDescription):
            return self.value >= other.value

    @property
    def spec(self):
        return self.__spec

    @property
    def description(self):
        return self.__description

class ModeTask(EnumSpecDescription):
    TASK = 'name, uuid, routing key, state and retries', False, False, False

    RESULT = 'TASK + results', False, True, False
    ERROR = 'TASK + exceptions', False, False, True
    OUTCOME = 'RESULT + ERROR', False, True, True

    SUCCESS = 'RESULT + args/kwargs', None, True, False
    FAILURE = 'ERROR + args/kwargs', None, False, True
    DONE = 'SUCCESS + FAILURE', None, True, True

    SENT = 'TASK + args/kwargs being sent', True, False, False
    FLOWING = 'SENT + SUCCESS', True, True, False
    BREAKING = 'SENT + FAILURE', True, False, True
    ALL = 'SENT + DONE', True, True, True

class ModeWorker(EnumSpecDescription):
    WORKER = 'name, status and pid', False
    STATS = 'WORKER + version, load and heartbeats', True

def find_mode(constant: Union[None, int, ModeTask, ModeWorker]) \
        -> Union[None, ModeTask, ModeWorker]:
    if isinstance(constant, (type(None), ModeTask, ModeWorker)):
        return constant
    if isinstance(constant, int) and constant in _VALUES:
        return _VALUES[constant]
    raise UserWarning('Invalid mode constant.')
<|endoftext|>


<commit_before>from django.test import TestCase

# Create your tests here.
<commit_msg>Created two passing tests<commit_after>from django.test import TestCase, Client, override_settings
from announce.models import Group, Person

def make_post(user_name='bob', user_id='test_id', token='1234abc', text=''):
    return {'token': token,
            'user_name': user_name,
            'user_id': user_id,
            'text': text,
            }

@override_settings(SLACK_TOKEN='1234abc')
class AnnounceTestCase(TestCase):
    def setUp(self):
        Group.objects.create(group_name='test_group')
        Person.objects.create(person_name='bob', person_id='test_id')

    def test_group_listing(self):
        c = Client()
        response = c.post('/announce/', make_post(text='list-groups'))
        self.assertTrue('test_group' in response.content)
        self.assertEqual(1, 1)
        print response.content

    def test_groups(self):
        g = Group.objects.get(group_name='test_group')
        self.assertFalse(g.person_set.all())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
The MIT License (MIT)

Copyright (c) 2017 SML

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import os
import discord
import datetime as dt
from collections import defaultdict
from discord.ext import commands

from __main__ import send_cmd_help
from cogs.utils import checks
from cogs.utils.chat_formatting import pagify
from cogs.utils.dataIO import dataIO

PATH = os.path.join("data", "reactionmanager")
JSON = os.path.join(PATH, "settings.json")

def nested_dict():
    """Recursively nested defaultdict."""
    return defaultdict(nested_dict)

class ReactionManager:

    def __init__(self, bot):
        """Reaction Management."""
        self.bot = bot
        self.settings = dataIO.load_json(JSON)

    @commands.group(aliases=['rm'], pass_context=True, no_pm=True)
    async def reactionmanager(self, ctx):
        """Reaction Manager."""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @reactionmanager.command(name='add', pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_add(self, ctx, *args):
        """Add reactions to a message by message id.

        Add reactions to a specific message id
        [p]rm add 123456 :white_check_mark: :x: :zzz:

        Add reactions to the last message in channel
        [p]rm add :white_check_mark: :x: :zzz:
        """
        channel = ctx.message.channel

        if not len(args):
            await send_cmd_help(ctx)
            return

        has_message_id = args[0].isdigit()

        emojis = args[1:] if has_message_id else args
        message_id = args[0] if has_message_id else None

        if has_message_id:
            try:
                message = await self.bot.get_message(channel, message_id)
            except discord.NotFound:
                await self.bot.say("Cannot find message with that id.")
                return
        else:
            # use the 2nd last message because the last message would be the command
            messages = []
            async for m in self.bot.logs_from(channel, limit=2):
                messages.append(m)

            # messages = [m async for m in self.bot.logs_from(channel, limit=2)]
            message = messages[1]

        for emoji in emojis:
            try:
                await self.bot.add_reaction(message, emoji)
            except discord.HTTPException:
                # reaction add failed
                pass
            except discord.Forbidden:
                await self.bot.say(
                    "I don’t have permission to react to that message.")
                break
            except discord.InvalidArgument:
                await self.bot.say("Invalid arguments for emojis")
                break

        await self.bot.delete_message(ctx.message)

    @reactionmanager.command(name="remove", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_remove(self, ctx, number: int):
        """Remove reactions from last X messages in the channel."""
        channel = ctx.message.channel
        author = ctx.message.author
        server = author.server
        has_permissions = channel.permissions_for(server.me).manage_messages
        to_manage = []

        if not has_permissions:
            await self.bot.say("I’m not allowed to remove reactions.")
            return

        async for message in self.bot.logs_from(channel, limit=number + 1):
            to_manage.append(message)

        await self.remove_reactions(to_manage)

    async def remove_reactions(self, messages):
        """Remove reactions."""
        for message in messages:
            await self.bot.clear_reactions(message)

    @reactionmanager.command(name="get", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_get(self, ctx, message_id: int):
        """Display list of reactions added by users."""
        channel = ctx.message.channel
        message = await self.bot.get_message(channel, message_id)

        if message is None:
            await self.bot.say("Cannot find that message id.")
            return

        title = message.channel.name
        description = message.content
        em = discord.Embed(
            title=title,
            description=description)

        for reaction in message.reactions:
            if reaction.custom_emoji:
                # <:emoji_name:emoji_id>
                emoji = '<:{}:{}>'.format(
                    reaction.emoji.name,
                    reaction.emoji.id)
            else:
                emoji = reaction.emoji

            reaction_users = await self.bot.get_reaction_users(reaction)
            users = ' '.join([m.mention for m in reaction_users])
            name = emoji
            count = reaction.count
            value = '{}: {}'.format(count, users)

            em.add_field(name=name, value=value, inline=True)

        em.set_footer(
            text='ID: {} | Updated: {}'.format(
                message.id,
                dt.datetime.utcnow().isoformat()))

        await self.bot.say(embed=em)

def check_folder():
    """Check folder."""
    if not os.path.exists(PATH):
        os.makedirs(PATH)

def check_file():
    """Check files."""
    defaults = {}
    if not dataIO.is_valid_json(JSON):
        dataIO.save_json(JSON, defaults)

def setup(bot):
    """Setup bot."""
    check_folder()
    check_file()
    n = ReactionManager(bot)
    bot.add_cog(n)
<commit_msg>[reactionmanager] added option to exclude self from reaction count.<commit_after># -*- coding: utf-8 -*-

"""
The MIT License (MIT)

Copyright (c) 2017 SML

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import os
import discord
import datetime as dt
from collections import defaultdict
from discord.ext import commands

from __main__ import send_cmd_help
from cogs.utils import checks
from cogs.utils.chat_formatting import pagify, bold, escape_mass_mentions
from cogs.utils.dataIO import dataIO

PATH = os.path.join("data", "reactionmanager")
JSON = os.path.join(PATH, "settings.json")

def nested_dict():
    """Recursively nested defaultdict."""
    return defaultdict(nested_dict)

class ReactionManager:

    def __init__(self, bot):
        """Reaction Management."""
        self.bot = bot
        self.settings = dataIO.load_json(JSON)

    @commands.group(aliases=['rm'], pass_context=True, no_pm=True)
    async def reactionmanager(self, ctx):
        """Reaction Manager."""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @reactionmanager.command(name='add', pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_add(self, ctx, *args):
        """Add reactions to a message by message id.

        Add reactions to a specific message id
        [p]rm add 123456 :white_check_mark: :x: :zzz:

        Add reactions to the last message in channel
        [p]rm add :white_check_mark: :x: :zzz:
        """
        channel = ctx.message.channel

        if not len(args):
            await send_cmd_help(ctx)
            return

        has_message_id = args[0].isdigit()

        emojis = args[1:] if has_message_id else args
        message_id = args[0] if has_message_id else None

        if has_message_id:
            try:
                message = await self.bot.get_message(channel, message_id)
            except discord.NotFound:
                await self.bot.say("Cannot find message with that id.")
                return
        else:
            # use the 2nd last message because the last message would be the command
            messages = []
            async for m in self.bot.logs_from(channel, limit=2):
                messages.append(m)

            # messages = [m async for m in self.bot.logs_from(channel, limit=2)]
            message = messages[1]

        for emoji in emojis:
            try:
                await self.bot.add_reaction(message, emoji)
            except discord.HTTPException:
                # reaction add failed
                pass
            except discord.Forbidden:
                await self.bot.say(
                    "I don’t have permission to react to that message.")
                break
            except discord.InvalidArgument:
                await self.bot.say("Invalid arguments for emojis")
                break

        await self.bot.delete_message(ctx.message)

    @reactionmanager.command(name="remove", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_remove(self, ctx, number: int):
        """Remove reactions from last X messages in the channel."""
        channel = ctx.message.channel
        author = ctx.message.author
        server = author.server
        has_permissions = channel.permissions_for(server.me).manage_messages
        to_manage = []

        if not has_permissions:
            await self.bot.say("I’m not allowed to remove reactions.")
            return

        async for message in self.bot.logs_from(channel, limit=number + 1):
            to_manage.append(message)

        await self.remove_reactions(to_manage)

    async def remove_reactions(self, messages):
        """Remove reactions."""
        for message in messages:
            await self.bot.clear_reactions(message)

    @reactionmanager.command(name="get", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_get(self, ctx, channel: discord.Channel, message_id, exclude_self=True):
        """Display list of reactions added by users."""
        message = await self.bot.get_message(channel, message_id)

        if message is None:
            await self.bot.say("Cannot find that message id.")
            return

        title = message.channel.name
        description = message.content

        out = [
            bold('Channel: {}'.format(title)),
            escape_mass_mentions(description)
        ]

        for reaction in message.reactions:
            if reaction.custom_emoji:
                # <:emoji_name:emoji_id>
                emoji = '<:{}:{}>'.format(
                    reaction.emoji.name,
                    reaction.emoji.id)
            else:
                emoji = reaction.emoji

            reaction_users = await self.bot.get_reaction_users(reaction)
            valid_users = []
            for u in reaction_users:
                if exclude_self and u == self.bot.user:
                    continue
                valid_users.append(u)
            users = ', '.join([u.display_name for u in valid_users])
            name = emoji
            count = reaction.count
            value = '{}: {}'.format(count, users)

            out.append(value)

        for page in pagify('\n'.join(out), shorten_by=24):
            await self.bot.say(page)

def check_folder():
    """Check folder."""
    if not os.path.exists(PATH):
        os.makedirs(PATH)

def check_file():
    """Check files."""
    defaults = {}
    if not dataIO.is_valid_json(JSON):
        dataIO.save_json(JSON, defaults)

def setup(bot):
    """Setup bot."""
    check_folder()
    check_file()
    n = ReactionManager(bot)
    bot.add_cog(n)
<|endoftext|>


<commit_before>"""Defines serializers for each of our models."""

from __future__ import absolute_import

from builtins import object

from allauth.socialaccount.models import SocialAccount
from rest_framework import serializers

from readthedocs.builds.models import Build, BuildCommandResult, Version
from readthedocs.oauth.models import RemoteOrganization, RemoteRepository
from readthedocs.projects.models import Project, Domain

class ProjectSerializer(serializers.ModelSerializer):
    canonical_url = serializers.ReadOnlyField(source='get_docs_url')

    class Meta(object):
        model = Project
        fields = (
            'id',
            'name', 'slug', 'description', 'language',
            'programming_language', 'repo', 'repo_type',
            'default_version', 'default_branch',
            'documentation_type',
            'users',
            'canonical_url',
        )

class ProjectAdminSerializer(ProjectSerializer):

    """
    Project serializer for admin only access.

    Includes special internal fields that don't need to be exposed through the
    general API, mostly for fields used in the build process
    """

    features = serializers.SlugRelatedField(
        many=True,
        read_only=True,
        slug_field='feature_id',
    )

    class Meta(ProjectSerializer.Meta):
        fields = ProjectSerializer.Meta.fields + (
            'enable_epub_build',
            'enable_pdf_build',
            'conf_py_file',
            'analytics_code',
            'cdn_enabled',
            'container_image',
            'container_mem_limit',
            'container_time_limit',
            'install_project',
            'use_system_packages',
            'skip',
            'requirements_file',
            'python_interpreter',
            'features',
            'has_valid_clone',
            'has_valid_webhook',
            'show_advertising',
            'environment_variables',
        )

class VersionSerializer(serializers.ModelSerializer):
    project = ProjectSerializer()
    downloads = serializers.DictField(source='get_downloads', read_only=True)

    class Meta(object):
        model = Version
        fields = (
            'id',
            'project', 'slug',
            'identifier', 'verbose_name',
            'active', 'built',
            'downloads',
            'type',
        )

class VersionAdminSerializer(VersionSerializer):

    """Version serializer that returns admin project data."""

    project = ProjectAdminSerializer()

class BuildCommandSerializer(serializers.ModelSerializer):

    run_time = serializers.ReadOnlyField()

    class Meta(object):
        model = BuildCommandResult
        exclude = ('')

class BuildSerializer(serializers.ModelSerializer):

    """Build serializer for user display, doesn't display internal fields."""

    commands = BuildCommandSerializer(many=True, read_only=True)
    project_slug = serializers.ReadOnlyField(source='project.slug')
    version_slug = serializers.ReadOnlyField(source='version.slug')
    docs_url = serializers.ReadOnlyField(source='version.get_absolute_url')
    state_display = serializers.ReadOnlyField(source='get_state_display')
    # Jsonfield needs an explicit serializer
    # https://github.com/dmkoch/django-jsonfield/issues/188#issuecomment-300439829
    config = serializers.JSONField(required=False)

    class Meta(object):
        model = Build
        # `_config` should be excluded to avoid conflicts with `config`
        exclude = ('builder', '_config')

class BuildAdminSerializer(BuildSerializer):

    """Build serializer for display to admin users and build instances."""

    class Meta(BuildSerializer.Meta):
        # `_config` should be excluded to avoid conflicts with `config`
        exclude = ('_config',)

class SearchIndexSerializer(serializers.Serializer):
    q = serializers.CharField(max_length=500)
    project = serializers.CharField(max_length=500, required=False)
    version = serializers.CharField(max_length=500, required=False)
    page = serializers.CharField(max_length=500, required=False)

class DomainSerializer(serializers.ModelSerializer):
    project = ProjectSerializer()

    class Meta(object):
        model = Domain
        fields = (
            'id',
            'project',
            'domain',
            'canonical',
            'machine',
            'cname',
        )

class RemoteOrganizationSerializer(serializers.ModelSerializer):

    class Meta(object):
        model = RemoteOrganization
        exclude = ('json', 'email', 'users')

class RemoteRepositorySerializer(serializers.ModelSerializer):

    """Remote service repository serializer."""

    organization = RemoteOrganizationSerializer()
    matches = serializers.SerializerMethodField()

    class Meta(object):
        model = RemoteRepository
        exclude = ('json', 'users')

    def get_matches(self, obj):
        request = self.context['request']
        if request.user is not None and request.user.is_authenticated:
            return obj.matches(request.user)

class ProviderSerializer(serializers.Serializer):

    id = serializers.CharField(max_length=20)
    name = serializers.CharField(max_length=20)

class SocialAccountSerializer(serializers.ModelSerializer):

    username = serializers.SerializerMethodField()
    avatar_url = serializers.URLField(source='get_avatar_url')
    provider = ProviderSerializer(source='get_provider')

    class Meta(object):
        model = SocialAccount
        exclude = ('extra_data',)

    def get_username(self, obj):
        return (
            obj.extra_data.get('username') or
            obj.extra_data.get('login')
            # FIXME: which one is GitLab?
        )
<commit_msg>Use project detail endpoint with admin serializer<commit_after>"""Defines serializers for each of our models."""

from __future__ import absolute_import

from builtins import object

from allauth.socialaccount.models import SocialAccount
from rest_framework import serializers

from readthedocs.builds.models import Build, BuildCommandResult, Version
from readthedocs.oauth.models import RemoteOrganization, RemoteRepository
from readthedocs.projects.models import Project, Domain

class ProjectSerializer(serializers.ModelSerializer):
    canonical_url = serializers.ReadOnlyField(source='get_docs_url')

    class Meta(object):
        model = Project
        fields = (
            'id',
            'name', 'slug', 'description', 'language',
            'programming_language', 'repo', 'repo_type',
            'default_version', 'default_branch',
            'documentation_type',
            'users',
            'canonical_url',
        )

class ProjectAdminSerializer(ProjectSerializer):

    """
    Project serializer for admin only access.

    Includes special internal fields that don't need to be exposed through the
    general API, mostly for fields used in the build process
    """

    features = serializers.SlugRelatedField(
        many=True,
        read_only=True,
        slug_field='feature_id',
    )

    def get_environment_variables(self, obj):
        return {
            variable.name: variable.value
            for variable in obj.environmentvariable_set.all()
        }

    class Meta(ProjectSerializer.Meta):
        fields = ProjectSerializer.Meta.fields + (
            'enable_epub_build',
            'enable_pdf_build',
            'conf_py_file',
            'analytics_code',
            'cdn_enabled',
            'container_image',
            'container_mem_limit',
            'container_time_limit',
            'install_project',
            'use_system_packages',
            'skip',
            'requirements_file',
            'python_interpreter',
            'features',
            'has_valid_clone',
            'has_valid_webhook',
            'show_advertising',
            'environment_variables',
        )

class VersionSerializer(serializers.ModelSerializer):
    project = ProjectSerializer()
    downloads = serializers.DictField(source='get_downloads', read_only=True)

    class Meta(object):
        model = Version
        fields = (
            'id',
            'project', 'slug',
            'identifier', 'verbose_name',
            'active', 'built',
            'downloads',
            'type',
        )

class VersionAdminSerializer(VersionSerializer):

    """Version serializer that returns admin project data."""

    project = ProjectAdminSerializer()

class BuildCommandSerializer(serializers.ModelSerializer):

    run_time = serializers.ReadOnlyField()

    class Meta(object):
        model = BuildCommandResult
        exclude = ('')

class BuildSerializer(serializers.ModelSerializer):

    """Build serializer for user display, doesn't display internal fields."""

    commands = BuildCommandSerializer(many=True, read_only=True)
    project_slug = serializers.ReadOnlyField(source='project.slug')
    version_slug = serializers.ReadOnlyField(source='version.slug')
    docs_url = serializers.ReadOnlyField(source='version.get_absolute_url')
    state_display = serializers.ReadOnlyField(source='get_state_display')
    # Jsonfield needs an explicit serializer
    # https://github.com/dmkoch/django-jsonfield/issues/188#issuecomment-300439829
    config = serializers.JSONField(required=False)

    class Meta(object):
        model = Build
        # `_config` should be excluded to avoid conflicts with `config`
        exclude = ('builder', '_config')

class BuildAdminSerializer(BuildSerializer):

    """Build serializer for display to admin users and build instances."""

    class Meta(BuildSerializer.Meta):
        # `_config` should be excluded to avoid conflicts with `config`
        exclude = ('_config',)

class SearchIndexSerializer(serializers.Serializer):
    q = serializers.CharField(max_length=500)
    project = serializers.CharField(max_length=500, required=False)
    version = serializers.CharField(max_length=500, required=False)
    page = serializers.CharField(max_length=500, required=False)

class DomainSerializer(serializers.ModelSerializer):
    project = ProjectSerializer()

    class Meta(object):
        model = Domain
        fields = (
            'id',
            'project',
            'domain',
            'canonical',
            'machine',
            'cname',
        )

class RemoteOrganizationSerializer(serializers.ModelSerializer):

    class Meta(object):
        model = RemoteOrganization
        exclude = ('json', 'email', 'users')

class RemoteRepositorySerializer(serializers.ModelSerializer):

    """Remote service repository serializer."""

    organization = RemoteOrganizationSerializer()
    matches = serializers.SerializerMethodField()

    class Meta(object):
        model = RemoteRepository
        exclude = ('json', 'users')

    def get_matches(self, obj):
        request = self.context['request']
        if request.user is not None and request.user.is_authenticated:
            return obj.matches(request.user)

class ProviderSerializer(serializers.Serializer):

    id = serializers.CharField(max_length=20)
    name = serializers.CharField(max_length=20)

class SocialAccountSerializer(serializers.ModelSerializer):

    username = serializers.SerializerMethodField()
    avatar_url = serializers.URLField(source='get_avatar_url')
    provider = ProviderSerializer(source='get_provider')

    class Meta(object):
        model = SocialAccount
        exclude = ('extra_data',)

    def get_username(self, obj):
        return (
            obj.extra_data.get('username') or
            obj.extra_data.get('login')
            # FIXME: which one is GitLab?
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# An attempt at re-implementing LZJB compression in native Python.
#

import math

NBBY = 8
MATCH_BITS = 6
MATCH_MIN = 3
MATCH_MAX = ((1 << MATCH_BITS) + (MATCH_MIN - 1))
OFFSET_MASK = ((1 << (16 - MATCH_BITS)) - 1)
LEMPEL_SIZE_BASE = 1024

def compress(s, with_size = True):
	LEMPEL_SIZE = LEMPEL_SIZE_BASE

	# During compression, treat output string as list of code points.
	dst = []

	# Encode input size. This uses a variable-length encoding.
	if with_size:
		size = len(s)
		if size:
			sbytes = []
			size += 1
			while True:
				sbytes.append(size & 0x7f)
				size = int(math.floor(size / 128))
				if size == 0:
					break
			sbytes[0] |= 0x80
			for sb in reversed(sbytes):
				dst.append(sb)

	lempel = [0] * LEMPEL_SIZE
	copymask = 1 << (NBBY - 1)
	src = 0 # Current input offset.
	while src < len(s):
		copymask <<= 1
		if (copymask == (1 << NBBY)):
			copymask = 1
			copymap = len(dst)
			dst.append(0)
		if src > len(s) - MATCH_MAX:
			dst.append(ord(s[src]))
			src += 1
			continue
		hsh = (ord(s[src]) << 16) + (ord(s[src + 1]) << 8) + ord(s[src + 2])
		hsh += hsh >> 9
		hsh += hsh >> 5
		hsh &= LEMPEL_SIZE - 1
		offset = (src - lempel[hsh]) & OFFSET_MASK
		print "src=%u hsh=0x%04x offset=%u" % (src, hsh, offset)
		lempel[hsh] = src
		cpy = src - offset
		if cpy >= 0 and cpy != src and s[src:src+3] == s[cpy:cpy+3]:
			dst[copymap] = dst[copymap] | copymask
			for mlen in xrange(MATCH_MIN, MATCH_MAX):
				if s[src + mlen] != s[cpy + mlen]:
					break
			dst.append(((mlen - MATCH_MIN) << (NBBY - MATCH_BITS)) | (offset >> NBBY))
			dst.append(offset & 0xff)
			src += mlen
		else:
			dst.append(ord(s[src]))
			src += 1
	# Now implode the list of codepoints into an actual string.
	return "".join(map(chr, dst))

def decompressed_size(s):
	dstSize = 0
	src = 0
	# Extract prefixed encoded size, if present.
	while True:
		c = ord(s[src])
		src += 1
		if (c & 0x80):
			dstSize |= c & 0x7f
			break
		dstSize = (dstSize | c) << 7
	dstSize -= 1	# -1 means "not known".
	return (dstSize, src)

def decompress(s, with_size = True):
	src = 0
	dstSize = 0
	if with_size:
		dstSize, src = decompressed_size(s)
		if dstSize < 0:
			return None

	dst = ""
	copymask = 1 << (NBBY - 1)
	while src < len(s):
		copymask <<= 1
		if copymask == (1 << NBBY):
			copymask = 1
			copymap = ord(s[src])
			src += 1
		if copymap & copymask:
			mlen = (ord(s[src]) >> (NBBY - MATCH_BITS)) + MATCH_MIN
			offset = ((ord(s[src]) << NBBY) | ord(s[src + 1])) & OFFSET_MASK
			src += 2
			cpy = len(dst) - offset
			print "src=%u mlen=%u offset=%u cpy=%u" % (src, mlen, offset, cpy)
			if cpy < 0:
				print "Decompression failed"
				return None
			while mlen > 0:
				dst += dst[cpy]
				cpy += 1
				mlen -= 1
		else:
			dst += s[src]
			src += 1
	return dst

if __name__ == "__main__":
	data = 18 * "whatever ever is what this ever?"
	compressed = compress(data)
	print "%u -> %u (%.2f%%)" % (len(data), len(compressed), 100.0 * len(compressed) / len(data))
	print "data size was %u (encoded in %u bytes)" % decompressed_size(compressed)
	decompressed = decompress(compressed)
	print len(decompressed)
	print data == decompressed
<commit_msg>Silence.<commit_after>#!/usr/bin/env python
#
# An attempt at re-implementing LZJB compression in native Python.
#

import math

NBBY = 8
MATCH_BITS = 6
MATCH_MIN = 3
MATCH_MAX = ((1 << MATCH_BITS) + (MATCH_MIN - 1))
OFFSET_MASK = ((1 << (16 - MATCH_BITS)) - 1)
LEMPEL_SIZE_BASE = 1024

def compress(s, with_size = True):
	LEMPEL_SIZE = LEMPEL_SIZE_BASE

	# During compression, treat output string as list of code points.
	dst = []

	# Encode input size. This uses a variable-length encoding.
	if with_size:
		size = len(s)
		if size:
			sbytes = []
			size += 1
			while True:
				sbytes.append(size & 0x7f)
				size = int(math.floor(size / 128))
				if size == 0:
					break
			sbytes[0] |= 0x80
			for sb in reversed(sbytes):
				dst.append(sb)

	lempel = [0] * LEMPEL_SIZE
	copymask = 1 << (NBBY - 1)
	src = 0 # Current input offset.
	while src < len(s):
		copymask <<= 1
		if (copymask == (1 << NBBY)):
			copymask = 1
			copymap = len(dst)
			dst.append(0)
		if src > len(s) - MATCH_MAX:
			dst.append(ord(s[src]))
			src += 1
			continue
		hsh = (ord(s[src]) << 16) + (ord(s[src + 1]) << 8) + ord(s[src + 2])
		hsh += hsh >> 9
		hsh += hsh >> 5
		hsh &= LEMPEL_SIZE - 1
		offset = (src - lempel[hsh]) & OFFSET_MASK
		lempel[hsh] = src
		cpy = src - offset
		if cpy >= 0 and cpy != src and s[src:src+3] == s[cpy:cpy+3]:
			dst[copymap] = dst[copymap] | copymask
			for mlen in xrange(MATCH_MIN, MATCH_MAX):
				if s[src + mlen] != s[cpy + mlen]:
					break
			dst.append(((mlen - MATCH_MIN) << (NBBY - MATCH_BITS)) | (offset >> NBBY))
			dst.append(offset & 0xff)
			src += mlen
		else:
			dst.append(ord(s[src]))
			src += 1
	# Now implode the list of codepoints into an actual string.
	return "".join(map(chr, dst))

def decompressed_size(s):
	dstSize = 0
	src = 0
	# Extract prefixed encoded size, if present.
	while True:
		c = ord(s[src])
		src += 1
		if (c & 0x80):
			dstSize |= c & 0x7f
			break
		dstSize = (dstSize | c) << 7
	dstSize -= 1	# -1 means "not known".
	return (dstSize, src)

def decompress(s, with_size = True):
	src = 0
	dstSize = 0
	if with_size:
		dstSize, src = decompressed_size(s)
		if dstSize < 0:
			return None

	dst = ""
	copymask = 1 << (NBBY - 1)
	while src < len(s):
		copymask <<= 1
		if copymask == (1 << NBBY):
			copymask = 1
			copymap = ord(s[src])
			src += 1
		if copymap & copymask:
			mlen = (ord(s[src]) >> (NBBY - MATCH_BITS)) + MATCH_MIN
			offset = ((ord(s[src]) << NBBY) | ord(s[src + 1])) & OFFSET_MASK
			src += 2
			cpy = len(dst) - offset
			if cpy < 0:
				return None
			while mlen > 0:
				dst += dst[cpy]
				cpy += 1
				mlen -= 1
		else:
			dst += s[src]
			src += 1
	return dst

if __name__ == "__main__":
	data = 18 * "whatever ever is what this ever?"
	compressed = compress(data)
	print "%u -> %u (%.2f%%)" % (len(data), len(compressed), 100.0 * len(compressed) / len(data))
	print "data size was %u (encoded in %u bytes)" % decompressed_size(compressed)
	decompressed = decompress(compressed)
	print len(decompressed)
	print data == decompressed
<|endoftext|>