<commit_before>import os
import sys
import pkg_resources
import threading

from fanstatic.checksum import checksum

DEFAULT_SIGNATURE = 'fanstatic'

EXTENSIONS = ['.css', '.kss', '.js']

NEEDED = 'fanstatic.needed'

class UnknownResourceExtension(Exception):
    """Unknown resource extension"""

class Library(object):

    _signature = None

    def __init__(self, name, rootpath):
        self.name = name
        self.rootpath = rootpath
        self.path = os.path.join(caller_dir(), rootpath)

    def signature(self, devmode=False):
        if devmode:
            # Always re-compute.
            sig = checksum(self.path)
        elif self._signature is None:
            # Only compute if not conputed before.
            sig = self._signature = checksum(self.path)
        else:
            # Use cached value.
            sig = self._signature
        return ':hash:%s' % sig

# Total hack to be able to get the dir the resources will be in.
def caller_dir():
    return os.path.dirname(sys._getframe(2).f_globals['__file__'])

def _libraries(libs={}):
    if not libs:
        for entry_point in pkg_resources.iter_entry_points(
            'fanstatic.libraries'):
            libs[entry_point.name] = entry_point.load()
    return libs

def libraries():
    return _libraries().itervalues()

def library_by_name(name):
    return _libraries()[name]

class InclusionBase(object):
    pass

class ResourceInclusion(InclusionBase):
    """Resource inclusion

    A resource inclusion specifies how to include a single resource in
    a library.
    """

    def __init__(self, library, relpath, depends=None,
                 supersedes=None, eager_superseder=False,
                 bottom=False, **kw):
        """Create a resource inclusion

        library  - the library this resource is in
        relpath  - the relative path from the root of the library indicating
                   the actual resource
        depends  - optionally, a list of resources that this resource depends
                   on. Entries in the list can be
                   ResourceInclusions or strings indicating the path.
                   In case of a string, a ResourceInclusion assumed based
                   on the same library as this inclusion.
        supersedes - optionally, a list of resources that this resource
                   supersedes as a rollup resource. If all these
                   resources are required, the superseding resource
                   instead will show up.
        eager_superseder - even if only part of the requirements are
                           met, supersede anyway
        bottom - optionally, indicate that this resource can be
                 safely included on the bottom of the page (just
                 before ````). This can be used to
                 improve the performance of page loads when javascript
                 resources are in use. Not all javascript-based resources
                 can however be safely included that way.
        keyword arguments - different paths that represent the same
                  resource in different modes (debug, minified, etc),
                  or alternatively a fully specified ResourceInclusion.
        """
        self.library = library
        self.relpath = relpath
        self.bottom = bottom

        assert not isinstance(depends, basestring)
        depends = depends or []
        self.depends = normalize_inclusions(library, depends)

        self.rollups = []

        normalized_modes = {}
        for mode_name, inclusion in kw.items():
            normalized_modes[mode_name] = normalize_inclusion(
                library, inclusion)
        self.modes = normalized_modes

        assert not isinstance(supersedes, basestring)
        self.supersedes = supersedes or []
        self.eager_superseder = eager_superseder

        # create a reference to the superseder in the superseded inclusion
        for inclusion in self.supersedes:
            inclusion.rollups.append(self)
        # also create a reference to the superseding mode in the superseded
        # mode
        # XXX what if mode is full-fledged resource inclusion which lists
        # supersedes itself?
        for mode_name, mode in self.modes.items():
            for inclusion in self.supersedes:
                superseded_mode = inclusion.mode(mode_name)
                # if there is no such mode, let's skip it
                if superseded_mode is inclusion:
                    continue
                mode.supersedes.append(superseded_mode)
                superseded_mode.rollups.append(mode)

    def __repr__(self):
        return "<ResourceInclusion '%s' in library '%s'>" % (
            self.relpath, self.library.name)

    def ext(self):
        name, ext = os.path.splitext(self.relpath)
        return ext

    def mode(self, mode):
        if mode is None:
            return self
        # try getting the alternative
        try:
            return self.modes[mode]
        except KeyError:
            # fall back on the default mode if mode not found
            return self

    def key(self):
        return self.library.name, self.relpath

    def need(self):
        needed = get_current_needed_inclusions()
        needed.need(self)

    def inclusions(self):
        """Get all inclusions needed by this inclusion, including itself.
        """
        result = []
        for depend in self.depends:
            result.extend(depend.inclusions())
        result.append(self)
        return result

class GroupInclusion(InclusionBase):
    """An inclusion used to group resources together.

    It doesn't define a resource itself.
    """
    def __init__(self, depends):
        self.depends = depends

    def need(self):
        needed = get_current_needed_inclusions()
        needed.need(self)

    def inclusions(self):
        """Get all inclusions needed by this inclusion.
        """
        result = []
        for depend in self.depends:
            result.extend(depend.inclusions())
        return result

def normalize_inclusions(library, inclusions):
    return [normalize_inclusion(library, inclusion)
            for inclusion in inclusions]

def normalize_inclusion(library, inclusion):
    if isinstance(inclusion, InclusionBase):
        return inclusion
    assert isinstance(inclusion, basestring)
    return ResourceInclusion(library, inclusion)

class NeededInclusions(object):
    def __init__(self,
                 base_url='',
                 devmode=False,
                 hashing=False,
                 bottom=False,
                 force_bottom=False,
                 inclusions=None,
                 mode=None,
                 publisher_signature=DEFAULT_SIGNATURE,
                 rollup=False,
                 ):
        self.base_url = base_url
        self.devmode = devmode
        self.hashing = hashing

        self._bottom = bottom
        self._force_bottom = force_bottom
        self._inclusions = inclusions or []
        self._mode = mode
        self._publisher_signature = publisher_signature
        self._rollup = rollup

    def __len__(self):
        return len(self._inclusions)

    def need(self, inclusion):
        self._inclusions.append(inclusion)

    def _sorted_inclusions(self):
        return reversed(sorted(self._inclusions, key=lambda i: i.depth()))

    def inclusions(self):
        inclusions = []
        for inclusion in self._inclusions:
            inclusions.extend(inclusion.inclusions())

        inclusions = apply_mode(inclusions, self._mode)
        if self._rollup:
            inclusions = consolidate(inclusions)
        # sort only by extension, not dependency, as we can rely on
        # python's stable sort to keep inclusion order intact
        inclusions = sort_inclusions_by_extension(inclusions)
        inclusions = remove_duplicates(inclusions)

        return inclusions

    def library_url(self, library):
        segments = [self.base_url, self._publisher_signature]
        if self.hashing:
            segments.append(library.signature(devmode=self.devmode))
        segments.append(library.name)
        result = segments.pop(0)
        for segment in segments:
            if not result.endswith('/'):
                result += '/'
            result += segment
        return result

    def render(self):
        """Render a set of inclusions.
        """
        return self.render_inclusions(self.inclusions())

    def render_inclusions(self, inclusions):
        result = []
        url_cache = {} # prevent multiple computations for a library in one request
        for inclusion in inclusions:
            library = inclusion.library
            library_url = url_cache.get(library.name)
            if library_url is None:
                library_url = url_cache[library.name] = self.library_url(
                    library)
            result.append(
                render_inclusion(
                    inclusion, '%s/%s' %(library_url, inclusion.relpath)))
        return '\n'.join(result)

    def render_into_html(self, html):
        to_insert = self.render()
        return html.replace('', '\n    %s\n' % to_insert, 1)

    def render_topbottom(self):
        inclusions = self.inclusions()

        # seperate inclusions in top and bottom inclusions if this is needed
        if self._bottom:
            top_inclusions = []
            bottom_inclusions = []
            if not self._force_bottom:
                for inclusion in inclusions:
                    if inclusion.bottom:
                        bottom_inclusions.append(inclusion)
                    else:
                        top_inclusions.append(inclusion)
            else:
                for inclusion in inclusions:
                    if inclusion.ext() == '.js':
                        bottom_inclusions.append(inclusion)
                    else:
                        top_inclusions.append(inclusion)
        else:
            top_inclusions = inclusions
            bottom_inclusions = []

        return (self.render_inclusions(top_inclusions),
                self.render_inclusions(bottom_inclusions))

    def render_topbottom_into_html(self, html):
        top, bottom = self.render_topbottom()
        if top:
            html = html.replace('', '\n    %s\n' % top, 1)
        if bottom:
            html = html.replace('', '%s' % bottom, 1)
        return html

class NoNeededInclusions(Exception):
    pass

thread_local_needed_data = threading.local()

def init_current_needed_inclusions(*args, **kw):
    needed = NeededInclusions(*args, **kw)
    thread_local_needed_data.__dict__[NEEDED] = needed
    return needed

def get_current_needed_inclusions():
    needed = thread_local_needed_data.__dict__.get(NEEDED)
    if needed is None:
        raise NoNeededInclusions('No NeededInclusions object initialized.')
    return needed

def apply_mode(inclusions, mode):
    return [inclusion.mode(mode) for inclusion in inclusions]

def remove_duplicates(inclusions):
    """Given a set of inclusions, consolidate them so each only occurs once.
    """
    seen = set()
    result = []
    for inclusion in inclusions:
        if inclusion.key() in seen:
            continue
        seen.add(inclusion.key())
        result.append(inclusion)
    return result

def consolidate(inclusions):
    # keep track of rollups: rollup key -> set of inclusion keys
    potential_rollups = {}
    for inclusion in inclusions:
        for rollup in inclusion.rollups:
            s = potential_rollups.setdefault(rollup.key(), set())
            s.add(inclusion.key())

    # now go through inclusions, replacing them with rollups if
    # conditions match
    result = []
    for inclusion in inclusions:
        eager_superseders = []
        exact_superseders = []
        for rollup in inclusion.rollups:
            s = potential_rollups[rollup.key()]
            if rollup.eager_superseder:
                eager_superseders.append(rollup)
            if len(s) == len(rollup.supersedes):
                exact_superseders.append(rollup)
        if eager_superseders:
            # use the eager superseder that rolls up the most
            eager_superseders = sorted(eager_superseders,
                                       key=lambda i: len(i.supersedes))
            result.append(eager_superseders[-1])
        elif exact_superseders:
            # use the exact superseder that rolls up the most
            exact_superseders = sorted(exact_superseders,
                                       key=lambda i: len(i.supersedes))
            result.append(exact_superseders[-1])
        else:
            # nothing to supersede resource so use it directly
            result.append(inclusion)
    return result

def sort_inclusions_by_extension(inclusions):

    def key(inclusion):
        return EXTENSIONS.index(inclusion.ext())

    return sorted(inclusions, key=key)

def sort_inclusions_topological(inclusions):
    """Sort inclusions by dependency and supersedes.
    """
    dead = {}
    result = []
    for inclusion in inclusions:
        dead[inclusion.key()] = False

    for inclusion in inclusions:
        _visit(inclusion, result, dead)
    return result

def _visit(inclusion, result, dead):
    if dead[inclusion.key()]:
        return
    dead[inclusion.key()] = True
    for depend in inclusion.depends:
        _visit(depend, result, dead)
    for depend in inclusion.supersedes:
        _visit(depend ,result, dead)
    result.append(inclusion)

def render_css(url):
    return ('<link rel="stylesheet" type="text/css" href="%s" />' %
            url)

def render_kss(url):
    raise NotImplementedError

def render_js(url):
    return ('' %
            url)

inclusion_renderers = {
    '.css': render_css,
    '.kss': render_kss,
    '.js': render_js,
    }

def render_inclusion(inclusion, url):
    renderer = inclusion_renderers.get(inclusion.ext(), None)
    if renderer is None:
        raise UnknownResourceExtension(
            "Unknown resource extension %s for resource inclusion: %s" %
            (inclusion.ext(), repr(inclusion)))
    return renderer(url)

def generate_code(**kw):
    name_to_inclusion = kw
    inclusion_to_name = {}
    inclusions = []
    for name, inclusion in kw.items():
        inclusion_to_name[inclusion.key()] = name
        inclusions.append(inclusion)

    # libraries with the same name are the same libraries
    libraries = {}
    for inclusion in inclusions:
        libraries[inclusion.library.name] = inclusion.library
    libraries = sorted(libraries.values())

    result = []
    # import on top
    result.append("from fanstatic import Library, ResourceInclusion")
    result.append("")
    # define libraries
    for library in libraries:
        result.append("%s = Library('%s', '%s')" %
                      (library.name, library.name, library.rootpath))
    result.append("")

    # sort inclusions in the order we want them to be
    inclusions = sort_inclusions_by_extension(
        sort_inclusions_topological(inclusions))

    # now generate inclusion code
    for inclusion in inclusions:
        s = "%s = ResourceInclusion(%s, '%s'" % (
            inclusion_to_name[inclusion.key()],
            inclusion.library.name,
            inclusion.relpath)
        if inclusion.depends:
            depends_s = ', depends=[%s]' % ', '.join(
                [inclusion_to_name[d.key()] for d in inclusion.depends])
            s += depends_s
        if inclusion.supersedes:
            supersedes_s = ', supersedes=[%s]' % ', '.join(
                [inclusion_to_name[i.key()] for i in inclusion.supersedes])
            s += supersedes_s
        if inclusion.modes:
            items = []
            for mode_name, mode in inclusion.modes.items():
                items.append((mode_name,
                              generate_inline_inclusion(mode, inclusion)))
            items = sorted(items)
            modes_s = ', %s' % ', '.join(["%s=%s" % (name, mode) for
                                          (name, mode) in items])
            s += modes_s
        s += ')'
        result.append(s)
    return '\n'.join(result)

def generate_inline_inclusion(inclusion, associated_inclusion):
    if inclusion.library.name == associated_inclusion.library.name:
        return "'%s'" % inclusion.relpath
    else:
        return "ResourceInclusion(%s, '%s')" % (inclusion.library.name,
                                                inclusion.relpath)
<commit_msg>implement a LibraryRegistry class to have an "official" API for adding and removing libraries, for example from tests<commit_after>import os
import sys
import pkg_resources
import threading

from fanstatic.checksum import checksum

DEFAULT_SIGNATURE = 'fanstatic'

EXTENSIONS = ['.css', '.kss', '.js']

NEEDED = 'fanstatic.needed'

# Total hack to be able to get the dir the resources will be in.
def caller_dir():
    return os.path.dirname(sys._getframe(2).f_globals['__file__'])

class UnknownResourceExtension(Exception):
    """Unknown resource extension"""

class Library(object):

    _signature = None

    def __init__(self, name, rootpath):
        self.name = name
        self.rootpath = rootpath
        self.path = os.path.join(caller_dir(), rootpath)

    def signature(self, devmode=False):
        if devmode:
            # Always re-compute.
            sig = checksum(self.path)
        elif self._signature is None:
            # Only compute if not conputed before.
            sig = self._signature = checksum(self.path)
        else:
            # Use cached value.
            sig = self._signature
        return ':hash:%s' % sig

class LibraryRegistry(dict):

    def __init__(self):
        super(LibraryRegistry, self).__init__()
        self.reset()

    def reset(self):
        # Load all fanstatic.libraries entry points.
        for entry_point in pkg_resources.iter_entry_points(
            'fanstatic.libraries'):
            self[entry_point.name] = entry_point.load()

    def add(self, library):
        self[library.name] = library

library_registry = LibraryRegistry()

libraries = library_registry.itervalues # poor man's dict.viewvalues()

def library_by_name(name):
    return library_registry[name]

class InclusionBase(object):
    pass

class ResourceInclusion(InclusionBase):
    """Resource inclusion

    A resource inclusion specifies how to include a single resource in
    a library.
    """

    def __init__(self, library, relpath, depends=None,
                 supersedes=None, eager_superseder=False,
                 bottom=False, **kw):
        """Create a resource inclusion

        library  - the library this resource is in
        relpath  - the relative path from the root of the library indicating
                   the actual resource
        depends  - optionally, a list of resources that this resource depends
                   on. Entries in the list can be
                   ResourceInclusions or strings indicating the path.
                   In case of a string, a ResourceInclusion assumed based
                   on the same library as this inclusion.
        supersedes - optionally, a list of resources that this resource
                   supersedes as a rollup resource. If all these
                   resources are required, the superseding resource
                   instead will show up.
        eager_superseder - even if only part of the requirements are
                           met, supersede anyway
        bottom - optionally, indicate that this resource can be
                 safely included on the bottom of the page (just
                 before ````). This can be used to
                 improve the performance of page loads when javascript
                 resources are in use. Not all javascript-based resources
                 can however be safely included that way.
        keyword arguments - different paths that represent the same
                  resource in different modes (debug, minified, etc),
                  or alternatively a fully specified ResourceInclusion.
        """
        self.library = library
        self.relpath = relpath
        self.bottom = bottom

        assert not isinstance(depends, basestring)
        depends = depends or []
        self.depends = normalize_inclusions(library, depends)

        self.rollups = []

        normalized_modes = {}
        for mode_name, inclusion in kw.items():
            normalized_modes[mode_name] = normalize_inclusion(
                library, inclusion)
        self.modes = normalized_modes

        assert not isinstance(supersedes, basestring)
        self.supersedes = supersedes or []
        self.eager_superseder = eager_superseder

        # create a reference to the superseder in the superseded inclusion
        for inclusion in self.supersedes:
            inclusion.rollups.append(self)
        # also create a reference to the superseding mode in the superseded
        # mode
        # XXX what if mode is full-fledged resource inclusion which lists
        # supersedes itself?
        for mode_name, mode in self.modes.items():
            for inclusion in self.supersedes:
                superseded_mode = inclusion.mode(mode_name)
                # if there is no such mode, let's skip it
                if superseded_mode is inclusion:
                    continue
                mode.supersedes.append(superseded_mode)
                superseded_mode.rollups.append(mode)

    def __repr__(self):
        return "<ResourceInclusion '%s' in library '%s'>" % (
            self.relpath, self.library.name)

    def ext(self):
        name, ext = os.path.splitext(self.relpath)
        return ext

    def mode(self, mode):
        if mode is None:
            return self
        # try getting the alternative
        try:
            return self.modes[mode]
        except KeyError:
            # fall back on the default mode if mode not found
            return self

    def key(self):
        return self.library.name, self.relpath

    def need(self):
        needed = get_current_needed_inclusions()
        needed.need(self)

    def inclusions(self):
        """Get all inclusions needed by this inclusion, including itself.
        """
        result = []
        for depend in self.depends:
            result.extend(depend.inclusions())
        result.append(self)
        return result

class GroupInclusion(InclusionBase):
    """An inclusion used to group resources together.

    It doesn't define a resource itself.
    """
    def __init__(self, depends):
        self.depends = depends

    def need(self):
        needed = get_current_needed_inclusions()
        needed.need(self)

    def inclusions(self):
        """Get all inclusions needed by this inclusion.
        """
        result = []
        for depend in self.depends:
            result.extend(depend.inclusions())
        return result

def normalize_inclusions(library, inclusions):
    return [normalize_inclusion(library, inclusion)
            for inclusion in inclusions]

def normalize_inclusion(library, inclusion):
    if isinstance(inclusion, InclusionBase):
        return inclusion
    assert isinstance(inclusion, basestring)
    return ResourceInclusion(library, inclusion)

class NeededInclusions(object):
    def __init__(self,
                 base_url='',
                 devmode=False,
                 hashing=False,
                 bottom=False,
                 force_bottom=False,
                 inclusions=None,
                 mode=None,
                 publisher_signature=DEFAULT_SIGNATURE,
                 rollup=False,
                 ):
        self.base_url = base_url
        self.devmode = devmode
        self.hashing = hashing

        self._bottom = bottom
        self._force_bottom = force_bottom
        self._inclusions = inclusions or []
        self._mode = mode
        self._publisher_signature = publisher_signature
        self._rollup = rollup

    def __len__(self):
        return len(self._inclusions)

    def need(self, inclusion):
        self._inclusions.append(inclusion)

    def _sorted_inclusions(self):
        return reversed(sorted(self._inclusions, key=lambda i: i.depth()))

    def inclusions(self):
        inclusions = []
        for inclusion in self._inclusions:
            inclusions.extend(inclusion.inclusions())

        inclusions = apply_mode(inclusions, self._mode)
        if self._rollup:
            inclusions = consolidate(inclusions)
        # sort only by extension, not dependency, as we can rely on
        # python's stable sort to keep inclusion order intact
        inclusions = sort_inclusions_by_extension(inclusions)
        inclusions = remove_duplicates(inclusions)

        return inclusions

    def library_url(self, library):
        segments = [self.base_url, self._publisher_signature]
        if self.hashing:
            segments.append(library.signature(devmode=self.devmode))
        segments.append(library.name)
        result = segments.pop(0)
        for segment in segments:
            if not result.endswith('/'):
                result += '/'
            result += segment
        return result

    def render(self):
        """Render a set of inclusions.
        """
        return self.render_inclusions(self.inclusions())

    def render_inclusions(self, inclusions):
        result = []
        url_cache = {} # prevent multiple computations for a library in one request
        for inclusion in inclusions:
            library = inclusion.library
            library_url = url_cache.get(library.name)
            if library_url is None:
                library_url = url_cache[library.name] = self.library_url(
                    library)
            result.append(
                render_inclusion(
                    inclusion, '%s/%s' %(library_url, inclusion.relpath)))
        return '\n'.join(result)

    def render_into_html(self, html):
        to_insert = self.render()
        return html.replace('', '\n    %s\n' % to_insert, 1)

    def render_topbottom(self):
        inclusions = self.inclusions()

        # seperate inclusions in top and bottom inclusions if this is needed
        if self._bottom:
            top_inclusions = []
            bottom_inclusions = []
            if not self._force_bottom:
                for inclusion in inclusions:
                    if inclusion.bottom:
                        bottom_inclusions.append(inclusion)
                    else:
                        top_inclusions.append(inclusion)
            else:
                for inclusion in inclusions:
                    if inclusion.ext() == '.js':
                        bottom_inclusions.append(inclusion)
                    else:
                        top_inclusions.append(inclusion)
        else:
            top_inclusions = inclusions
            bottom_inclusions = []

        return (self.render_inclusions(top_inclusions),
                self.render_inclusions(bottom_inclusions))

    def render_topbottom_into_html(self, html):
        top, bottom = self.render_topbottom()
        if top:
            html = html.replace('', '\n    %s\n' % top, 1)
        if bottom:
            html = html.replace('', '%s' % bottom, 1)
        return html

class NoNeededInclusions(Exception):
    pass

thread_local_needed_data = threading.local()

def init_current_needed_inclusions(*args, **kw):
    needed = NeededInclusions(*args, **kw)
    thread_local_needed_data.__dict__[NEEDED] = needed
    return needed

def get_current_needed_inclusions():
    needed = thread_local_needed_data.__dict__.get(NEEDED)
    if needed is None:
        raise NoNeededInclusions('No NeededInclusions object initialized.')
    return needed

def apply_mode(inclusions, mode):
    return [inclusion.mode(mode) for inclusion in inclusions]

def remove_duplicates(inclusions):
    """Given a set of inclusions, consolidate them so each only occurs once.
    """
    seen = set()
    result = []
    for inclusion in inclusions:
        if inclusion.key() in seen:
            continue
        seen.add(inclusion.key())
        result.append(inclusion)
    return result

def consolidate(inclusions):
    # keep track of rollups: rollup key -> set of inclusion keys
    potential_rollups = {}
    for inclusion in inclusions:
        for rollup in inclusion.rollups:
            s = potential_rollups.setdefault(rollup.key(), set())
            s.add(inclusion.key())

    # now go through inclusions, replacing them with rollups if
    # conditions match
    result = []
    for inclusion in inclusions:
        eager_superseders = []
        exact_superseders = []
        for rollup in inclusion.rollups:
            s = potential_rollups[rollup.key()]
            if rollup.eager_superseder:
                eager_superseders.append(rollup)
            if len(s) == len(rollup.supersedes):
                exact_superseders.append(rollup)
        if eager_superseders:
            # use the eager superseder that rolls up the most
            eager_superseders = sorted(eager_superseders,
                                       key=lambda i: len(i.supersedes))
            result.append(eager_superseders[-1])
        elif exact_superseders:
            # use the exact superseder that rolls up the most
            exact_superseders = sorted(exact_superseders,
                                       key=lambda i: len(i.supersedes))
            result.append(exact_superseders[-1])
        else:
            # nothing to supersede resource so use it directly
            result.append(inclusion)
    return result

def sort_inclusions_by_extension(inclusions):

    def key(inclusion):
        return EXTENSIONS.index(inclusion.ext())

    return sorted(inclusions, key=key)

def sort_inclusions_topological(inclusions):
    """Sort inclusions by dependency and supersedes.
    """
    dead = {}
    result = []
    for inclusion in inclusions:
        dead[inclusion.key()] = False

    for inclusion in inclusions:
        _visit(inclusion, result, dead)
    return result

def _visit(inclusion, result, dead):
    if dead[inclusion.key()]:
        return
    dead[inclusion.key()] = True
    for depend in inclusion.depends:
        _visit(depend, result, dead)
    for depend in inclusion.supersedes:
        _visit(depend ,result, dead)
    result.append(inclusion)

def render_css(url):
    return ('<link rel="stylesheet" type="text/css" href="%s" />' %
            url)

def render_kss(url):
    raise NotImplementedError

def render_js(url):
    return ('' %
            url)

inclusion_renderers = {
    '.css': render_css,
    '.kss': render_kss,
    '.js': render_js,
    }

def render_inclusion(inclusion, url):
    renderer = inclusion_renderers.get(inclusion.ext(), None)
    if renderer is None:
        raise UnknownResourceExtension(
            "Unknown resource extension %s for resource inclusion: %s" %
            (inclusion.ext(), repr(inclusion)))
    return renderer(url)

def generate_code(**kw):
    name_to_inclusion = kw
    inclusion_to_name = {}
    inclusions = []
    for name, inclusion in kw.items():
        inclusion_to_name[inclusion.key()] = name
        inclusions.append(inclusion)

    # libraries with the same name are the same libraries
    libraries = {}
    for inclusion in inclusions:
        libraries[inclusion.library.name] = inclusion.library
    libraries = sorted(libraries.values())

    result = []
    # import on top
    result.append("from fanstatic import Library, ResourceInclusion")
    result.append("")
    # define libraries
    for library in libraries:
        result.append("%s = Library('%s', '%s')" %
                      (library.name, library.name, library.rootpath))
    result.append("")

    # sort inclusions in the order we want them to be
    inclusions = sort_inclusions_by_extension(
        sort_inclusions_topological(inclusions))

    # now generate inclusion code
    for inclusion in inclusions:
        s = "%s = ResourceInclusion(%s, '%s'" % (
            inclusion_to_name[inclusion.key()],
            inclusion.library.name,
            inclusion.relpath)
        if inclusion.depends:
            depends_s = ', depends=[%s]' % ', '.join(
                [inclusion_to_name[d.key()] for d in inclusion.depends])
            s += depends_s
        if inclusion.supersedes:
            supersedes_s = ', supersedes=[%s]' % ', '.join(
                [inclusion_to_name[i.key()] for i in inclusion.supersedes])
            s += supersedes_s
        if inclusion.modes:
            items = []
            for mode_name, mode in inclusion.modes.items():
                items.append((mode_name,
                              generate_inline_inclusion(mode, inclusion)))
            items = sorted(items)
            modes_s = ', %s' % ', '.join(["%s=%s" % (name, mode) for
                                          (name, mode) in items])
            s += modes_s
        s += ')'
        result.append(s)
    return '\n'.join(result)

def generate_inline_inclusion(inclusion, associated_inclusion):
    if inclusion.library.name == associated_inclusion.library.name:
        return "'%s'" % inclusion.relpath
    else:
        return "ResourceInclusion(%s, '%s')" % (inclusion.library.name,
                                                inclusion.relpath)
<|endoftext|>


<commit_before>from __future__ import division, unicode_literals

import os
import re
import sys
import time
import random

from ..compat import compat_os_name
from ..utils import (
    decodeArgument,
    encodeFilename,
    error_to_compat_str,
    format_bytes,
    shell_quote,
    timeconvert,
)

class FileDownloader(object):
    """File Downloader class.

    File downloader objects are the ones responsible of downloading the
    actual video file and writing it to disk.

    File downloaders accept a lot of parameters. In order not to saturate
    the object constructor with arguments, it receives a dictionary of
    options instead.

    Available options:

    verbose:            Print additional info to stdout.
    quiet:              Do not print messages to stdout.
    ratelimit:          Download speed limit, in bytes/sec.
    retries:            Number of times to retry for HTTP error 5xx
    buffersize:         Size of download buffer in bytes.
    noresizebuffer:     Do not automatically resize the download buffer.
    continuedl:         Try to continue downloads if possible.
    noprogress:         Do not print the progress bar.
    logtostderr:        Log messages to stderr instead of stdout.
    consoletitle:       Display progress in console window's titlebar.
    nopart:             Do not use temporary .part files.
    updatetime:         Use the Last-modified header to set output file timestamps.
    test:               Download only first bytes to test the downloader.
    min_filesize:       Skip files smaller than this size
    max_filesize:       Skip files larger than this size
    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
    external_downloader_args:  A list of additional command-line arguments for the
                        external downloader.
    hls_use_mpegts:     Use the mpegts container for HLS videos.
    http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
                        useful for bypassing bandwidth throttling imposed by
                        a webserver (experimental)

    Subclasses of this one must re-define the real_download method.
    """

    _TEST_FILE_SIZE = 10241
    params = None

    def __init__(self, ydl, params):
        """Create a FileDownloader object with the given options."""
        self.ydl = ydl
        self._progress_hooks = []
        self.params = params
        self.add_progress_hook(self.report_progress)

    @staticmethod
    def format_seconds(seconds):
        (mins, secs) = divmod(seconds, 60)
        (hours, mins) = divmod(mins, 60)
        if hours > 99:
            return '--:--:--'
        if hours == 0:
            return '%02d:%02d' % (mins, secs)
        else:
            return '%02d:%02d:%02d' % (hours, mins, secs)

    @staticmethod
    def calc_percent(byte_counter, data_len):
        if data_len is None:
            return None
        return float(byte_counter) / float(data_len) * 100.0

    @staticmethod
    def format_percent(percent):
        if percent is None:
            return '---.-%'
        return '%6s' % ('%3.1f%%' % percent)

    @staticmethod
    def calc_eta(start, now, total, current):
        if total is None:
            return None
        if now is None:
            now = time.time()
        dif = now - start
        if current == 0 or dif < 0.001:  # One millisecond
            return None
        rate = float(current) / dif
        return int((float(total) - float(current)) / rate)

    @staticmethod
    def format_eta(eta):
        if eta is None:
            return '--:--'
        return FileDownloader.format_seconds(eta)

    @staticmethod
    def calc_speed(start, now, bytes):
        dif = now - start
        if bytes == 0 or dif < 0.001:  # One millisecond
            return None
        return float(bytes) / dif

    @staticmethod
    def format_speed(speed):
        if speed is None:
            return '%10s' % '---b/s'
        return '%10s' % ('%s/s' % format_bytes(speed))

    @staticmethod
    def format_retries(retries):
        return 'inf' if retries == float('inf') else '%.0f' % retries

    @staticmethod
    def best_block_size(elapsed_time, bytes):
        new_min = max(bytes / 2.0, 1.0)
        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
        if elapsed_time < 0.001:
            return int(new_max)
        rate = bytes / elapsed_time
        if rate > new_max:
            return int(new_max)
        if rate < new_min:
            return int(new_min)
        return int(rate)

    @staticmethod
    def parse_bytes(bytestr):
        """Parse a string indicating a byte quantity into an integer."""
        matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
        if matchobj is None:
            return None
        number = float(matchobj.group(1))
        multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
        return int(round(number * multiplier))

    def to_screen(self, *args, **kargs):
        self.ydl.to_screen(*args, **kargs)

    def to_stderr(self, message):
        self.ydl.to_screen(message)

    def to_console_title(self, message):
        self.ydl.to_console_title(message)

    def trouble(self, *args, **kargs):
        self.ydl.trouble(*args, **kargs)

    def report_warning(self, *args, **kargs):
        self.ydl.report_warning(*args, **kargs)

    def report_error(self, *args, **kargs):
        self.ydl.report_error(*args, **kargs)

    def slow_down(self, start_time, now, byte_counter):
        """Sleep if the download speed is over the rate limit."""
        rate_limit = self.params.get('ratelimit')
        if rate_limit is None or byte_counter == 0:
            return
        if now is None:
            now = time.time()
        elapsed = now - start_time
        if elapsed <= 0.0:
            return
        speed = float(byte_counter) / elapsed
        if speed > rate_limit:
            time.sleep(max((byte_counter // rate_limit) - elapsed, 0))

    def temp_name(self, filename):
        """Returns a temporary filename for the given filename."""
        if self.params.get('nopart', False) or filename == '-' or \
                (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
            return filename
        return filename + '.part'

    def undo_temp_name(self, filename):
        if filename.endswith('.part'):
            return filename[:-len('.part')]
        return filename

    def ytdl_filename(self, filename):
        return filename + '.ytdl'

    def try_rename(self, old_filename, new_filename):
        try:
            if old_filename == new_filename:
                return
            os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
        except (IOError, OSError) as err:
            self.report_error('unable to rename file: %s' % error_to_compat_str(err))

    def try_utime(self, filename, last_modified_hdr):
        """Try to set the last-modified time of the given file."""
        if last_modified_hdr is None:
            return
        if not os.path.isfile(encodeFilename(filename)):
            return
        timestr = last_modified_hdr
        if timestr is None:
            return
        filetime = timeconvert(timestr)
        if filetime is None:
            return filetime
        # Ignore obviously invalid dates
        if filetime == 0:
            return
        try:
            os.utime(filename, (time.time(), filetime))
        except Exception:
            pass
        return filetime

    def report_destination(self, filename):
        """Report destination filename."""
        self.to_screen('[download] Destination: ' + filename)

    def _report_progress_status(self, msg, is_last_line=False):
        fullmsg = '[download] ' + msg
        if self.params.get('progress_with_newline', False):
            self.to_screen(fullmsg)
        else:
            if compat_os_name == 'nt':
                prev_len = getattr(self, '_report_progress_prev_line_length',
                                   0)
                if prev_len > len(fullmsg):
                    fullmsg += ' ' * (prev_len - len(fullmsg))
                self._report_progress_prev_line_length = len(fullmsg)
                clear_line = '\r'
            else:
                clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
            self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
        self.to_console_title('youtube-dl ' + msg)

    def report_progress(self, s):
        if s['status'] == 'finished':
            if self.params.get('noprogress', False):
                self.to_screen('[download] Download completed')
            else:
                msg_template = '100%%'
                if s.get('total_bytes') is not None:
                    s['_total_bytes_str'] = format_bytes(s['total_bytes'])
                    msg_template += ' of %(_total_bytes_str)s'
                if s.get('elapsed') is not None:
                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                    msg_template += ' in %(_elapsed_str)s'
                self._report_progress_status(
                    msg_template % s, is_last_line=True)

        if self.params.get('noprogress'):
            return

        if s['status'] != 'downloading':
            return

        if s.get('eta') is not None:
            s['_eta_str'] = self.format_eta(s['eta'])
        else:
            s['_eta_str'] = 'Unknown ETA'

        if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
        elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
        else:
            if s.get('downloaded_bytes') == 0:
                s['_percent_str'] = self.format_percent(0)
            else:
                s['_percent_str'] = 'Unknown %'

        if s.get('speed') is not None:
            s['_speed_str'] = self.format_speed(s['speed'])
        else:
            s['_speed_str'] = 'Unknown speed'

        if s.get('total_bytes') is not None:
            s['_total_bytes_str'] = format_bytes(s['total_bytes'])
            msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
        elif s.get('total_bytes_estimate') is not None:
            s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
            msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
        else:
            if s.get('downloaded_bytes') is not None:
                s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
                if s.get('elapsed'):
                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
                else:
                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
            else:
                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'

        self._report_progress_status(msg_template % s)

    def report_resuming_byte(self, resume_len):
        """Report attempt to resume at given byte."""
        self.to_screen('[download] Resuming download at byte %s' % resume_len)

    def report_retry(self, err, count, retries):
        """Report retry in case of HTTP error 5xx"""
        self.to_screen(
            '[download] Got server HTTP error: %s. Retrying (attempt %d of %s)...'
            % (error_to_compat_str(err), count, self.format_retries(retries)))

    def report_file_already_downloaded(self, file_name):
        """Report file has already been fully downloaded."""
        try:
            self.to_screen('[download] %s has already been downloaded' % file_name)
        except UnicodeEncodeError:
            self.to_screen('[download] The file has already been downloaded')

    def report_unable_to_resume(self):
        """Report it was impossible to resume download."""
        self.to_screen('[download] Unable to resume')

    def download(self, filename, info_dict):
        """Download to a filename using the info from info_dict
        Return True on success and False otherwise
        """

        nooverwrites_and_exists = (
            self.params.get('nooverwrites', False)
            and os.path.exists(encodeFilename(filename))
        )

        if not hasattr(filename, 'write'):
            continuedl_and_exists = (
                self.params.get('continuedl', True)
                and os.path.isfile(encodeFilename(filename))
                and not self.params.get('nopart', False)
            )

            # Check file already present
            if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
                self.report_file_already_downloaded(filename)
                self._hook_progress({
                    'filename': filename,
                    'status': 'finished',
                    'total_bytes': os.path.getsize(encodeFilename(filename)),
                })
                return True

        min_sleep_interval = self.params.get('sleep_interval')
        if min_sleep_interval:
            max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
            sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
            self.to_screen(
                '[download] Sleeping %s seconds...' % (
                    int(sleep_interval) if sleep_interval.is_integer()
                    else '%.2f' % sleep_interval))
            time.sleep(sleep_interval)

        return self.real_download(filename, info_dict)

    def real_download(self, filename, info_dict):
        """Real download process. Redefine in subclasses."""
        raise NotImplementedError('This method must be implemented by subclasses')

    def _hook_progress(self, status):
        for ph in self._progress_hooks:
            ph(status)

    def add_progress_hook(self, ph):
        # See YoutubeDl.py (search for progress_hooks) for a description of
        # this interface
        self._progress_hooks.append(ph)

    def _debug_cmd(self, args, exe=None):
        if not self.params.get('verbose', False):
            return

        str_args = [decodeArgument(a) for a in args]

        if exe is None:
            exe = os.path.basename(str_args[0])

        self.to_screen('[debug] %s command line: %s' % (
            exe, shell_quote(str_args)))
<commit_msg>[downloader/common] Improve rate limit (#21301)<commit_after>from __future__ import division, unicode_literals

import os
import re
import sys
import time
import random

from ..compat import compat_os_name
from ..utils import (
    decodeArgument,
    encodeFilename,
    error_to_compat_str,
    format_bytes,
    shell_quote,
    timeconvert,
)

class FileDownloader(object):
    """File Downloader class.

    File downloader objects are the ones responsible of downloading the
    actual video file and writing it to disk.

    File downloaders accept a lot of parameters. In order not to saturate
    the object constructor with arguments, it receives a dictionary of
    options instead.

    Available options:

    verbose:            Print additional info to stdout.
    quiet:              Do not print messages to stdout.
    ratelimit:          Download speed limit, in bytes/sec.
    retries:            Number of times to retry for HTTP error 5xx
    buffersize:         Size of download buffer in bytes.
    noresizebuffer:     Do not automatically resize the download buffer.
    continuedl:         Try to continue downloads if possible.
    noprogress:         Do not print the progress bar.
    logtostderr:        Log messages to stderr instead of stdout.
    consoletitle:       Display progress in console window's titlebar.
    nopart:             Do not use temporary .part files.
    updatetime:         Use the Last-modified header to set output file timestamps.
    test:               Download only first bytes to test the downloader.
    min_filesize:       Skip files smaller than this size
    max_filesize:       Skip files larger than this size
    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
    external_downloader_args:  A list of additional command-line arguments for the
                        external downloader.
    hls_use_mpegts:     Use the mpegts container for HLS videos.
    http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
                        useful for bypassing bandwidth throttling imposed by
                        a webserver (experimental)

    Subclasses of this one must re-define the real_download method.
    """

    _TEST_FILE_SIZE = 10241
    params = None

    def __init__(self, ydl, params):
        """Create a FileDownloader object with the given options."""
        self.ydl = ydl
        self._progress_hooks = []
        self.params = params
        self.add_progress_hook(self.report_progress)

    @staticmethod
    def format_seconds(seconds):
        (mins, secs) = divmod(seconds, 60)
        (hours, mins) = divmod(mins, 60)
        if hours > 99:
            return '--:--:--'
        if hours == 0:
            return '%02d:%02d' % (mins, secs)
        else:
            return '%02d:%02d:%02d' % (hours, mins, secs)

    @staticmethod
    def calc_percent(byte_counter, data_len):
        if data_len is None:
            return None
        return float(byte_counter) / float(data_len) * 100.0

    @staticmethod
    def format_percent(percent):
        if percent is None:
            return '---.-%'
        return '%6s' % ('%3.1f%%' % percent)

    @staticmethod
    def calc_eta(start, now, total, current):
        if total is None:
            return None
        if now is None:
            now = time.time()
        dif = now - start
        if current == 0 or dif < 0.001:  # One millisecond
            return None
        rate = float(current) / dif
        return int((float(total) - float(current)) / rate)

    @staticmethod
    def format_eta(eta):
        if eta is None:
            return '--:--'
        return FileDownloader.format_seconds(eta)

    @staticmethod
    def calc_speed(start, now, bytes):
        dif = now - start
        if bytes == 0 or dif < 0.001:  # One millisecond
            return None
        return float(bytes) / dif

    @staticmethod
    def format_speed(speed):
        if speed is None:
            return '%10s' % '---b/s'
        return '%10s' % ('%s/s' % format_bytes(speed))

    @staticmethod
    def format_retries(retries):
        return 'inf' if retries == float('inf') else '%.0f' % retries

    @staticmethod
    def best_block_size(elapsed_time, bytes):
        new_min = max(bytes / 2.0, 1.0)
        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
        if elapsed_time < 0.001:
            return int(new_max)
        rate = bytes / elapsed_time
        if rate > new_max:
            return int(new_max)
        if rate < new_min:
            return int(new_min)
        return int(rate)

    @staticmethod
    def parse_bytes(bytestr):
        """Parse a string indicating a byte quantity into an integer."""
        matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
        if matchobj is None:
            return None
        number = float(matchobj.group(1))
        multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
        return int(round(number * multiplier))

    def to_screen(self, *args, **kargs):
        self.ydl.to_screen(*args, **kargs)

    def to_stderr(self, message):
        self.ydl.to_screen(message)

    def to_console_title(self, message):
        self.ydl.to_console_title(message)

    def trouble(self, *args, **kargs):
        self.ydl.trouble(*args, **kargs)

    def report_warning(self, *args, **kargs):
        self.ydl.report_warning(*args, **kargs)

    def report_error(self, *args, **kargs):
        self.ydl.report_error(*args, **kargs)

    def slow_down(self, start_time, now, byte_counter):
        """Sleep if the download speed is over the rate limit."""
        rate_limit = self.params.get('ratelimit')
        if rate_limit is None or byte_counter == 0:
            return
        if now is None:
            now = time.time()
        elapsed = now - start_time
        if elapsed <= 0.0:
            return
        speed = float(byte_counter) / elapsed
        if speed > rate_limit:
            sleep_time = float(byte_counter) / rate_limit - elapsed
            if sleep_time > 0:
                time.sleep(sleep_time)

    def temp_name(self, filename):
        """Returns a temporary filename for the given filename."""
        if self.params.get('nopart', False) or filename == '-' or \
                (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
            return filename
        return filename + '.part'

    def undo_temp_name(self, filename):
        if filename.endswith('.part'):
            return filename[:-len('.part')]
        return filename

    def ytdl_filename(self, filename):
        return filename + '.ytdl'

    def try_rename(self, old_filename, new_filename):
        try:
            if old_filename == new_filename:
                return
            os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
        except (IOError, OSError) as err:
            self.report_error('unable to rename file: %s' % error_to_compat_str(err))

    def try_utime(self, filename, last_modified_hdr):
        """Try to set the last-modified time of the given file."""
        if last_modified_hdr is None:
            return
        if not os.path.isfile(encodeFilename(filename)):
            return
        timestr = last_modified_hdr
        if timestr is None:
            return
        filetime = timeconvert(timestr)
        if filetime is None:
            return filetime
        # Ignore obviously invalid dates
        if filetime == 0:
            return
        try:
            os.utime(filename, (time.time(), filetime))
        except Exception:
            pass
        return filetime

    def report_destination(self, filename):
        """Report destination filename."""
        self.to_screen('[download] Destination: ' + filename)

    def _report_progress_status(self, msg, is_last_line=False):
        fullmsg = '[download] ' + msg
        if self.params.get('progress_with_newline', False):
            self.to_screen(fullmsg)
        else:
            if compat_os_name == 'nt':
                prev_len = getattr(self, '_report_progress_prev_line_length',
                                   0)
                if prev_len > len(fullmsg):
                    fullmsg += ' ' * (prev_len - len(fullmsg))
                self._report_progress_prev_line_length = len(fullmsg)
                clear_line = '\r'
            else:
                clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
            self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
        self.to_console_title('youtube-dl ' + msg)

    def report_progress(self, s):
        if s['status'] == 'finished':
            if self.params.get('noprogress', False):
                self.to_screen('[download] Download completed')
            else:
                msg_template = '100%%'
                if s.get('total_bytes') is not None:
                    s['_total_bytes_str'] = format_bytes(s['total_bytes'])
                    msg_template += ' of %(_total_bytes_str)s'
                if s.get('elapsed') is not None:
                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                    msg_template += ' in %(_elapsed_str)s'
                self._report_progress_status(
                    msg_template % s, is_last_line=True)

        if self.params.get('noprogress'):
            return

        if s['status'] != 'downloading':
            return

        if s.get('eta') is not None:
            s['_eta_str'] = self.format_eta(s['eta'])
        else:
            s['_eta_str'] = 'Unknown ETA'

        if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
        elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
        else:
            if s.get('downloaded_bytes') == 0:
                s['_percent_str'] = self.format_percent(0)
            else:
                s['_percent_str'] = 'Unknown %'

        if s.get('speed') is not None:
            s['_speed_str'] = self.format_speed(s['speed'])
        else:
            s['_speed_str'] = 'Unknown speed'

        if s.get('total_bytes') is not None:
            s['_total_bytes_str'] = format_bytes(s['total_bytes'])
            msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
        elif s.get('total_bytes_estimate') is not None:
            s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
            msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
        else:
            if s.get('downloaded_bytes') is not None:
                s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
                if s.get('elapsed'):
                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
                else:
                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
            else:
                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'

        self._report_progress_status(msg_template % s)

    def report_resuming_byte(self, resume_len):
        """Report attempt to resume at given byte."""
        self.to_screen('[download] Resuming download at byte %s' % resume_len)

    def report_retry(self, err, count, retries):
        """Report retry in case of HTTP error 5xx"""
        self.to_screen(
            '[download] Got server HTTP error: %s. Retrying (attempt %d of %s)...'
            % (error_to_compat_str(err), count, self.format_retries(retries)))

    def report_file_already_downloaded(self, file_name):
        """Report file has already been fully downloaded."""
        try:
            self.to_screen('[download] %s has already been downloaded' % file_name)
        except UnicodeEncodeError:
            self.to_screen('[download] The file has already been downloaded')

    def report_unable_to_resume(self):
        """Report it was impossible to resume download."""
        self.to_screen('[download] Unable to resume')

    def download(self, filename, info_dict):
        """Download to a filename using the info from info_dict
        Return True on success and False otherwise
        """

        nooverwrites_and_exists = (
            self.params.get('nooverwrites', False)
            and os.path.exists(encodeFilename(filename))
        )

        if not hasattr(filename, 'write'):
            continuedl_and_exists = (
                self.params.get('continuedl', True)
                and os.path.isfile(encodeFilename(filename))
                and not self.params.get('nopart', False)
            )

            # Check file already present
            if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
                self.report_file_already_downloaded(filename)
                self._hook_progress({
                    'filename': filename,
                    'status': 'finished',
                    'total_bytes': os.path.getsize(encodeFilename(filename)),
                })
                return True

        min_sleep_interval = self.params.get('sleep_interval')
        if min_sleep_interval:
            max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
            sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
            self.to_screen(
                '[download] Sleeping %s seconds...' % (
                    int(sleep_interval) if sleep_interval.is_integer()
                    else '%.2f' % sleep_interval))
            time.sleep(sleep_interval)

        return self.real_download(filename, info_dict)

    def real_download(self, filename, info_dict):
        """Real download process. Redefine in subclasses."""
        raise NotImplementedError('This method must be implemented by subclasses')

    def _hook_progress(self, status):
        for ph in self._progress_hooks:
            ph(status)

    def add_progress_hook(self, ph):
        # See YoutubeDl.py (search for progress_hooks) for a description of
        # this interface
        self._progress_hooks.append(ph)

    def _debug_cmd(self, args, exe=None):
        if not self.params.get('verbose', False):
            return

        str_args = [decodeArgument(a) for a in args]

        if exe is None:
            exe = os.path.basename(str_args[0])

        self.to_screen('[debug] %s command line: %s' % (
            exe, shell_quote(str_args)))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from importlib import import_module

from fabric.api import env

def init():
    for plugin in env.plugins:
        mod = import_module(plugin)
        mod.init()

def init_cli(conf):
    for key, value in conf.items():
        setattr(env, key, value)
    init()
<commit_msg>Init nothing if there is no plugin.<commit_after># -*- coding: utf-8 -*-

from importlib import import_module

from fabric.api import env

def init():
    if not hasattr(env, 'plugins'):
        return
    for plugin in env.plugins:
        mod = import_module(plugin)
        mod.init()

def init_cli(conf):
    for key, value in conf.items():
        setattr(env, key, value)
    init()
<|endoftext|>


<commit_before>"""Manages the spawning of mpi processes to send to the various solvers.
"""
# TODO: don't send whole matrix? only marginal time savings but memory better.

import os
import functools
from .slepc_linalg import (
    slepc_seigsys,
    slepc_svds,
    slepc_mfn_multiply,
)

# Work out if already running as mpi
if ('OMPI_COMM_WORLD_SIZE' in os.environ) or ('PMI_SIZE' in os.environ):
    ALREADY_RUNNING_AS_MPI = True
    if '_QUIMB_MPI_LAUNCHED' not in os.environ:
        raise RuntimeError(
            "For the moment, quimb programs launched explicitly"
            " using MPI need to use `quimb-mpiexec`."
        )

else:
    ALREADY_RUNNING_AS_MPI = False

# Work out the desired total number of workers
for _NUM_MPI_WORKERS_VAR in ['QUIMB_NUM_MPI_WORKERS',
                             'QUIMB_NUM_PROCS',
                             'OMP_NUM_THREADS',
                             ]:
    if _NUM_MPI_WORKERS_VAR in os.environ:
        NUM_MPI_WORKERS = int(os.environ[_NUM_MPI_WORKERS_VAR])
        NUM_MPI_WORKERS_SET = True
        break
    NUM_MPI_WORKERS_SET = False

if not NUM_MPI_WORKERS_SET:
    import psutil
    _NUM_MPI_WORKERS_VAR = 'psutil'
    NUM_MPI_WORKERS = psutil.cpu_count(logical=False)

class CachedPoolWithShutdown(object):
    """
    """

    def __init__(self, pool_fn):
        self._settings = '__UNINITIALIZED__'
        self._pool_fn = pool_fn

    def __call__(self, num_workers=None, num_threads=1):
        # convert None to default so the cache the same
        if num_workers is None:
            num_workers = NUM_MPI_WORKERS
        elif ALREADY_RUNNING_AS_MPI and (num_workers != NUM_MPI_WORKERS):
            raise ValueError("Can't specify number of processes when running "
                             "under MPI rather than spawning processes.")

        # first call
        if self._settings == '__UNINITIALIZED__':
            self._pool = self._pool_fn(num_workers, num_threads)
            self._settings = (num_workers, num_threads)
        # new type of pool requested
        elif self._settings != (num_workers, num_threads):
            self._pool.shutdown()
            self._pool = self._pool_fn(num_workers, num_threads)
            self._settings = (num_workers, num_threads)
        return self._pool

@CachedPoolWithShutdown
def get_mpi_pool(num_workers=None, num_threads=1):
    """
    """
    from mpi4py.futures import MPIPoolExecutor
    return MPIPoolExecutor(num_workers, main=False, delay=1e-2,
                           env={'OMP_NUM_THREADS': str(num_threads),
                                'QUIMB_NUM_MPI_WORKERS': str(num_workers),
                                '_QUIMB_MPI_LAUNCHED': 'SPAWNED'})

class GetMPIBeforeCall(object):
    """Wrap a function to automatically get the correct communicator before
    its called, and to set the `comm_self` kwarg to allow forced self mode.

    This is called by every mpi process before the function evaluation.
    """

    def __init__(self, fn):
        self.fn = fn

    def __call__(self, *args,
                 comm_self=False,
                 wait_for_workers=None,
                 **kwargs):
        """
        Parameters
        ----------
        *args :
            Supplied to self.fn
        comm_self : bool, optional
            Whether to force use of MPI.COMM_SELF
        wait_for_workers : int, optional
            If set, wait for the communicator to have this many workers, this
            can help to catch some errors regarding expected worker numbers.
        **kwargs :
            Supplied to self.fn
        """
        from mpi4py import MPI

        if not comm_self:
            comm = MPI.COMM_WORLD
        else:
            comm = MPI.COMM_SELF

        if wait_for_workers is not None:
            from time import time
            t0 = time()
            while comm.Get_size() != wait_for_workers:
                if time() - t0 > 2:
                    raise RuntimeError(
                        "Timeout while waiting for {} workers "
                        "to join comm {}.".format(wait_for_workers, comm)
                    )

        comm.Barrier()
        res = self.fn(*args, comm=comm, **kwargs)
        comm.Barrier()
        return res

class SpawnMPIProcessesFunc(object):
    """Automatically wrap a function to be executed in parallel by a
    pool of mpi workers.

    This is only called by the master mpi process in manual mode, or only by
    the (non-mpi) spawning process in automatic mode.
    """

    def __init__(self, fn):
        self.fn = fn

    def __call__(self, *args,
                 num_workers=None,
                 num_threads=1,
                 mpi_pool=None,
                 spawn_all=not ALREADY_RUNNING_AS_MPI,
                 **kwargs):
        """
        Parameters
        ----------
            *args
                Supplied to `self.fn`.
            num_workers : int, optional
                How many total process should run function in parallel.
            num_threads : int, optional
                How many (OMP) threads each process should use
            mpi_pool : pool-like, optional
                If not None (default), submit function to this pool.
            spawn_all : bool, optional
                Whether all the parallel processes should be spawned (True), or
                num_workers - 1, so that the current process can also do work.
            **kwargs
                Supplied to `self.fn`.

        Returns
        -------
            `fn` output from the master process.
        """
        if num_workers is None:
            num_workers = NUM_MPI_WORKERS

        if num_workers == 1:  # no pool or communicator required
            return self.fn(*args, comm_self=True, **kwargs)

        kwargs['wait_for_workers'] = num_workers

        if mpi_pool is not None:
            pool = mpi_pool
        else:
            pool = get_mpi_pool(num_workers, num_threads)

        # the (non mpi) main process is idle while the workers compute.
        if spawn_all:
            futures = [pool.submit(self.fn, *args, **kwargs)
                       for _ in range(num_workers)]
            results = [f.result() for f in futures]

        # the master process is the master mpi process and contributes
        else:
            futures = [pool.submit(self.fn, *args, **kwargs)
                       for _ in range(num_workers - 1)]
            results = ([self.fn(*args, **kwargs)] +
                       [f.result() for f in futures])

        # Get master result, (not always first submitted)
        return next(r for r in results if r is not None)

# ---------------------------------- SLEPC ---------------------------------- #

seigsys_slepc_mpi = functools.wraps(slepc_seigsys)(
    GetMPIBeforeCall(slepc_seigsys))
seigsys_slepc_spawn = functools.wraps(slepc_seigsys)(
    SpawnMPIProcessesFunc(seigsys_slepc_mpi))

svds_slepc_mpi = functools.wraps(slepc_svds)(
    GetMPIBeforeCall(slepc_svds))
svds_slepc_spawn = functools.wraps(slepc_svds)(
    SpawnMPIProcessesFunc(svds_slepc_mpi))

mfn_multiply_slepc_mpi = functools.wraps(slepc_mfn_multiply)(
    GetMPIBeforeCall(slepc_mfn_multiply))
mfn_multiply_slepc_spawn = functools.wraps(slepc_mfn_multiply)(
    SpawnMPIProcessesFunc(mfn_multiply_slepc_mpi))
<commit_msg>doc mpi pool stuff<commit_after>"""Manages the spawning of mpi processes to send to the various solvers.
"""
# TODO: don't send whole matrix? only marginal time savings but memory better.

import os
import functools
from .slepc_linalg import (
    slepc_seigsys,
    slepc_svds,
    slepc_mfn_multiply,
)

# Work out if already running as mpi
if ('OMPI_COMM_WORLD_SIZE' in os.environ) or ('PMI_SIZE' in os.environ):
    ALREADY_RUNNING_AS_MPI = True
    if '_QUIMB_MPI_LAUNCHED' not in os.environ:
        raise RuntimeError(
            "For the moment, quimb programs launched explicitly"
            " using MPI need to use `quimb-mpiexec`."
        )

else:
    ALREADY_RUNNING_AS_MPI = False

# Work out the desired total number of workers
for _NUM_MPI_WORKERS_VAR in ['QUIMB_NUM_MPI_WORKERS',
                             'QUIMB_NUM_PROCS',
                             'OMP_NUM_THREADS',
                             ]:
    if _NUM_MPI_WORKERS_VAR in os.environ:
        NUM_MPI_WORKERS = int(os.environ[_NUM_MPI_WORKERS_VAR])
        NUM_MPI_WORKERS_SET = True
        break
    NUM_MPI_WORKERS_SET = False

if not NUM_MPI_WORKERS_SET:
    import psutil
    _NUM_MPI_WORKERS_VAR = 'psutil'
    NUM_MPI_WORKERS = psutil.cpu_count(logical=False)

class CachedPoolWithShutdown(object):
    """Decorator for caching the mpi pool when called with the equivalent args,
    and shutting down previous ones when not needed.
    """

    def __init__(self, pool_fn):
        self._settings = '__UNINITIALIZED__'
        self._pool_fn = pool_fn

    def __call__(self, num_workers=None, num_threads=1):
        # convert None to default so the cache the same
        if num_workers is None:
            num_workers = NUM_MPI_WORKERS
        elif ALREADY_RUNNING_AS_MPI and (num_workers != NUM_MPI_WORKERS):
            raise ValueError("Can't specify number of processes when running "
                             "under MPI rather than spawning processes.")

        # first call
        if self._settings == '__UNINITIALIZED__':
            self._pool = self._pool_fn(num_workers, num_threads)
            self._settings = (num_workers, num_threads)
        # new type of pool requested
        elif self._settings != (num_workers, num_threads):
            self._pool.shutdown()
            self._pool = self._pool_fn(num_workers, num_threads)
            self._settings = (num_workers, num_threads)
        return self._pool

@CachedPoolWithShutdown
def get_mpi_pool(num_workers=None, num_threads=1):
    """Get the MPI executor pool, with specified number of processes and
    threads per process.
    """
    from mpi4py.futures import MPIPoolExecutor
    return MPIPoolExecutor(num_workers, main=False, delay=1e-2,
                           env={'OMP_NUM_THREADS': str(num_threads),
                                'QUIMB_NUM_MPI_WORKERS': str(num_workers),
                                '_QUIMB_MPI_LAUNCHED': 'SPAWNED'})

class GetMPIBeforeCall(object):
    """Wrap a function to automatically get the correct communicator before
    its called, and to set the `comm_self` kwarg to allow forced self mode.

    This is called by every mpi process before the function evaluation.
    """

    def __init__(self, fn):
        self.fn = fn

    def __call__(self, *args,
                 comm_self=False,
                 wait_for_workers=None,
                 **kwargs):
        """
        Parameters
        ----------
        *args :
            Supplied to self.fn
        comm_self : bool, optional
            Whether to force use of MPI.COMM_SELF
        wait_for_workers : int, optional
            If set, wait for the communicator to have this many workers, this
            can help to catch some errors regarding expected worker numbers.
        **kwargs :
            Supplied to self.fn
        """
        from mpi4py import MPI

        if not comm_self:
            comm = MPI.COMM_WORLD
        else:
            comm = MPI.COMM_SELF

        if wait_for_workers is not None:
            from time import time
            t0 = time()
            while comm.Get_size() != wait_for_workers:
                if time() - t0 > 2:
                    raise RuntimeError(
                        "Timeout while waiting for {} workers "
                        "to join comm {}.".format(wait_for_workers, comm)
                    )

        comm.Barrier()
        res = self.fn(*args, comm=comm, **kwargs)
        comm.Barrier()
        return res

class SpawnMPIProcessesFunc(object):
    """Automatically wrap a function to be executed in parallel by a
    pool of mpi workers.

    This is only called by the master mpi process in manual mode, or only by
    the (non-mpi) spawning process in automatic mode.
    """

    def __init__(self, fn):
        self.fn = fn

    def __call__(self, *args,
                 num_workers=None,
                 num_threads=1,
                 mpi_pool=None,
                 spawn_all=not ALREADY_RUNNING_AS_MPI,
                 **kwargs):
        """
        Parameters
        ----------
            *args
                Supplied to `self.fn`.
            num_workers : int, optional
                How many total process should run function in parallel.
            num_threads : int, optional
                How many (OMP) threads each process should use
            mpi_pool : pool-like, optional
                If not None (default), submit function to this pool.
            spawn_all : bool, optional
                Whether all the parallel processes should be spawned (True), or
                num_workers - 1, so that the current process can also do work.
            **kwargs
                Supplied to `self.fn`.

        Returns
        -------
            `fn` output from the master process.
        """
        if num_workers is None:
            num_workers = NUM_MPI_WORKERS

        if num_workers == 1:  # no pool or communicator required
            return self.fn(*args, comm_self=True, **kwargs)

        kwargs['wait_for_workers'] = num_workers

        if mpi_pool is not None:
            pool = mpi_pool
        else:
            pool = get_mpi_pool(num_workers, num_threads)

        # the (non mpi) main process is idle while the workers compute.
        if spawn_all:
            futures = [pool.submit(self.fn, *args, **kwargs)
                       for _ in range(num_workers)]
            results = [f.result() for f in futures]

        # the master process is the master mpi process and contributes
        else:
            futures = [pool.submit(self.fn, *args, **kwargs)
                       for _ in range(num_workers - 1)]
            results = ([self.fn(*args, **kwargs)] +
                       [f.result() for f in futures])

        # Get master result, (not always first submitted)
        return next(r for r in results if r is not None)

# ---------------------------------- SLEPC ---------------------------------- #

seigsys_slepc_mpi = functools.wraps(slepc_seigsys)(
    GetMPIBeforeCall(slepc_seigsys))
seigsys_slepc_spawn = functools.wraps(slepc_seigsys)(
    SpawnMPIProcessesFunc(seigsys_slepc_mpi))

svds_slepc_mpi = functools.wraps(slepc_svds)(
    GetMPIBeforeCall(slepc_svds))
svds_slepc_spawn = functools.wraps(slepc_svds)(
    SpawnMPIProcessesFunc(svds_slepc_mpi))

mfn_multiply_slepc_mpi = functools.wraps(slepc_mfn_multiply)(
    GetMPIBeforeCall(slepc_mfn_multiply))
mfn_multiply_slepc_spawn = functools.wraps(slepc_mfn_multiply)(
    SpawnMPIProcessesFunc(mfn_multiply_slepc_mpi))
<|endoftext|>


<commit_before>import logging
import re

import requests
from twisted.internet import defer
from twisted.internet.threads import deferToThread

from cardinal import util
from cardinal.bot import user_info
from cardinal.decorators import command, regex, help
from cardinal.util import F

# CoinMarketCap API Endpoint
CMC_QUOTE_API_URL = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest"  # noqa: E501

# Regex pattern that matches PyLink relay bots (copied from ticker plugin)
RELAY_REGEX = r'^(?:<(.+?)>\s+)'

# For 'crypto' command - checking cryptocurrency price
CRYPTO_REGEX = RELAY_REGEX + r'(\.crypto.*?)$'

# Max simultaneous cryptocurrency results
MAX_LOOKUPS = 5

def colorize(percentage):
    message = '{:.2f}%'.format(percentage)
    if percentage > 0:
        return F.C.light_green(message)
    else:
        return F.C.light_red(message)

class CMCError(Exception):
    """Represents a CoinMarketCap error"""
    pass

class CryptoPlugin:
    def __init__(self, cardinal, config):
        self.logger = logging.getLogger(__name__)
        self.cardinal = cardinal

        self.config = config or {}
        self.config.setdefault('cmc_api_key', None)
        self.config.setdefault('default_crypto_price_currency', 'USD')
        self.config.setdefault('relay_bots', [])

        if not self.config["cmc_api_key"]:
            raise KeyError(
                "Missing cmc_api_key (CoinMarketCap) - crypto functions will "
                "be unavailable")

        self.relay_bots = []
        for relay_bot in self.config['relay_bots']:
            user = user_info(
                relay_bot['nick'],
                relay_bot['user'],
                relay_bot['vhost'])
            self.relay_bots.append(user)

    @command('crypto')
    @help('Check the price of a cryptocurrency')
    @help('Syntax: .crypto <cryptocurrency,...> [price currency]')
    @defer.inlineCallbacks
    def crypto(self, cardinal, user, channel, message):
        nick = user.nick

        try:
            parts = message.split(' ', 2)
            coin = parts[1]
        except IndexError:
            cardinal.sendMsg(
                channel, "Syntax: .crypto <cryptocurrency> [price currency]")
            return

        if len(coin.split(',')) > MAX_LOOKUPS:
            cardinal.sendMsg(
                channel, "Please request no more than {} coins at once".format(
                    MAX_LOOKUPS))
            return

        try:
            currency = parts[2]
        except IndexError:
            currency = self.config['default_crypto_price_currency']

        try:
            resp = yield self.make_cmc_request(coin, currency)
        except CMCError as e:
            self.logger.warning(e)

            # Probably safe to send?
            cardinal.sendMsg(
                channel, "{}".format(e))
            return
        except Exception as exc:
            self.logger.warning(
                "Error trying to look up coin {} in currency {}: {}"
                .format(coin, currency, exc))
            cardinal.sendMsg(
                channel, "{}: I couldn't look that coin up".format(nick))
            return

        for coin in resp.values():
            name = coin['name']
            symbol = coin['symbol']
            for quote_currency, quote in coin['quote'].items():
                price = quote['price']
                if price >= 1:
                    price = round(price, 2)
                else:
                    price = float("%.4g" % price)

                cardinal.sendMsg(
                    channel,
                    "{} ({}) = {} {} - Daily Change (24h): {} "
                    "(Market Cap: {:,.2f})".format(
                        name, F.bold(symbol),
                        price, quote_currency,
                        colorize(quote['percent_change_24h']),
                        quote['market_cap']))

    @regex(CRYPTO_REGEX)
    @defer.inlineCallbacks
    def crypto_regex(self, cardinal, user, channel, message):
        """Hack to support relayed messages"""
        match = re.match(CRYPTO_REGEX, message)

        # this group should only be present when a relay bot is relaying a
        # message for another user
        if not match.group(1):
            return
        if not self.is_relay_bot(user):
            return

        user = user_info(util.strip_formatting(match.group(1)),
                         user.user,
                         user.vhost,
                         )

        yield self.crypto(cardinal, user, channel, match.group(2))

    @defer.inlineCallbacks
    def make_cmc_request(self, coin, currency):
        r = yield deferToThread(requests.get, CMC_QUOTE_API_URL, params={
            'convert': currency,
            'symbol': coin,
        }, headers={
            'Accepts': 'application/json',
            'X-CMC_PRO_API_KEY': self.config['cmc_api_key'],
        })

        resp = r.json()
        if resp['status']['error_code']:
            raise CMCError(
                "CoinMarketCap gave error {error_code}: {error_message}"
                .format(
                    error_code=resp['status']['error_code'],
                    error_message=resp['status']['error_message'],
                ))

        return resp['data']

    def is_relay_bot(self, user):
        """Compares a user against the registered relay bots."""
        for bot in self.relay_bots:
            if (bot.nick is None or bot.nick == user.nick) and \
                    (bot.user is None or bot.user == user.user) and \
                    (bot.vhost is None or bot.vhost == user.vhost):
                return True

        return False

entrypoint = CryptoPlugin
<commit_msg>Add market cap rank to crypto output<commit_after>import logging
import re

import requests
from twisted.internet import defer
from twisted.internet.threads import deferToThread

from cardinal import util
from cardinal.bot import user_info
from cardinal.decorators import command, regex, help
from cardinal.util import F

# CoinMarketCap API Endpoint
CMC_QUOTE_API_URL = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest"  # noqa: E501

# Regex pattern that matches PyLink relay bots (copied from ticker plugin)
RELAY_REGEX = r'^(?:<(.+?)>\s+)'

# For 'crypto' command - checking cryptocurrency price
CRYPTO_REGEX = RELAY_REGEX + r'(\.crypto.*?)$'

# Max simultaneous cryptocurrency results
MAX_LOOKUPS = 5

def colorize(percentage):
    message = '{:.2f}%'.format(percentage)
    if percentage > 0:
        return F.C.light_green(message)
    else:
        return F.C.light_red(message)

class CMCError(Exception):
    """Represents a CoinMarketCap error"""
    pass

class CryptoPlugin:
    def __init__(self, cardinal, config):
        self.logger = logging.getLogger(__name__)
        self.cardinal = cardinal

        self.config = config or {}
        self.config.setdefault('cmc_api_key', None)
        self.config.setdefault('default_crypto_price_currency', 'USD')
        self.config.setdefault('relay_bots', [])

        if not self.config["cmc_api_key"]:
            raise KeyError(
                "Missing cmc_api_key (CoinMarketCap) - crypto functions will "
                "be unavailable")

        self.relay_bots = []
        for relay_bot in self.config['relay_bots']:
            user = user_info(
                relay_bot['nick'],
                relay_bot['user'],
                relay_bot['vhost'])
            self.relay_bots.append(user)

    @command('crypto')
    @help('Check the price of a cryptocurrency')
    @help('Syntax: .crypto <cryptocurrency,...> [price currency]')
    @defer.inlineCallbacks
    def crypto(self, cardinal, user, channel, message):
        nick = user.nick

        try:
            parts = message.split(' ', 2)
            coin = parts[1]
        except IndexError:
            cardinal.sendMsg(
                channel, "Syntax: .crypto <cryptocurrency> [price currency]")
            return

        if len(coin.split(',')) > MAX_LOOKUPS:
            cardinal.sendMsg(
                channel, "Please request no more than {} coins at once".format(
                    MAX_LOOKUPS))
            return

        try:
            currency = parts[2]
        except IndexError:
            currency = self.config['default_crypto_price_currency']

        try:
            resp = yield self.make_cmc_request(coin, currency)
        except CMCError as e:
            self.logger.warning(e)

            # Probably safe to send?
            cardinal.sendMsg(
                channel, "{}".format(e))
            return
        except Exception as exc:
            self.logger.warning(
                "Error trying to look up coin {} in currency {}: {}"
                .format(coin, currency, exc))
            cardinal.sendMsg(
                channel, "{}: I couldn't look that coin up".format(nick))
            return

        for coin in resp.values():
            name = coin['name']
            symbol = coin['symbol']
            cmc_rank = coin['cmc_rank']
            for quote_currency, quote in coin['quote'].items():
                price = quote['price']
                if price >= 1:
                    price = round(price, 2)
                else:
                    price = float("%.4g" % price)

                cardinal.sendMsg(
                    channel,
                    "{} ({}) = {} {} - Daily Change (24h): {} "
                    "(Market Cap: {:,.2f} - Ranked #{})".format(
                        name, F.bold(symbol),
                        price, quote_currency,
                        colorize(quote['percent_change_24h']),
                        quote['market_cap'],
                        cmc_rank))

    @regex(CRYPTO_REGEX)
    @defer.inlineCallbacks
    def crypto_regex(self, cardinal, user, channel, message):
        """Hack to support relayed messages"""
        match = re.match(CRYPTO_REGEX, message)

        # this group should only be present when a relay bot is relaying a
        # message for another user
        if not match.group(1):
            return
        if not self.is_relay_bot(user):
            return

        user = user_info(util.strip_formatting(match.group(1)),
                         user.user,
                         user.vhost,
                         )

        yield self.crypto(cardinal, user, channel, match.group(2))

    @defer.inlineCallbacks
    def make_cmc_request(self, coin, currency):
        r = yield deferToThread(requests.get, CMC_QUOTE_API_URL, params={
            'convert': currency,
            'symbol': coin,
        }, headers={
            'Accepts': 'application/json',
            'X-CMC_PRO_API_KEY': self.config['cmc_api_key'],
        })

        resp = r.json()
        if resp['status']['error_code']:
            raise CMCError(
                "CoinMarketCap gave error {error_code}: {error_message}"
                .format(
                    error_code=resp['status']['error_code'],
                    error_message=resp['status']['error_message'],
                ))

        return resp['data']

    def is_relay_bot(self, user):
        """Compares a user against the registered relay bots."""
        for bot in self.relay_bots:
            if (bot.nick is None or bot.nick == user.nick) and \
                    (bot.user is None or bot.user == user.user) and \
                    (bot.vhost is None or bot.vhost == user.vhost):
                return True

        return False

entrypoint = CryptoPlugin
<|endoftext|>


<commit_before>import datetime
from google.appengine.ext import ndb

from models.award import Award
from models.district import District
from models.event import Event
from models.event_details import EventDetails
from models.location import Location
from models.match import Match
from models.media import Media
from models.robot import Robot
from models.team import Team

class ModelSerializer(object):
    """
    Converts a model to and from a JSON serializable object.
    """
    KINDS = {
        'Award': Award,
        'District': District,
        'Event': Event,
        'EventDetails': EventDetails,
        'Location': Location,
        'Match': Match,
        'Media': Media,
        'Robot': Robot,
        'Team': Team,
    }

    @classmethod
    def to_json(cls, o):
        if isinstance(o, list):
            return [cls.to_json(l) for l in o]
        if isinstance(o, dict):
            x = {}
            for l in o:
                x[l] = cls.to_json(o[l])
            return x
        if isinstance(o, datetime.datetime):
            return o.strftime('%Y-%m-%dT%H:%M:%S.%f')
        if isinstance(o, ndb.GeoPt):
            return {'lat': o.lat, 'lon': o.lon}
        if isinstance(o, ndb.Key):
            return {'__kind__': o.kind(), 'id': o.id()}
        if isinstance(o, ndb.Model):
            dct = {
                '__kind__': type(o).__name__,
            }
            if o.key:
                dct['__id__'] = o.key.id()
            for attr_name in o.to_dict():
                dct[attr_name] = cls.to_json(getattr(o, attr_name))
            return dct
        return o

    @classmethod
    def to_obj(cls, o, obj_type=None):
        if o is None:
            return None
        if isinstance(o, list):
            return [cls.to_obj(l, obj_type) for l in o]
        if isinstance(obj_type, ndb.DateTimeProperty):
            return datetime.datetime.strptime(o, '%Y-%m-%dT%H:%M:%S.%f')
        if isinstance(obj_type, ndb.GeoPtProperty):
            return ndb.GeoPt(o['lat'], o['lon'])
        if isinstance(obj_type, ndb.KeyProperty):
            return ndb.Key(o['__kind__'], o['id'])
        if obj_type is None or isinstance(obj_type, ndb.StructuredProperty):
            obj_type = cls.KINDS[o['__kind__']]
            if obj_type is not None:
                obj = obj_type(id=o.get('__id__'))
                obj._write_disabled = True  # Disable writing to DB from deserialized models for safety
                for attr_name in obj.to_dict():
                    attr_type = obj_type.__dict__[attr_name]
                    setattr(
                        obj,
                        attr_name,
                        cls.to_obj(o.get(attr_name), attr_type)
                    )
                return obj
        return o
<commit_msg>Fix deprecated properties causing serializer to fail<commit_after>import datetime
from google.appengine.ext import ndb

from models.award import Award
from models.district import District
from models.event import Event
from models.event_details import EventDetails
from models.location import Location
from models.match import Match
from models.media import Media
from models.robot import Robot
from models.team import Team

class ModelSerializer(object):
    """
    Converts a model to and from a JSON serializable object.
    """
    KINDS = {
        'Award': Award,
        'District': District,
        'Event': Event,
        'EventDetails': EventDetails,
        'Location': Location,
        'Match': Match,
        'Media': Media,
        'Robot': Robot,
        'Team': Team,
    }

    @classmethod
    def to_json(cls, o):
        if isinstance(o, list):
            return [cls.to_json(l) for l in o]
        if isinstance(o, dict):
            x = {}
            for l in o:
                x[l] = cls.to_json(o[l])
            return x
        if isinstance(o, datetime.datetime):
            return o.strftime('%Y-%m-%dT%H:%M:%S.%f')
        if isinstance(o, ndb.GeoPt):
            return {'lat': o.lat, 'lon': o.lon}
        if isinstance(o, ndb.Key):
            return {'__kind__': o.kind(), 'id': o.id()}
        if isinstance(o, ndb.Model):
            dct = {
                '__kind__': type(o).__name__,
            }
            if o.key:
                dct['__id__'] = o.key.id()
            for attr_name in o.to_dict():
                if hasattr(o, attr_name):
                    dct[attr_name] = cls.to_json(getattr(o, attr_name))
            return dct
        return o

    @classmethod
    def to_obj(cls, o, obj_type=None):
        if o is None:
            return None
        if isinstance(o, list):
            return [cls.to_obj(l, obj_type) for l in o]
        if isinstance(obj_type, ndb.DateTimeProperty):
            return datetime.datetime.strptime(o, '%Y-%m-%dT%H:%M:%S.%f')
        if isinstance(obj_type, ndb.GeoPtProperty):
            return ndb.GeoPt(o['lat'], o['lon'])
        if isinstance(obj_type, ndb.KeyProperty):
            return ndb.Key(o['__kind__'], o['id'])
        if obj_type is None or isinstance(obj_type, ndb.StructuredProperty):
            obj_type = cls.KINDS[o['__kind__']]
            if obj_type is not None:
                obj = obj_type(id=o.get('__id__'))
                obj._write_disabled = True  # Disable writing to DB from deserialized models for safety
                for attr_name in obj.to_dict():
                    attr_type = obj_type.__dict__[attr_name]
                    setattr(
                        obj,
                        attr_name,
                        cls.to_obj(o.get(attr_name), attr_type)
                    )
                return obj
        return o
<|endoftext|>