<commit_before>"""
Manage ruby gems.
"""

__opts__ = {
    "rvm.runas": None,
}

import re

def _gem(command, ruby=None, runas=None):
    cmdline = "gem {command}".format(command=command)
    if __salt__["rvm.is_installed"]:
        return __salt__["rvm.do"](ruby, cmdline, runas=runas)

    ret = __salt__["cmd.run_all"](
        cmdline, runas=runas or __opts__["rvm.runas"])

    if ret["retcode"] == 0:
        return ret["stdout"]
    else:
        return False

def install(gems, ruby=None, runas=None):
    """
    Installs one or several gems.

    gems
        The gems to install.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("install {gems}".format(gems=gems), ruby, runas=runas)

def uninstall(gems, ruby=None, runas=None):
    """
    Uninstall one or several gems.

    gems
        The gems to uninstall.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("uninstall {gems}".format(gems=gems), ruby, runas=runas)

def update(gems, ruby=None, runas=None):
    """
    Update one or several gems.

    gems
        The gems to update.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("update {gems}".format(gems=gems), ruby, runas=runas)

def update_system(version="", ruby=None, runas=None):
    """
    Update rubygems.

    version : (newest)
        The version of rubygems to install.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("update --system {version}".
                format(version=version), ruby, runas=runas)

def list(prefix="", ruby=None, runas=None):
    """
    List locally installed gems.

    prefix :
        Only list gems when the name matches this prefix.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    gems = {}
    for line in _gem("list {prefix}".format(prefix=prefix),
                     ruby, runas=runas).splitlines():
        m = re.match("^([^ ]+) \((.+)\)", line)
        if m:
            gem = m.group(1)
            versions = m.group(2).split(", ")
            gems[gem] = versions
    return gems

def sources_add(source_uri, ruby=None, runas=None):
    """
    Add a gem source.

    source_uri
        The source URI to add.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("sources --add {source_uri}".
                format(source_uri=source_uri), ruby, runas=runas)

def sources_remove(source_uri, ruby=None, runas=None):
    """
    Remove a gem source.

    source_uri
        The source URI to remove.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("sources --remove {source_uri}".
                format(source_uri=source_uri), ruby, runas=runas)

def sources_list(ruby=None, runas=None):
    """
    List the configured gem sources.

    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("sources", ruby, runas=runas).splitlines()[2:]
<commit_msg>Fixing some bugs in gem module<commit_after>"""
Manage ruby gems.
"""

__opts__ = {
    "rvm.runas": None,
}

import re

def _gem(command, ruby=None, runas=None):
    cmdline = "gem {command}".format(command=command)
    if __salt__["rvm.is_installed"]():
        return __salt__["rvm.do"](ruby, cmdline, runas=runas)

    ret = __salt__["cmd.run_all"](
        cmdline, runas=runas or __opts__["rvm.runas"])

    if ret["retcode"] == 0:
        return ret["stdout"]
    else:
        return False

def install(gems, ruby=None, runas=None):
    """
    Installs one or several gems.

    gems
        The gems to install.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("install {gems}".format(gems=gems), ruby, runas=runas)

def uninstall(gems, ruby=None, runas=None):
    """
    Uninstall one or several gems.

    gems
        The gems to uninstall.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("uninstall {gems}".format(gems=gems), ruby, runas=runas)

def update(gems, ruby=None, runas=None):
    """
    Update one or several gems.

    gems
        The gems to update.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("update {gems}".format(gems=gems), ruby, runas=runas)

def update_system(version="", ruby=None, runas=None):
    """
    Update rubygems.

    version : (newest)
        The version of rubygems to install.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("update --system {version}".
                format(version=version), ruby, runas=runas)

def list(prefix="", ruby=None, runas=None):
    """
    List locally installed gems.

    prefix :
        Only list gems when the name matches this prefix.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    gems = {}
    stdout = _gem("list {prefix}".format(prefix=prefix),
                     ruby, runas=runas)
    lines = []
    if isinstance(stdout, str):
        lines = stdout.splitlines()
    for line in lines:
        m = re.match("^([^ ]+) \((.+)\)", line)
        if m:
            gem = m.group(1)
            versions = m.group(2).split(", ")
            gems[gem] = versions
    return gems

def sources_add(source_uri, ruby=None, runas=None):
    """
    Add a gem source.

    source_uri
        The source URI to add.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("sources --add {source_uri}".
                format(source_uri=source_uri), ruby, runas=runas)

def sources_remove(source_uri, ruby=None, runas=None):
    """
    Remove a gem source.

    source_uri
        The source URI to remove.
    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("sources --remove {source_uri}".
                format(source_uri=source_uri), ruby, runas=runas)

def sources_list(ruby=None, runas=None):
    """
    List the configured gem sources.

    ruby : None
        If RVM is installed, the ruby version and gemset to use.
    runas : None
        The user to run gem as.
    """
    return _gem("sources", ruby, runas=runas).splitlines()[2:]
<|endoftext|>


<commit_before>from setuptools import setup

setup(
    name="fake-django-messaging-contentstore",
    version="0.1.1a",
    url='http://github.com/praekelt/django-messaging-contentstore',
    license='BSD',
    description="A verified fake implementation of \
                django-messaging-contentstore for testing.",
    long_description=open('README.rst', 'r').read(),
    author='Praekelt Foundation',
    author_email='dev@praekeltfoundation.org',
    py_modules=[],
    install_requires=[],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: POSIX',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Topic :: Internet :: WWW/HTTP',
    ],
)
<commit_msg>Bump fake version<commit_after>from setuptools import setup

setup(
    name="fake-django-messaging-contentstore",
    version="0.1.2",
    url='http://github.com/praekelt/django-messaging-contentstore',
    license='BSD',
    description="A verified fake implementation of \
                django-messaging-contentstore for testing.",
    long_description=open('README.rst', 'r').read(),
    author='Praekelt Foundation',
    author_email='dev@praekeltfoundation.org',
    py_modules=[],
    install_requires=[],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: POSIX',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Topic :: Internet :: WWW/HTTP',
    ],
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Virtual machine image management tools
'''

# Import python libs
import logging

# Set up logging
log = logging.getLogger(__name__)

def mount_image(location):
    '''
    Mount the named image and return the mount point

    CLI Example:

    .. code-block:: bash

        salt '*' img.mount_image /tmp/foo
    '''
    if 'guestfs.mount' in __salt__:
        return __salt__['guestfs.mount'](location)
    elif 'qemu_nbd.init' in __salt__:
        mnt = __salt__['qemu_nbd.init'](location)
        if not mnt:
            return ''
        first = mnt.iterkeys().next()
        __context__['img.mnt_{0}'.format(first)] = mnt
        return first
    return ''

#compatibility for api change
mnt_image = mount_image

def umount_image(mnt):
    '''
    Unmount an image mountpoint

    CLI Example:

    .. code-block:: bash

        salt '*' img.umount_image /mnt/foo
    '''
    if 'qemu_nbd.clear' in __salt__:
        if 'img.mnt_{0}'.format(mnt) in __context__:
            __salt__['qemu_nbd.clear'](__context__['img.mnt_{0}'.format(mnt)])
            return
    __salt__['mount.umount'](mnt)

#def get_image(name):
#    '''
#    Download a vm image from a remote source and add it to the image cache
#    system
#    '''
#    cache_dir = os.path.join(__salt__['config.option']('img.cache'), 'src')
#    parse = urlparse.urlparse(name)
#    if __salt__['config.valid_file_proto'](parse.scheme):
#        # Valid scheme to download
#        dest = os.path.join(cache_dir, parse.netloc)
#        sfn = __salt__['file.get_managed'](dest, None, name, )

def bootstrap(location, size, fmt):
    '''
    HIGHLY EXPERIMENTAL
    Bootstrap a virtual machine image

    location:
        The location to create the image

    size:
        The size of the image to create in megabytes

    fmt:
        The image format, raw or qcow2

    CLI Example:

    .. code-block:: bash

        salt '*' img.bootstrap /srv/salt-images/host.qcow 4096 qcow2
    '''
    location = __salt__['img.make_image'](location, size, fmt)
    if not location:
        return ''
    nbd = __salt__['qemu_nbd.connect'](location)
    __salt__['partition.mklabel'](nbd, 'msdos')
    __salt__['partition.mkpart'](nbd, 'primary', 'ext4', 1, -1)
    __salt__['partition.probe'](nbd)
    __salt__['partition.mkfs']('{0}p1'.format(nbd), 'ext4')
    mnt = __salt__['qemu_nbd.mount'](nbd)
    #return __salt__['pkg.bootstrap'](nbd, mnt.iterkeys().next())
<commit_msg>Making salt.modules.img python3 compatible<commit_after># -*- coding: utf-8 -*-
'''
Virtual machine image management tools
'''
from __future__ import absolute_import

# Import python libs
import logging

# Set up logging
log = logging.getLogger(__name__)

def mount_image(location):
    '''
    Mount the named image and return the mount point

    CLI Example:

    .. code-block:: bash

        salt '*' img.mount_image /tmp/foo
    '''
    if 'guestfs.mount' in __salt__:
        return __salt__['guestfs.mount'](location)
    elif 'qemu_nbd.init' in __salt__:
        mnt = __salt__['qemu_nbd.init'](location)
        if not mnt:
            return ''
        first = next(mnt.iterkeys())
        __context__['img.mnt_{0}'.format(first)] = mnt
        return first
    return ''

#compatibility for api change
mnt_image = mount_image

def umount_image(mnt):
    '''
    Unmount an image mountpoint

    CLI Example:

    .. code-block:: bash

        salt '*' img.umount_image /mnt/foo
    '''
    if 'qemu_nbd.clear' in __salt__:
        if 'img.mnt_{0}'.format(mnt) in __context__:
            __salt__['qemu_nbd.clear'](__context__['img.mnt_{0}'.format(mnt)])
            return
    __salt__['mount.umount'](mnt)

#def get_image(name):
#    '''
#    Download a vm image from a remote source and add it to the image cache
#    system
#    '''
#    cache_dir = os.path.join(__salt__['config.option']('img.cache'), 'src')
#    parse = urlparse.urlparse(name)
#    if __salt__['config.valid_file_proto'](parse.scheme):
#        # Valid scheme to download
#        dest = os.path.join(cache_dir, parse.netloc)
#        sfn = __salt__['file.get_managed'](dest, None, name, )

def bootstrap(location, size, fmt):
    '''
    HIGHLY EXPERIMENTAL
    Bootstrap a virtual machine image

    location:
        The location to create the image

    size:
        The size of the image to create in megabytes

    fmt:
        The image format, raw or qcow2

    CLI Example:

    .. code-block:: bash

        salt '*' img.bootstrap /srv/salt-images/host.qcow 4096 qcow2
    '''
    location = __salt__['img.make_image'](location, size, fmt)
    if not location:
        return ''
    nbd = __salt__['qemu_nbd.connect'](location)
    __salt__['partition.mklabel'](nbd, 'msdos')
    __salt__['partition.mkpart'](nbd, 'primary', 'ext4', 1, -1)
    __salt__['partition.probe'](nbd)
    __salt__['partition.mkfs']('{0}p1'.format(nbd), 'ext4')
    mnt = __salt__['qemu_nbd.mount'](nbd)
    #return __salt__['pkg.bootstrap'](nbd, mnt.iterkeys().next())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import re
import logging
import numpy  # this is needed by the doctests, don't remove it
from openquake.hazardlib import InvalidFile
from openquake.baselib.node import scientificformat
from openquake.baselib.python3compat import encode

FIVEDIGITS = '%.5E'

class HeaderTranslator(object):
    r"""
    An utility to convert the headers in CSV files. When reading,
    the column names are converted into column descriptions with the
    method .read, when writing column descriptions are converted
    into column names with the method .write. The usage is

    >>> htranslator = HeaderTranslator(
    ...     '(asset_ref):\|S100',
    ...     '(eid):uint32',
    ...     '(taxonomy):object')
    >>> htranslator.write('asset_ref:|S100 value:5'.split())
    ['asset_ref', 'value:5']
    >>> htranslator.read('asset_ref value:5'.split())
    ['asset_ref:|S100', 'value:5']
    """
    def __init__(self, *regexps):
        self.suffix = []
        short_regexps = []
        for regex in regexps:
            prefix, suffix = regex.split(')')
            short_regexps.append(prefix + ')$')
            self.suffix.append(suffix)
        self.short_regex = '|'.join(short_regexps)
        self.long_regex = '|'.join(regexps)

    def read(self, names):
        """
        Convert names into descriptions
        """
        descrs = []
        for name in names:
            mo = re.match(self.short_regex, name)
            if mo:
                idx = mo.lastindex  # matching group index, starting from 1
                suffix = self.suffix[idx - 1].replace(r':\|', ':|')
                descrs.append(mo.group(mo.lastindex) + suffix +
                              name[mo.end():])
            else:
                descrs.append(name)
        return descrs

    def write(self, descrs):
        """
        Convert descriptions into names
        """
        # example: '(poe-[\d\.]+):float32' -> 'poe-[\d\.]+'
        names = []
        for descr in descrs:
            mo = re.match(self.long_regex, descr)
            if mo:
                names.append(mo.group(mo.lastindex) + descr[mo.end():])
            else:
                names.append(descr)
        return names

htranslator = HeaderTranslator(
    '(rlzi):uint16',
    '(sid):uint32',
    '(eid):uint64',
    '(imti):uint8',
    '(gmv_.+):float32',
    '(aid):uint32',
    '(boundary):object',
    '(tectonic_region_type):object',
    '(asset_ref):\|S100',
    '(rup_id):uint32',
    '(event_id):uint64',
    '(event_set):uint32',
    '(eid):uint32',
    '(eid-\d+):float32',
    '(year):uint32',
    '(return_period):uint32',
    '(site_id):uint32',
    '(taxonomy):\|S100',
    '(tag):\|S100',
    '(multiplicity):uint16',
    '(magnitude):float32',
    '(centroid_lon):float32',
    '(centroid_lat):float32',
    '(centroid_depth):float32',
    '(numsites):uint32',
    '(losses):float32',
    '(poes):float32',
    '(avg):float32',
    '(poe-[\d\.]+):float32',
    '(lon):float32',
    '(lat):float32',
    '(depth):float32',
    '(structural.*):float32',
    '(nonstructural.*):float32',
    '(business_interruption.*):float32',
    '(contents.*):float32',
    '(occupants):float32',
    '(occupants~.+):float32',
    '(occupants_ins):float32',
    '(no_damage):float32',
    '(slight):float32',
    '(moderate):float32',
    '(extensive):float32',
    '(extreme):float32',
    '(complete):float32',
    '(\d+):float32',  # realization column, used in the GMF scenario exporter
)

# recursive function used internally by build_header
def _build_header(dtype, root):
    header = []
    if dtype.fields is None:
        if not root:
            return []
        return [root + (str(dtype), dtype.shape)]
    for field in dtype.names:
        dt = dtype.fields[field][0]
        if dt.subdtype is None:  # nested
            header.extend(_build_header(dt, root + (field,)))
        else:
            numpytype = str(dt.subdtype[0])
            header.append(root + (field, numpytype, dt.shape))
    return header

# NB: builds an header that can be read by parse_header
def build_header(dtype):
    """
    Convert a numpy nested dtype into a list of strings suitable as header
    of csv file.

    >>> imt_dt = numpy.dtype([('PGA', float, 3), ('PGV', float, 4)])
    >>> build_header(imt_dt)
    ['PGA:3', 'PGV:4']
    >>> gmf_dt = numpy.dtype([('A', imt_dt), ('B', imt_dt),
    ...                       ('idx', numpy.uint32)])
    >>> build_header(gmf_dt)
    ['A~PGA:3', 'A~PGV:4', 'B~PGA:3', 'B~PGV:4', 'idx:uint32']
    """
    header = _build_header(dtype, ())
    h = []
    for col in header:
        name = '~'.join(col[:-2])
        numpytype = col[-2]
        shape = col[-1]
        coldescr = name
        if numpytype != 'float64':
            coldescr += ':' + numpytype
        if shape:
            coldescr += ':' + ':'.join(map(str, shape))
        h.append(coldescr)
    return h

def extract_from(data, fields):
    """
    Extract data from numpy arrays with nested records.

    >>> imt_dt = numpy.dtype([('PGA', float, 3), ('PGV', float, 4)])
    >>> a = numpy.array([([1, 2, 3], [4, 5, 6, 7])], imt_dt)
    >>> extract_from(a, ['PGA'])
    array([[ 1.,  2.,  3.]])

    >>> gmf_dt = numpy.dtype([('A', imt_dt), ('B', imt_dt),
    ...                       ('idx', numpy.uint32)])
    >>> b = numpy.array([(([1, 2, 3], [4, 5, 6, 7]),
    ...                  ([1, 2, 4], [3, 5, 6, 7]), 8)], gmf_dt)
    >>> extract_from(b, ['idx'])
    array([8], dtype=uint32)
    >>> extract_from(b, ['B', 'PGV'])
    array([[ 3.,  5.,  6.,  7.]])
    """
    for f in fields:
        data = data[f]
    return data

def write_csv(dest, data, sep=',', fmt='%.6E', header=None, comment=None):
    """
    :param dest: file, filename or io.StringIO instance
    :param data: array to save
    :param sep: separator to use (default comma)
    :param fmt: formatting string (default '%12.8E')
    :param header:
       optional list with the names of the columns to display
    :param comment:
       optional first line starting with a # character
    """
    close = True
    if len(data) == 0:
        logging.warn('%s is empty', dest)
    if hasattr(dest, 'write'):
        # file-like object in append mode
        # it must be closed by client code
        close = False
    elif not hasattr(dest, 'getvalue'):
        # not a StringIO, assume dest is a filename
        dest = open(dest, 'wb')
    try:
        # see if data is a composite numpy array
        data.dtype.fields
    except AttributeError:
        # not a composite array
        autoheader = []
    else:
        autoheader = build_header(data.dtype)

    if comment:
        dest.write(encode('# %s\n' % comment))

    someheader = header or autoheader
    if header != 'no-header' and someheader:
        dest.write(encode(sep.join(htranslator.write(someheader)) + u'\n'))

    if autoheader:
        all_fields = [col.split(':', 1)[0].split('~')
                      for col in autoheader]
        for record in data:
            row = []
            for fields in all_fields:
                val = extract_from(record, fields)
                if fields[0] in ('lon', 'lat', 'depth'):
                    row.append('%.5f' % val)
                else:
                    row.append(scientificformat(val, fmt))
            dest.write(encode(sep.join(row) + u'\n'))
    else:
        for row in data:
            dest.write(encode(sep.join(scientificformat(col, fmt)
                                       for col in row) + u'\n'))
    if hasattr(dest, 'getvalue'):
        return dest.getvalue()[:-1]  # a newline is strangely added
    elif close:
        dest.close()
    return dest.name

class CsvWriter(object):
    """
    Class used in the exporters to save a bunch of CSV files
    """
    def __init__(self, sep=',', fmt='%12.8E'):
        self.sep = sep
        self.fmt = fmt
        self.fnames = set()

    def save(self, data, fname, header=None):
        """
        Save data on fname.

        :param data: numpy array or list of lists
        :param fname: path name
        :param header: header to use
        """
        write_csv(fname, data, self.sep, self.fmt, header)
        self.fnames.add(getattr(fname, 'name', fname))

    def getsaved(self):
        """
        Returns the list of files saved by this CsvWriter
        """
        return sorted(self.fnames)

def castable_to_int(s):
    """
    Return True if the string `s` can be interpreted as an integer
    """
    try:
        int(s)
    except ValueError:
        return False
    else:
        return True

def parse_header(header):
    """
    Convert a list of the form `['fieldname:fieldtype:fieldsize',...]`
    into a numpy composite dtype. The parser understands headers generated
    by :func:`openquake.commonlib.writers.build_header`.
    Here is an example:

    >>> parse_header(['PGA:float32', 'PGV', 'avg:float32:2'])
    (['PGA', 'PGV', 'avg'], dtype([('PGA', '<f4'), ('PGV', '<f8'), ('avg', '<f4', (2,))]))

    :params header: a list of type descriptions
    :returns: column names and the corresponding composite dtype
    """
    triples = []
    fields = []
    for col_str in header:
        col = col_str.strip().split(':')
        n = len(col)
        if n == 1:  # default dtype and no shape
            col = [col[0], 'float64', '']
        elif n == 2:
            if castable_to_int(col[1]):  # default dtype and shape
                col = [col[0], 'float64', col[1]]
            else:  # dtype and no shape
                col = [col[0], col[1], '']
        elif n > 3:
            raise ValueError('Invalid column description: %s' % col_str)
        field = col[0]
        numpytype = col[1]
        shape = () if not col[2].strip() else (int(col[2]),)
        triples.append((field, numpytype, shape))
        fields.append(field)
    return fields, numpy.dtype(triples)

def _cast(col, ntype, shape, lineno, fname):
    # convert strings into tuples or numbers, used inside read_composite_array
    if shape:
        return tuple(map(ntype, col.split()))
    else:
        return ntype(col)

# NB: this only works with flat composite arrays
def read_composite_array(fname, sep=','):
    r"""
    Convert a CSV file with header into a numpy array of records.

    >>> from openquake.baselib.general import writetmp
    >>> fname = writetmp('PGA:3,PGV:2,avg:1\n'
    ...                  '.1 .2 .3,.4 .5,.6\n')
    >>> print(read_composite_array(fname))  # array of shape (1,)
    [([0.1, 0.2, 0.3], [0.4, 0.5], [0.6])]
    """
    with open(fname) as f:
        header = next(f)
        transheader = htranslator.read(header.split(sep))
        fields, dtype = parse_header(transheader)
        ts_pairs = []  # [(type, shape), ...]
        for name in fields:
            dt = dtype.fields[name][0]
            ts_pairs.append((dt.subdtype[0].type if dt.subdtype else dt.type,
                             dt.shape))
        col_ids = list(range(1, len(ts_pairs) + 1))
        num_columns = len(col_ids)
        records = []
        col, col_id = '', 0
        for i, line in enumerate(f, 2):
            row = line.split(sep)
            if len(row) != num_columns:
                raise InvalidFile(
                    'expected %d columns, found %d in file %s, line %d' %
                    (num_columns, len(row), fname, i))
            try:
                record = []
                for (ntype, shape), col, col_id in zip(ts_pairs, row, col_ids):
                    record.append(_cast(col, ntype, shape, i, fname))
                records.append(tuple(record))
            except Exception as e:
                raise InvalidFile(
                    'Could not cast %r in file %s, line %d, column %d '
                    'using %s: %s' % (col, fname, i, col_id,
                                      (ntype.__name__,) + shape, e))
        return numpy.array(records, dtype)

# this is simple and without error checking for the moment
def read_array(fname, sep=','):
    r"""
    Convert a CSV file without header into a numpy array of floats.

    >>> from openquake.baselib.general import writetmp
    >>> print(read_array(writetmp('.1 .2, .3 .4, .5 .6\n')))
    [[[ 0.1  0.2]
      [ 0.3  0.4]
      [ 0.5  0.6]]]
    """
    with open(fname) as f:
        records = []
        for line in f:
            row = line.split(sep)
            record = [list(map(float, col.split())) for col in row]
            records.append(record)
        return numpy.array(records)

if __name__ == '__main__':  # pretty print of NRML files
    import sys
    import shutil
    from openquake.hazardlib import nrml
    nrmlfiles = sys.argv[1:]
    for fname in nrmlfiles:
        node = nrml.read(fname)
        shutil.copy(fname, fname + '.bak')
        with open(fname, 'w') as out:
            nrml.write(list(node), out)
<commit_msg>Fixed docstring [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import re
import logging
import numpy  # this is needed by the doctests, don't remove it
from openquake.hazardlib import InvalidFile
from openquake.baselib.node import scientificformat
from openquake.baselib.python3compat import encode

FIVEDIGITS = '%.5E'

class HeaderTranslator(object):
    r"""
    An utility to convert the headers in CSV files. When reading,
    the column names are converted into column descriptions with the
    method .read, when writing column descriptions are converted
    into column names with the method .write. The usage is

    >>> htranslator = HeaderTranslator(
    ...     '(asset_ref):\|S100',
    ...     '(eid):uint32',
    ...     '(taxonomy):object')
    >>> htranslator.write('asset_ref:|S100 value:5'.split())
    ['asset_ref', 'value:5']
    >>> htranslator.read('asset_ref value:5'.split())
    ['asset_ref:|S100', 'value:5']
    """
    def __init__(self, *regexps):
        self.suffix = []
        short_regexps = []
        for regex in regexps:
            prefix, suffix = regex.split(')')
            short_regexps.append(prefix + ')$')
            self.suffix.append(suffix)
        self.short_regex = '|'.join(short_regexps)
        self.long_regex = '|'.join(regexps)

    def read(self, names):
        """
        Convert names into descriptions
        """
        descrs = []
        for name in names:
            mo = re.match(self.short_regex, name)
            if mo:
                idx = mo.lastindex  # matching group index, starting from 1
                suffix = self.suffix[idx - 1].replace(r':\|', ':|')
                descrs.append(mo.group(mo.lastindex) + suffix +
                              name[mo.end():])
            else:
                descrs.append(name)
        return descrs

    def write(self, descrs):
        """
        Convert descriptions into names
        """
        # example: '(poe-[\d\.]+):float32' -> 'poe-[\d\.]+'
        names = []
        for descr in descrs:
            mo = re.match(self.long_regex, descr)
            if mo:
                names.append(mo.group(mo.lastindex) + descr[mo.end():])
            else:
                names.append(descr)
        return names

htranslator = HeaderTranslator(
    '(rlzi):uint16',
    '(sid):uint32',
    '(eid):uint64',
    '(imti):uint8',
    '(gmv_.+):float32',
    '(aid):uint32',
    '(boundary):object',
    '(tectonic_region_type):object',
    '(asset_ref):\|S100',
    '(rup_id):uint32',
    '(event_id):uint64',
    '(event_set):uint32',
    '(eid):uint32',
    '(eid-\d+):float32',
    '(year):uint32',
    '(return_period):uint32',
    '(site_id):uint32',
    '(taxonomy):\|S100',
    '(tag):\|S100',
    '(multiplicity):uint16',
    '(magnitude):float32',
    '(centroid_lon):float32',
    '(centroid_lat):float32',
    '(centroid_depth):float32',
    '(numsites):uint32',
    '(losses):float32',
    '(poes):float32',
    '(avg):float32',
    '(poe-[\d\.]+):float32',
    '(lon):float32',
    '(lat):float32',
    '(depth):float32',
    '(structural.*):float32',
    '(nonstructural.*):float32',
    '(business_interruption.*):float32',
    '(contents.*):float32',
    '(occupants):float32',
    '(occupants~.+):float32',
    '(occupants_ins):float32',
    '(no_damage):float32',
    '(slight):float32',
    '(moderate):float32',
    '(extensive):float32',
    '(extreme):float32',
    '(complete):float32',
    '(\d+):float32',  # realization column, used in the GMF scenario exporter
)

# recursive function used internally by build_header
def _build_header(dtype, root):
    header = []
    if dtype.fields is None:
        if not root:
            return []
        return [root + (str(dtype), dtype.shape)]
    for field in dtype.names:
        dt = dtype.fields[field][0]
        if dt.subdtype is None:  # nested
            header.extend(_build_header(dt, root + (field,)))
        else:
            numpytype = str(dt.subdtype[0])
            header.append(root + (field, numpytype, dt.shape))
    return header

# NB: builds an header that can be read by parse_header
def build_header(dtype):
    """
    Convert a numpy nested dtype into a list of strings suitable as header
    of csv file.

    >>> imt_dt = numpy.dtype([('PGA', float, 3), ('PGV', float, 4)])
    >>> build_header(imt_dt)
    ['PGA:3', 'PGV:4']
    >>> gmf_dt = numpy.dtype([('A', imt_dt), ('B', imt_dt),
    ...                       ('idx', numpy.uint32)])
    >>> build_header(gmf_dt)
    ['A~PGA:3', 'A~PGV:4', 'B~PGA:3', 'B~PGV:4', 'idx:uint32']
    """
    header = _build_header(dtype, ())
    h = []
    for col in header:
        name = '~'.join(col[:-2])
        numpytype = col[-2]
        shape = col[-1]
        coldescr = name
        if numpytype != 'float64':
            coldescr += ':' + numpytype
        if shape:
            coldescr += ':' + ':'.join(map(str, shape))
        h.append(coldescr)
    return h

def extract_from(data, fields):
    """
    Extract data from numpy arrays with nested records.

    >>> imt_dt = numpy.dtype([('PGA', float, 3), ('PGV', float, 4)])
    >>> a = numpy.array([([1, 2, 3], [4, 5, 6, 7])], imt_dt)
    >>> extract_from(a, ['PGA'])
    array([[ 1.,  2.,  3.]])

    >>> gmf_dt = numpy.dtype([('A', imt_dt), ('B', imt_dt),
    ...                       ('idx', numpy.uint32)])
    >>> b = numpy.array([(([1, 2, 3], [4, 5, 6, 7]),
    ...                  ([1, 2, 4], [3, 5, 6, 7]), 8)], gmf_dt)
    >>> extract_from(b, ['idx'])
    array([8], dtype=uint32)
    >>> extract_from(b, ['B', 'PGV'])
    array([[ 3.,  5.,  6.,  7.]])
    """
    for f in fields:
        data = data[f]
    return data

def write_csv(dest, data, sep=',', fmt='%.6E', header=None, comment=None):
    """
    :param dest: file, filename or io.BytesIO instance
    :param data: array to save
    :param sep: separator to use (default comma)
    :param fmt: formatting string (default '%12.8E')
    :param header:
       optional list with the names of the columns to display
    :param comment:
       optional first line starting with a # character
    """
    close = True
    if len(data) == 0:
        logging.warn('%s is empty', dest)
    if hasattr(dest, 'write'):
        # file-like object in append mode
        # it must be closed by client code
        close = False
    elif not hasattr(dest, 'getvalue'):
        # not a BytesIO, assume dest is a filename
        dest = open(dest, 'wb')
    try:
        # see if data is a composite numpy array
        data.dtype.fields
    except AttributeError:
        # not a composite array
        autoheader = []
    else:
        autoheader = build_header(data.dtype)

    if comment:
        dest.write(encode('# %s\n' % comment))

    someheader = header or autoheader
    if header != 'no-header' and someheader:
        dest.write(encode(sep.join(htranslator.write(someheader)) + u'\n'))

    if autoheader:
        all_fields = [col.split(':', 1)[0].split('~')
                      for col in autoheader]
        for record in data:
            row = []
            for fields in all_fields:
                val = extract_from(record, fields)
                if fields[0] in ('lon', 'lat', 'depth'):
                    row.append('%.5f' % val)
                else:
                    row.append(scientificformat(val, fmt))
            dest.write(encode(sep.join(row) + u'\n'))
    else:
        for row in data:
            dest.write(encode(sep.join(scientificformat(col, fmt)
                                       for col in row) + u'\n'))
    if hasattr(dest, 'getvalue'):
        return dest.getvalue()[:-1]  # a newline is strangely added
    elif close:
        dest.close()
    return dest.name

class CsvWriter(object):
    """
    Class used in the exporters to save a bunch of CSV files
    """
    def __init__(self, sep=',', fmt='%12.8E'):
        self.sep = sep
        self.fmt = fmt
        self.fnames = set()

    def save(self, data, fname, header=None):
        """
        Save data on fname.

        :param data: numpy array or list of lists
        :param fname: path name
        :param header: header to use
        """
        write_csv(fname, data, self.sep, self.fmt, header)
        self.fnames.add(getattr(fname, 'name', fname))

    def getsaved(self):
        """
        Returns the list of files saved by this CsvWriter
        """
        return sorted(self.fnames)

def castable_to_int(s):
    """
    Return True if the string `s` can be interpreted as an integer
    """
    try:
        int(s)
    except ValueError:
        return False
    else:
        return True

def parse_header(header):
    """
    Convert a list of the form `['fieldname:fieldtype:fieldsize',...]`
    into a numpy composite dtype. The parser understands headers generated
    by :func:`openquake.commonlib.writers.build_header`.
    Here is an example:

    >>> parse_header(['PGA:float32', 'PGV', 'avg:float32:2'])
    (['PGA', 'PGV', 'avg'], dtype([('PGA', '<f4'), ('PGV', '<f8'), ('avg', '<f4', (2,))]))

    :params header: a list of type descriptions
    :returns: column names and the corresponding composite dtype
    """
    triples = []
    fields = []
    for col_str in header:
        col = col_str.strip().split(':')
        n = len(col)
        if n == 1:  # default dtype and no shape
            col = [col[0], 'float64', '']
        elif n == 2:
            if castable_to_int(col[1]):  # default dtype and shape
                col = [col[0], 'float64', col[1]]
            else:  # dtype and no shape
                col = [col[0], col[1], '']
        elif n > 3:
            raise ValueError('Invalid column description: %s' % col_str)
        field = col[0]
        numpytype = col[1]
        shape = () if not col[2].strip() else (int(col[2]),)
        triples.append((field, numpytype, shape))
        fields.append(field)
    return fields, numpy.dtype(triples)

def _cast(col, ntype, shape, lineno, fname):
    # convert strings into tuples or numbers, used inside read_composite_array
    if shape:
        return tuple(map(ntype, col.split()))
    else:
        return ntype(col)

# NB: this only works with flat composite arrays
def read_composite_array(fname, sep=','):
    r"""
    Convert a CSV file with header into a numpy array of records.

    >>> from openquake.baselib.general import writetmp
    >>> fname = writetmp('PGA:3,PGV:2,avg:1\n'
    ...                  '.1 .2 .3,.4 .5,.6\n')
    >>> print(read_composite_array(fname))  # array of shape (1,)
    [([0.1, 0.2, 0.3], [0.4, 0.5], [0.6])]
    """
    with open(fname) as f:
        header = next(f)
        transheader = htranslator.read(header.split(sep))
        fields, dtype = parse_header(transheader)
        ts_pairs = []  # [(type, shape), ...]
        for name in fields:
            dt = dtype.fields[name][0]
            ts_pairs.append((dt.subdtype[0].type if dt.subdtype else dt.type,
                             dt.shape))
        col_ids = list(range(1, len(ts_pairs) + 1))
        num_columns = len(col_ids)
        records = []
        col, col_id = '', 0
        for i, line in enumerate(f, 2):
            row = line.split(sep)
            if len(row) != num_columns:
                raise InvalidFile(
                    'expected %d columns, found %d in file %s, line %d' %
                    (num_columns, len(row), fname, i))
            try:
                record = []
                for (ntype, shape), col, col_id in zip(ts_pairs, row, col_ids):
                    record.append(_cast(col, ntype, shape, i, fname))
                records.append(tuple(record))
            except Exception as e:
                raise InvalidFile(
                    'Could not cast %r in file %s, line %d, column %d '
                    'using %s: %s' % (col, fname, i, col_id,
                                      (ntype.__name__,) + shape, e))
        return numpy.array(records, dtype)

# this is simple and without error checking for the moment
def read_array(fname, sep=','):
    r"""
    Convert a CSV file without header into a numpy array of floats.

    >>> from openquake.baselib.general import writetmp
    >>> print(read_array(writetmp('.1 .2, .3 .4, .5 .6\n')))
    [[[ 0.1  0.2]
      [ 0.3  0.4]
      [ 0.5  0.6]]]
    """
    with open(fname) as f:
        records = []
        for line in f:
            row = line.split(sep)
            record = [list(map(float, col.split())) for col in row]
            records.append(record)
        return numpy.array(records)

if __name__ == '__main__':  # pretty print of NRML files
    import sys
    import shutil
    from openquake.hazardlib import nrml
    nrmlfiles = sys.argv[1:]
    for fname in nrmlfiles:
        node = nrml.read(fname)
        shutil.copy(fname, fname + '.bak')
        with open(fname, 'w') as out:
            nrml.write(list(node), out)
<|endoftext|>


<commit_before>from viaduct import db
from viaduct.models import BaseEntity

class News(db.Model, BaseEntity):
    __tablename__ = 'news'

    title = db.Column(db.Text)
    content = db.Column(db.Text)

    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    user = db.relationship('User', backref=db.backref('news', lazy='dynamic'))

    end_time = db.Column(db.Date)

    def __init__(self, title='', content='', user_id=None, end_time=None):

        self.title = title
        self.content = content
        self.user_id = user_id
        self.end_time = end_time

    def get_short_content(self, characters):
        """ Get a shortened version of the total post """
        if len(self.content) > characters:
            short_content = self.content[:characters].strip()

            # Remove the last word
            words = short_content.split(' ')[:-1]

            return ' '.join(words) + '...'

        return self.content
<commit_msg>Removed end_date to archive_date and added publish_date<commit_after>from datetime import date
from viaduct import db
from viaduct.models import BaseEntity

class News(db.Model, BaseEntity):
    __tablename__ = 'news'

    title = db.Column(db.Text)
    content = db.Column(db.Text)

    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    user = db.relationship('User', backref=db.backref('news', lazy='dynamic'))

    publish_date = db.Column(db.Date)
    archive_date = db.Column(db.Date)

    def __init__(self, title='', content='', user_id=None,
                 archive_date=None, publish_date=date.today()):

        self.title = title
        self.content = content
        self.user_id = user_id
        self.archive_date = archive_date
        self.publish_date = publish_date

    def get_short_content(self, characters):
        """ Get a shortened version of the total post """
        if len(self.content) > characters:
            short_content = self.content[:characters].strip()

            # Remove the last word
            words = short_content.split(' ')[:-1]

            return ' '.join(words) + '...'

        return self.content
<|endoftext|>


<commit_before>#coding=utf-8
import time
from mq import MessageQueue
from threading import Thread, Lock

from utils import get_time

addr = {
    "host":"127.0.0.1",
    "port":6379,
}

r = MessageQueue(**addr)

class SchedulerWorker(object):

    def __init__(self,queue, thread_num=5):
        queue = 'queue'
        r.set_queue(queue)
        self.mutex= Lock()
        self.thread_num = thread_num

    def run(self):
        self.spawn_worker()

    def spawn_worker(self):
        t_list = []
        for th_i in range(self.thread_num):
            t_threading = Thread(target=self.spawn_run,args=())
            t_list.append(t_threading)

        for th_i in t_list:
            th_i.start()

    def spawn_run(self):
        while True:
            if self.mutex.acquire(1):
                res = r.zrangebyscore()
                for i in res:
                    r.zrem(i)
                    r.rpush(i)
                self.mutex.release()
            time.sleep(0.1)

if __name__ == '__main__':
    queue = 'queue'
    scheduler = SchedulerWorker(queue)
    scheduler.run()
<commit_msg>format code<commit_after>#coding=utf-8
import time
from mq import MessageQueue
from threading import Thread, Lock

from utils import get_time

addr = {
    "host":"127.0.0.1",
    "port":6379,
}

r = MessageQueue(**addr)

class SchedulerWorker(object):

    def __init__(self,queue, thread_num=5):
        r.set_queue(queue)
        self.mutex= Lock()
        self.thread_num = thread_num

    def run(self):
        self.spawn_worker()

    def spawn_worker(self):
        t_list = []
        for th_i in range(self.thread_num):
            t_threading = Thread(target=self.spawn_run,args=())
            t_list.append(t_threading)

        for th_i in t_list:
            th_i.start()

    def spawn_run(self):
        while True:
            if self.mutex.acquire(1):
                res = r.zrangebyscore()
                for i in res:
                    r.zrem(i)
                    r.rpush(i)
                self.mutex.release()
            time.sleep(0.1)

if __name__ == '__main__':
    queue = 'queue'
    scheduler = SchedulerWorker(queue)
    scheduler.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import re
import logging
import unicodedata

from trovebox import Trovebox
from trovebox.errors import TroveboxError, TroveboxDuplicateError

from progressbar import ProgressBar, Percentage, Bar, ETA, FileTransferSpeed

def is_image(filename):
    # Trovebox supports .jpg, .gif, and .png files
    return bool(re.search(r'\.(jpg|jpeg|gif|png)$', filename, flags=re.IGNORECASE))

def main():
    try:
        client = Trovebox()
        client.configure(api_version=2)

    except IOError, e:
        print
        print '!! Could not initialize Trovebox connection.'
        print '!! Check that ~/.config/trovebox/default exists and contains proper information.'
        print
        raise e

    logger = logging.getLogger('trovebox-upload')
    hdlr = logging.FileHandler('upload.log')
    formatter = logging.Formatter('%(asctime)s\t%(levelname)s\t%(message)s')
    hdlr.setFormatter(formatter)

    logger.addHandler(hdlr)
    logger.setLevel(logging.DEBUG)

    files_count = size_count = uploaded_count = 0

    for root, _, files in os.walk(u'.'):
        for filename in files:
            if not is_image(filename):
                continue

            files_count += 1
            size_count += os.path.getsize(os.path.join(root, filename))

    print 'Found %d files, total %d Mib' % (files_count, size_count / (1024 * 1024))

    widgets = [Percentage(), Bar(), ETA(), ' ', FileTransferSpeed()]
    size = ProgressBar(maxval=size_count, widgets=widgets).start()

    for root, folders, files in os.walk(u'.'):
        folder_name = album = None

        for filename in files:
            if not is_image(filename):
                continue

            if not folder_name:
                # Convert decomposed string into a composed string.
                # Mac OS uses decomposed unicode filenames, while the Trovebox
                # album name font only supports precomposed filenames.
                folder_name = unicodedata.normalize('NFC', root.split('/')[-1])
                logger.info('Entering folder %s' % root)

                try:
                    album = client.album.create(folder_name)

                except TroveboxError, e:
                    logger.warning('Album "%s" already exists, falling back on path' % folder_name)
                    album = client.album.create(unicodedata.normalize('NFC', root))

            path = os.path.join(root, filename)

            try:
                logger.info('Uploading %s' % filename)
                client.photo.upload(path, albums=[album.id])
            except TroveboxDuplicateError:
                logger.warning('File %s was already uploaded' % path)
                pass

            uploaded_count += os.path.getsize(path)
            size.update(uploaded_count)
    size.finish()

if __name__ == "__main__":
    main()
<commit_msg>Catch and log Trovebox errors<commit_after># -*- coding: utf-8 -*-
import os
import re
import logging
import unicodedata

from trovebox import Trovebox
from trovebox.errors import TroveboxError, TroveboxDuplicateError

from progressbar import ProgressBar, Percentage, Bar, ETA, FileTransferSpeed

def is_image(filename):
    # Trovebox supports .jpg, .gif, and .png files
    return bool(re.search(r'\.(jpg|jpeg|gif|png)$', filename, flags=re.IGNORECASE))

def main():
    try:
        client = Trovebox()
        client.configure(api_version=2)

    except IOError, e:
        print
        print '!! Could not initialize Trovebox connection.'
        print '!! Check that ~/.config/trovebox/default exists and contains proper information.'
        print
        raise e

    logger = logging.getLogger('trovebox-upload')
    hdlr = logging.FileHandler('upload.log')
    formatter = logging.Formatter('%(asctime)s\t%(levelname)s\t%(message)s')
    hdlr.setFormatter(formatter)

    logger.addHandler(hdlr)
    logger.setLevel(logging.DEBUG)

    files_count = size_count = uploaded_count = 0

    for root, _, files in os.walk(u'.'):
        for filename in files:
            if not is_image(filename):
                continue

            files_count += 1
            size_count += os.path.getsize(os.path.join(root, filename))

    print 'Found %d files, total %d Mib' % (files_count, size_count / (1024 * 1024))

    widgets = [Percentage(), Bar(), ETA(), ' ', FileTransferSpeed()]
    size = ProgressBar(maxval=size_count, widgets=widgets).start()

    for root, folders, files in os.walk(u'.'):
        folder_name = album = None

        for filename in files:
            if not is_image(filename):
                continue

            if not folder_name:
                # Convert decomposed string into a composed string.
                # Mac OS uses decomposed unicode filenames, while the Trovebox
                # album name font only supports precomposed filenames.
                folder_name = unicodedata.normalize('NFC', root.split('/')[-1])
                logger.info('Entering folder %s' % root)

                try:
                    album = client.album.create(folder_name)

                except TroveboxError, e:
                    logger.warning('Album "%s" already exists, falling back on path' % folder_name)
                    album = client.album.create(unicodedata.normalize('NFC', root))

            path = os.path.join(root, filename)

            try:
                logger.info('Uploading %s' % filename)
                client.photo.upload(path, albums=[album.id])
            except TroveboxDuplicateError:
                logger.warning('File %s was already uploaded' % path)
            except TroveboxError, e:
                logger.error(e.message)

            uploaded_count += os.path.getsize(path)
            size.update(uploaded_count)
    size.finish()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from vidscraper import errors
from vidscraper.sites import (
    vimeo, google_video, youtube)

AUTOSCRAPE_SUITES = [
    vimeo.SUITE, google_video.SUITE, youtube.SUITE,
    blip.SUITE]

def scrape_suite(url, suite, fields=None):
    scraped_data = {}

    funcs_map = suite['funcs']
    fields = fields or funcs_map.keys()
    order = suite.get('order')
    if order:
        # remove items in the order that are not in the fields
        for field in set(order).difference(fields):
            order.pop(field)

        # add items that may have been missing from the order but
        # which are in the fields
        order.extend(set(fields).difference(order))
        fields = order

    shortmem = {}
    for field in fields:
        func = funcs_map[field]
        scraped_data[field] = func(url, shortmem=shortmem)

    return scraped_data

def auto_scrape(url, fields=None):
    for suite in AUTOSCRAPE_SUITES:
        if suite['regex'].match(url):
            return scrape_suite(url, suite, fields)

    # If we get here that means that none of the regexes matched, so
    # throw an error
    raise errors.CantIdentifyUrl(
        "No video scraping suite was found that can scrape that url")
<commit_msg>Fix the blip autoscrape.  Missed an import.<commit_after>from vidscraper import errors
from vidscraper.sites import (
    vimeo, google_video, youtube, blip)

AUTOSCRAPE_SUITES = [
    vimeo.SUITE, google_video.SUITE, youtube.SUITE,
    blip.SUITE]

def scrape_suite(url, suite, fields=None):
    scraped_data = {}

    funcs_map = suite['funcs']
    fields = fields or funcs_map.keys()
    order = suite.get('order')
    if order:
        # remove items in the order that are not in the fields
        for field in set(order).difference(fields):
            order.pop(field)

        # add items that may have been missing from the order but
        # which are in the fields
        order.extend(set(fields).difference(order))
        fields = order

    shortmem = {}
    for field in fields:
        func = funcs_map[field]
        scraped_data[field] = func(url, shortmem=shortmem)

    return scraped_data

def auto_scrape(url, fields=None):
    for suite in AUTOSCRAPE_SUITES:
        if suite['regex'].match(url):
            return scrape_suite(url, suite, fields)

    # If we get here that means that none of the regexes matched, so
    # throw an error
    raise errors.CantIdentifyUrl(
        "No video scraping suite was found that can scrape that url")
<|endoftext|>


<commit_before># Copyright 2020 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests whether modules produce similar output given np.ndarray inputs."""

from absl.testing import absltest
from absl.testing import parameterized
import haiku as hk
from haiku._src import test_utils
from haiku._src.integration import descriptors
import jax
import jax.numpy as jnp
import numpy as np

ModuleFn = descriptors.ModuleFn

class NumpyInputsTest(parameterized.TestCase):

  @test_utils.combined_named_parameters(descriptors.ALL_MODULES)
  def test_numpy_and_jax_results_close(self, module_fn: ModuleFn, shape, dtype):

    f = hk.transform_with_state(lambda x: module_fn()(x))  # pylint: disable=unnecessary-lambda

    rng = jax.random.PRNGKey(42)
    x = jnp.ones(shape, dtype)
    params, state = f.init(rng, x)
    out, new_state = f.apply(params, state, rng, x)

    np_rng = np.asarray(rng)
    np_x = np.asarray(x)

    with self.subTest('init'):
      params2, state2 = f.init(np_rng, np_x)
      jax.tree_multimap(np.testing.assert_allclose, params, params2)
      jax.tree_multimap(
          lambda x, y: np.testing.assert_allclose(x, y, atol=1e-9),
          state, state2)

    with self.subTest('apply'):
      np_params = jax.tree_map(np.asarray, params)
      np_state = jax.tree_map(np.asarray, state)
      out2, new_state2 = f.apply(np_params, np_state, np_rng, np_x)
      jax.tree_multimap(np.testing.assert_allclose, out, out2)
      jax.tree_multimap(
          lambda x, y: np.testing.assert_allclose(x, y, atol=1e-9),
          new_state, new_state2)

if __name__ == '__main__':
  absltest.main()
<commit_msg>Ensure NumPy inputs test covers case where params/state are closed over.<commit_after># Copyright 2020 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests whether modules produce similar output given np.ndarray inputs."""

import functools
from typing import Tuple

from absl.testing import absltest
from absl.testing import parameterized
import haiku as hk
from haiku._src import test_utils
from haiku._src.integration import descriptors
import jax
import jax.numpy as jnp
import numpy as np

ModuleFn = descriptors.ModuleFn

def tree_assert_allclose(a, b, *, atol=1e-6):
  jax.tree_multimap(
      functools.partial(np.testing.assert_allclose, atol=atol), a, b)

class NumpyInputsTest(parameterized.TestCase):

  @test_utils.combined_named_parameters(
      descriptors.ALL_MODULES,
      test_utils.named_bools('np_inputs'),
      test_utils.named_bools('np_params'),
      test_utils.named_bools('close_over_params'))
  def test_numpy_and_jax_results_close(
      self,
      module_fn: ModuleFn,
      shape: Tuple[int, ...],
      dtype: jnp.dtype,
      np_params: bool,
      np_inputs: bool,
      close_over_params: bool,
  ):
    if not (np_params or np_inputs):
      self.skipTest('Pure JAX variants tested elsewhere')

    f = hk.transform_with_state(lambda x: module_fn()(x))  # pylint: disable=unnecessary-lambda

    rng = jax.random.PRNGKey(42)
    x = jnp.ones(shape, dtype)
    params, state = f.init(rng, x)
    if close_over_params:
      apply_fn = functools.partial(f.apply, params, state)
      out, new_state = jax.jit(apply_fn)(rng, x)
    else:
      out, new_state = jax.jit(f.apply)(params, state, rng, x)

    if np_inputs:
      rng, x = jax.device_get((rng, x))

      with self.subTest('init'):
        params2, state2 = f.init(rng, x)
        tree_assert_allclose(params, params2)
        tree_assert_allclose(state, state2)

    with self.subTest('apply'):
      if np_params:
        params, state = jax.device_get((params, state))

      if close_over_params:
        apply_fn = functools.partial(f.apply, params, state)
        out2, new_state2 = jax.jit(apply_fn)(rng, x)
      else:
        out2, new_state2 = jax.jit(f.apply)(params, state, rng, x)

      tree_assert_allclose(out, out2)
      tree_assert_allclose(new_state, new_state2)

if __name__ == '__main__':
  absltest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Upload a file, or directory, to an Overview server.
# From https://github.com/overview/overview-upload-directory
import requests
import sys
import os
import uuid
import json
import argparse
import hashlib
import traceback

files_uploaded = 0

# check if a file is already on the server, via sha1 hash
def file_already_on_server(server_url, api_token, bytes):
	sha1 = hashlib.sha1(bytes).hexdigest()
	url = server_url + '/api/v1/document-sets/files/' + sha1
	r = requests.head(url, auth=(api_token,'x-auth-token'))

	return r.status_code == 204  # 204 = already got it, 404 = don't got it

# may need to escape file names in some way, not sure...
def content_disposition(filename):
	return 'attachment; filename="' + filename + '"'

# per http://docs.overviewproject.apiary.io/#reference/files/file/upload-file?console=1
def upload_single_file(server_url, api_token, filename, skip_existing=False):
	global files_uploaded

	bytes = open(filename, 'rb').read()

	if skip_existing and file_already_on_server(server_url, api_token, bytes):
		print("Skipping " + filename + ", already on server")
		return
	else:
		print("Uploading " + filename + "...")

	url = server_url + '/api/v1/files/' + str(uuid.uuid4())
	headers = { 'content-disposition' : content_disposition(filename),
				'content-length' : str(os.path.getsize(filename)) }
	r = requests.post(url, auth=(api_token,'x-auth-token'), headers=headers, data=bytes)
	r.raise_for_status()

	files_uploaded += 1

# per http://docs.overviewproject.apiary.io/#reference/files/finish-uploading-files/add-files-to-document-set?console=1
def finish_upload(server_url, api_token):
	url = server_url + '/api/v1/files/finish'
	headers = {'Content-Type':'application/json'}
	data = { 'lang' : 'en' }

	r = requests.post(url, auth=(api_token,'x-auth-token'), headers=headers, data=json.dumps(data))
	r.raise_for_status()

def upload_directory(server_url, api_token, filename, skip_existing=False):
	for subdir, dirs, files in os.walk(filename):
	    for file in files:

	    	if file[0]=='.':
	    		continue		# don't upload normally hidden files (I'm looking at you, .DS_store)

	    	fullname = os.path.join(subdir, file)
	    	try:
	    		upload_single_file(server_url, api_token, fullname, skip_existing=skip_existing)
	    	except Exception as e:
	    		print("Error: " + str(e) + ", skipping " + fullname)
	    		traceback.print_exc()

# ---- Main ----

parser = argparse.ArgumentParser(description='Upload a file or directory to an Overview server.')
parser.add_argument('file', help='file or directory to upload')
parser.add_argument('-t', '--token', help='API token corresponding to document set', required=True)
parser.add_argument('-s', '--server', help='url of Overview server, defaults to http://localhost:9000', default='http://localhost:9000')
parser.add_argument('-n', '--noskip', help='Don\'t skip files already on server', action="store_true")
args = parser.parse_args()

skip_existing = not args.noskip

filename = args.file.rstrip('/')  # removing trailing slash important for consistent directory handling

if not os.path.exists(filename):
	print("Cannot find file or directory " + filename)
else:
	ispath = os.path.isdir(filename)

	# do this all from directory containing specified file, to get consistent file titles
	# if user specified a file, the filename (no path) is a title
	# if user specified a directory, that directory name (no path) is the base of the title
	basename = os.path.basename(filename)
	dirname = os.path.dirname(filename)
	if dirname!='':
		os.chdir(dirname)

	if ispath:
		upload_directory(args.server, args.token, basename, skip_existing=skip_existing)
	else:
		upload_single_file(args.server, args.token, basename, skip_existing=skip_existing)

	# Send a finish only if we actually uploaded something (may have skipped all)
	if files_uploaded > 0:
		finish_upload(args.server, args.token,)

<commit_msg>Nicer formatting<commit_after>#!/usr/bin/env python3
# Upload a file, or directory, to an Overview server.
# From https://github.com/overview/overview-upload-directory
import requests
import sys
import os
import uuid
import json
import argparse
import hashlib
import traceback

files_uploaded = 0

# check if a file is already on the server, via sha1 hash
def file_already_on_server(server_url, api_token, bytes):
	sha1 = hashlib.sha1(bytes).hexdigest()
	url = server_url + '/api/v1/document-sets/files/' + sha1
	r = requests.head(url, auth=(api_token,'x-auth-token'))

	return r.status_code == 204  # 204 = already got it, 404 = don't got it

# may need to escape file names in some way, not sure...
def content_disposition(filename):
	return 'attachment; filename="' + filename + '"'

# per http://docs.overviewproject.apiary.io/#reference/files/file/upload-file?console=1
def upload_single_file(server_url, api_token, filename, skip_existing=False):
	global files_uploaded

	bytes = open(filename, 'rb').read()

	if skip_existing and file_already_on_server(server_url, api_token, bytes):
		print("Skipping " + filename + ", already on server")
		return
	else:
		print("Uploading " + filename + "...")

	url = server_url + '/api/v1/files/' + str(uuid.uuid4())
	headers = { 'content-disposition' : content_disposition(filename),
				'content-length' : str(os.path.getsize(filename)) }
	r = requests.post(url, auth=(api_token,'x-auth-token'), headers=headers, data=bytes)
	r.raise_for_status()

	files_uploaded += 1

# per http://docs.overviewproject.apiary.io/#reference/files/finish-uploading-files/add-files-to-document-set?console=1
def finish_upload(server_url, api_token):
	url = server_url + '/api/v1/files/finish'
	headers = {'Content-Type':'application/json'}
	data = { 'lang' : 'en' }

	r = requests.post(url, auth=(api_token,'x-auth-token'), headers=headers, data=json.dumps(data))
	r.raise_for_status()

def upload_directory(server_url, api_token, filename, skip_existing=False):
	for subdir, dirs, files in os.walk(filename):
	    for file in files:

	    	if file[0]=='.':
	    		continue		# don't upload normally hidden files (I'm looking at you, .DS_store)

	    	fullname = os.path.join(subdir, file)
	    	try:
	    		upload_single_file(server_url, api_token, fullname, skip_existing=skip_existing)
	    	except Exception as e:
	    		print("Error, skipping " + fullname)
	    		traceback.print_exc()
	    		print('')

# ---- Main ----

parser = argparse.ArgumentParser(description='Upload a file or directory to an Overview server.')
parser.add_argument('file', help='file or directory to upload')
parser.add_argument('-t', '--token', help='API token corresponding to document set', required=True)
parser.add_argument('-s', '--server', help='url of Overview server, defaults to http://localhost:9000', default='http://localhost:9000')
parser.add_argument('-n', '--noskip', help='Don\'t skip files already on server', action="store_true")
args = parser.parse_args()

skip_existing = not args.noskip

filename = args.file.rstrip('/')  # removing trailing slash important for consistent directory handling

if not os.path.exists(filename):
	print("Cannot find file or directory " + filename)
else:
	ispath = os.path.isdir(filename)

	# do this all from directory containing specified file, to get consistent file titles
	# if user specified a file, the filename (no path) is a title
	# if user specified a directory, that directory name (no path) is the base of the title
	basename = os.path.basename(filename)
	dirname = os.path.dirname(filename)
	if dirname!='':
		os.chdir(dirname)

	if ispath:
		upload_directory(args.server, args.token, basename, skip_existing=skip_existing)
	else:
		upload_single_file(args.server, args.token, basename, skip_existing=skip_existing)

	# Send a finish only if we actually uploaded something (may have skipped all)
	if files_uploaded > 0:
		finish_upload(args.server, args.token,)

<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from openstack.message.v1 import claim
from openstack.message.v1 import message
from openstack.message.v1 import queue
from openstack import proxy

class Proxy(proxy.BaseProxy):

    def create_queue(self, **attrs):
        """Create a new queue from attributes

        :param dict attrs: Keyword arguments which will be used to create
                           a :class:`~openstack.message.v1.queue.Queue`,
                           comprised of the properties on the Queue class.

        :returns: The results of queue creation
        :rtype: :class:`~openstack.message.v1.queue.Queue`
        """
        return self._create(queue.Queue, **attrs)

    def delete_queue(self, value, ignore_missing=True):
        """Delete a queue

        :param value: The value can be either the name of a queue or a
                      :class:`~openstack.message.v1.queue.Queue` instance.
        :param bool ignore_missing: When set to ``False``
                    :class:`~openstack.exceptions.ResourceNotFound` will be
                    raised when the queue does not exist.
                    When set to ``True``, no exception will be set when
                    attempting to delete a nonexistent queue.

        :returns: ``None``
        """
        return self._delete(queue.Queue, value, ignore_missing=ignore_missing)

    def create_messages(self, values):
        """Create new messages

        :param values: The list of
                       :class:`~openstack.message.v1.message.Message` objects
                       to create.
        :type values: :py:class:`list`

        :returns: The list of
                  :class:`~openstack.message.v1.message.Message` objects
                  that were created.
        """
        return message.Message.create_messages(self._session, values)

    def claim_messages(self, value):
        """Claims a set of messages.

        :param value: The value must be a
                      :class:`~openstack.message.v1.claim.Claim` instance.

        :returns: The list of
                  :class:`~openstack.message.v1.message.Message` objects
                  that were claimed.
        """
        return claim.Claim.claim_messages(self._session, value)

    def delete_message(self, value):
        """Delete a message

        :param value: The value must be a
                      :class:`~openstack.message.v1.message.Message` instance.

        :returns: ``None``
        """
        message.Message.delete_by_id(self._session, value)
<commit_msg>Deprecate Message v1<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from openstack.message.v1 import claim
from openstack.message.v1 import message
from openstack.message.v1 import queue
from openstack import proxy
from openstack import utils

class Proxy(proxy.BaseProxy):

    @utils.deprecated(deprecated_in="0.9.16", removed_in="0.9.17",
                      details="Message v1 is deprecated since 2014. Use v2.")
    def create_queue(self, **attrs):
        """Create a new queue from attributes

        :param dict attrs: Keyword arguments which will be used to create
                           a :class:`~openstack.message.v1.queue.Queue`,
                           comprised of the properties on the Queue class.

        :returns: The results of queue creation
        :rtype: :class:`~openstack.message.v1.queue.Queue`
        """
        return self._create(queue.Queue, **attrs)

    @utils.deprecated(deprecated_in="0.9.16", removed_in="0.9.17",
                      details="Message v1 is deprecated since 2014. Use v2.")
    def delete_queue(self, value, ignore_missing=True):
        """Delete a queue

        :param value: The value can be either the name of a queue or a
                      :class:`~openstack.message.v1.queue.Queue` instance.
        :param bool ignore_missing: When set to ``False``
                    :class:`~openstack.exceptions.ResourceNotFound` will be
                    raised when the queue does not exist.
                    When set to ``True``, no exception will be set when
                    attempting to delete a nonexistent queue.

        :returns: ``None``
        """
        return self._delete(queue.Queue, value, ignore_missing=ignore_missing)

    @utils.deprecated(deprecated_in="0.9.16", removed_in="0.9.17",
                      details="Message v1 is deprecated since 2014. Use v2.")
    def create_messages(self, values):
        """Create new messages

        :param values: The list of
                       :class:`~openstack.message.v1.message.Message` objects
                       to create.
        :type values: :py:class:`list`

        :returns: The list of
                  :class:`~openstack.message.v1.message.Message` objects
                  that were created.
        """
        return message.Message.create_messages(self._session, values)

    @utils.deprecated(deprecated_in="0.9.16", removed_in="0.9.17",
                      details="Message v1 is deprecated since 2014. Use v2.")
    def claim_messages(self, value):
        """Claims a set of messages.

        :param value: The value must be a
                      :class:`~openstack.message.v1.claim.Claim` instance.

        :returns: The list of
                  :class:`~openstack.message.v1.message.Message` objects
                  that were claimed.
        """
        return claim.Claim.claim_messages(self._session, value)

    @utils.deprecated(deprecated_in="0.9.16", removed_in="0.9.17",
                      details="Message v1 is deprecated since 2014. Use v2.")
    def delete_message(self, value):
        """Delete a message

        :param value: The value must be a
                      :class:`~openstack.message.v1.message.Message` instance.

        :returns: ``None``
        """
        message.Message.delete_by_id(self._session, value)
<|endoftext|>


<commit_before>'''
Read in the roster from a flat file using the renderer system
'''

# Import python libs
import os
import fnmatch
import re

# Import Salt libs
import salt.loader
from salt.template import compile_template

def targets(tgt, tgt_type='glob', **kwargs):
    '''
    Return the targets from the flat yaml file, checks opts for location but
    defaults to /etc/salt/roster
    '''
    if os.path.isfile(__opts__['conf_file']):
        template = os.path.join(
                os.path.dirname(__opts__['conf_file']),
                'roster')
    else:
        template = os.path.join(__opts__['conf_file'], 'roster')
    rend = salt.loader.render(__opts__, {})
    raw = compile_template(template, rend, __opts__['renderer'], **kwargs)
    rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4')
    return rmatcher.targets()

class RosterMatcher(object):
    '''
    Matcher for the roster data structure
    '''
    def __init__(self, raw, tgt, tgt_type, ipv='ipv4'):
        self.tgt = tgt
        self.tgt_type = tgt_type
        self.raw = raw
        self.ipv = ipv

    def targets(self):
        '''
        Execute the correct tgt_type routine and return
        '''
        try:
            return getattr(self, 'ret_{0}_minions'.format(self.tgt_type))()
        except AttributeError:
            return {}

    def ret_glob_minions(self):
        '''
        Return minions that match via glob
        '''
        minions = {}
        for minion in self.raw:
            if fnmatch.fnmatch(minion, self.tgt):
                ip = self.get_ip(minion)
                if ip:
                    minions[minion] = ip

    def ret_pcre_minions(self):
        '''
        Return minions that match via pcre
        '''
        minions = {}
        for minion in self.raw:
            if re.match(self.tgt, minion):
                ip = self.get_ip(minion)
                if ip:
                    minions[minion] = ip

    def get_data(self, minion):
        '''
        Return the configured ip
        '''
        if isinstance(self.raw[minion], basestring):
            return {'host': self.raw[minion]}
        if isinstance(self.raw[minion], dict):
            return self.raw[minion]
        return False

<commit_msg>Patch up switch from ip to data<commit_after>'''
Read in the roster from a flat file using the renderer system
'''

# Import python libs
import os
import fnmatch
import re

# Import Salt libs
import salt.loader
from salt.template import compile_template

def targets(tgt, tgt_type='glob', **kwargs):
    '''
    Return the targets from the flat yaml file, checks opts for location but
    defaults to /etc/salt/roster
    '''
    if os.path.isfile(__opts__['conf_file']):
        template = os.path.join(
                os.path.dirname(__opts__['conf_file']),
                'roster')
    else:
        template = os.path.join(__opts__['conf_file'], 'roster')
    rend = salt.loader.render(__opts__, {})
    raw = compile_template(template, rend, __opts__['renderer'], **kwargs)
    rmatcher = RosterMatcher(raw, tgt, tgt_type, 'ipv4')
    return rmatcher.targets()

class RosterMatcher(object):
    '''
    Matcher for the roster data structure
    '''
    def __init__(self, raw, tgt, tgt_type, ipv='ipv4'):
        self.tgt = tgt
        self.tgt_type = tgt_type
        self.raw = raw
        self.ipv = ipv

    def targets(self):
        '''
        Execute the correct tgt_type routine and return
        '''
        try:
            return getattr(self, 'ret_{0}_minions'.format(self.tgt_type))()
        except AttributeError:
            return {}

    def ret_glob_minions(self):
        '''
        Return minions that match via glob
        '''
        minions = {}
        for minion in self.raw:
            if fnmatch.fnmatch(minion, self.tgt):
                data = self.get_data(minion)
                if data:
                    minions[minion] = data
        return minions

    def ret_pcre_minions(self):
        '''
        Return minions that match via pcre
        '''
        minions = {}
        for minion in self.raw:
            if re.match(self.tgt, minion):
                data = self.get_data(minion)
                if data:
                    minions[minion] = data
        return minions

    def get_data(self, minion):
        '''
        Return the configured ip
        '''
        if isinstance(self.raw[minion], basestring):
            return {'host': self.raw[minion]}
        if isinstance(self.raw[minion], dict):
            return self.raw[minion]
        return False

<|endoftext|>


<commit_before>"""Support for Greenwave Reality (TCP Connected) lights."""
from __future__ import annotations

from datetime import timedelta
import logging
import os

import greenwavereality as greenwave
import voluptuous as vol

from homeassistant.components.light import (
    ATTR_BRIGHTNESS,
    PLATFORM_SCHEMA,
    SUPPORT_BRIGHTNESS,
    LightEntity,
)
from homeassistant.const import CONF_HOST
from homeassistant.core import HomeAssistant
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
from homeassistant.util import Throttle

_LOGGER = logging.getLogger(__name__)

CONF_VERSION = "version"

SUPPORTED_FEATURES = SUPPORT_BRIGHTNESS

PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
    {vol.Required(CONF_HOST): cv.string, vol.Required(CONF_VERSION): cv.positive_int}
)

MIN_TIME_BETWEEN_UPDATES = timedelta(minutes=1)

def setup_platform(
    hass: HomeAssistant,
    config: ConfigType,
    add_entities: AddEntitiesCallback,
    discovery_info: DiscoveryInfoType | None = None,
) -> None:
    """Set up the Greenwave Reality Platform."""
    host = config.get(CONF_HOST)
    tokenfilename = hass.config.path(".greenwave")
    if config.get(CONF_VERSION) == 3:
        if os.path.exists(tokenfilename):
            with open(tokenfilename, encoding="utf8") as tokenfile:
                token = tokenfile.read()
        else:
            try:
                token = greenwave.grab_token(host, "hass", "homeassistant")
            except PermissionError:
                _LOGGER.error("The Gateway Is Not In Sync Mode")
                raise
            with open(tokenfilename, "w+", encoding="utf8") as tokenfile:
                tokenfile.write(token)
    else:
        token = None
    bulbs = greenwave.grab_bulbs(host, token)
    add_entities(
        GreenwaveLight(device, host, token, GatewayData(host, token))
        for device in bulbs.values()
    )

class GreenwaveLight(LightEntity):
    """Representation of an Greenwave Reality Light."""

    def __init__(self, light, host, token, gatewaydata):
        """Initialize a Greenwave Reality Light."""
        self._did = int(light["did"])
        self._name = light["name"]
        self._state = int(light["state"])
        self._brightness = greenwave.hass_brightness(light)
        self._host = host
        self._online = greenwave.check_online(light)
        self._token = token
        self._gatewaydata = gatewaydata

    @property
    def supported_features(self):
        """Flag supported features."""
        return SUPPORTED_FEATURES

    @property
    def available(self):
        """Return True if entity is available."""
        return self._online

    @property
    def name(self):
        """Return the display name of this light."""
        return self._name

    @property
    def brightness(self):
        """Return the brightness of the light."""
        return self._brightness

    @property
    def is_on(self):
        """Return true if light is on."""
        return self._state

    def turn_on(self, **kwargs):
        """Instruct the light to turn on."""
        temp_brightness = int((kwargs.get(ATTR_BRIGHTNESS, 255) / 255) * 100)
        greenwave.set_brightness(self._host, self._did, temp_brightness, self._token)
        greenwave.turn_on(self._host, self._did, self._token)

    def turn_off(self, **kwargs):
        """Instruct the light to turn off."""
        greenwave.turn_off(self._host, self._did, self._token)

    def update(self):
        """Fetch new state data for this light."""
        self._gatewaydata.update()
        bulbs = self._gatewaydata.greenwave

        self._state = int(bulbs[self._did]["state"])
        self._brightness = greenwave.hass_brightness(bulbs[self._did])
        self._online = greenwave.check_online(bulbs[self._did])
        self._name = bulbs[self._did]["name"]

class GatewayData:
    """Handle Gateway data and limit updates."""

    def __init__(self, host, token):
        """Initialize the data object."""
        self._host = host
        self._token = token
        self._greenwave = greenwave.grab_bulbs(host, token)

    @property
    def greenwave(self):
        """Return Gateway API object."""
        return self._greenwave

    @Throttle(MIN_TIME_BETWEEN_UPDATES)
    def update(self):
        """Get the latest data from the gateway."""
        self._greenwave = greenwave.grab_bulbs(self._host, self._token)
        return self._greenwave
<commit_msg>Migrate greenwave light to color_mode (#69258)<commit_after>"""Support for Greenwave Reality (TCP Connected) lights."""
from __future__ import annotations

from datetime import timedelta
import logging
import os

import greenwavereality as greenwave
import voluptuous as vol

from homeassistant.components.light import (
    ATTR_BRIGHTNESS,
    COLOR_MODE_BRIGHTNESS,
    PLATFORM_SCHEMA,
    LightEntity,
)
from homeassistant.const import CONF_HOST
from homeassistant.core import HomeAssistant
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
from homeassistant.util import Throttle

_LOGGER = logging.getLogger(__name__)

CONF_VERSION = "version"

PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
    {vol.Required(CONF_HOST): cv.string, vol.Required(CONF_VERSION): cv.positive_int}
)

MIN_TIME_BETWEEN_UPDATES = timedelta(minutes=1)

def setup_platform(
    hass: HomeAssistant,
    config: ConfigType,
    add_entities: AddEntitiesCallback,
    discovery_info: DiscoveryInfoType | None = None,
) -> None:
    """Set up the Greenwave Reality Platform."""
    host = config.get(CONF_HOST)
    tokenfilename = hass.config.path(".greenwave")
    if config.get(CONF_VERSION) == 3:
        if os.path.exists(tokenfilename):
            with open(tokenfilename, encoding="utf8") as tokenfile:
                token = tokenfile.read()
        else:
            try:
                token = greenwave.grab_token(host, "hass", "homeassistant")
            except PermissionError:
                _LOGGER.error("The Gateway Is Not In Sync Mode")
                raise
            with open(tokenfilename, "w+", encoding="utf8") as tokenfile:
                tokenfile.write(token)
    else:
        token = None
    bulbs = greenwave.grab_bulbs(host, token)
    add_entities(
        GreenwaveLight(device, host, token, GatewayData(host, token))
        for device in bulbs.values()
    )

class GreenwaveLight(LightEntity):
    """Representation of an Greenwave Reality Light."""

    _attr_color_mode = COLOR_MODE_BRIGHTNESS
    _attr_supported_color_modes = {COLOR_MODE_BRIGHTNESS}

    def __init__(self, light, host, token, gatewaydata):
        """Initialize a Greenwave Reality Light."""
        self._did = int(light["did"])
        self._name = light["name"]
        self._state = int(light["state"])
        self._brightness = greenwave.hass_brightness(light)
        self._host = host
        self._online = greenwave.check_online(light)
        self._token = token
        self._gatewaydata = gatewaydata

    @property
    def available(self):
        """Return True if entity is available."""
        return self._online

    @property
    def name(self):
        """Return the display name of this light."""
        return self._name

    @property
    def brightness(self):
        """Return the brightness of the light."""
        return self._brightness

    @property
    def is_on(self):
        """Return true if light is on."""
        return self._state

    def turn_on(self, **kwargs):
        """Instruct the light to turn on."""
        temp_brightness = int((kwargs.get(ATTR_BRIGHTNESS, 255) / 255) * 100)
        greenwave.set_brightness(self._host, self._did, temp_brightness, self._token)
        greenwave.turn_on(self._host, self._did, self._token)

    def turn_off(self, **kwargs):
        """Instruct the light to turn off."""
        greenwave.turn_off(self._host, self._did, self._token)

    def update(self):
        """Fetch new state data for this light."""
        self._gatewaydata.update()
        bulbs = self._gatewaydata.greenwave

        self._state = int(bulbs[self._did]["state"])
        self._brightness = greenwave.hass_brightness(bulbs[self._did])
        self._online = greenwave.check_online(bulbs[self._did])
        self._name = bulbs[self._did]["name"]

class GatewayData:
    """Handle Gateway data and limit updates."""

    def __init__(self, host, token):
        """Initialize the data object."""
        self._host = host
        self._token = token
        self._greenwave = greenwave.grab_bulbs(host, token)

    @property
    def greenwave(self):
        """Return Gateway API object."""
        return self._greenwave

    @Throttle(MIN_TIME_BETWEEN_UPDATES)
    def update(self):
        """Get the latest data from the gateway."""
        self._greenwave = greenwave.grab_bulbs(self._host, self._token)
        return self._greenwave
<|endoftext|>