<commit_before># -*- coding: utf-8 -*-
# pylint: disable=bad-continuation
""" ‘Double Click’ – Extensions to Click.
"""
# Copyright ©  2015 Jürgen Hermann <jh@web.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals, print_function

import os
import re

import configobj
from bunch import bunchify
from click import *  # pylint: disable=wildcard-import
from click import __all__

__all__ = tuple(__all__) + (
    'pretty_path', 'serror',
    'LoggedFailure', 'AliasedGroup', 'Configuration',
)

def pretty_path(path, _home_re=re.compile('^' + re.escape(os.path.expanduser('~') + os.sep))):
    """Prettify path for humans, and make it Unicode."""
    path = format_filename(path)
    path = _home_re.sub('~' + os.sep, path)
    return path

def serror(message, *args, **kwargs):
    """Print a styled error message, while using any arguments to format the message."""
    if args or kwargs:
        message = message.format(*args, **kwargs)
    return secho(message, fg='white', bg='red', bold=True)

class LoggedFailure(UsageError):
    """Report a failure condition to the user."""

    def __init__(self, message):
        message = style(message, fg='white', bg='red', bold=True)
        UsageError.__init__(self, message)

class AliasedGroup(Group):
    """ A command group with alias names.

        Inherit from this class and define a ``MAP`` class variable,
        which is a mapping from alias names to canonical command names.
        Then use that derived class as the ``cls`` parameter for a
        ``click.group`` decorator.
    """

    MAP = {}

    def get_command(self, ctx, cmd_name):
        """Map some aliases to their 'real' names."""
        cmd_name = self.MAP.get(cmd_name, cmd_name)
        return Group.get_command(self, ctx, cmd_name)

class Configuration(object):
    """ Configuration container that is initialized early in the main command.

        The default instance is available via the Click context as ``ctx.obj.cfg``.
        Configuration is lazily loaded, on first access.
    """

    NO_DEFAULT = object()

    @classmethod
    def from_context(cls, ctx, config_paths=None):
        """Create a configuration object, and initialize the Click context with it."""
        if ctx.obj is None:
            ctx.obj = Bunch()
        ctx.obj.cfg = cls(ctx.info_name, config_paths)
        return ctx.obj.cfg

    def __init__(self, name, config_paths=None):
        """ Set up static knowledge about configuration.

            ``config_paths`` is a list of PATHs to config files or directories.
            If that is non-empty, exactly those paths will be considered; an empty
            path element represents the default locations (just like in MANPATH).

            If the environment variable ``‹prefix›_CONFIG`` is set, its value will
            be appended to the default locations.
        """
        self.values = configobj.ConfigObj({}, encoding='utf-8', default_encoding='utf-8')
        self.name = name
        self.config_paths = []
        self.loaded = False

        env_config = os.environ.get((self.name + '-config').upper().replace('-', '_'), '')
        defaults = [
            '/etc/{}.conf'.format(self.name),
            os.path.join(get_app_dir(self.name) + '.conf'),
        ] + [i for i in env_config.split(os.pathsep) if i]

        for path in config_paths or []:
            for name in path.split(os.pathsep):
                if name:
                    self.config_paths.append(name)
                else:
                    self.config_paths.extend(defaults)

        if not self.config_paths:
            self.config_paths = defaults

    def locations(self, exists=True):
        """ Return the location of the config file(s).

            A given directory will be scanned for ``*.conf`` files, in alphabetical order.
            Any duplicates will be eliminated.

            If ``exists`` is True, only existing configuration locations are returned.
        """
        result = []
        for config_files in self.config_paths:
            if not config_files:
                continue
            if os.path.isdir(config_files):
                config_files = [os.path.join(config_files, i)
                                for i in os.listdir(config_files)
                                if i.endswith('.conf')]
            else:
                config_files = [config_files]
            for config_file in config_files:
                if not exists or os.path.exists(config_file):
                    config_file = os.path.abspath(config_file)
                    if config_file in result:
                        result.remove(config_file)
                    result.append(config_file)
        return result

    def load(self):
        """Load configuration from the defined locations."""
        if not self.loaded:
            self.values = configobj.ConfigObj({}, encoding='utf-8', default_encoding='utf-8')
            for path in self.locations():
                part = configobj.ConfigObj(infile=path, encoding='utf-8', default_encoding='utf-8')
                self.values.merge(part)
            self.loaded = True
        return self.values

    def section(self, ctx):
        """Return section of the config for a specific context (sub-command)."""
        values = self.load()
        try:
            return values[ctx.info_name]
        except KeyError:
            raise LoggedFailure("Configuration section '{}' not found!".format(ctx.info_name))

    def get(self, name, default=NO_DEFAULT):
        """Return the specified key from the root section."""
        values = self.load()
        try:
            return values[name]
        except KeyError:
            if default is self.NO_DEFAULT:
                raise LoggedFailure("Configuration value '{}' not found in root section!".format(name))
            return default
<commit_msg>:mag: import 'Bunch'<commit_after># -*- coding: utf-8 -*-
# pylint: disable=bad-continuation
""" ‘Double Click’ – Extensions to Click.
"""
# Copyright ©  2015 Jürgen Hermann <jh@web.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals, print_function

import os
import re

import configobj
from bunch import bunchify, Bunch
from click import *  # pylint: disable=wildcard-import
from click import __all__

__all__ = tuple(__all__) + (
    'pretty_path', 'serror',
    'LoggedFailure', 'AliasedGroup', 'Configuration',
)

def pretty_path(path, _home_re=re.compile('^' + re.escape(os.path.expanduser('~') + os.sep))):
    """Prettify path for humans, and make it Unicode."""
    path = format_filename(path)
    path = _home_re.sub('~' + os.sep, path)
    return path

def serror(message, *args, **kwargs):
    """Print a styled error message, while using any arguments to format the message."""
    if args or kwargs:
        message = message.format(*args, **kwargs)
    return secho(message, fg='white', bg='red', bold=True)

class LoggedFailure(UsageError):
    """Report a failure condition to the user."""

    def __init__(self, message):
        message = style(message, fg='white', bg='red', bold=True)
        UsageError.__init__(self, message)

class AliasedGroup(Group):
    """ A command group with alias names.

        Inherit from this class and define a ``MAP`` class variable,
        which is a mapping from alias names to canonical command names.
        Then use that derived class as the ``cls`` parameter for a
        ``click.group`` decorator.
    """

    MAP = {}

    def get_command(self, ctx, cmd_name):
        """Map some aliases to their 'real' names."""
        cmd_name = self.MAP.get(cmd_name, cmd_name)
        return Group.get_command(self, ctx, cmd_name)

class Configuration(object):
    """ Configuration container that is initialized early in the main command.

        The default instance is available via the Click context as ``ctx.obj.cfg``.
        Configuration is lazily loaded, on first access.
    """

    NO_DEFAULT = object()

    @classmethod
    def from_context(cls, ctx, config_paths=None):
        """Create a configuration object, and initialize the Click context with it."""
        if ctx.obj is None:
            ctx.obj = Bunch()
        ctx.obj.cfg = cls(ctx.info_name, config_paths)
        return ctx.obj.cfg

    def __init__(self, name, config_paths=None):
        """ Set up static knowledge about configuration.

            ``config_paths`` is a list of PATHs to config files or directories.
            If that is non-empty, exactly those paths will be considered; an empty
            path element represents the default locations (just like in MANPATH).

            If the environment variable ``‹prefix›_CONFIG`` is set, its value will
            be appended to the default locations.
        """
        self.values = configobj.ConfigObj({}, encoding='utf-8', default_encoding='utf-8')
        self.name = name
        self.config_paths = []
        self.loaded = False

        env_config = os.environ.get((self.name + '-config').upper().replace('-', '_'), '')
        defaults = [
            '/etc/{}.conf'.format(self.name),
            os.path.join(get_app_dir(self.name) + '.conf'),
        ] + [i for i in env_config.split(os.pathsep) if i]

        for path in config_paths or []:
            for name in path.split(os.pathsep):
                if name:
                    self.config_paths.append(name)
                else:
                    self.config_paths.extend(defaults)

        if not self.config_paths:
            self.config_paths = defaults

    def locations(self, exists=True):
        """ Return the location of the config file(s).

            A given directory will be scanned for ``*.conf`` files, in alphabetical order.
            Any duplicates will be eliminated.

            If ``exists`` is True, only existing configuration locations are returned.
        """
        result = []
        for config_files in self.config_paths:
            if not config_files:
                continue
            if os.path.isdir(config_files):
                config_files = [os.path.join(config_files, i)
                                for i in os.listdir(config_files)
                                if i.endswith('.conf')]
            else:
                config_files = [config_files]
            for config_file in config_files:
                if not exists or os.path.exists(config_file):
                    config_file = os.path.abspath(config_file)
                    if config_file in result:
                        result.remove(config_file)
                    result.append(config_file)
        return result

    def load(self):
        """Load configuration from the defined locations."""
        if not self.loaded:
            self.values = configobj.ConfigObj({}, encoding='utf-8', default_encoding='utf-8')
            for path in self.locations():
                part = configobj.ConfigObj(infile=path, encoding='utf-8', default_encoding='utf-8')
                self.values.merge(part)
            self.loaded = True
        return self.values

    def section(self, ctx):
        """Return section of the config for a specific context (sub-command)."""
        values = self.load()
        try:
            return values[ctx.info_name]
        except KeyError:
            raise LoggedFailure("Configuration section '{}' not found!".format(ctx.info_name))

    def get(self, name, default=NO_DEFAULT):
        """Return the specified key from the root section."""
        values = self.load()
        try:
            return values[name]
        except KeyError:
            if default is self.NO_DEFAULT:
                raise LoggedFailure("Configuration value '{}' not found in root section!".format(name))
            return default
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand
from django.utils import translation
from django.conf import settings

class Command(BaseCommand):
    help = "Counts PulbicBodies for Topics"

    def handle(self, *args, **options):
        translation.activate(settings.LANGUAGE_CODE)

        from froide.publicbody.models import PublicBodyTopic

        for topic in PublicBodyTopic.objects.all():
            topic.count = len(topic.publicbody_set.all())
            topic.save()
<commit_msg>Remove obsolete count_topic command<commit_after><|endoftext|>


<commit_before>"""
Test converting the builtin Potential classes to other packages
"""

# Third-party
from astropy.coordinates import CylindricalRepresentation
from astropy.tests.helper import catch_warnings
import astropy.units as u
import numpy as np
import pytest

# This project
import gala.potential as gp
from gala.units import galactic
from gala.tests.optional_deps import HAS_GALPY
from gala.potential.potential.interop import galpy_to_gala_potential

# Set these globally!
ro = 8.122 * u.kpc
vo = 245 * u.km/u.s

if HAS_GALPY:
    import galpy.potential as galpy_gp

    from gala.potential.potential.interop import (
        _gala_to_galpy,
        _galpy_to_gala
    )

def pytest_generate_tests(metafunc):
    # Some magic, semi-random numbers below!
    gala_pots = []
    galpy_pots = []

    if not HAS_GALPY:
        return

    # Test the Gala -> Galpy direction
    for Potential in _gala_to_galpy.keys():
        init = {}
        len_scale = 1.
        for k, par in Potential._parameters.items():
            if k == 'm':
                val = 1.43e10 * u.Msun
            elif par.physical_type == 'length':
                val = 5.12 * u.kpc * len_scale
                len_scale *= 0.5
            elif par.physical_type == 'dimensionless':
                val = 1.
            elif par.physical_type == 'speed':
                val = 201.41 * u.km/u.s
            else:
                continue

            init[k] = val

        pot = Potential(**init, units=galactic)
        galpy_pot = pot.to_galpy_potential(ro=ro, vo=vo)

        gala_pots.append(pot)
        galpy_pots.append(galpy_pot)

    # Make a composite potential too:
    gala_pots.append(gala_pots[0] + gala_pots[1])
    galpy_pots.append([galpy_pots[0], galpy_pots[1]])

    # Test the Galpy -> Gala direction
    for Potential in _galpy_to_gala.keys():
        galpy_pot = Potential(ro=ro, vo=vo)  # use defaults

        with catch_warnings(RuntimeWarning) as warns:
            pot = galpy_to_gala_potential(galpy_pot, ro=ro, vo=vo)

        if isinstance(galpy_pot, galpy_gp.MN3ExponentialDiskPotential):
            assert len(warns) > 0

        gala_pots.append(pot)
        galpy_pots.append(galpy_pot)

    test_names = [f'{g1.__class__.__name__}:{g2.__class__.__name__}'
                  for g1, g2 in zip(gala_pots, galpy_pots)]

    metafunc.parametrize(['gala_pot', 'galpy_pot'],
                         list(zip(gala_pots, galpy_pots)),
                         ids=test_names)

@pytest.mark.skipif(not HAS_GALPY,
                    reason="must have galpy installed to run these tests")
class TestGalpy:

    def setup(self):
        # Test points:
        rng = np.random.default_rng(42)
        ntest = 4

        Rs = rng.uniform(1, 15, size=ntest) * u.kpc
        phis = rng.uniform(0, 2*np.pi, size=ntest) * u.radian
        zs = rng.uniform(1, 15, size=ntest) * u.kpc

        cyl = CylindricalRepresentation(Rs, phis, zs)
        xyz = cyl.to_cartesian().xyz

        self.Rs = Rs.to_value(ro)
        self.phis = phis.to_value(u.rad)
        self.zs = zs.to_value(ro)
        self.Rpz_iter = np.array(list(zip(self.Rs, self.phis, self.zs))).copy()

        self.xyz = xyz.copy()

        Jac = np.zeros((len(cyl), 3, 3))
        Jac[:, 0, 0] = xyz[0] / cyl.rho
        Jac[:, 0, 1] = xyz[1] / cyl.rho
        Jac[:, 1, 0] = (-xyz[1] / cyl.rho**2).to_value(1 / ro)
        Jac[:, 1, 1] = (xyz[0] / cyl.rho**2).to_value(1 / ro)
        Jac[:, 2, 2] = 1.
        self.Jac = Jac

    def test_density(self, gala_pot, galpy_pot):
        if isinstance(gala_pot, gp.LogarithmicPotential):
            pytest.skip()

        gala_val = gala_pot.density(self.xyz).to_value(u.Msun / u.pc**3)
        galpy_val = np.array([galpy_gp.evaluateDensities(galpy_pot,
                                                         R=RR, z=zz, phi=pp)
                              for RR, pp, zz in self.Rpz_iter])
        assert np.allclose(gala_val, galpy_val)

    def test_energy(self, gala_pot, galpy_pot):
        gala_val = gala_pot.energy(self.xyz).to_value(u.km**2 / u.s**2)
        galpy_val = np.array([galpy_gp.evaluatePotentials(galpy_pot,
                                                          R=RR, z=zz, phi=pp)
                              for RR, pp, zz in self.Rpz_iter])

        if isinstance(gala_pot, gp.LogarithmicPotential):
            # Logarithms are weird
            gala_val -= (0.5 * gala_pot.parameters['v_c']**2 *
                         np.log(ro.value**2)).to_value((u.km / u.s)**2)

        assert np.allclose(gala_val, galpy_val)

    def test_gradient(self, gala_pot, galpy_pot):
        gala_grad = gala_pot.gradient(self.xyz)
        gala_grad = gala_grad.to_value(u.km/u.s/u.Myr)

        # TODO: Starting with galpy 1.7, this has been failing because of a
        # units issue with dPhi/dphi
        if isinstance(gala_pot, gp.LongMuraliBarPotential):
            pytest.skip()

        galpy_dR = np.array([-galpy_gp.evaluateRforces(galpy_pot,
                                                       R=RR, z=zz, phi=pp)
                            for RR, pp, zz in self.Rpz_iter])
        galpy_dp = np.array([-galpy_gp.evaluatephiforces(galpy_pot,
                                                         R=RR, z=zz, phi=pp)
                            for RR, pp, zz in self.Rpz_iter])
        galpy_dp = (galpy_dp*(u.km/u.s)**2).to_value(vo**2)

        galpy_dz = np.array([-galpy_gp.evaluatezforces(galpy_pot,
                                                       R=RR, z=zz, phi=pp)
                            for RR, pp, zz in self.Rpz_iter])
        galpy_dRpz = np.stack((galpy_dR, galpy_dp, galpy_dz),
                              axis=1)

        galpy_grad = np.einsum('nij,ni->nj', self.Jac, galpy_dRpz).T

        assert np.allclose(gala_grad, galpy_grad)

    def test_vcirc(self, gala_pot, galpy_pot):
        tmp = self.xyz.copy()
        tmp[2] = 0.

        if (not hasattr(galpy_pot, 'vcirc')
                or isinstance(gala_pot, gp.LongMuraliBarPotential)):
            pytest.skip()

        gala_vcirc = gala_pot.circular_velocity(tmp).to_value(u.km/u.s)
        galpy_vcirc = np.array([galpy_pot.vcirc(R=RR)
                                for RR, *_ in self.Rpz_iter])
        assert np.allclose(gala_vcirc, galpy_vcirc)
<commit_msg>add one other test with sech2_z=False<commit_after>"""
Test converting the builtin Potential classes to other packages
"""

# Third-party
from astropy.coordinates import CylindricalRepresentation
from astropy.tests.helper import catch_warnings
import astropy.units as u
import numpy as np
import pytest

# This project
import gala.potential as gp
from gala.units import galactic
from gala.tests.optional_deps import HAS_GALPY
from gala.potential.potential.interop import galpy_to_gala_potential

# Set these globally!
ro = 8.122 * u.kpc
vo = 245 * u.km/u.s

if HAS_GALPY:
    import galpy.potential as galpy_gp

    from gala.potential.potential.interop import (
        _gala_to_galpy,
        _galpy_to_gala
    )

def pytest_generate_tests(metafunc):
    # Some magic, semi-random numbers below!
    gala_pots = []
    galpy_pots = []

    if not HAS_GALPY:
        return

    # Test the Gala -> Galpy direction
    for Potential in _gala_to_galpy.keys():
        init = {}
        len_scale = 1.
        for k, par in Potential._parameters.items():
            if k == 'm':
                val = 1.43e10 * u.Msun
            elif par.physical_type == 'length':
                val = 5.12 * u.kpc * len_scale
                len_scale *= 0.5
            elif par.physical_type == 'dimensionless':
                val = 1.
            elif par.physical_type == 'speed':
                val = 201.41 * u.km/u.s
            else:
                continue

            init[k] = val

        pot = Potential(**init, units=galactic)
        galpy_pot = pot.to_galpy_potential(ro=ro, vo=vo)

        gala_pots.append(pot)
        galpy_pots.append(galpy_pot)

        # Custom settings in the MN3 potential:
        if isinstance(pot, gp.MN3ExponentialDiskPotential):
            pot = Potential(**init, units=galactic, sech2_z=False)
            galpy_pot = pot.to_galpy_potential(ro=ro, vo=vo)
            gala_pots.append(pot)
            galpy_pots.append(galpy_pot)

    # Make a composite potential too:
    gala_pots.append(gala_pots[0] + gala_pots[1])
    galpy_pots.append([galpy_pots[0], galpy_pots[1]])

    # Test the Galpy -> Gala direction
    for Potential in _galpy_to_gala.keys():
        galpy_pot = Potential(ro=ro, vo=vo)  # use defaults

        with catch_warnings(RuntimeWarning) as warns:
            pot = galpy_to_gala_potential(galpy_pot, ro=ro, vo=vo)

        if isinstance(galpy_pot, galpy_gp.MN3ExponentialDiskPotential):
            assert len(warns) > 0

        gala_pots.append(pot)
        galpy_pots.append(galpy_pot)

    test_names = [f'{g1.__class__.__name__}:{g2.__class__.__name__}'
                  for g1, g2 in zip(gala_pots, galpy_pots)]

    metafunc.parametrize(['gala_pot', 'galpy_pot'],
                         list(zip(gala_pots, galpy_pots)),
                         ids=test_names)

@pytest.mark.skipif(not HAS_GALPY,
                    reason="must have galpy installed to run these tests")
class TestGalpy:

    def setup(self):
        # Test points:
        rng = np.random.default_rng(42)
        ntest = 4

        Rs = rng.uniform(1, 15, size=ntest) * u.kpc
        phis = rng.uniform(0, 2*np.pi, size=ntest) * u.radian
        zs = rng.uniform(1, 15, size=ntest) * u.kpc

        cyl = CylindricalRepresentation(Rs, phis, zs)
        xyz = cyl.to_cartesian().xyz

        self.Rs = Rs.to_value(ro)
        self.phis = phis.to_value(u.rad)
        self.zs = zs.to_value(ro)
        self.Rpz_iter = np.array(list(zip(self.Rs, self.phis, self.zs))).copy()

        self.xyz = xyz.copy()

        Jac = np.zeros((len(cyl), 3, 3))
        Jac[:, 0, 0] = xyz[0] / cyl.rho
        Jac[:, 0, 1] = xyz[1] / cyl.rho
        Jac[:, 1, 0] = (-xyz[1] / cyl.rho**2).to_value(1 / ro)
        Jac[:, 1, 1] = (xyz[0] / cyl.rho**2).to_value(1 / ro)
        Jac[:, 2, 2] = 1.
        self.Jac = Jac

    def test_density(self, gala_pot, galpy_pot):
        if isinstance(gala_pot, gp.LogarithmicPotential):
            pytest.skip()

        gala_val = gala_pot.density(self.xyz).to_value(u.Msun / u.pc**3)
        galpy_val = np.array([galpy_gp.evaluateDensities(galpy_pot,
                                                         R=RR, z=zz, phi=pp)
                              for RR, pp, zz in self.Rpz_iter])
        assert np.allclose(gala_val, galpy_val)

    def test_energy(self, gala_pot, galpy_pot):
        gala_val = gala_pot.energy(self.xyz).to_value(u.km**2 / u.s**2)
        galpy_val = np.array([galpy_gp.evaluatePotentials(galpy_pot,
                                                          R=RR, z=zz, phi=pp)
                              for RR, pp, zz in self.Rpz_iter])

        if isinstance(gala_pot, gp.LogarithmicPotential):
            # Logarithms are weird
            gala_val -= (0.5 * gala_pot.parameters['v_c']**2 *
                         np.log(ro.value**2)).to_value((u.km / u.s)**2)

        assert np.allclose(gala_val, galpy_val)

    def test_gradient(self, gala_pot, galpy_pot):
        gala_grad = gala_pot.gradient(self.xyz)
        gala_grad = gala_grad.to_value(u.km/u.s/u.Myr)

        # TODO: Starting with galpy 1.7, this has been failing because of a
        # units issue with dPhi/dphi
        if isinstance(gala_pot, gp.LongMuraliBarPotential):
            pytest.skip()

        galpy_dR = np.array([-galpy_gp.evaluateRforces(galpy_pot,
                                                       R=RR, z=zz, phi=pp)
                            for RR, pp, zz in self.Rpz_iter])
        galpy_dp = np.array([-galpy_gp.evaluatephiforces(galpy_pot,
                                                         R=RR, z=zz, phi=pp)
                            for RR, pp, zz in self.Rpz_iter])
        galpy_dp = (galpy_dp*(u.km/u.s)**2).to_value(vo**2)

        galpy_dz = np.array([-galpy_gp.evaluatezforces(galpy_pot,
                                                       R=RR, z=zz, phi=pp)
                            for RR, pp, zz in self.Rpz_iter])
        galpy_dRpz = np.stack((galpy_dR, galpy_dp, galpy_dz),
                              axis=1)

        galpy_grad = np.einsum('nij,ni->nj', self.Jac, galpy_dRpz).T

        assert np.allclose(gala_grad, galpy_grad)

    def test_vcirc(self, gala_pot, galpy_pot):
        tmp = self.xyz.copy()
        tmp[2] = 0.

        if (not hasattr(galpy_pot, 'vcirc')
                or isinstance(gala_pot, gp.LongMuraliBarPotential)):
            pytest.skip()

        gala_vcirc = gala_pot.circular_velocity(tmp).to_value(u.km/u.s)
        galpy_vcirc = np.array([galpy_pot.vcirc(R=RR)
                                for RR, *_ in self.Rpz_iter])
        assert np.allclose(gala_vcirc, galpy_vcirc)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Copyright 2017 Dean Hall.  See LICENSE for details.

This is a tool to generate a HeyMac identity configuration file.
An asymmetric cryptographic keypair is generated
such that the SHA512(SHA512(pub_key)) starts with 0xFC or 0xFD
"""

import codecs
import hashlib
import json
import os.path
import random

from heymac import utl

def gen_keypair():
    """Generates 256 bit keypair
    """
    # TODO: run system() to call real crypto tool to generate a key pair
    prv_key = bytes(bytearray(32))
    pub_key = random.getrandbits(256).to_bytes(32, 'big')
    return (prv_key, pub_key)

def gen_personal_keypair():
    return _gen_specific_keypair("fc")

def gen_device_keypair():
    return _gen_specific_keypair("fd")

def gen_feb12b_keypair():
    """Generates a feb/12 keypair.
    Note that feb/12 is a subset of fe8/10
    the well-known link-local prefix.
    """
    return _gen_specific_keypair("feb")

def _gen_specific_keypair(prefix):
    """Generates keypairs and forgets them
    until one is made with a specific prefix (nibble, byte or bytes).
    """
    done = False
    while not done:
        prv_key, pub_key = gen_keypair()
        h = hashlib.sha512()
        h.update(pub_key)
        h.update(h.digest())
        done = h.hexdigest().startswith(prefix)
    return (prv_key, pub_key)

if __name__ == "__main__":
    name = input("Full name: ")
    callsign = input("Callsign: ")
    _, pub_key = gen_feb12b_keypair()

    fn = os.path.join(utl.get_app_data_path("HeyMac"), "mac_identity.json")
    with open(fn, 'w') as f:
        json_str = json.dumps({
            "name": name,
            "callsign": callsign,
            # "pub_key": pub_key.hex()}) # Python 3.5 and later
            "pub_key": codecs.encode(pub_key, 'hex').decode() # Python 3.4
            })
        f.write(json_str)
    print("Wrote: %s" % fn)
<commit_msg>Fixed gen_identity script<commit_after>#!/usr/bin/env python3

"""
Copyright 2017 Dean Hall.  See LICENSE for details.

This is a tool to generate a HeyMac identity configuration file.
An asymmetric cryptographic keypair is generated
such that the SHA512(SHA512(pub_key)) starts with 0xFC or 0xFD
"""

import codecs
import hashlib
import json
import os.path
import random

from heymac import utl

def gen_keypair():
    """Generates 256 bit keypair
    """
    # TODO: run system() to call real crypto tool to generate a key pair
    prv_key = bytes(bytearray(32))
    pub_key = random.getrandbits(256).to_bytes(32, 'big')
    return (prv_key, pub_key)

def gen_personal_keypair():
    return _gen_specific_keypair("fc")

def gen_device_keypair():
    return _gen_specific_keypair("fd")

def gen_feb12b_keypair():
    """Generates a feb/12 keypair.
    Note that feb/12 is a subset of fe8/10
    the well-known link-local prefix.
    """
    return _gen_specific_keypair("feb")

def _gen_specific_keypair(prefix):
    """Generates keypairs and forgets them
    until one is made with a specific prefix (nibble, byte or bytes).
    """
    done = False
    while not done:
        prv_key, pub_key = gen_keypair()
        h = hashlib.sha512()
        h.update(pub_key)
        h.update(h.digest())
        done = h.hexdigest().startswith(prefix)
    return (prv_key, pub_key)

if __name__ == "__main__":
    name = input("Full name: ")
    callsign = input("Callsign: ")
    _, pub_key = gen_personal_keypair()

    fn = os.path.join(utl.get_app_data_path("HeyMac"), "mac_identity.json")
    with open(fn, 'w') as f:
        json_str = json.dumps({
            "name": name,
            "callsign": callsign,
            # "pub_key": pub_key.hex()}) # Python 3.5 and later
            "pub_key": codecs.encode(pub_key, 'hex').decode() # Python 3.4
            })
        f.write(json_str)
    print("Wrote: %s" % fn)
<|endoftext|>


<commit_before>"""Merge the 1km Q2 24 hour precip data estimates"""
import datetime
import sys
import os

import pytz
import numpy as np
from PIL import Image
import pyiem.mrms as mrms
from pyiem import iemre
from pyiem.util import ncopen

def findfile(ts):
    """See if we can find a file to use"""
    for hr in [0, -1, 1, -2, 2, -3, 3]:
        ts2 = ts + datetime.timedelta(hours=hr)
        fn = ts2.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/q2/"
                           "p24h_%Y%m%d%H00.png"))
        if os.path.isfile(fn):
            return fn

def run(ts):
    """Actually do the work, please"""
    nc = ncopen(iemre.get_daily_mrms_ncname(ts.year), 'a', timeout=300)
    offset = iemre.daily_offset(ts)
    ncprecip = nc.variables['p01d']
    ts += datetime.timedelta(hours=24)
    gmtts = ts.astimezone(pytz.utc)
    fn = findfile(gmtts)
    if fn is None:
        print("merge_mrms_q2 failed to find file for time: %s" % (gmtts, ))
        return
    img = Image.open(fn)
    data = np.asarray(img)
    # data is 3500,7000 , starting at upper L
    data = np.flipud(data)
    # Anything over 254 is bad
    res = np.where(data > 254, 0, data)
    res = np.where(np.logical_and(data >= 0, data < 100), data * 0.25, res)
    res = np.where(np.logical_and(data >= 100, data < 180),
                   25. + ((data - 100) * 1.25), res)
    res = np.where(np.logical_and(data >= 180, data < 255),
                   125. + ((data - 180) * 5.), res)

    y1 = int((iemre.NORTH - mrms.SOUTH) * 100.0)
    y0 = int((iemre.SOUTH - mrms.SOUTH) * 100.0)
    x0 = int((iemre.WEST - mrms.WEST) * 100.0)
    x1 = int((iemre.EAST - mrms.WEST) * 100.0)
    ncprecip[offset, :, :] = res[y0:y1, x0:x1]
    nc.close()

def main(argv):
    """Go Main Go"""
    if len(argv) == 4:
        # 12 noon to prevent ugliness with timezones
        ts = datetime.datetime(int(argv[1]), int(argv[2]),
                               int(argv[3]), 12, 0)
    else:
        ts = datetime.datetime.now() - datetime.timedelta(hours=24)
        ts = ts.replace(hour=12)

    ts = ts.replace(tzinfo=pytz.utc)
    ts = ts.astimezone(pytz.timezone("America/Chicago"))
    ts = ts.replace(hour=0, minute=0, second=0, microsecond=0)
    run(ts)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>rectify archive GIS file location<commit_after>"""Merge the 1km Q2 24 hour precip data estimates"""
import datetime
import sys
import os

import pytz
import numpy as np
from PIL import Image
import pyiem.mrms as mrms
from pyiem import iemre
from pyiem.util import ncopen

def findfile(ts):
    """See if we can find a file to use"""
    for hr in [0, -1, 1, -2, 2, -3, 3]:
        ts2 = ts + datetime.timedelta(hours=hr)
        fn = ts2.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/mrms/"
                           "p24h_%Y%m%d%H00.png"))
        if os.path.isfile(fn):
            return fn

def run(ts):
    """Actually do the work, please"""
    nc = ncopen(iemre.get_daily_mrms_ncname(ts.year), 'a', timeout=300)
    offset = iemre.daily_offset(ts)
    ncprecip = nc.variables['p01d']
    ts += datetime.timedelta(hours=24)
    gmtts = ts.astimezone(pytz.utc)
    fn = findfile(gmtts)
    if fn is None:
        print("merge_mrms_q2 failed to find file for time: %s" % (gmtts, ))
        return
    img = Image.open(fn)
    data = np.asarray(img)
    # data is 3500,7000 , starting at upper L
    data = np.flipud(data)
    # Anything over 254 is bad
    res = np.where(data > 254, 0, data)
    res = np.where(np.logical_and(data >= 0, data < 100), data * 0.25, res)
    res = np.where(np.logical_and(data >= 100, data < 180),
                   25. + ((data - 100) * 1.25), res)
    res = np.where(np.logical_and(data >= 180, data < 255),
                   125. + ((data - 180) * 5.), res)

    y1 = int((iemre.NORTH - mrms.SOUTH) * 100.0)
    y0 = int((iemre.SOUTH - mrms.SOUTH) * 100.0)
    x0 = int((iemre.WEST - mrms.WEST) * 100.0)
    x1 = int((iemre.EAST - mrms.WEST) * 100.0)
    ncprecip[offset, :, :] = res[y0:y1, x0:x1]
    nc.close()

def main(argv):
    """Go Main Go"""
    if len(argv) == 4:
        # 12 noon to prevent ugliness with timezones
        ts = datetime.datetime(int(argv[1]), int(argv[2]),
                               int(argv[3]), 12, 0)
    else:
        ts = datetime.datetime.now() - datetime.timedelta(hours=24)
        ts = ts.replace(hour=12)

    ts = ts.replace(tzinfo=pytz.utc)
    ts = ts.astimezone(pytz.timezone("America/Chicago"))
    ts = ts.replace(hour=0, minute=0, second=0, microsecond=0)
    run(ts)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>''' -- imports from installed packages -- '''
from django.http import HttpResponseRedirect
from django.http import HttpResponse
from django.shortcuts import render_to_response #, render  uncomment when to use
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required

from gnowsys_ndf.ndf.models import *
from gnowsys_ndf.ndf.views.methods import *

db = get_database()
collection = db['Nodes']

def custom_app_view(request, group_id, app_name, app_id, app_set_id=None, app_set_instance_id=None):
    """
    custom view for custom GAPPS
    """
    app_collection_set = []
    atlist = []
    rtlist = []
    app = collection.Node.find_one({"_id":ObjectId(app_id)})
    app_set = ""
    nodes = ""
    nodes_dict = ""
    app_menu = ""
    app_set_template = ""
    app_set_instance_template = ""
    app_set_instance_name = ""
    app_set_name = ""
    title = ""
    tags = ""
    content = ""
    for eachset in app.collection_set:
	 app_set = collection.Node.find_one({"_id":eachset})
	 app_collection_set.append({"id":str(app_set._id),"name":app_set.name})
    if app_set_id:
        classtype = ""
        app_set_template = "yes"
        systemtype = collection.Node.find_one({"_id":ObjectId(app_set_id)})
        systemtype_name = systemtype.name
        title = systemtype_name
        if request.method=="POST":
            search = request.POST.get("search","")
            classtype = request.POST.get("class","")
            nodes = list(collection.Node.find({'name':{'$regex':search, '$options': 'i'},'member_of': {'$all': [systemtype._id]},'_type':'GSystem'}))
        else :
            nodes = list(collection.Node.find({'member_of': {'$all': [systemtype._id]},'_type':'GSystem','group_set':{'$all': [ObjectId(group_id)]}}))
        nodes_dict = []
        for each in nodes:
            nodes_dict.append({"id":str(each._id), "name":each.name, "created_by":User.objects.get(id=each.created_by).username, "created_at":each.created_at})
    else :
        app_menu = "yes"
        title = app_name
    if app_set_instance_id :
        app_set_instance_template = "yes"
        app_set_template = ""
        systemtype_attributetype_set = []
        systemtype_relationtype_set = []
        system = collection.Node.find_one({"_id":ObjectId(app_set_instance_id)})
        systemtype = collection.Node.find_one({"_id":ObjectId(app_set_id)})
        for each in systemtype.attribute_type_set:
            systemtype_attributetype_set.append({"type":each.name,"type_id":str(each._id),"value":each.data_type})
        for each in systemtype.relation_type_set:
            systemtype_relationtype_set.append({"rt_name":each.name,"type_id":str(each._id)})

        for eachatset in systemtype_attributetype_set :
            for eachattribute in collection.Node.find({"_type":"GAttribute", "subject":system._id, "attribute_type.$id":ObjectId(eachatset["type_id"])}):
                atlist.append({"type":eachatset["type"],"type_id":eachatset["type_id"],"value":eachattribute.object_value})
        for eachrtset in systemtype_relationtype_set :
            for eachrelation in collection.Node.find({"_type":"GRelation", "subject":system._id, "relation_type.$id":ObjectId(eachrtset["type_id"])}):
                right_subject = collection.Node.find_one({"_id":ObjectId(eachrelation.right_subject)})
                rtlist.append({"type":eachrtset["rt_name"],"type_id":eachrtset["type_id"],"value_name": right_subject.name,"value_id":str(right_subject._id)})

        tags = ",".join(system.tags)
        content = system.content
        app_set_name = systemtype.name
        app_set_instance_name = system.name
        title =  systemtype.name +"-" +system.name
    template = "ndf/custom_template_for_app.html"
    variable = RequestContext(request, {'groupid':group_id, 'app_name':app_name, 'app_id':app_id, "app_collection_set":app_collection_set,"app_set_id":app_set_id,"nodes":nodes_dict, "app_menu":app_menu, "app_set_template":app_set_template, "app_set_instance_template":app_set_instance_template, "app_set_name":app_set_name, "app_set_instance_name":app_set_instance_name, "title":title, "app_set_instance_atlist":atlist, "app_set_instance_rtlist":rtlist, 'tags':tags, "content":content})
    return render_to_response(template, variable)

@login_required
def custom_app_new_view(request, group_id, app_name, app_id, app_set_id=None):
    """
    create new instance of app_set of apps view for custom GAPPS
    """
    app_collection_set = []
    app = collection.Node.find_one({"_id":ObjectId(app_id)})
    app_set = ""
    nodes = ""
    systemtype = ""
    systemtype_name = ""
    systemtype_attributetype_set = []
    systemtype_relationtype_set = []
    title = ""
    file_st_ids = []
    app_type_of_id = ""
    File = 'False'
    print app_set_id,"test-app-id"
    for eachset in app.collection_set:
	 app_set = collection.Node.find_one({"_id":eachset})
	 app_collection_set.append({"id":str(app_set._id),"name":app_set.name})
    if app_set_id:
        systemtype = collection.Node.find_one({"_id":ObjectId(app_set_id)})
        systemtype_name = systemtype.name
        title = systemtype_name + " - new"
        for each in systemtype.attribute_type_set:
            systemtype_attributetype_set.append({"type":each.name,"type_id":str(each._id),"value":each.data_type})
        for eachrt in systemtype.relation_type_set:
            object_type = [ {"name":rtot.name, "id":str(rtot._id)} for rtot in collection.Node.find({'member_of': {'$all': [ collection.Node.find_one({"_id":eachrt.object_type[0]})._id]}}) ]
            systemtype_relationtype_set.append({"rt_name":eachrt.name,"type_id":str(eachrt._id),"object_type":object_type})

    request_at_dict = {}
    request_rt_dict = {}

    files_sts = ['File','Image','Video']
    if app_set_id:
        app = collection.Node.one({'_id':ObjectId(app_set_id)})
        for each in files_sts:
            node_id = collection.Node.one({'name':each})._id
            if node_id in app.type_of:
                File = 'True'

    print app.type_of,"test",file_st_ids
    if request.method=="POST":
        tags = request.POST.get("tags","")
        content_org = unicode(request.POST.get("content_org",""))
        name = request.POST.get("name","")
        for each in systemtype_attributetype_set:
            request_at_dict[each["type_id"]] = request.POST.get(each["type_id"],"")
        for eachrtset in systemtype_relationtype_set:
            request_rt_dict[eachrtset["type_id"]] = request.POST.get(eachrtset["type_id"],"")

        if app.type_of in file_st_ids:
            newgsystem = collection.File() #creating File object ot store files
        else:
            newgsystem = collection.GSystem()

        newgsystem.name = name
        newgsystem.member_of=[ObjectId(app_set_id)]
        newgsystem.created_by = request.user.id
        newgsystem.group_set.append(ObjectId(group_id))
        if tags:
             newgsystem.tags = tags.split(",")
        if content_org:
            usrname = request.user.username
            filename = slugify(newgsystem.name) + "-" + usrname
            newgsystem.content = org2html(content_org, file_prefix=filename)
        newgsystem.save()
        for key,value in request_at_dict.items():
            attributetype_key = collection.Node.find_one({"_id":ObjectId(key)})
            newattribute = collection.GAttribute()
            newattribute.subject = newgsystem._id
            newattribute.attribute_type = attributetype_key
            newattribute.object_value = value
 #           newattribute.name = unicode(newgsystem.name+"- "+attributetype_key.name+"-"+value)
            newattribute.save()
        for key,value in request_rt_dict.items():
            relationtype_key = collection.Node.find_one({"_id":ObjectId(key)})
            right_subject = collection.Node.find_one({"_id":ObjectId(value)})
            newrelation = collection.GRelation()
            newrelation.subject = newgsystem._id
            newrelation.relation_type = relationtype_key
            newrelation.right_subject = right_subject._id
#            newrelation.name = unicode(newgsystem.name+"- "+relationtype_key.name+"-"+right_subject.name)
            newrelation.save()
        return HttpResponseRedirect(reverse('GAPPS_set', kwargs={'group_id': group_id, 'app_name': app_name, "app_id":app_id, "app_set_id":app_set_id}))

    template = "ndf/custom_template_for_app.html"
    variable = RequestContext(request, {'groupid':group_id, 'app_name':app_name, 'app_id':app_id, "app_collection_set":app_collection_set, "app_set_id":app_set_id, "nodes":nodes, "systemtype_attributetype_set":systemtype_attributetype_set, "systemtype_relationtype_set":systemtype_relationtype_set, "create_new":"yes", "app_set_name":systemtype_name, 'title':title, 'File':File})
    return render_to_response(template, variable)

<commit_msg>bug resolved while adding instance to app via designer module<commit_after>''' -- imports from installed packages -- '''
from django.http import HttpResponseRedirect
from django.http import HttpResponse
from django.shortcuts import render_to_response #, render  uncomment when to use
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required

from gnowsys_ndf.ndf.models import *
from gnowsys_ndf.ndf.views.methods import *

db = get_database()
collection = db['Nodes']

def custom_app_view(request, group_id, app_name, app_id, app_set_id=None, app_set_instance_id=None):
    """
    custom view for custom GAPPS
    """
    app_collection_set = []
    atlist = []
    rtlist = []
    app = collection.Node.find_one({"_id":ObjectId(app_id)})
    app_set = ""
    nodes = ""
    nodes_dict = ""
    app_menu = ""
    app_set_template = ""
    app_set_instance_template = ""
    app_set_instance_name = ""
    app_set_name = ""
    title = ""
    tags = ""
    content = ""
    for eachset in app.collection_set:
	 app_set = collection.Node.find_one({"_id":eachset})
	 app_collection_set.append({"id":str(app_set._id),"name":app_set.name})
    if app_set_id:
        classtype = ""
        app_set_template = "yes"
        systemtype = collection.Node.find_one({"_id":ObjectId(app_set_id)})
        systemtype_name = systemtype.name
        title = systemtype_name
        if request.method=="POST":
            search = request.POST.get("search","")
            classtype = request.POST.get("class","")
            nodes = list(collection.Node.find({'name':{'$regex':search, '$options': 'i'},'member_of': {'$all': [systemtype._id]},'_type':'GSystem'}))
        else :
            nodes = list(collection.Node.find({'member_of': {'$all': [systemtype._id]},'_type':'GSystem','group_set':{'$all': [ObjectId(group_id)]}}))
        nodes_dict = []
        for each in nodes:
            nodes_dict.append({"id":str(each._id), "name":each.name, "created_by":User.objects.get(id=each.created_by).username, "created_at":each.created_at})
    else :
        app_menu = "yes"
        title = app_name
    if app_set_instance_id :
        app_set_instance_template = "yes"
        app_set_template = ""
        systemtype_attributetype_set = []
        systemtype_relationtype_set = []
        system = collection.Node.find_one({"_id":ObjectId(app_set_instance_id)})
        systemtype = collection.Node.find_one({"_id":ObjectId(app_set_id)})
        for each in systemtype.attribute_type_set:
            systemtype_attributetype_set.append({"type":each.name,"type_id":str(each._id),"value":each.data_type})
        for each in systemtype.relation_type_set:
            systemtype_relationtype_set.append({"rt_name":each.name,"type_id":str(each._id)})

        for eachatset in systemtype_attributetype_set :
            for eachattribute in collection.Node.find({"_type":"GAttribute", "subject":system._id, "attribute_type.$id":ObjectId(eachatset["type_id"])}):
                atlist.append({"type":eachatset["type"],"type_id":eachatset["type_id"],"value":eachattribute.object_value})
        for eachrtset in systemtype_relationtype_set :
            for eachrelation in collection.Node.find({"_type":"GRelation", "subject":system._id, "relation_type.$id":ObjectId(eachrtset["type_id"])}):
                right_subject = collection.Node.find_one({"_id":ObjectId(eachrelation.right_subject)})
                rtlist.append({"type":eachrtset["rt_name"],"type_id":eachrtset["type_id"],"value_name": right_subject.name,"value_id":str(right_subject._id)})

        tags = ",".join(system.tags)
        content = system.content
        app_set_name = systemtype.name
        app_set_instance_name = system.name
        title =  systemtype.name +"-" +system.name
    template = "ndf/custom_template_for_app.html"
    variable = RequestContext(request, {'groupid':group_id, 'app_name':app_name, 'app_id':app_id, "app_collection_set":app_collection_set,"app_set_id":app_set_id,"nodes":nodes_dict, "app_menu":app_menu, "app_set_template":app_set_template, "app_set_instance_template":app_set_instance_template, "app_set_name":app_set_name, "app_set_instance_name":app_set_instance_name, "title":title, "app_set_instance_atlist":atlist, "app_set_instance_rtlist":rtlist, 'tags':tags, "content":content})
    return render_to_response(template, variable)

@login_required
def custom_app_new_view(request, group_id, app_name, app_id, app_set_id=None):
    """
    create new instance of app_set of apps view for custom GAPPS
    """
    app_collection_set = []
    app = collection.Node.find_one({"_id":ObjectId(app_id)})
    app_set = ""
    nodes = ""
    systemtype = ""
    systemtype_name = ""
    systemtype_attributetype_set = []
    systemtype_relationtype_set = []
    title = ""
    file_st_ids = []
    app_type_of_id = ""
    File = 'False'
    print app_set_id,"test-app-id"
    for eachset in app.collection_set:
	 app_set = collection.Node.find_one({"_id":eachset})
	 app_collection_set.append({"id":str(app_set._id),"name":app_set.name})
    if app_set_id:
        systemtype = collection.Node.find_one({"_id":ObjectId(app_set_id)})
        systemtype_name = systemtype.name
        title = systemtype_name + " - new"
        for each in systemtype.attribute_type_set:
            systemtype_attributetype_set.append({"type":each.name,"type_id":str(each._id),"value":each.data_type})
        for eachrt in systemtype.relation_type_set:
            object_type = [ {"name":rtot.name, "id":str(rtot._id)} for rtot in collection.Node.find({'member_of': {'$all': [ collection.Node.find_one({"_id":eachrt.object_type[0]})._id]}}) ]
            systemtype_relationtype_set.append({"rt_name":eachrt.name,"type_id":str(eachrt._id),"object_type":object_type})

    request_at_dict = {}
    request_rt_dict = {}

    files_sts = ['File','Image','Video']
    if app_set_id:
        app = collection.Node.one({'_id':ObjectId(app_set_id)})
        for each in files_sts:
            node_id = collection.Node.one({'name':each})._id
            if node_id in app.type_of:
                File = 'True'

    print app.type_of,"test",file_st_ids
    if request.method=="POST":
        tags = request.POST.get("tags","")
        content_org = unicode(request.POST.get("content_org",""))
        name = request.POST.get("name","")
        for each in systemtype_attributetype_set:
            request_at_dict[each["type_id"]] = request.POST.get(each["type_id"],"")
        for eachrtset in systemtype_relationtype_set:
            request_rt_dict[eachrtset["type_id"]] = request.POST.get(eachrtset["type_id"],"")

        if app.type_of in file_st_ids:
            newgsystem = collection.File() #creating File object ot store files
        else:
            newgsystem = collection.GSystem()

        newgsystem.name = name
        newgsystem.member_of=[ObjectId(app_set_id)]
        newgsystem.created_by = request.user.id
        newgsystem.group_set.append(ObjectId(group_id))
        if tags:
             newgsystem.tags = tags.split(",")
        if content_org:
            usrname = request.user.username
            filename = slugify(newgsystem.name) + "-" + usrname
            newgsystem.content = org2html(content_org, file_prefix=filename)
        newgsystem.save()
        for key,value in request_at_dict.items():
            attributetype_key = collection.Node.find_one({"_id":ObjectId(key)})
            newattribute = collection.GAttribute()
            newattribute.subject = newgsystem._id
            newattribute.attribute_type = attributetype_key
            newattribute.object_value = value
 #           newattribute.name = unicode(newgsystem.name+"- "+attributetype_key.name+"-"+value)
            newattribute.save()
        for key,value in request_rt_dict.items():
            if key:
                relationtype_key = collection.Node.find_one({"_id":ObjectId(key)})
            if value:
                right_subject = collection.Node.find_one({"_id":ObjectId(value)})
                newrelation = collection.GRelation()
                newrelation.subject = newgsystem._id
                newrelation.relation_type = relationtype_key
                newrelation.right_subject = right_subject._id
#            newrelation.name = unicode(newgsystem.name+"- "+relationtype_key.name+"-"+right_subject.name)
                newrelation.save()
        return HttpResponseRedirect(reverse('GAPPS_set', kwargs={'group_id': group_id, 'app_name': app_name, "app_id":app_id, "app_set_id":app_set_id}))

    template = "ndf/custom_template_for_app.html"
    variable = RequestContext(request, {'groupid':group_id, 'app_name':app_name, 'app_id':app_id, "app_collection_set":app_collection_set, "app_set_id":app_set_id, "nodes":nodes, "systemtype_attributetype_set":systemtype_attributetype_set, "systemtype_relationtype_set":systemtype_relationtype_set, "create_new":"yes", "app_set_name":systemtype_name, 'title':title, 'File':File})
    return render_to_response(template, variable)

<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Created on 30 Apr 2018

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The led_controller utility operates the a two-colour (red / green) LED mounted on the South Coast Science free-to-air
SHT board. LEDs can be made to flash or have steady state. The utility is intended to run as a systemd service, or as
an (un-managed) background process.

The led_controller waits for data on stdin or the named Unix domain socket, then updates the state of the LEDs
accordingly. Updates are in the form of a JSON array containing two single-character strings - the first represents
70% of the duty cycle, the second 30%. If a steady state is required, the two values should be the same.

When the led_controller starts, the LEDs remain in their previous state. When the led_controller terminates,
the LEDs remain in their last state.

SYNOPSIS
led_controller -v -u scs_led_control.uds

EXAMPLES
( tail -f ~/SCS/pipes/led_control_pipe & ) | ./led_controller.py -v &
./led_controller.py -v -u /home/scs/SCS/pipes/scs_led_control.uds

DOCUMENT EXAMPLE
{"colour0": "R", "colour1": "G"}

SEE ALSO
scs_dev/interface_power
scs_dev/led

RESOURCES
https://unix.stackexchange.com/questions/139490/continuous-reading-from-named-pipe-cat-or-tail-f
"""

import json
import sys

from scs_core.comms.uds_reader import UDSReader

from scs_core.data.json import JSONify

from scs_core.sys.signalled_exit import SignalledExit

from scs_dev.cmd.cmd_led_controller import CmdLEDController

from scs_dfe.interface.interface_conf import InterfaceConf

from scs_dfe.led.led_controller import LEDController
from scs_dfe.led.led_state import LEDState

from scs_host.bus.i2c import I2C
from scs_host.sys.host import Host

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    controller = None
    reader = None
    prev_state = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdLEDController()

    if cmd.verbose:
        print("led_controller: %s" % cmd, file=sys.stderr)
        sys.stderr.flush()

    try:
        I2C.open(Host.I2C_SENSORS)

        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # UDSReader...
        reader = UDSReader(cmd.uds)

        if cmd.verbose:
            print("led_controller: %s" % reader, file=sys.stderr)

        # Interface...
        interface_conf = InterfaceConf.load(Host)

        if interface_conf is None:
            print("led_controller: InterfaceConf not available.", file=sys.stderr)
            exit(1)

        interface = interface_conf.interface()

        if interface is None:
            print("led_controller: Interface not available.", file=sys.stderr)
            exit(1)

        # LEDController..
        controller = LEDController(interface.led())

        if cmd.verbose:
            print("led_controller: %s" % controller, file=sys.stderr)
            sys.stderr.flush()

        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("led_controller", cmd.verbose)

        controller.start()

        reader.connect()

        for message in reader.messages():
            try:
                jdict = json.loads(message)
            except ValueError:
                continue

            state = LEDState.construct_from_jdict(jdict)

            if state is None or state == prev_state:
                continue

            if cmd.verbose:
                print("led_controller: %s" % state, file=sys.stderr)
                sys.stderr.flush()

            if not state.is_valid():
                continue

            controller.set_state(state)

            prev_state = state

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except ConnectionError as ex:
        print("led_controller: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("led_controller: finishing", file=sys.stderr)

        if reader:
            reader.close()

        if controller:
            controller.stop()

        I2C.close()
<commit_msg>Improved reporting for led_controller.py<commit_after>#!/usr/bin/env python3

"""
Created on 30 Apr 2018

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The led_controller utility operates the a two-colour (red / green) LED mounted on the South Coast Science free-to-air
SHT board. LEDs can be made to flash or have steady state. The utility is intended to run as a systemd service, or as
an (un-managed) background process.

The led_controller waits for data on stdin or the named Unix domain socket, then updates the state of the LEDs
accordingly. Updates are in the form of a JSON array containing two single-character strings - the first represents
70% of the duty cycle, the second 30%. If a steady state is required, the two values should be the same.

When the led_controller starts, the LEDs remain in their previous state. When the led_controller terminates,
the LEDs remain in their last state.

SYNOPSIS
led_controller -v -u scs_led_control.uds

EXAMPLES
( tail -f ~/SCS/pipes/led_control_pipe & ) | ./led_controller.py -v &
./led_controller.py -v -u /home/scs/SCS/pipes/scs_led_control.uds

DOCUMENT EXAMPLE
{"colour0": "R", "colour1": "G"}

SEE ALSO
scs_dev/interface_power
scs_dev/led

RESOURCES
https://unix.stackexchange.com/questions/139490/continuous-reading-from-named-pipe-cat-or-tail-f
"""

import json
import sys

from scs_core.comms.uds_reader import UDSReader

from scs_core.sys.signalled_exit import SignalledExit

from scs_dev.cmd.cmd_led_controller import CmdLEDController

from scs_dfe.interface.interface_conf import InterfaceConf

from scs_dfe.led.led_controller import LEDController
from scs_dfe.led.led_state import LEDState

from scs_host.bus.i2c import I2C
from scs_host.sys.host import Host

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    controller = None
    reader = None
    prev_state = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdLEDController()

    if cmd.verbose:
        print("led_controller: %s" % cmd, file=sys.stderr)
        sys.stderr.flush()

    try:
        I2C.open(Host.I2C_SENSORS)

        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # UDSReader...
        reader = UDSReader(cmd.uds)

        if cmd.verbose:
            print("led_controller: %s" % reader, file=sys.stderr)

        # Interface...
        interface_conf = InterfaceConf.load(Host)

        if interface_conf is None:
            print("led_controller: InterfaceConf not available.", file=sys.stderr)
            exit(1)

        interface = interface_conf.interface()

        if interface is None:
            print("led_controller: Interface not available.", file=sys.stderr)
            exit(1)

        # LEDController..
        controller = LEDController(interface.led())

        if cmd.verbose:
            print("led_controller: %s" % controller, file=sys.stderr)
            sys.stderr.flush()

        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("led_controller", cmd.verbose)

        controller.start()

        reader.connect()

        for message in reader.messages():
            try:
                jdict = json.loads(message)
            except ValueError:
                continue

            state = LEDState.construct_from_jdict(jdict)

            if state is None or state == prev_state:
                continue

            if cmd.verbose:
                print("led_controller: %s" % state, file=sys.stderr)
                sys.stderr.flush()

            if not state.is_valid():
                continue

            controller.set_state(state)

            prev_state = state

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except ConnectionError as ex:
        print("led_controller: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("led_controller: finishing", file=sys.stderr)

        if reader:
            reader.close()

        if controller:
            controller.stop()

        I2C.close()
<|endoftext|>


<commit_before>'''
 Consume a REST service of DOT Snowplow locations and data
'''
import urllib2
import json
import psycopg2
import pytz
import datetime

URI = ("https://geonexusr.iowadot.gov/ArcGIS/rest/services/Operations/"
       +"Realtime_Trucks/MapServer/1/query?text=&geometry=&"
       +"geometryType=esriGeometryPoint&inSR="
       +"&spatialRel=esriSpatialRelIntersects&relationParam=&objectIds="
       +"&where=label+is+not+null&time=&returnCountOnly=false&"
       +"returnIdsOnly=false&returnGeometry=true&maxAllowableOffset="
       +"&outSR=&outFields=*&f=json")

def workflow():
    ''' Do stuff '''

    postgis = psycopg2.connect(database='postgis', host='iemdb')
    cursor = postgis.cursor()

    current = {}
    cursor.execute("""SELECT label, valid from idot_snowplow_current""")
    for row in cursor:
        current[ row[0] ] = row[1]

    valid = datetime.datetime.now()
    valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"),microsecond=0)

    data = json.loads( urllib2.urlopen(URI, timeout=30).read() )
    for feat in data.get('features', []):
        logdt = feat['attributes']['LOGDT']
        ts = datetime.datetime.utcfromtimestamp(logdt/1000.)
        valid = valid.replace(year=ts.year, month=ts.month, day=ts.day,
                              hour=ts.hour,minute=ts.minute,second=ts.second)
        label = feat['attributes']['LABEL']

        if current.get(label, None) is None:
            #print 'New IDOT Snowplow #%s "%s" Valid: %s' % (len(current)+1,
            #                                                label, valid)
            cursor.execute("""INSERT into idot_snowplow_current(label, valid)
            VALUES (%s,%s)""", (label, valid))
        if current.get(label, None) is None or current[label] < valid:
            cursor.execute("""UPDATE idot_snowplow_current
            SET
            valid = %s,
            heading = %s,
            velocity = %s,
            roadtemp = %s,
            airtemp = %s,
            solidmaterial = %s,
            liquidmaterial = %s,
            prewetmaterial = %s,
            solidsetrate = %s,
            liquidsetrate = %s,
            prewetsetrate = %s,
            leftwingplowstate = %s,
            rightwingplowstate = %s,
            frontplowstate = %s,
            underbellyplowstate = %s,
            solid_spread_code = %s,
            road_temp_code = %s,
            geom = 'SRID=4326;POINT(%s %s)'
            WHERE label = %s""", (valid, feat['attributes']['HEADING'],
                feat['attributes']['VELOCITY'],
                feat['attributes']['ROADTEMP'],
                feat['attributes']['AIRTEMP'],
                feat['attributes']['SOLIDMATERIAL'],
                feat['attributes']['LIQUIDMATERIAL'],
                feat['attributes']['PREWETMATERIAL'],
                feat['attributes']['SOLIDSETRATE'],
                feat['attributes']['LIQUIDSETRATE'],
                feat['attributes']['PREWETSETRATE'],
                feat['attributes']['LEFTWINGPLOWSTATE'],
                feat['attributes']['RIGHTWINGPLOWSTATE'],
                feat['attributes']['FRONTPLOWSTATE'],
                feat['attributes']['UNDERBELLYPLOWSTATE'],
                feat['attributes']['SOLID_SPREAD_CODE'],
                feat['attributes']['ROAD_TEMP_CODE'],
                feat['attributes']['XPOSITION'],
                feat['attributes']['YPOSITION'],
                                  label))
            # Archive it too
            cursor.execute("""INSERT into idot_snowplow_2013_2014
            (label, valid, heading, velocity, roadtemp, airtemp, solidmaterial,
            liquidmaterial, prewetmaterial, solidsetrate, liquidsetrate,
            prewetsetrate, leftwingplowstate, rightwingplowstate,
            frontplowstate , underbellyplowstate, solid_spread_code,
            road_temp_code, geom)
             values
            (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
            , %s, 'SRID=4326;POINT(%s %s)')""" , (label, valid, feat['attributes']['HEADING'],
                feat['attributes']['VELOCITY'],
                feat['attributes']['ROADTEMP'],
                feat['attributes']['AIRTEMP'],
                feat['attributes']['SOLIDMATERIAL'],
                feat['attributes']['LIQUIDMATERIAL'],
                feat['attributes']['PREWETMATERIAL'],
                feat['attributes']['SOLIDSETRATE'],
                feat['attributes']['LIQUIDSETRATE'],
                feat['attributes']['PREWETSETRATE'],
                feat['attributes']['LEFTWINGPLOWSTATE'],
                feat['attributes']['RIGHTWINGPLOWSTATE'],
                feat['attributes']['FRONTPLOWSTATE'],
                feat['attributes']['UNDERBELLYPLOWSTATE'],
                feat['attributes']['SOLID_SPREAD_CODE'],
                feat['attributes']['ROAD_TEMP_CODE'],
                feat['attributes']['XPOSITION'],
                feat['attributes']['YPOSITION']))

    postgis.commit()
    postgis.close()

if __name__ == '__main__':
    workflow()
<commit_msg>prevent dup plows in feed from causing a database insert fault<commit_after>'''
 Consume a REST service of DOT Snowplow locations and data
'''
import urllib2
import json
import psycopg2
import pytz
import datetime

URI = ("https://geonexusr.iowadot.gov/ArcGIS/rest/services/Operations/"
       +"Realtime_Trucks/MapServer/1/query?text=&geometry=&"
       +"geometryType=esriGeometryPoint&inSR="
       +"&spatialRel=esriSpatialRelIntersects&relationParam=&objectIds="
       +"&where=label+is+not+null&time=&returnCountOnly=false&"
       +"returnIdsOnly=false&returnGeometry=true&maxAllowableOffset="
       +"&outSR=&outFields=*&f=json")

def workflow():
    ''' Do stuff '''

    postgis = psycopg2.connect(database='postgis', host='iemdb')
    cursor = postgis.cursor()

    current = {}
    cursor.execute("""SELECT label, valid from idot_snowplow_current""")
    for row in cursor:
        current[ row[0] ] = row[1]

    valid = datetime.datetime.now()
    valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"),microsecond=0)

    data = json.loads( urllib2.urlopen(URI, timeout=30).read() )
    newplows = {}
    for feat in data.get('features', []):
        logdt = feat['attributes']['LOGDT']
        ts = datetime.datetime.utcfromtimestamp(logdt/1000.)
        valid = valid.replace(year=ts.year, month=ts.month, day=ts.day,
                              hour=ts.hour,minute=ts.minute,second=ts.second)
        label = feat['attributes']['LABEL']

        if current.get(label, None) is None and not newplows.has_key(label):
            #print 'New IDOT Snowplow #%s "%s" Valid: %s' % (len(current)+1,
            #                                                label, valid)
            newplows[label] = 1
            cursor.execute("""INSERT into idot_snowplow_current(label, valid)
            VALUES (%s,%s)""", (label, valid))
        if current.get(label, None) is None or current[label] < valid:
            cursor.execute("""UPDATE idot_snowplow_current
            SET
            valid = %s,
            heading = %s,
            velocity = %s,
            roadtemp = %s,
            airtemp = %s,
            solidmaterial = %s,
            liquidmaterial = %s,
            prewetmaterial = %s,
            solidsetrate = %s,
            liquidsetrate = %s,
            prewetsetrate = %s,
            leftwingplowstate = %s,
            rightwingplowstate = %s,
            frontplowstate = %s,
            underbellyplowstate = %s,
            solid_spread_code = %s,
            road_temp_code = %s,
            geom = 'SRID=4326;POINT(%s %s)'
            WHERE label = %s""", (valid, feat['attributes']['HEADING'],
                feat['attributes']['VELOCITY'],
                feat['attributes']['ROADTEMP'],
                feat['attributes']['AIRTEMP'],
                feat['attributes']['SOLIDMATERIAL'],
                feat['attributes']['LIQUIDMATERIAL'],
                feat['attributes']['PREWETMATERIAL'],
                feat['attributes']['SOLIDSETRATE'],
                feat['attributes']['LIQUIDSETRATE'],
                feat['attributes']['PREWETSETRATE'],
                feat['attributes']['LEFTWINGPLOWSTATE'],
                feat['attributes']['RIGHTWINGPLOWSTATE'],
                feat['attributes']['FRONTPLOWSTATE'],
                feat['attributes']['UNDERBELLYPLOWSTATE'],
                feat['attributes']['SOLID_SPREAD_CODE'],
                feat['attributes']['ROAD_TEMP_CODE'],
                feat['attributes']['XPOSITION'],
                feat['attributes']['YPOSITION'],
                                  label))
            # Archive it too
            cursor.execute("""INSERT into idot_snowplow_2013_2014
            (label, valid, heading, velocity, roadtemp, airtemp, solidmaterial,
            liquidmaterial, prewetmaterial, solidsetrate, liquidsetrate,
            prewetsetrate, leftwingplowstate, rightwingplowstate,
            frontplowstate , underbellyplowstate, solid_spread_code,
            road_temp_code, geom)
             values
            (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
            , %s, 'SRID=4326;POINT(%s %s)')""" , (label, valid, feat['attributes']['HEADING'],
                feat['attributes']['VELOCITY'],
                feat['attributes']['ROADTEMP'],
                feat['attributes']['AIRTEMP'],
                feat['attributes']['SOLIDMATERIAL'],
                feat['attributes']['LIQUIDMATERIAL'],
                feat['attributes']['PREWETMATERIAL'],
                feat['attributes']['SOLIDSETRATE'],
                feat['attributes']['LIQUIDSETRATE'],
                feat['attributes']['PREWETSETRATE'],
                feat['attributes']['LEFTWINGPLOWSTATE'],
                feat['attributes']['RIGHTWINGPLOWSTATE'],
                feat['attributes']['FRONTPLOWSTATE'],
                feat['attributes']['UNDERBELLYPLOWSTATE'],
                feat['attributes']['SOLID_SPREAD_CODE'],
                feat['attributes']['ROAD_TEMP_CODE'],
                feat['attributes']['XPOSITION'],
                feat['attributes']['YPOSITION']))

    postgis.commit()
    postgis.close()

if __name__ == '__main__':
    workflow()
<|endoftext|>


<commit_before>import asyncio
import logging
import traceback

class Server(object):
    def __init__(self, loop=None, logger=None):
        self.logger = logger if logger is not None else logging.getLogger('root')
        self._loop = loop if loop is not None else asyncio.get_event_loop()

        self.handler = None
        self._server = None     # asyncio.Server
        self._clients = {}      # asyncio.Task -> (asyncio.StreamReader, asyncio.StreamWriter)

    def _accept_client(self, client_reader, client_writer):
        if not self._clients and self.handler is not None:               # first client connects
            self.handler.set_up()

        def client_disconnected(task):
            if task.exception():
                ex = task.exception()
                output = traceback.format_exception(ex.__class__, ex, ex.__traceback__)
                self.logger.critical(''.join(output))

            del self._clients[task]
            if not self._clients and self.handler is not None:           # last client disconnected
                self.handler.tear_down()

        # schedule a new Task to handle this specific client connection
        # TODO use self._loop.create_task once Python 3.4.2 is available on OS X via brew
        task = asyncio.Task(self._handle_client(client_reader, client_writer), loop=self._loop)
        task.add_done_callback(client_disconnected)

        self._clients[task] = (client_reader, client_writer)

    def client_ip(self, client_writer):
        return client_writer.get_extra_info('peername')[0]

    @asyncio.coroutine
    def _handle_client(self, client_reader, client_writer):
        client_ip = self.client_ip(client_writer)
        self.logger.info("connection from {0}".format(client_ip))
        try:
            while True:
                data = yield from client_reader.readexactly(4)
                message_size = int.from_bytes(data, byteorder='big')

                message = yield from client_reader.readexactly(message_size)

                if self.handler is not None:
                    # HINT this could (probably?) be parallelized by creating a task -> multiple messages would be processed in parallel
                    yield from self.handler.process(message)
        except asyncio.CancelledError:
            self.logger.info('disconnecting client %s', client_ip)
        except asyncio.IncompleteReadError:
            self.logger.info('client %s disconnected', client_ip)

    def start(self, ip='', port=10000):
        # start asyncio.Server
        future_server = asyncio.start_server(self._accept_client, ip, port, loop=self._loop)
        # wait until server socket is set up
        self._server = self._loop.run_until_complete(future_server)
        self.logger.info('server started, listening on %s:%s', ip, port)

    def stop(self):
        if self._server is None:
            return

        for task in self._clients:
            task.cancel()

        # wait for clients to be disconnected
        if self._clients:
            done, pending = self._loop.run_until_complete(asyncio.wait(self._clients.keys(), loop=self._loop, timeout=2))

            if pending:
                for task in pending:
                    client_writer = self._clients[task][1]
                    self.logger.error("could not disconnect client %s", self.client_ip(client_writer))

        self._server.close()
        self._loop.run_until_complete(self._server.wait_closed())
        self._server = None
        self.logger.info('server stopped')
<commit_msg>Processing messages in parallel. There's an asyncio task for every received message.<commit_after>import asyncio
import logging
import traceback

class Server(object):
    def __init__(self, loop=None, logger=None):
        self.logger = logger if logger is not None else logging.getLogger('root')
        self._loop = loop if loop is not None else asyncio.get_event_loop()

        self.handler = None
        self._server = None     # asyncio.Server
        self._clients = {}      # asyncio.Task -> (asyncio.StreamReader, asyncio.StreamWriter)
        self._workers = set()   # asyncio.Task, each running handler.process()

    def _accept_client(self, client_reader, client_writer):
        if not self._clients and self.handler is not None:               # first client connects
            self.handler.set_up()

        def client_disconnected(task):
            if task.exception():
                ex = task.exception()
                output = traceback.format_exception(ex.__class__, ex, ex.__traceback__)
                self.logger.critical(''.join(output))

            del self._clients[task]
            if not self._clients and self.handler is not None:           # last client disconnected
                self.handler.tear_down()

        # schedule a new Task to handle this specific client connection
        # TODO use self._loop.create_task once Python 3.4.2 is available on OS X via brew
        task = asyncio.Task(self._handle_client(client_reader, client_writer), loop=self._loop)
        task.add_done_callback(client_disconnected)

        self._clients[task] = (client_reader, client_writer)

    def client_ip(self, client_writer):
        return client_writer.get_extra_info('peername')[0]

    @asyncio.coroutine
    def _handle_client(self, client_reader, client_writer):
        def worker_finished(task):
            if task.exception():
                ex = task.exception()
                output = traceback.format_exception(ex.__class__, ex, ex.__traceback__)
                self.logger.critical(''.join(output))

            self._workers.remove(task)

        client_ip = self.client_ip(client_writer)
        self.logger.info("connection from {0}".format(client_ip))
        try:
            while True:
                data = yield from client_reader.readexactly(4)
                message_size = int.from_bytes(data, byteorder='big')

                message = yield from client_reader.readexactly(message_size)

                if self.handler is not None:
                    # start async task to process message
                    # TODO use create_task (see above)
                    worker = asyncio.Task(self.handler.process(message), loop=self._loop)
                    worker.add_done_callback(worker_finished)
                    self._workers.add(worker)
        except asyncio.CancelledError:
            self.logger.info('disconnecting client %s', client_ip)
        except asyncio.IncompleteReadError:
            self.logger.info('client %s disconnected', client_ip)

    def start(self, ip='', port=10000):
        # start asyncio.Server
        future_server = asyncio.start_server(self._accept_client, ip, port, loop=self._loop)
        # wait until server socket is set up
        self._server = self._loop.run_until_complete(future_server)
        self.logger.info('server started, listening on %s:%s', ip, port)

    def stop(self):
        if self._server is None:
            return

        # wait for worker to finish
        if self._workers:
            for task in self._workers:
                task.cancel()

            done, pending = self._loop.run_until_complete(asyncio.wait(self._workers, loop=self._loop, timeout=2))

            if pending:
                self.logger.error("could not cancel processing of %d messages", len(pending))

        # wait for clients to be disconnected
        if self._clients:
            for task in self._clients:
                task.cancel()

            done, pending = self._loop.run_until_complete(asyncio.wait(self._clients.keys(), loop=self._loop, timeout=2))

            if pending:
                for task in pending:
                    client_writer = self._clients[task][1]
                    self.logger.error("could not disconnect client %s", self.client_ip(client_writer))

        self._server.close()
        self._loop.run_until_complete(self._server.wait_closed())
        self._server = None
        self.logger.info('server stopped')
<|endoftext|>


<commit_before>from __future__ import print_function
import datetime
import warnings
import re
from itertools import chain, repeat, islice

from .config import Configuration
from .utils import trace, string_types

from pkg_resources import iter_entry_points

from pkg_resources import parse_version as pkg_parse_version

SEMVER_MINOR = 2
SEMVER_PATCH = 3
SEMVER_LEN = 3

def _pad(iterable, size, padding=None):
    padded = chain(iterable, repeat(padding))
    return list(islice(padded, size))

def _parse_version_tag(tag, config):
    tagstring = tag if not isinstance(tag, string_types) else '%s' % tag
    match = config.tag_regex.match(tagstring)

    result = None
    if match:
        if len(match.groups()) == 1:
            key = 1
        else:
            key = 'version'

        result = {
            'version': match.group(key),
            'prefix': match.group(0)[:match.start(key)],
            'suffix': match.group(0)[match.end(key):],
        }

    trace("tag '%s' parsed to %s" % (tag, result))
    return result

def _get_version_class():
    modern_version = pkg_parse_version("1.0")
    if isinstance(modern_version, tuple):
        return None
    else:
        return type(modern_version)

VERSION_CLASS = _get_version_class()

class SetuptoolsOutdatedWarning(Warning):
    pass

# append so integrators can disable the warning
warnings.simplefilter("error", SetuptoolsOutdatedWarning, append=True)

def _warn_if_setuptools_outdated():
    if VERSION_CLASS is None:
        warnings.warn("your setuptools is too old (<12)", SetuptoolsOutdatedWarning)

def callable_or_entrypoint(group, callable_or_name):
    trace("ep", (group, callable_or_name))

    if callable(callable_or_name):
        return callable_or_name

    for ep in iter_entry_points(group, callable_or_name):
        trace("ep found:", ep.name)
        return ep.load()

def tag_to_version(tag, config=None):
    """
    take a tag that might be prefixed with a keyword and return only the version part
    :param config: optional configuration object
    """
    trace("tag", tag)

    if not config:
        config = Configuration()

    tagdict = _parse_version_tag(tag, config)
    if not isinstance(tagdict, dict) or not tagdict.get('version', None):
        warnings.warn("tag %r no version found" % (tag,))
        return None

    version = tagdict['version']
    trace("version pre parse", version)

    if tagdict.get('suffix', ''):
        warnings.warn("tag %r will be stripped of its suffix '%s'" % (tag, tagdict['suffix']))

    if VERSION_CLASS is not None:
        version = pkg_parse_version(version)
        trace("version", repr(version))

    return version

def tags_to_versions(tags, config=None):
    """
    take tags that might be prefixed with a keyword and return only the version part
    :param tags: an iterable of tags
    :param config: optional configuration object
    """
    result = []
    for tag in tags:
        version = tag_to_version(tag, config)
        if version is not None:
            result.append(version)
    return result

class ScmVersion(object):

    def __init__(
        self,
        tag_version,
        distance=None,
        node=None,
        dirty=False,
        preformatted=False,
        branch=None,
        **kw
    ):
        if kw:
            trace("unknown args", kw)
        self.tag = tag_version
        if dirty and distance is None:
            distance = 0
        self.distance = distance
        self.node = node
        self.time = datetime.datetime.now()
        self.extra = kw
        self.dirty = dirty
        self.preformatted = preformatted
        self.branch = branch

    @property
    def exact(self):
        return self.distance is None

    def __repr__(self):
        return self.format_with(
            "<ScmVersion {tag} d={distance}" " n={node} d={dirty} b={branch} x={extra}>"
        )

    def format_with(self, fmt, **kw):
        return fmt.format(
            time=self.time,
            tag=self.tag,
            distance=self.distance,
            node=self.node,
            dirty=self.dirty,
            extra=self.extra,
            branch=self.branch,
            **kw
        )

    def format_choice(self, clean_format, dirty_format, **kw):
        return self.format_with(dirty_format if self.dirty else clean_format, **kw)

    def format_next_version(self, guess_next, fmt="{guessed}.dev{distance}", **kw):
        guessed = guess_next(self.tag, **kw)
        return self.format_with(fmt, guessed=guessed)

def _parse_tag(tag, preformatted, config):
    if preformatted:
        return tag
    if VERSION_CLASS is None or not isinstance(tag, VERSION_CLASS):
        tag = tag_to_version(tag, config)
    return tag

def meta(tag, distance=None, dirty=False, node=None, preformatted=False, config=None, **kw):
    if not config:
        warnings.warn("meta invoked without explicit configuration, will use defaults where required.")
    parsed_version = _parse_tag(tag, preformatted, config)
    trace("version", tag, "->", parsed_version)
    assert parsed_version is not None, "cant parse version %s" % tag
    return ScmVersion(parsed_version, distance, node, dirty, preformatted, **kw)

def guess_next_version(tag_version):
    version = _strip_local(str(tag_version))
    return _bump_dev(version) or _bump_regex(version)

def _strip_local(version_string):
    public, sep, local = version_string.partition("+")
    return public

def _bump_dev(version):
    if ".dev" not in version:
        return

    prefix, tail = version.rsplit(".dev", 1)
    assert tail == "0", "own dev numbers are unsupported"
    return prefix

def _bump_regex(version):
    prefix, tail = re.match(r"(.*?)(\d+)$", version).groups()
    return "%s%d" % (prefix, int(tail) + 1)

def guess_next_dev_version(version):
    if version.exact:
        return version.format_with("{tag}")
    else:
        return version.format_next_version(guess_next_version)

def guess_next_simple_semver(version, retain, increment=True):
    parts = map(int, str(version).split("."))
    parts = _pad(parts, retain, 0)
    if increment:
        parts[-1] += 1
    parts = _pad(parts, SEMVER_LEN, 0)
    return ".".join(map(str, parts))

def simplified_semver_version(version):
    if version.exact:
        return guess_next_simple_semver(version.tag, retain=SEMVER_LEN, increment=False)
    else:
        if version.branch is not None and "feature" in version.branch:
            return version.format_next_version(
                guess_next_simple_semver, retain=SEMVER_MINOR
            )
        else:
            return version.format_next_version(
                guess_next_simple_semver, retain=SEMVER_PATCH
            )

def _format_local_with_time(version, time_format):

    if version.exact or version.node is None:
        return version.format_choice(
            "", "+d{time:{time_format}}", time_format=time_format
        )
    else:
        return version.format_choice(
            "+{node}", "+{node}.d{time:{time_format}}", time_format=time_format
        )

def get_local_node_and_date(version):
    return _format_local_with_time(version, time_format="%Y%m%d")

def get_local_node_and_timestamp(version, fmt="%Y%m%d%H%M%S"):
    return _format_local_with_time(version, time_format=fmt)

def get_local_dirty_tag(version):
    return version.format_choice("", "+dirty")

def postrelease_version(version):
    if version.exact:
        return version.format_with("{tag}")
    else:
        return version.format_with("{tag}.post{distance}")

def format_version(version, **config):
    trace("scm version", version)
    trace("config", config)
    if version.preformatted:
        return version.tag
    version_scheme = callable_or_entrypoint(
        "setuptools_scm.version_scheme", config["version_scheme"]
    )
    local_scheme = callable_or_entrypoint(
        "setuptools_scm.local_scheme", config["local_scheme"]
    )
    main_version = version_scheme(version)
    trace("version", main_version)
    local_version = local_scheme(version)
    trace("local_version", local_version)
    return version_scheme(version) + local_scheme(version)
<commit_msg>removed string formatting<commit_after>from __future__ import print_function
import datetime
import warnings
import re
from itertools import chain, repeat, islice

from .config import Configuration
from .utils import trace, string_types

from pkg_resources import iter_entry_points

from pkg_resources import parse_version as pkg_parse_version

SEMVER_MINOR = 2
SEMVER_PATCH = 3
SEMVER_LEN = 3

def _pad(iterable, size, padding=None):
    padded = chain(iterable, repeat(padding))
    return list(islice(padded, size))

def _parse_version_tag(tag, config):
    tagstring = tag if not isinstance(tag, string_types) else str(tag)
    match = config.tag_regex.match(tagstring)

    result = None
    if match:
        if len(match.groups()) == 1:
            key = 1
        else:
            key = 'version'

        result = {
            'version': match.group(key),
            'prefix': match.group(0)[:match.start(key)],
            'suffix': match.group(0)[match.end(key):],
        }

    trace("tag '%s' parsed to %s" % (tag, result))
    return result

def _get_version_class():
    modern_version = pkg_parse_version("1.0")
    if isinstance(modern_version, tuple):
        return None
    else:
        return type(modern_version)

VERSION_CLASS = _get_version_class()

class SetuptoolsOutdatedWarning(Warning):
    pass

# append so integrators can disable the warning
warnings.simplefilter("error", SetuptoolsOutdatedWarning, append=True)

def _warn_if_setuptools_outdated():
    if VERSION_CLASS is None:
        warnings.warn("your setuptools is too old (<12)", SetuptoolsOutdatedWarning)

def callable_or_entrypoint(group, callable_or_name):
    trace("ep", (group, callable_or_name))

    if callable(callable_or_name):
        return callable_or_name

    for ep in iter_entry_points(group, callable_or_name):
        trace("ep found:", ep.name)
        return ep.load()

def tag_to_version(tag, config=None):
    """
    take a tag that might be prefixed with a keyword and return only the version part
    :param config: optional configuration object
    """
    trace("tag", tag)

    if not config:
        config = Configuration()

    tagdict = _parse_version_tag(tag, config)
    if not isinstance(tagdict, dict) or not tagdict.get('version', None):
        warnings.warn("tag %r no version found" % (tag,))
        return None

    version = tagdict['version']
    trace("version pre parse", version)

    if tagdict.get('suffix', ''):
        warnings.warn("tag %r will be stripped of its suffix '%s'" % (tag, tagdict['suffix']))

    if VERSION_CLASS is not None:
        version = pkg_parse_version(version)
        trace("version", repr(version))

    return version

def tags_to_versions(tags, config=None):
    """
    take tags that might be prefixed with a keyword and return only the version part
    :param tags: an iterable of tags
    :param config: optional configuration object
    """
    result = []
    for tag in tags:
        version = tag_to_version(tag, config)
        if version is not None:
            result.append(version)
    return result

class ScmVersion(object):

    def __init__(
        self,
        tag_version,
        distance=None,
        node=None,
        dirty=False,
        preformatted=False,
        branch=None,
        **kw
    ):
        if kw:
            trace("unknown args", kw)
        self.tag = tag_version
        if dirty and distance is None:
            distance = 0
        self.distance = distance
        self.node = node
        self.time = datetime.datetime.now()
        self.extra = kw
        self.dirty = dirty
        self.preformatted = preformatted
        self.branch = branch

    @property
    def exact(self):
        return self.distance is None

    def __repr__(self):
        return self.format_with(
            "<ScmVersion {tag} d={distance}" " n={node} d={dirty} b={branch} x={extra}>"
        )

    def format_with(self, fmt, **kw):
        return fmt.format(
            time=self.time,
            tag=self.tag,
            distance=self.distance,
            node=self.node,
            dirty=self.dirty,
            extra=self.extra,
            branch=self.branch,
            **kw
        )

    def format_choice(self, clean_format, dirty_format, **kw):
        return self.format_with(dirty_format if self.dirty else clean_format, **kw)

    def format_next_version(self, guess_next, fmt="{guessed}.dev{distance}", **kw):
        guessed = guess_next(self.tag, **kw)
        return self.format_with(fmt, guessed=guessed)

def _parse_tag(tag, preformatted, config):
    if preformatted:
        return tag
    if VERSION_CLASS is None or not isinstance(tag, VERSION_CLASS):
        tag = tag_to_version(tag, config)
    return tag

def meta(tag, distance=None, dirty=False, node=None, preformatted=False, config=None, **kw):
    if not config:
        warnings.warn("meta invoked without explicit configuration, will use defaults where required.")
    parsed_version = _parse_tag(tag, preformatted, config)
    trace("version", tag, "->", parsed_version)
    assert parsed_version is not None, "cant parse version %s" % tag
    return ScmVersion(parsed_version, distance, node, dirty, preformatted, **kw)

def guess_next_version(tag_version):
    version = _strip_local(str(tag_version))
    return _bump_dev(version) or _bump_regex(version)

def _strip_local(version_string):
    public, sep, local = version_string.partition("+")
    return public

def _bump_dev(version):
    if ".dev" not in version:
        return

    prefix, tail = version.rsplit(".dev", 1)
    assert tail == "0", "own dev numbers are unsupported"
    return prefix

def _bump_regex(version):
    prefix, tail = re.match(r"(.*?)(\d+)$", version).groups()
    return "%s%d" % (prefix, int(tail) + 1)

def guess_next_dev_version(version):
    if version.exact:
        return version.format_with("{tag}")
    else:
        return version.format_next_version(guess_next_version)

def guess_next_simple_semver(version, retain, increment=True):
    parts = map(int, str(version).split("."))
    parts = _pad(parts, retain, 0)
    if increment:
        parts[-1] += 1
    parts = _pad(parts, SEMVER_LEN, 0)
    return ".".join(map(str, parts))

def simplified_semver_version(version):
    if version.exact:
        return guess_next_simple_semver(version.tag, retain=SEMVER_LEN, increment=False)
    else:
        if version.branch is not None and "feature" in version.branch:
            return version.format_next_version(
                guess_next_simple_semver, retain=SEMVER_MINOR
            )
        else:
            return version.format_next_version(
                guess_next_simple_semver, retain=SEMVER_PATCH
            )

def _format_local_with_time(version, time_format):

    if version.exact or version.node is None:
        return version.format_choice(
            "", "+d{time:{time_format}}", time_format=time_format
        )
    else:
        return version.format_choice(
            "+{node}", "+{node}.d{time:{time_format}}", time_format=time_format
        )

def get_local_node_and_date(version):
    return _format_local_with_time(version, time_format="%Y%m%d")

def get_local_node_and_timestamp(version, fmt="%Y%m%d%H%M%S"):
    return _format_local_with_time(version, time_format=fmt)

def get_local_dirty_tag(version):
    return version.format_choice("", "+dirty")

def postrelease_version(version):
    if version.exact:
        return version.format_with("{tag}")
    else:
        return version.format_with("{tag}.post{distance}")

def format_version(version, **config):
    trace("scm version", version)
    trace("config", config)
    if version.preformatted:
        return version.tag
    version_scheme = callable_or_entrypoint(
        "setuptools_scm.version_scheme", config["version_scheme"]
    )
    local_scheme = callable_or_entrypoint(
        "setuptools_scm.local_scheme", config["local_scheme"]
    )
    main_version = version_scheme(version)
    trace("version", main_version)
    local_version = local_scheme(version)
    trace("local_version", local_version)
    return version_scheme(version) + local_scheme(version)
<|endoftext|>