<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import time

from django.db import models
from django.db.models import Count
from django.db.models.query import QuerySet
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices
from model_utils.managers import PassThroughManager

from observations.models import Observation

CONSTELLATIONS = Choices(
    ('AND', _('Andromeda')),
    ('ANT', _('Antlia')),
    ('APS', _('Apus')),
    ('AQR', _('Aquarius')),
    ('AQL', _('Aquila')),
    ('ARA', _('Ara')),
    ('ARI', _('Aries')),
    ('AUR', _('Auriga')),
    ('BOO', _('Boötes')),
    ('CAE', _('Caelum')),
    ('CAM', _('Camelopardalis')),
    ('CNC', _('Cancer')),
    ('CVN', _('Canes Venatici')),
    ('CMA', _('Canis Major')),
    ('CMI', _('Canis Minor')),
    ('CAP', _('Capricornus')),
    ('CAR', _('Carina')),
    ('CAS', _('Cassiopeia')),
    ('CEN', _('Centaurus')),
    ('CEP', _('Cepheus')),
    ('CET', _('Cetus')),
    ('CHA', _('Chamaeleon')),
    ('CIR', _('Circinus')),
    ('COL', _('Columba')),
    ('COM', _('Coma Berenices')),
    ('CRA', _('Corona Australis')),
    ('CRB', _('Corona Borealis')),
    ('CRV', _('Corvus')),
    ('CRT', _('Crater')),
    ('CRU', _('Crux')),
    ('CYG', _('Cygnus')),
    ('DEL', _('Delphinus')),
    ('DOR', _('Dorado')),
    ('DRA', _('Draco')),
    ('EQU', _('Equuleus')),
    ('ERI', _('Eridanus')),
    ('FOR', _('Fornax')),
    ('GEM', _('Gemini')),
    ('GRU', _('Grus')),
    ('HER', _('Hercules')),
    ('HOR', _('Horologium')),
    ('HYA', _('Hydra')),
    ('HYI', _('Hydrus')),
    ('IND', _('Indus')),
    ('LAC', _('Lacerta')),
    ('LEO', _('Leo')),
    ('LMI', _('Leo Minor')),
    ('LEP', _('Lepus')),
    ('LIB', _('Libra')),
    ('LUP', _('Lupus')),
    ('LYN', _('Lynx')),
    ('LYR', _('Lyra')),
    ('MEN', _('Mensa')),
    ('MIC', _('Microscopium')),
    ('MON', _('Monoceros')),
    ('MUS', _('Musca')),
    ('NOR', _('Norma')),
    ('OCT', _('Octans')),
    ('OPH', _('Ophiuchus')),
    ('ORI', _('Orion')),
    ('PAV', _('Pavo')),
    ('PEG', _('Pegasus')),
    ('PER', _('Perseus')),
    ('PHE', _('Phoenix')),
    ('PIC', _('Pictor')),
    ('PSC', _('Pisces')),
    ('PSA', _('Piscis Austrinus')),
    ('PUP', _('Puppis')),
    ('PYX', _('Pyxis')),
    ('RET', _('Reticulum')),
    ('SGE', _('Sagitta')),
    ('SGR', _('Sagittarius')),
    ('SCO', _('Scorpius')),
    ('SCL', _('Sculptor')),
    ('SCT', _('Scutum')),
    ('SER', _('Serpens')),
    ('SEX', _('Sextans')),
    ('TAU', _('Taurus')),
    ('TEL', _('Telescopium')),
    ('TRI', _('Triangulum')),
    ('TRA', _('Triangulum Australe')),
    ('TUC', _('Tucana')),
    ('UMA', _('Ursa Major')),
    ('UMI', _('Ursa Minor')),
    ('VEL', _('Vela')),
    ('VIR', _('Virgo')),
    ('VOL', _('Volans')),
    ('VUL', _('Vulpecula')),
)

CONSTELLATIONS_DICT = dict(CONSTELLATIONS)

class StarQuerySet(QuerySet):
    def get_total_stats(self):
        current_jd = 2440587.5 + time.time() / 86400.0
        last_month = current_jd - 30
        queryset = Observation.objects.filter(jd__gt=last_month)
        queryset = queryset.aggregate(star_count=Count('star', distinct=True))
        return {
            'total_star_count': self.count(),
            'observed_last_month_count': queryset['star_count'],
        }

@python_2_unicode_compatible
class Star(models.Model):
    """
    A variable star.
    """
    constellation = models.CharField(
        _("Constellation"), max_length=3, choices=CONSTELLATIONS, db_index=True
    )
    name = models.CharField(_("Name"), max_length=20, db_index=True)
    ra = models.CharField(_("Right Ascension"), max_length=15, default='')
    dec = models.CharField(_("Declination"), max_length=15, default='')
    variability_type = models.ForeignKey(
        'stars.VariabilityType', null=True, blank=True,
        verbose_name=_("Type of variability"),
    )
    max_magnitude = models.FloatField(_("Maximum brightness"), null=True)
    min_magnitude = models.FloatField(_("Minimum brightness"), null=True)
    epoch = models.FloatField(_("Epoch"), null=True)
    period = models.FloatField(_("Period"), null=True)

    objects = PassThroughManager.for_queryset_class(StarQuerySet)()

    class Meta:
        verbose_name = _("Variable star")
        verbose_name_plural = _("Variable stars")
        ordering = ('constellation', 'name',)

    def __str__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return('stars:star_detail', [], {'pk': self.pk})

    def is_periodic(self):
        """
        Returns True if the star is periodic (has a defined period).
        """
        return self.period is not None

    def get_gcvs_search_name(self):
        """
        Fixes GCVS inconsequence in handling escape characters.
        """
        return self.name.replace(' ', '+')

@python_2_unicode_compatible
class VariabilityType(models.Model):
    """
    A short description of variability type from GCVS.
    """
    code = models.CharField(_("Letter code"), max_length=12, db_index=True)
    short_description = models.CharField(
        _("Short description"), max_length=100, blank=True, default=''
    )
    long_description = models.TextField(_("Long description"), default='')

    class Meta:
        verbose_name = _("Variability type")
        verbose_name_plural = _("Variability types")
        ordering = ('code',)

    def __str__(self):
        return self.code

    @models.permalink
    def get_absolute_url(self):
        return ('stars:variabilitytype_detail', [], {'pk': self.pk})
<commit_msg>Refactored and fixed star stats.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.db.models import Count
from django.db.models.query import QuerySet
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices
from model_utils.managers import PassThroughManager

from observations.models import Observation
from observations.utils import jd_now

CONSTELLATIONS = Choices(
    ('AND', _('Andromeda')),
    ('ANT', _('Antlia')),
    ('APS', _('Apus')),
    ('AQR', _('Aquarius')),
    ('AQL', _('Aquila')),
    ('ARA', _('Ara')),
    ('ARI', _('Aries')),
    ('AUR', _('Auriga')),
    ('BOO', _('Boötes')),
    ('CAE', _('Caelum')),
    ('CAM', _('Camelopardalis')),
    ('CNC', _('Cancer')),
    ('CVN', _('Canes Venatici')),
    ('CMA', _('Canis Major')),
    ('CMI', _('Canis Minor')),
    ('CAP', _('Capricornus')),
    ('CAR', _('Carina')),
    ('CAS', _('Cassiopeia')),
    ('CEN', _('Centaurus')),
    ('CEP', _('Cepheus')),
    ('CET', _('Cetus')),
    ('CHA', _('Chamaeleon')),
    ('CIR', _('Circinus')),
    ('COL', _('Columba')),
    ('COM', _('Coma Berenices')),
    ('CRA', _('Corona Australis')),
    ('CRB', _('Corona Borealis')),
    ('CRV', _('Corvus')),
    ('CRT', _('Crater')),
    ('CRU', _('Crux')),
    ('CYG', _('Cygnus')),
    ('DEL', _('Delphinus')),
    ('DOR', _('Dorado')),
    ('DRA', _('Draco')),
    ('EQU', _('Equuleus')),
    ('ERI', _('Eridanus')),
    ('FOR', _('Fornax')),
    ('GEM', _('Gemini')),
    ('GRU', _('Grus')),
    ('HER', _('Hercules')),
    ('HOR', _('Horologium')),
    ('HYA', _('Hydra')),
    ('HYI', _('Hydrus')),
    ('IND', _('Indus')),
    ('LAC', _('Lacerta')),
    ('LEO', _('Leo')),
    ('LMI', _('Leo Minor')),
    ('LEP', _('Lepus')),
    ('LIB', _('Libra')),
    ('LUP', _('Lupus')),
    ('LYN', _('Lynx')),
    ('LYR', _('Lyra')),
    ('MEN', _('Mensa')),
    ('MIC', _('Microscopium')),
    ('MON', _('Monoceros')),
    ('MUS', _('Musca')),
    ('NOR', _('Norma')),
    ('OCT', _('Octans')),
    ('OPH', _('Ophiuchus')),
    ('ORI', _('Orion')),
    ('PAV', _('Pavo')),
    ('PEG', _('Pegasus')),
    ('PER', _('Perseus')),
    ('PHE', _('Phoenix')),
    ('PIC', _('Pictor')),
    ('PSC', _('Pisces')),
    ('PSA', _('Piscis Austrinus')),
    ('PUP', _('Puppis')),
    ('PYX', _('Pyxis')),
    ('RET', _('Reticulum')),
    ('SGE', _('Sagitta')),
    ('SGR', _('Sagittarius')),
    ('SCO', _('Scorpius')),
    ('SCL', _('Sculptor')),
    ('SCT', _('Scutum')),
    ('SER', _('Serpens')),
    ('SEX', _('Sextans')),
    ('TAU', _('Taurus')),
    ('TEL', _('Telescopium')),
    ('TRI', _('Triangulum')),
    ('TRA', _('Triangulum Australe')),
    ('TUC', _('Tucana')),
    ('UMA', _('Ursa Major')),
    ('UMI', _('Ursa Minor')),
    ('VEL', _('Vela')),
    ('VIR', _('Virgo')),
    ('VOL', _('Volans')),
    ('VUL', _('Vulpecula')),
)

CONSTELLATIONS_DICT = dict(CONSTELLATIONS)

class StarQuerySet(QuerySet):
    def get_total_stats(self):
        last_month = jd_now() - 30
        star_ids = Observation.objects.filter(jd__gt=last_month).values('star')
        return {
            'total_star_count': self.count(),
            'observed_last_month_count': self.filter(pk__in=star_ids).count(),
        }

@python_2_unicode_compatible
class Star(models.Model):
    """
    A variable star.
    """
    constellation = models.CharField(
        _("Constellation"), max_length=3, choices=CONSTELLATIONS, db_index=True
    )
    name = models.CharField(_("Name"), max_length=20, db_index=True)
    ra = models.CharField(_("Right Ascension"), max_length=15, default='')
    dec = models.CharField(_("Declination"), max_length=15, default='')
    variability_type = models.ForeignKey(
        'stars.VariabilityType', null=True, blank=True,
        verbose_name=_("Type of variability"),
    )
    max_magnitude = models.FloatField(_("Maximum brightness"), null=True)
    min_magnitude = models.FloatField(_("Minimum brightness"), null=True)
    epoch = models.FloatField(_("Epoch"), null=True)
    period = models.FloatField(_("Period"), null=True)

    objects = PassThroughManager.for_queryset_class(StarQuerySet)()

    class Meta:
        verbose_name = _("Variable star")
        verbose_name_plural = _("Variable stars")
        ordering = ('constellation', 'name',)

    def __str__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return('stars:star_detail', [], {'pk': self.pk})

    def is_periodic(self):
        """
        Returns True if the star is periodic (has a defined period).
        """
        return self.period is not None

    def get_gcvs_search_name(self):
        """
        Fixes GCVS inconsequence in handling escape characters.
        """
        return self.name.replace(' ', '+')

@python_2_unicode_compatible
class VariabilityType(models.Model):
    """
    A short description of variability type from GCVS.
    """
    code = models.CharField(_("Letter code"), max_length=12, db_index=True)
    short_description = models.CharField(
        _("Short description"), max_length=100, blank=True, default=''
    )
    long_description = models.TextField(_("Long description"), default='')

    class Meta:
        verbose_name = _("Variability type")
        verbose_name_plural = _("Variability types")
        ordering = ('code',)

    def __str__(self):
        return self.code

    @models.permalink
    def get_absolute_url(self):
        return ('stars:variabilitytype_detail', [], {'pk': self.pk})
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# tsuserver3, an Attorney Online server
#
# Copyright (C) 2016 argoneus <argoneuscze@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Install dependencies in case one is missing

import sys
import subprocess

def check_deps():
    py_version = sys.version_info
    if py_version.major < 3 or (py_version.major == 3 and py_version.minor < 7):
        print("tsuserver3 requires at least Python 3.7! Your version: {}.{}"
            .format(py_version.major, py_version.minor))
        sys.exit(1)

    try:
        import arrow
    except ModuleNotFoundError:
        print('Installing dependencies for you...')
        try:
            subprocess.check_call([
                sys.executable, '-m', 'pip', 'install', '--user', '-r',
                'requirements.txt'
                ])
            print('If an import error occurs after the installation, try '
                'restarting the server.')
        except subprocess.CalledProcessError:
            print(
                'Couldn\'t install it for you, because you don\'t have pip, '
                'or another error occurred.'
            )

def main():
    from server.tsuserver import TsuServer3
    server = TsuServer3()
    server.start()

if __name__ == '__main__':
    print('tsuserver3 - an Attorney Online server')
    check_deps()
    main()
<commit_msg>indentation<commit_after>#!/usr/bin/env python3

# tsuserver3, an Attorney Online server
#
# Copyright (C) 2016 argoneus <argoneuscze@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Install dependencies in case one is missing

import sys
import subprocess

def check_deps():
    py_version = sys.version_info
    if py_version.major < 3 or (py_version.major == 3 and py_version.minor < 7):
        print("tsuserver3 requires at least Python 3.7! Your version: {}.{}"
                .format(py_version.major, py_version.minor))
        sys.exit(1)

    try:
        import arrow
    except ModuleNotFoundError:
        print('Installing dependencies for you...')
        try:
            subprocess.check_call([
                sys.executable, '-m', 'pip', 'install', '--user', '-r',
                'requirements.txt'
                ])
            print('If an import error occurs after the installation, try '
                    'restarting the server.')
        except subprocess.CalledProcessError:
            print('Couldn\'t install it for you, because you don\'t have pip, '
                'or another error occurred.')

def main():
    from server.tsuserver import TsuServer3
    server = TsuServer3()
    server.start()

if __name__ == '__main__':
    print('tsuserver3 - an Attorney Online server')
    check_deps()
    main()
<|endoftext|>


<commit_before>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os

from wa.framework.configuration.core import JobSpec
from wa.framework.exception import ConfigError
from wa.utils.serializer import json, read_pod, SerializerSyntaxError
from wa.utils.types import toggle_set, counter

###############
### Parsers ###
###############

class ConfigParser(object):

    def load_from_path(self, state, filepath):
        self.load(state, _load_file(filepath, "Config"), filepath)

    def load(self, state, raw, source, wrap_exceptions=True):  # pylint: disable=too-many-branches
        try:
            state.plugin_cache.add_source(source)
            if 'run_name' in raw:
                msg = '"run_name" can only be specified in the config '\
                      'section of an agenda'
                raise ConfigError(msg)

            if 'id' in raw:
                raise ConfigError('"id" cannot be set globally')

            merge_result_processors_instruments(raw)

            # Get WA core configuration
            for cfg_point in state.settings.configuration.itervalues():
                value = pop_aliased_param(cfg_point, raw)
                if value is not None:
                    state.settings.set(cfg_point.name, value)

            # Get run specific configuration
            for cfg_point in state.run_config.configuration.itervalues():
                value = pop_aliased_param(cfg_point, raw)
                if value is not None:
                    state.run_config.set(cfg_point.name, value)

            # Get global job spec configuration
            for cfg_point in JobSpec.configuration.itervalues():
                value = pop_aliased_param(cfg_point, raw)
                if value is not None:
                    state.jobs_config.set_global_value(cfg_point.name, value)

            for name, values in raw.iteritems():
                # Assume that all leftover config is for a plug-in or a global
                # alias it is up to PluginCache to assert this assumption
                state.plugin_cache.add_configs(name, values, source)

        except ConfigError as e:
            if wrap_exceptions:
                raise ConfigError('Error in "{}":\n{}'.format(source, str(e)))
            else:
                raise e

class AgendaParser(object):

    def load_from_path(self, state, filepath):
        raw = _load_file(filepath, 'Agenda')
        self.load(state, raw, filepath)

    def load(self, state, raw, source):
        try:
            if not isinstance(raw, dict):
                raise ConfigError('Invalid agenda, top level entry must be a dict')

            self._populate_and_validate_config(state, raw, source)
            sections = self._pop_sections(raw)
            global_workloads = self._pop_workloads(raw)

            if raw:
                msg = 'Invalid top level agenda entry(ies): "{}"'
                raise ConfigError(msg.format('", "'.join(raw.keys())))

            sect_ids, wkl_ids = self._collect_ids(sections, global_workloads)
            self._process_global_workloads(state, global_workloads, wkl_ids)
            self._process_sections(state, sections, sect_ids, wkl_ids)

            state.agenda = source

        except (ConfigError, SerializerSyntaxError) as e:
            raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))

    def _populate_and_validate_config(self, state, raw, source):
        for name in ['config', 'global']:
            entry = raw.pop(name, None)
            if entry is None:
                continue

            if not isinstance(entry, dict):
                msg = 'Invalid entry "{}" - must be a dict'
                raise ConfigError(msg.format(name))

            if 'run_name' in entry:
                state.run_config.set('run_name', entry.pop('run_name'))

            state.load_config(entry, '{}/{}'.format(source, name), wrap_exceptions=False)

    def _pop_sections(self, raw):
        sections = raw.pop("sections", [])
        if not isinstance(sections, list):
            raise ConfigError('Invalid entry "sections" - must be a list')
        return sections

    def _pop_workloads(self, raw):
        workloads = raw.pop("workloads", [])
        if not isinstance(workloads, list):
            raise ConfigError('Invalid entry "workloads" - must be a list')
        return workloads

    def _collect_ids(self, sections, global_workloads):
        seen_section_ids = set()
        seen_workload_ids = set()

        for workload in global_workloads:
            workload = _get_workload_entry(workload)
            _collect_valid_id(workload.get("id"), seen_workload_ids, "workload")

        for section in sections:
            _collect_valid_id(section.get("id"), seen_section_ids, "section")
            for workload in section["workloads"] if "workloads" in section else []:
                workload = _get_workload_entry(workload)
                _collect_valid_id(workload.get("id"), seen_workload_ids,
                                  "workload")

        return seen_section_ids, seen_workload_ids

    def _process_global_workloads(self, state, global_workloads, seen_wkl_ids):
        for workload_entry in global_workloads:
            workload = _process_workload_entry(workload_entry, seen_wkl_ids,
                                               state.jobs_config)
            state.jobs_config.add_workload(workload)

    def _process_sections(self, state, sections, seen_sect_ids, seen_wkl_ids):
        for section in sections:
            workloads = []
            for workload_entry in section.pop("workloads", []):
                workload = _process_workload_entry(workload_entry, seen_wkl_ids,
                                                   state.jobs_config)
                workloads.append(workload)

            if 'params' in section:
                if 'runtime_params' in section:
                    msg = 'both "params" and "runtime_params" specified in a '\
                          'section: "{}"'
                    raise ConfigError(msg.format(json.dumps(section, indent=None)))
                section['runtime_params'] = section.pop('params')

            section = _construct_valid_entry(section, seen_sect_ids,
                                             "s", state.jobs_config)
            state.jobs_config.add_section(section, workloads)

########################
### Helper functions ###
########################

def pop_aliased_param(cfg_point, d, default=None):
    """
    Given a ConfigurationPoint and a dict, this function will search the dict for
    the ConfigurationPoint's name/aliases. If more than one is found it will raise
    a ConfigError. If one (and only one) is found then it will return the value
    for the ConfigurationPoint. If the name or aliases are present in the dict it will
    return the "default" parameter of this function.
    """
    aliases = [cfg_point.name] + cfg_point.aliases
    alias_map = [a for a in aliases if a in d]
    if len(alias_map) > 1:
        raise ConfigError(DUPLICATE_ENTRY_ERROR.format(aliases))
    elif alias_map:
        return d.pop(alias_map[0])
    else:
        return default

def _load_file(filepath, error_name):
    if not os.path.isfile(filepath):
        raise ValueError("{} does not exist".format(filepath))
    try:
        raw = read_pod(filepath)
    except SerializerSyntaxError as e:
        raise ConfigError('Error parsing {} {}: {}'.format(error_name, filepath, e))
    if not isinstance(raw, dict):
        message = '{} does not contain a valid {} structure; top level must be a dict.'
        raise ConfigError(message.format(filepath, error_name))
    return raw

def merge_result_processors_instruments(raw):
    instr_config = JobSpec.configuration['instrumentation']
    instruments = toggle_set(pop_aliased_param(instr_config, raw, default=[]))
    result_processors = toggle_set(raw.pop('result_processors', []))
    if instruments and result_processors:
        conflicts = instruments.conflicts_with(result_processors)
        if conflicts:
            msg = '"instrumentation" and "result_processors" have '\
                  'conflicting entries: {}'
            entires = ', '.join('"{}"'.format(c.strip("~")) for c in conflicts)
            raise ConfigError(msg.format(entires))
    raw['instrumentation'] = instruments.merge_with(result_processors)

def _pop_aliased(d, names, entry_id):
    name_count = sum(1 for n in names if n in d)
    if name_count > 1:
        names_list = ', '.join(names)
        msg = 'Invalid workload entry "{}": at most one of ({}}) must be specified.'
        raise ConfigError(msg.format(entry_id, names_list))
    for name in names:
        if name in d:
            return d.pop(name)
    return None

def _construct_valid_entry(raw, seen_ids, prefix, jobs_config):
    workload_entry = {}

    # Generate an automatic ID if the entry doesn't already have one
    if 'id' not in raw:
        while True:
            new_id = '{}{}'.format(prefix, counter(name=prefix))
            if new_id not in seen_ids:
                break
        workload_entry['id'] = new_id
        seen_ids.add(new_id)
    else:
        workload_entry['id'] = raw.pop('id')

    # Process instrumentation
    merge_result_processors_instruments(raw)

    # Validate all workload_entry
    for name, cfg_point in JobSpec.configuration.iteritems():
        value = pop_aliased_param(cfg_point, raw)
        if value is not None:
            value = cfg_point.kind(value)
            cfg_point.validate_value(name, value)
            workload_entry[name] = value

    if "instrumentation" in workload_entry:
        jobs_config.update_enabled_instruments(workload_entry["instrumentation"])

    # error if there are unknown workload_entry
    if raw:
        msg = 'Invalid entry(ies) in "{}": "{}"'
        raise ConfigError(msg.format(workload_entry['id'], ', '.join(raw.keys())))

    return workload_entry

def _collect_valid_id(entry_id, seen_ids, entry_type):
    if entry_id is None:
        return
    if entry_id in seen_ids:
        raise ConfigError('Duplicate {} ID "{}".'.format(entry_type, entry_id))
    # "-" is reserved for joining section and workload IDs
    if "-" in entry_id:
        msg = 'Invalid {} ID "{}"; IDs cannot contain a "-"'
        raise ConfigError(msg.format(entry_type, entry_id))
    if entry_id == "global":
        msg = 'Invalid {} ID "global"; is a reserved ID'
        raise ConfigError(msg.format(entry_type))
    seen_ids.add(entry_id)

def _get_workload_entry(workload):
    if isinstance(workload, basestring):
        workload = {'name': workload}
    elif not isinstance(workload, dict):
        raise ConfigError('Invalid workload entry: "{}"')
    return workload

def _process_workload_entry(workload, seen_workload_ids, jobs_config):
    workload = _get_workload_entry(workload)
    workload = _construct_valid_entry(workload, seen_workload_ids,
                                      "wk", jobs_config)
    return workload

<commit_msg>Parser: Fixes raising duplicate entry error.<commit_after>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os

from wa.framework.configuration.core import JobSpec
from wa.framework.exception import ConfigError
from wa.utils.serializer import json, read_pod, SerializerSyntaxError
from wa.utils.types import toggle_set, counter

###############
### Parsers ###
###############

class ConfigParser(object):

    def load_from_path(self, state, filepath):
        self.load(state, _load_file(filepath, "Config"), filepath)

    def load(self, state, raw, source, wrap_exceptions=True):  # pylint: disable=too-many-branches
        try:
            state.plugin_cache.add_source(source)
            if 'run_name' in raw:
                msg = '"run_name" can only be specified in the config '\
                      'section of an agenda'
                raise ConfigError(msg)

            if 'id' in raw:
                raise ConfigError('"id" cannot be set globally')

            merge_result_processors_instruments(raw)

            # Get WA core configuration
            for cfg_point in state.settings.configuration.itervalues():
                value = pop_aliased_param(cfg_point, raw)
                if value is not None:
                    state.settings.set(cfg_point.name, value)

            # Get run specific configuration
            for cfg_point in state.run_config.configuration.itervalues():
                value = pop_aliased_param(cfg_point, raw)
                if value is not None:
                    state.run_config.set(cfg_point.name, value)

            # Get global job spec configuration
            for cfg_point in JobSpec.configuration.itervalues():
                value = pop_aliased_param(cfg_point, raw)
                if value is not None:
                    state.jobs_config.set_global_value(cfg_point.name, value)

            for name, values in raw.iteritems():
                # Assume that all leftover config is for a plug-in or a global
                # alias it is up to PluginCache to assert this assumption
                state.plugin_cache.add_configs(name, values, source)

        except ConfigError as e:
            if wrap_exceptions:
                raise ConfigError('Error in "{}":\n{}'.format(source, str(e)))
            else:
                raise e

class AgendaParser(object):

    def load_from_path(self, state, filepath):
        raw = _load_file(filepath, 'Agenda')
        self.load(state, raw, filepath)

    def load(self, state, raw, source):
        try:
            if not isinstance(raw, dict):
                raise ConfigError('Invalid agenda, top level entry must be a dict')

            self._populate_and_validate_config(state, raw, source)
            sections = self._pop_sections(raw)
            global_workloads = self._pop_workloads(raw)

            if raw:
                msg = 'Invalid top level agenda entry(ies): "{}"'
                raise ConfigError(msg.format('", "'.join(raw.keys())))

            sect_ids, wkl_ids = self._collect_ids(sections, global_workloads)
            self._process_global_workloads(state, global_workloads, wkl_ids)
            self._process_sections(state, sections, sect_ids, wkl_ids)

            state.agenda = source

        except (ConfigError, SerializerSyntaxError) as e:
            raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))

    def _populate_and_validate_config(self, state, raw, source):
        for name in ['config', 'global']:
            entry = raw.pop(name, None)
            if entry is None:
                continue

            if not isinstance(entry, dict):
                msg = 'Invalid entry "{}" - must be a dict'
                raise ConfigError(msg.format(name))

            if 'run_name' in entry:
                state.run_config.set('run_name', entry.pop('run_name'))

            state.load_config(entry, '{}/{}'.format(source, name), wrap_exceptions=False)

    def _pop_sections(self, raw):
        sections = raw.pop("sections", [])
        if not isinstance(sections, list):
            raise ConfigError('Invalid entry "sections" - must be a list')
        return sections

    def _pop_workloads(self, raw):
        workloads = raw.pop("workloads", [])
        if not isinstance(workloads, list):
            raise ConfigError('Invalid entry "workloads" - must be a list')
        return workloads

    def _collect_ids(self, sections, global_workloads):
        seen_section_ids = set()
        seen_workload_ids = set()

        for workload in global_workloads:
            workload = _get_workload_entry(workload)
            _collect_valid_id(workload.get("id"), seen_workload_ids, "workload")

        for section in sections:
            _collect_valid_id(section.get("id"), seen_section_ids, "section")
            for workload in section["workloads"] if "workloads" in section else []:
                workload = _get_workload_entry(workload)
                _collect_valid_id(workload.get("id"), seen_workload_ids,
                                  "workload")

        return seen_section_ids, seen_workload_ids

    def _process_global_workloads(self, state, global_workloads, seen_wkl_ids):
        for workload_entry in global_workloads:
            workload = _process_workload_entry(workload_entry, seen_wkl_ids,
                                               state.jobs_config)
            state.jobs_config.add_workload(workload)

    def _process_sections(self, state, sections, seen_sect_ids, seen_wkl_ids):
        for section in sections:
            workloads = []
            for workload_entry in section.pop("workloads", []):
                workload = _process_workload_entry(workload_entry, seen_wkl_ids,
                                                   state.jobs_config)
                workloads.append(workload)

            if 'params' in section:
                if 'runtime_params' in section:
                    msg = 'both "params" and "runtime_params" specified in a '\
                          'section: "{}"'
                    raise ConfigError(msg.format(json.dumps(section, indent=None)))
                section['runtime_params'] = section.pop('params')

            section = _construct_valid_entry(section, seen_sect_ids,
                                             "s", state.jobs_config)
            state.jobs_config.add_section(section, workloads)

########################
### Helper functions ###
########################

def pop_aliased_param(cfg_point, d, default=None):
    """
    Given a ConfigurationPoint and a dict, this function will search the dict for
    the ConfigurationPoint's name/aliases. If more than one is found it will raise
    a ConfigError. If one (and only one) is found then it will return the value
    for the ConfigurationPoint. If the name or aliases are present in the dict it will
    return the "default" parameter of this function.
    """
    aliases = [cfg_point.name] + cfg_point.aliases
    alias_map = [a for a in aliases if a in d]
    if len(alias_map) > 1:
        raise ConfigError('Duplicate entry: {}'.format(aliases))
    elif alias_map:
        return d.pop(alias_map[0])
    else:
        return default

def _load_file(filepath, error_name):
    if not os.path.isfile(filepath):
        raise ValueError("{} does not exist".format(filepath))
    try:
        raw = read_pod(filepath)
    except SerializerSyntaxError as e:
        raise ConfigError('Error parsing {} {}: {}'.format(error_name, filepath, e))
    if not isinstance(raw, dict):
        message = '{} does not contain a valid {} structure; top level must be a dict.'
        raise ConfigError(message.format(filepath, error_name))
    return raw

def merge_result_processors_instruments(raw):
    instr_config = JobSpec.configuration['instrumentation']
    instruments = toggle_set(pop_aliased_param(instr_config, raw, default=[]))
    result_processors = toggle_set(raw.pop('result_processors', []))
    if instruments and result_processors:
        conflicts = instruments.conflicts_with(result_processors)
        if conflicts:
            msg = '"instrumentation" and "result_processors" have '\
                  'conflicting entries: {}'
            entires = ', '.join('"{}"'.format(c.strip("~")) for c in conflicts)
            raise ConfigError(msg.format(entires))
    raw['instrumentation'] = instruments.merge_with(result_processors)

def _pop_aliased(d, names, entry_id):
    name_count = sum(1 for n in names if n in d)
    if name_count > 1:
        names_list = ', '.join(names)
        msg = 'Invalid workload entry "{}": at most one of ({}}) must be specified.'
        raise ConfigError(msg.format(entry_id, names_list))
    for name in names:
        if name in d:
            return d.pop(name)
    return None

def _construct_valid_entry(raw, seen_ids, prefix, jobs_config):
    workload_entry = {}

    # Generate an automatic ID if the entry doesn't already have one
    if 'id' not in raw:
        while True:
            new_id = '{}{}'.format(prefix, counter(name=prefix))
            if new_id not in seen_ids:
                break
        workload_entry['id'] = new_id
        seen_ids.add(new_id)
    else:
        workload_entry['id'] = raw.pop('id')

    # Process instrumentation
    merge_result_processors_instruments(raw)

    # Validate all workload_entry
    for name, cfg_point in JobSpec.configuration.iteritems():
        value = pop_aliased_param(cfg_point, raw)
        if value is not None:
            value = cfg_point.kind(value)
            cfg_point.validate_value(name, value)
            workload_entry[name] = value

    if "instrumentation" in workload_entry:
        jobs_config.update_enabled_instruments(workload_entry["instrumentation"])

    # error if there are unknown workload_entry
    if raw:
        msg = 'Invalid entry(ies) in "{}": "{}"'
        raise ConfigError(msg.format(workload_entry['id'], ', '.join(raw.keys())))

    return workload_entry

def _collect_valid_id(entry_id, seen_ids, entry_type):
    if entry_id is None:
        return
    if entry_id in seen_ids:
        raise ConfigError('Duplicate {} ID "{}".'.format(entry_type, entry_id))
    # "-" is reserved for joining section and workload IDs
    if "-" in entry_id:
        msg = 'Invalid {} ID "{}"; IDs cannot contain a "-"'
        raise ConfigError(msg.format(entry_type, entry_id))
    if entry_id == "global":
        msg = 'Invalid {} ID "global"; is a reserved ID'
        raise ConfigError(msg.format(entry_type))
    seen_ids.add(entry_id)

def _get_workload_entry(workload):
    if isinstance(workload, basestring):
        workload = {'name': workload}
    elif not isinstance(workload, dict):
        raise ConfigError('Invalid workload entry: "{}"')
    return workload

def _process_workload_entry(workload, seen_workload_ids, jobs_config):
    workload = _get_workload_entry(workload)
    workload = _construct_valid_entry(workload, seen_workload_ids,
                                      "wk", jobs_config)
    return workload

<|endoftext|>


<commit_before>import threading as t
import instana.log as l
import resource
import os
import gc as gc_
import sys
import instana.agent_const as a
import copy
import time
import json
from types import ModuleType

class Snapshot(object):
    name = None
    version = None
    rlimit_core = (0, 0)
    rlimit_cpu = (0, 0)
    rlimit_fsize = (0, 0)
    rlimit_data = (0, 0)
    rlimit_stack = (0, 0)
    rlimit_rss = (0, 0)
    rlimit_nproc = (0, 0)
    rlimit_nofile = (0, 0)
    rlimit_memlock = (0, 0)
    rlimit_as = (0, 0)
    versions = None

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def to_dict(self):
        kvs = dict()
        kvs['name'] = self.name
        kvs['version'] = self.version
        kvs['versions'] = self.versions
        return kvs

class GC(object):
    collect0 = 0
    collect1 = 0
    collect2 = 0
    threshold0 = 0
    threshold1 = 0
    threshold2 = 0

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def to_dict(self):
        return self.__dict__

class Metrics(object):
    ru_utime = .0
    ru_stime = .0
    ru_maxrss = 0
    ru_ixrss = 0
    ru_idrss = 0
    ru_isrss = 0
    ru_minflt = 0
    ru_majflt = 0
    ru_nswap = 0
    ru_inblock = 0
    ru_oublock = 0
    ru_msgsnd = 0
    ru_msgrcv = 0
    ru_nsignals = 0
    ru_nvcs = 0
    ru_nivcsw = 0
    dead_threads = 0
    alive_threads = 0
    daemon_threads = 0
    gc = None

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def delta_data(self, delta):
        data = self.__dict__
        if delta is None:
            return data

        unchanged_items = set(data.items()) & set(delta.items())
        for x in unchanged_items:
            data.pop(x[0])

        return data

    def to_dict(self):
        return self.__dict__

class EntityData(object):
    pid = 0
    snapshot = None
    metrics = None

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def to_dict(self):
        return self.__dict__

class Meter(object):
    SNAPSHOT_PERIOD = 600
    snapshot_countdown = 25
    sensor = None
    last_usage = None
    last_collect = None
    timer = None
    last_metrics = None

    def __init__(self, sensor):
        self.sensor = sensor
        self.run()

    def run(self):
        self.timer = t.Thread(target=self.collect_and_report)
        self.timer.daemon = True
        self.timer.name = "Instana Metric Collection"
        self.timer.start()

    def collect_and_report(self):
        while 1:
            self.process()
            time.sleep(1)

    def process(self):
        if self.sensor.agent.can_send():
            self.snapshot_countdown = self.snapshot_countdown - 1
            ss = None
            cm = self.collect_metrics()

            l.debug("snapshot_countdown is:", str(self.snapshot_countdown))
            if self.snapshot_countdown == 0:
                self.snapshot_countdown = self.SNAPSHOT_PERIOD
                ss = self.collect_snapshot()
                md = copy.deepcopy(cm).delta_data(None)
            else:
                md = copy.deepcopy(cm).delta_data(self.last_metrics)

            ed = EntityData(pid=os.getpid(), snapshot=ss, metrics=md)
            url = self.sensor.agent.make_url(a.AGENT_DATA_URL)
            self.sensor.agent.request(url, "POST", ed)
            self.last_metrics = cm.__dict__

    def collect_snapshot(self):
        try:
            s = Snapshot(name=self.sensor.service_name,
                         version=sys.version,
                         rlimit_core=resource.getrlimit(resource.RLIMIT_CORE),
                         rlimit_cpu=resource.getrlimit(resource.RLIMIT_CPU),
                         rlimit_fsize=resource.getrlimit(
                             resource.RLIMIT_FSIZE),
                         rlimit_data=resource.getrlimit(resource.RLIMIT_DATA),
                         rlimit_stack=resource.getrlimit(
                             resource.RLIMIT_STACK),
                         rlimit_rss=resource.getrlimit(resource.RLIMIT_RSS),
                         rlimit_nproc=resource.getrlimit(
                             resource.RLIMIT_NPROC),
                         rlimit_nofile=resource.getrlimit(
                             resource.RLIMIT_NOFILE),
                         rlimit_memlock=resource.getrlimit(
                             resource.RLIMIT_MEMLOCK),
                         rlimit_as=resource.getrlimit(resource.RLIMIT_AS),
                         versions=self.collect_modules())

            return s
        except Exception as e:
            l.error("collect_snapshot: ", str(e))

            return None

    def jsonable(self, value):
        if callable(value):
            result = value()
        elif type(value) is ModuleType:
            result = str(value)
        else:
            result = value
        return result

    def collect_modules(self):
        try:
            m = sys.modules
            r = {}
            for k in m:
                if m[k]:
                    d = m[k].__dict__
                    if "version" in d and d["version"]:
                        r[k] = self.jsonable(d["version"])
                    elif "__version__" in d and d["__version__"]:
                        r[k] = self.jsonable(d["__version__"])
                    else:
                        r[k] = "builtin"

            return r
        except Exception as e:
            l.error("collect_modules: ", e)

            return None

    def collect_metrics(self):
        u = resource.getrusage(resource.RUSAGE_SELF)
        if gc_.isenabled():
            c = list(gc_.get_count())
            th = list(gc_.get_threshold())
            g = GC(collect0=c[0] if not self.last_collect else c[0] - self.last_collect[0],
                   collect1=c[1] if not self.last_collect else c[
                       1] - self.last_collect[1],
                   collect2=c[2] if not self.last_collect else c[
                       2] - self.last_collect[2],
                   threshold0=th[0],
                   threshold1=th[1],
                   threshold2=th[2])

        thr = t.enumerate()
        daemon_threads = len([tr.daemon and tr.is_alive() for tr in thr])
        alive_threads = len([not tr.daemon and tr.is_alive() for tr in thr])
        dead_threads = len([not tr.is_alive() for tr in thr])

        m = Metrics(ru_utime=u[0] if not self.last_usage else u[0] - self.last_usage[0],
                    ru_stime=u[1] if not self.last_usage else u[1] - self.last_usage[1],
                    ru_maxrss=u[2],
                    ru_ixrss=u[3],
                    ru_idrss=u[4],
                    ru_isrss=u[5],
                    ru_minflt=u[6] if not self.last_usage else u[6] - self.last_usage[6],
                    ru_majflt=u[7] if not self.last_usage else u[7] - self.last_usage[7],
                    ru_nswap=u[8] if not self.last_usage else u[8] - self.last_usage[8],
                    ru_inblock=u[9] if not self.last_usage else u[9] - self.last_usage[9],
                    ru_oublock=u[10] if not self.last_usage else u[10] - self.last_usage[10],
                    ru_msgsnd=u[11] if not self.last_usage else u[11] - self.last_usage[11],
                    ru_msgrcv=u[12] if not self.last_usage else u[12] - self.last_usage[12],
                    ru_nsignals=u[13] if not self.last_usage else u[13] - self.last_usage[13],
                    ru_nvcs=u[14] if not self.last_usage else u[14] - self.last_usage[14],
                    ru_nivcsw=u[15] if not self.last_usage else u[15] - self.last_usage[15],
                    alive_threads=alive_threads,
                    dead_threads=dead_threads,
                    daemon_threads=daemon_threads,
                    gc=g)

        self.last_usage = u
        if gc_.isenabled():
            self.last_collect = c

        return m
<commit_msg>Remove debug message<commit_after>import threading as t
import instana.log as l
import resource
import os
import gc as gc_
import sys
import instana.agent_const as a
import copy
import time
import json
from types import ModuleType

class Snapshot(object):
    name = None
    version = None
    rlimit_core = (0, 0)
    rlimit_cpu = (0, 0)
    rlimit_fsize = (0, 0)
    rlimit_data = (0, 0)
    rlimit_stack = (0, 0)
    rlimit_rss = (0, 0)
    rlimit_nproc = (0, 0)
    rlimit_nofile = (0, 0)
    rlimit_memlock = (0, 0)
    rlimit_as = (0, 0)
    versions = None

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def to_dict(self):
        kvs = dict()
        kvs['name'] = self.name
        kvs['version'] = self.version
        kvs['versions'] = self.versions
        return kvs

class GC(object):
    collect0 = 0
    collect1 = 0
    collect2 = 0
    threshold0 = 0
    threshold1 = 0
    threshold2 = 0

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def to_dict(self):
        return self.__dict__

class Metrics(object):
    ru_utime = .0
    ru_stime = .0
    ru_maxrss = 0
    ru_ixrss = 0
    ru_idrss = 0
    ru_isrss = 0
    ru_minflt = 0
    ru_majflt = 0
    ru_nswap = 0
    ru_inblock = 0
    ru_oublock = 0
    ru_msgsnd = 0
    ru_msgrcv = 0
    ru_nsignals = 0
    ru_nvcs = 0
    ru_nivcsw = 0
    dead_threads = 0
    alive_threads = 0
    daemon_threads = 0
    gc = None

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def delta_data(self, delta):
        data = self.__dict__
        if delta is None:
            return data

        unchanged_items = set(data.items()) & set(delta.items())
        for x in unchanged_items:
            data.pop(x[0])

        return data

    def to_dict(self):
        return self.__dict__

class EntityData(object):
    pid = 0
    snapshot = None
    metrics = None

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def to_dict(self):
        return self.__dict__

class Meter(object):
    SNAPSHOT_PERIOD = 600
    snapshot_countdown = 25
    sensor = None
    last_usage = None
    last_collect = None
    timer = None
    last_metrics = None

    def __init__(self, sensor):
        self.sensor = sensor
        self.run()

    def run(self):
        self.timer = t.Thread(target=self.collect_and_report)
        self.timer.daemon = True
        self.timer.name = "Instana Metric Collection"
        self.timer.start()

    def collect_and_report(self):
        while 1:
            self.process()
            time.sleep(1)

    def process(self):
        if self.sensor.agent.can_send():
            self.snapshot_countdown = self.snapshot_countdown - 1
            ss = None
            cm = self.collect_metrics()

            if self.snapshot_countdown == 0:
                self.snapshot_countdown = self.SNAPSHOT_PERIOD
                ss = self.collect_snapshot()
                md = copy.deepcopy(cm).delta_data(None)
            else:
                md = copy.deepcopy(cm).delta_data(self.last_metrics)

            ed = EntityData(pid=os.getpid(), snapshot=ss, metrics=md)
            url = self.sensor.agent.make_url(a.AGENT_DATA_URL)
            self.sensor.agent.request(url, "POST", ed)
            self.last_metrics = cm.__dict__

    def collect_snapshot(self):
        try:
            s = Snapshot(name=self.sensor.service_name,
                         version=sys.version,
                         rlimit_core=resource.getrlimit(resource.RLIMIT_CORE),
                         rlimit_cpu=resource.getrlimit(resource.RLIMIT_CPU),
                         rlimit_fsize=resource.getrlimit(
                             resource.RLIMIT_FSIZE),
                         rlimit_data=resource.getrlimit(resource.RLIMIT_DATA),
                         rlimit_stack=resource.getrlimit(
                             resource.RLIMIT_STACK),
                         rlimit_rss=resource.getrlimit(resource.RLIMIT_RSS),
                         rlimit_nproc=resource.getrlimit(
                             resource.RLIMIT_NPROC),
                         rlimit_nofile=resource.getrlimit(
                             resource.RLIMIT_NOFILE),
                         rlimit_memlock=resource.getrlimit(
                             resource.RLIMIT_MEMLOCK),
                         rlimit_as=resource.getrlimit(resource.RLIMIT_AS),
                         versions=self.collect_modules())

            return s
        except Exception as e:
            l.error("collect_snapshot: ", str(e))

            return None

    def jsonable(self, value):
        if callable(value):
            result = value()
        elif type(value) is ModuleType:
            result = str(value)
        else:
            result = value
        return result

    def collect_modules(self):
        try:
            m = sys.modules
            r = {}
            for k in m:
                if m[k]:
                    d = m[k].__dict__
                    if "version" in d and d["version"]:
                        r[k] = self.jsonable(d["version"])
                    elif "__version__" in d and d["__version__"]:
                        r[k] = self.jsonable(d["__version__"])
                    else:
                        r[k] = "builtin"

            return r
        except Exception as e:
            l.error("collect_modules: ", e)

            return None

    def collect_metrics(self):
        u = resource.getrusage(resource.RUSAGE_SELF)
        if gc_.isenabled():
            c = list(gc_.get_count())
            th = list(gc_.get_threshold())
            g = GC(collect0=c[0] if not self.last_collect else c[0] - self.last_collect[0],
                   collect1=c[1] if not self.last_collect else c[
                       1] - self.last_collect[1],
                   collect2=c[2] if not self.last_collect else c[
                       2] - self.last_collect[2],
                   threshold0=th[0],
                   threshold1=th[1],
                   threshold2=th[2])

        thr = t.enumerate()
        daemon_threads = len([tr.daemon and tr.is_alive() for tr in thr])
        alive_threads = len([not tr.daemon and tr.is_alive() for tr in thr])
        dead_threads = len([not tr.is_alive() for tr in thr])

        m = Metrics(ru_utime=u[0] if not self.last_usage else u[0] - self.last_usage[0],
                    ru_stime=u[1] if not self.last_usage else u[1] - self.last_usage[1],
                    ru_maxrss=u[2],
                    ru_ixrss=u[3],
                    ru_idrss=u[4],
                    ru_isrss=u[5],
                    ru_minflt=u[6] if not self.last_usage else u[6] - self.last_usage[6],
                    ru_majflt=u[7] if not self.last_usage else u[7] - self.last_usage[7],
                    ru_nswap=u[8] if not self.last_usage else u[8] - self.last_usage[8],
                    ru_inblock=u[9] if not self.last_usage else u[9] - self.last_usage[9],
                    ru_oublock=u[10] if not self.last_usage else u[10] - self.last_usage[10],
                    ru_msgsnd=u[11] if not self.last_usage else u[11] - self.last_usage[11],
                    ru_msgrcv=u[12] if not self.last_usage else u[12] - self.last_usage[12],
                    ru_nsignals=u[13] if not self.last_usage else u[13] - self.last_usage[13],
                    ru_nvcs=u[14] if not self.last_usage else u[14] - self.last_usage[14],
                    ru_nivcsw=u[15] if not self.last_usage else u[15] - self.last_usage[15],
                    alive_threads=alive_threads,
                    dead_threads=dead_threads,
                    daemon_threads=daemon_threads,
                    gc=g)

        self.last_usage = u
        if gc_.isenabled():
            self.last_collect = c

        return m
<|endoftext|>


<commit_before>"""K-Nearest neighbors code
"""

import numpy as np
import matplotlib.pyplot as plt
import warnings
from matplotlib import style
from collections import Counter

style.use('fivethirtyeight')

dataset = {'k': [[1, 2], [2, 3], [3, 1]], 'r': [[6, 5], [7, 7], [8, 6]]}
new_features = [5, 7]

def k_nearest_neighbors(data, predict, k=3):
    if len(data) >= k:
        warnings.warn('K set to value less than total voting groups ')
    distances = []
    for group in data:
        for features in data[group]:
            euclidean_distance = np.linalg.norm(np.array(features) - np.array(predict))
            distances.append([euclidean_distance, group])
    votes = [i[1] for i in sorted(distances)[:k]]
    vote_result = Counter(votes).most_common(1)[0][0]
    return vote_result

result = k_nearest_neighbors(dataset, new_features, k=3)
print(result)

[[plt.scatter(ii[0], ii[1], s=100, color=i) for ii in dataset[i]] for i in dataset]
plt.scatter(new_features[0], new_features[1], s=200, color=result)
plt.show()
<commit_msg>add test on the same data used in sci-kit learn k-nearest<commit_after>"""K-Nearest neighbors code
"""

import numpy as np
import warnings
from collections import Counter
import pandas as pd
import random

def k_nearest_neighbors(data, predict, k=3):
    if len(data) >= k:
        warnings.warn('K set to value less than total voting groups ')
    distances = []
    for groups in data:
        for features in data[groups]:
            euclidean_distance = np.linalg.norm(np.array(features) - np.array(predict))
            distances.append([euclidean_distance, groups])
    votes = [idx[1] for idx in sorted(distances)[:k]]
    vote_result = Counter(votes).most_common(1)[0][0]
    return vote_result

# read data, replace missing values, drop useless id column
df = pd.read_csv('breast-cancer-wisconsin.data')
df.replace('?', -99999, inplace=True)
df.drop(['id'], 1, inplace=True)

# Convert to list making sure values are float (some data seems to be strings)
full_data = df.astype(float).values.tolist()

random.shuffle(full_data)

# Split data
test_size = 0.2
train_set = {2: [], 4: []}
test_set = {2: [], 4: []}

train_data = full_data[:-int(test_size*len(full_data))]
test_data = full_data[-int(test_size*len(full_data)):]

for i in train_data:
    train_set[i[-1]].append(i[:-1])

for i in test_data:
    test_set[i[-1]].append(i[:-1])

correct = 0
total = 0

for group in test_set:
    for info in test_set[group]:
        vote = k_nearest_neighbors(train_set, info, 5)
        if group == vote:
            correct += 1
        total += 1

print('Accuracy: ', correct/total)
<|endoftext|>


<commit_before>__version__ = '0.4.5'
<commit_msg>Update version<commit_after>__version__ = '0.4.6'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from odoo import models, fields, api
import hashlib,base64, httplib2
import json, urllib
from odoo.tools.safe_eval import safe_eval
from odoo.exceptions import UserError
import pdfkit
import tempfile, os

class express_menu_config(models.Model):
    """
    快递面单设置 目前只支持 下面这几个快递
    EMS 顺丰 圆通 百世 中通 韵达 申通 德邦 宅急送 京东 信丰 全峰 跨越速运 安能 快捷 天天 国通 优速
    """
    _name = 'express.menu.config'
    name = fields.Char(u'快递公司名称', required=True)
    abbreviation = fields.Char(u'简称', required=True)
    customername = fields.Char(u'客户名称')
    customerpwd = fields.Char(u'客户密码')
    monthcode = fields.Char(u'月结号')
    sendsite = fields.Char(u'发送站点')
    logisticcode = fields.Char(u'快递单号')

class warehouse(models.Model):
    _name = 'warehouse'
    _inherit = ['warehouse', 'state.city.county']
    detail_address = fields.Char(u'详细地址')
    principal_id = fields.Many2one('staff', u'负责人')

class wh_move(models.Model):
    """
    生成快递电子面单
    """
    _name = 'wh.move'
    _inherit = ['wh.move', 'state.city.county']
    express_menu = fields.Text(u'快递面单', copy=False)
    express_code = fields.Char(u'快递单号', copy=False)

    def get_shipping_type_config(self, menu_type):
        """
        根据传入的快递方式返回相应的参数
        """
        express_menu = self.env['express.menu.config'].search([('abbreviation', '=', menu_type)])
        if not express_menu:
            raise UserError("单据%s,承运商暂不支持 或者承运商商户简称输入错误(%s)"%(self.name, menu_type))
        shipping_type_config = dict(ShipperCode=menu_type,
                                    CustomerName=express_menu.customername or '',
                                    CustomerPwd=express_menu.customerpwd or '',
                                    MonthCode=express_menu.monthcode or '',
                                    SendSite=express_menu.sendsite or '',
                                    )
        return shipping_type_config

    def get_sender(self, ware_hosue_row):
        sender = dict(Company=ware_hosue_row.company_id.name,
                      Name=ware_hosue_row.principal_id.name or ware_hosue_row.company_id.name or '',
                      Mobile=ware_hosue_row.principal_id.work_phone or
                      ware_hosue_row.company_id.phone,
                      ProvinceName=ware_hosue_row.province_id.name  or '上海',
                      CityName=ware_hosue_row.city_id.city_name or '上海',
                      ExpAreaName=ware_hosue_row.county_id.county_name  or '浦东新区',
                      Address=ware_hosue_row.detail_address or '金海路2588号B-213')
        return sender

    def get_receiver_goods_message(self):
        #TODO: 要发货的详细地址字段 (有疑问)
        ORIGIN_EXPLAIN = {
            'wh.internal': 'wh.internal',
            'wh.out.others': 'wh.out',
            'buy.receipt.return': 'buy.receipt',
            'sell.delivery.sell': 'sell.delivery',
        }
        address = False
        if ORIGIN_EXPLAIN.get(self.origin):
            model_row = self.env[ORIGIN_EXPLAIN.get(self.origin)
                                ].search([('sell_move_id', '=', self.id)])
            address = model_row.address
            mobile = model_row.mobile
        receiver = dict(Company=' ',
                        Name=self.partner_id.name,
                        Mobile=mobile or self.partner_id.main_mobile,
                        ProvinceName=self.province_id.name  or '上海',
                        CityName=self.city_id.city_name or '上海',
                        ExpAreaName=self.county_id.county_name or '浦东新区',
                        Address=address or '金海路2588号B-213')

        goods = []
        qty = 0
        for line in self.line_out_ids:
            goods.append(dict(GoodsName=line.goods_id.name, #产品名称
                              Goodsquantity=int(line.goods_qty), #产品数量
                              GoodsWeight=1.0, #产品重量
                              GoodsCode=line.goods_id.code or '', # 产品编码
                              GoodsPrice=0.0, #产品价格
                             ))
            qty += 1
        return receiver, goods, qty

    @api.model
    def get_express_menu(self):
        expressconfigparam = self.env['ir.config_parameter']
        appid = expressconfigparam.get_param('express_menu_app_id', default='')
        appkey = expressconfigparam.get_param('express_menu_app_key', default='')
        path = expressconfigparam.get_param('express_menu_oder_url', default='')
        header = safe_eval(expressconfigparam.get_param('express_menu_request_headers',
                                                        default=''))
        order_code = self.name
        sender = self.get_sender(self.warehouse_id)
        remark = self.note or '小心轻放'
        shipping_type = self.express_type or 'YTO'
        receiver, commodity, qty = self.get_receiver_goods_message()
        request_data = dict(OrderCode=order_code, PayType=1, ExpType=1, Cost=1.0, OtherCost=1.0,
                            Sender=sender, Receiver=receiver, Commodity=commodity, Weight=1.0,
                            Quantity=qty, Volume=0.0, Remark=remark, IsReturnPrintTemplate=1)
        request_data.update(self.get_shipping_type_config(shipping_type))
        request_data = json.dumps(request_data)
        data = {'RequestData': request_data,
                'EBusinessID': appid,
                'RequestType': '1007',
                'DataType': '2',
                'DataSign': self.encrypt_kdn(request_data, appkey)}
        http = httplib2.Http()
        response, content = http.request(path, 'POST', headers=header, body=urllib.urlencode(data))
        content = content.replace('true', 'True').replace('false', 'False')
        self.express_code = (safe_eval(content).get('Order', {})).get('LogisticCode', "")
        self.express_menu = str(safe_eval(content).get('PrintTemplate'))
        if not self.express_code:
            raise UserError("获取快递面单失败!\n原因:%s"%str(content))
        return str(safe_eval(content).get('PrintTemplate'))

    def encrypt_kdn(self, data, appkey):
        """
        数据加密
        """
        key = base64.b64encode(hashlib.md5("%s%s" % (data, appkey)).hexdigest(), altchars=None)
        return urllib.quote(key, safe='/')

    @api.model
    def get_moves_html(self, move_ids):
        move_rows = self.browse(move_ids)
        return_html_list = []
        for move_row in move_rows:
            if move_row.express_code:
                return_html_list.append(move_row.express_menu)
            else:
                return_html_list.append(move_row.get_express_menu())
        return return_html_list
        <commit_msg>[FIX]ci fail<commit_after># -*- coding: utf-8 -*-

from odoo import models, fields, api
import hashlib,base64, httplib2
import json, urllib
from odoo.tools.safe_eval import safe_eval
from odoo.exceptions import UserError
import pdfkit
import tempfile, os

class express_menu_config(models.Model):
    """
    快递面单设置 目前只支持 下面这几个快递
    EMS 顺丰 圆通 百世 中通 韵达 申通 德邦 宅急送 京东 信丰 全峰 跨越速运 安能 快捷 天天 国通 优速
    """
    _name = 'express.menu.config'
    name = fields.Char(u'快递公司名称', required=True)
    abbreviation = fields.Char(u'简称', required=True)
    customername = fields.Char(u'客户名称')
    customerpwd = fields.Char(u'客户密码')
    monthcode = fields.Char(u'月结号')
    sendsite = fields.Char(u'发送站点')
    logisticcode = fields.Char(u'快递单号')

class warehouse(models.Model):
    _name = 'warehouse'
    _inherit = ['warehouse', 'state.city.county']
    detail_address = fields.Char(u'详细地址')
    principal_id = fields.Many2one('staff', u'负责人')

class wh_move(models.Model):
    """
    生成快递电子面单
    """
    _name = 'wh.move'
    _inherit = ['wh.move', 'state.city.county']
    express_menu = fields.Text(u'快递面单', copy=False)
    express_code = fields.Char(u'快递单号', copy=False)

    def get_shipping_type_config(self, menu_type):
        """
        根据传入的快递方式返回相应的参数
        """
        express_menu = self.env['express.menu.config'].search([('abbreviation', '=', menu_type)])
        if not express_menu:
            raise UserError(u"单据%s,承运商暂不支持 或者承运商商户简称输入错误(%s)"%(self.name, menu_type))
        shipping_type_config = dict(ShipperCode=menu_type,
                                    CustomerName=express_menu.customername or '',
                                    CustomerPwd=express_menu.customerpwd or '',
                                    MonthCode=express_menu.monthcode or '',
                                    SendSite=express_menu.sendsite or '',
                                    )
        return shipping_type_config

    def get_sender(self, ware_hosue_row):
        sender = dict(Company=ware_hosue_row.company_id.name,
                      Name=ware_hosue_row.principal_id.name or ware_hosue_row.company_id.name or '',
                      Mobile=ware_hosue_row.principal_id.work_phone or
                      ware_hosue_row.company_id.phone,
                      ProvinceName=ware_hosue_row.province_id.name  or '上海',
                      CityName=ware_hosue_row.city_id.city_name or '上海',
                      ExpAreaName=ware_hosue_row.county_id.county_name  or '浦东新区',
                      Address=ware_hosue_row.detail_address or '金海路2588号B-213')
        return sender

    def get_receiver_goods_message(self):
        #TODO: 要发货的详细地址字段 (有疑问)
        ORIGIN_EXPLAIN = {
            'wh.internal': 'wh.internal',
            'wh.out.others': 'wh.out',
            'buy.receipt.return': 'buy.receipt',
            'sell.delivery.sell': 'sell.delivery',
        }
        address = False
        if ORIGIN_EXPLAIN.get(self.origin):
            model_row = self.env[ORIGIN_EXPLAIN.get(self.origin)
                                ].search([('sell_move_id', '=', self.id)])
            address = model_row.address
            mobile = model_row.mobile
        receiver = dict(Company=' ',
                        Name=self.partner_id.name,
                        Mobile=mobile or self.partner_id.main_mobile,
                        ProvinceName=self.province_id.name  or '上海',
                        CityName=self.city_id.city_name or '上海',
                        ExpAreaName=self.county_id.county_name or '浦东新区',
                        Address=address or '金海路2588号B-213')

        goods = []
        qty = 0
        for line in self.line_out_ids:
            goods.append(dict(GoodsName=line.goods_id.name, #产品名称
                              Goodsquantity=int(line.goods_qty), #产品数量
                              GoodsWeight=1.0, #产品重量
                              GoodsCode=line.goods_id.code or '', # 产品编码
                              GoodsPrice=0.0, #产品价格
                             ))
            qty += 1
        return receiver, goods, qty

    @api.model
    def get_express_menu(self):
        expressconfigparam = self.env['ir.config_parameter']
        appid = expressconfigparam.get_param('express_menu_app_id', default='')
        appkey = expressconfigparam.get_param('express_menu_app_key', default='')
        path = expressconfigparam.get_param('express_menu_oder_url', default='')
        header = safe_eval(expressconfigparam.get_param('express_menu_request_headers',
                                                        default=''))
        order_code = self.name
        sender = self.get_sender(self.warehouse_id)
        remark = self.note or '小心轻放'
        shipping_type = self.express_type or 'YTO'
        receiver, commodity, qty = self.get_receiver_goods_message()
        request_data = dict(OrderCode=order_code, PayType=1, ExpType=1, Cost=1.0, OtherCost=1.0,
                            Sender=sender, Receiver=receiver, Commodity=commodity, Weight=1.0,
                            Quantity=qty, Volume=0.0, Remark=remark, IsReturnPrintTemplate=1)
        request_data.update(self.get_shipping_type_config(shipping_type))
        request_data = json.dumps(request_data)
        data = {'RequestData': request_data,
                'EBusinessID': appid,
                'RequestType': '1007',
                'DataType': '2',
                'DataSign': self.encrypt_kdn(request_data, appkey)}
        http = httplib2.Http()
        response, content = http.request(path, 'POST', headers=header, body=urllib.urlencode(data))
        content = content.replace('true', 'True').replace('false', 'False')
        self.express_code = (safe_eval(content).get('Order', {})).get('LogisticCode', "")
        self.express_menu = str(safe_eval(content).get('PrintTemplate'))
        if not self.express_code:
            raise UserError(u"获取快递面单失败!\n原因:%s"%str(content))
        return str(safe_eval(content).get('PrintTemplate'))

    def encrypt_kdn(self, data, appkey):
        """
        数据加密
        """
        key = base64.b64encode(hashlib.md5("%s%s" % (data, appkey)).hexdigest(), altchars=None)
        return urllib.quote(key, safe='/')

    @api.model
    def get_moves_html(self, move_ids):
        move_rows = self.browse(move_ids)
        return_html_list = []
        for move_row in move_rows:
            if move_row.express_code:
                return_html_list.append(move_row.express_menu)
            else:
                return_html_list.append(move_row.get_express_menu())
        return return_html_list
        <|endoftext|>