<commit_before># -*- coding: utf-8 -*-

"""Test settings."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os.path as op

from pytest import raises, yield_fixture, mark

from ..settings import (BaseSettings,
                        Settings,
                        _load_default_settings,
                        _recursive_dirs,
                        _phy_user_dir,
                        )

#------------------------------------------------------------------------------
# Fixtures
#------------------------------------------------------------------------------

@yield_fixture(params=['py', 'json'])
def settings(request):
    if request.param == 'py':
        yield ('py', '''a = 4\nb = 5\nd = {'k1': 2, 'k2': '3'}''')
    elif request.param == 'json':
        yield ('json', '''{"a": 4, "b": 5, "d": {"k1": 2, "k2": "3"}}''')

#------------------------------------------------------------------------------
# Test settings
#------------------------------------------------------------------------------

def test_phy_user_dir():
    assert op.exists(_phy_user_dir())

def test_recursive_dirs():
    dirs = list(_recursive_dirs())
    assert len(dirs) >= 5
    root = op.join(op.realpath(op.dirname(__file__)), '../../')
    for dir in dirs:
        dir = op.relpath(dir, root)
        assert '.' not in dir
        assert '_' not in dir

def test_load_default_settings():
    settings = _load_default_settings()
    keys = settings.keys()
    # assert 'log_file_level' in keys
    # assert 'on_open' in keys
    # assert 'spikedetekt' in keys
    # assert 'klustakwik2' in keys
    assert 'traces' in keys
    # assert 'cluster_manual_config' in keys

def test_base_settings():
    s = BaseSettings()

    # Namespaces are mandatory.
    with raises(KeyError):
        s['a']

    s['a'] = 3
    assert s['a'] == 3

def test_base_settings_file(tempdir, settings):
    ext, settings = settings
    path = op.join(tempdir, 'test.' + ext)
    with open(path, 'w') as f:
        f.write(settings)

    s = BaseSettings()

    s['a'] = 3
    s['c'] = 6
    assert s['a'] == 3

    s.load(path=None)

    # Now, load the settings file.
    s.load(path=path)
    assert s['a'] == 4
    assert s['b'] == 5
    assert s['c'] == 6
    assert s['d'] == {'k1': 2, 'k2': '3'}

    s = BaseSettings()
    s['d'] = {'k2': 30, 'k3': 40}
    s.load(path=path)
    assert s['d'] == {'k1': 2, 'k2': '3', 'k3': 40}

def test_base_settings_invalid(tempdir, settings):
    ext, settings = settings
    settings = settings[:-2]
    path = op.join(tempdir, 'test.' + ext)
    with open(path, 'w') as f:
        f.write(settings)

    s = BaseSettings()
    s.load(path)
    assert 'a' not in s

def test_internal_settings(tempdir):
    path = op.join(tempdir, 'test.json')

    s = BaseSettings()

    # Set the 'test' namespace.
    s['a'] = 3
    s['c'] = 6
    assert s['a'] == 3
    assert s['c'] == 6

    s.save(path)
    assert s['a'] == 3
    assert s['c'] == 6

    s = BaseSettings()
    with raises(KeyError):
        s['a']

    s.load(path)
    assert s['a'] == 3
    assert s['c'] == 6

def test_settings_nodir():
    Settings()

def test_settings_manager(tempdir, tempdir_bis):
    tempdir_exp = tempdir_bis
    sm = Settings(tempdir)

    # Check paths.
    assert sm.phy_user_dir == tempdir
    assert sm.internal_settings_path == op.join(tempdir,
                                                'internal_settings.json')
    assert sm.user_settings_path == op.join(tempdir, 'user_settings.py')

    # User settings.
    with raises(KeyError):
        sm['a']
    assert sm.get('a', None) is None
    # Artificially populate the user settings.
    sm._bs._store['a'] = 3
    assert sm['a'] == 3
    assert sm.get('a') == 3

    # Internal settings.
    sm['c'] = 5
    assert sm['c'] == 5

    # Set an experiment path.
    path = op.join(tempdir_exp, 'myexperiment.dat')
    sm.on_open(path)
    assert op.realpath(sm.exp_path) == op.realpath(path)
    assert sm.exp_name == 'myexperiment'
    assert (op.realpath(sm.exp_settings_dir) ==
            op.realpath(op.join(tempdir_exp, 'myexperiment.phy')))
    assert (op.realpath(sm.exp_settings_path) ==
            op.realpath(op.join(tempdir_exp, 'myexperiment.phy/'
                                             'user_settings.py')))

    # User settings.
    assert sm['a'] == 3
    sm._bs._store['a'] = 30
    assert sm['a'] == 30

    # Internal settings.
    sm['c'] = 50
    assert sm['c'] == 50

    # Check persistence.
    sm.save()
    sm = Settings(tempdir)
    sm.on_open(path)
    assert sm['c'] == 50
    assert 'a' not in sm

    assert len(sm.keys()) >= 10
    assert str(sm).startswith('<Settings')
<commit_msg>Increase coverage in settings<commit_after># -*- coding: utf-8 -*-

"""Test settings."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os.path as op

from pytest import raises, yield_fixture, mark

from ..settings import (BaseSettings,
                        Settings,
                        _load_default_settings,
                        _recursive_dirs,
                        _phy_user_dir,
                        )

#------------------------------------------------------------------------------
# Fixtures
#------------------------------------------------------------------------------

@yield_fixture(params=['py', 'json'])
def settings(request):
    if request.param == 'py':
        yield ('py', '''a = 4\nb = 5\nd = {'k1': 2, 'k2': '3'}''')
    elif request.param == 'json':
        yield ('json', '''{"a": 4, "b": 5, "d": {"k1": 2, "k2": "3"}}''')

#------------------------------------------------------------------------------
# Test settings
#------------------------------------------------------------------------------

def test_phy_user_dir():
    assert op.exists(_phy_user_dir())

def test_recursive_dirs():
    dirs = list(_recursive_dirs())
    assert len(dirs) >= 5
    root = op.join(op.realpath(op.dirname(__file__)), '../../')
    for dir in dirs:
        dir = op.relpath(dir, root)
        assert '.' not in dir
        assert '_' not in dir

def test_load_default_settings():
    settings = _load_default_settings()
    keys = settings.keys()
    # assert 'log_file_level' in keys
    # assert 'on_open' in keys
    # assert 'spikedetekt' in keys
    # assert 'klustakwik2' in keys
    assert 'traces' in keys
    # assert 'cluster_manual_config' in keys

def test_base_settings():
    s = BaseSettings()

    # Namespaces are mandatory.
    with raises(KeyError):
        s['a']

    s['a'] = 3
    assert s['a'] == 3

def test_base_settings_wrong_extension(tempdir):
    path = op.join(tempdir, 'test')
    with open(path, 'w'):
        pass
    s = BaseSettings()
    s.load(path=path)

def test_base_settings_file(tempdir, settings):
    ext, settings = settings
    path = op.join(tempdir, 'test.' + ext)
    with open(path, 'w') as f:
        f.write(settings)

    s = BaseSettings()

    s['a'] = 3
    s['c'] = 6
    assert s['a'] == 3

    # Warning: wrong path.
    s.load(path=None)

    # Now, load the settings file.
    s.load(path=path)
    assert s['a'] == 4
    assert s['b'] == 5
    assert s['c'] == 6
    assert s['d'] == {'k1': 2, 'k2': '3'}

    s = BaseSettings()
    s['d'] = {'k2': 30, 'k3': 40}
    s.load(path=path)
    assert s['d'] == {'k1': 2, 'k2': '3', 'k3': 40}

def test_base_settings_invalid(tempdir, settings):
    ext, settings = settings
    settings = settings[:-2]
    path = op.join(tempdir, 'test.' + ext)
    with open(path, 'w') as f:
        f.write(settings)

    s = BaseSettings()
    s.load(path)
    assert 'a' not in s

def test_internal_settings(tempdir):
    path = op.join(tempdir, 'test.json')

    s = BaseSettings()

    # Set the 'test' namespace.
    s['a'] = 3
    s['c'] = 6
    assert s['a'] == 3
    assert s['c'] == 6

    s.save(path)
    assert s['a'] == 3
    assert s['c'] == 6

    s = BaseSettings()
    with raises(KeyError):
        s['a']

    s.load(path)
    assert s['a'] == 3
    assert s['c'] == 6

def test_settings_nodir():
    Settings()

def test_settings_manager(tempdir, tempdir_bis):
    tempdir_exp = tempdir_bis
    sm = Settings(tempdir)

    # Check paths.
    assert sm.phy_user_dir == tempdir
    assert sm.internal_settings_path == op.join(tempdir,
                                                'internal_settings.json')
    assert sm.user_settings_path == op.join(tempdir, 'user_settings.py')

    # User settings.
    with raises(KeyError):
        sm['a']
    assert sm.get('a', None) is None
    # Artificially populate the user settings.
    sm._bs._store['a'] = 3
    assert sm['a'] == 3
    assert sm.get('a') == 3

    # Internal settings.
    sm['c'] = 5
    assert sm['c'] == 5

    # Set an experiment path.
    path = op.join(tempdir_exp, 'myexperiment.dat')
    sm.on_open(path)
    assert op.realpath(sm.exp_path) == op.realpath(path)
    assert sm.exp_name == 'myexperiment'
    assert (op.realpath(sm.exp_settings_dir) ==
            op.realpath(op.join(tempdir_exp, 'myexperiment.phy')))
    assert (op.realpath(sm.exp_settings_path) ==
            op.realpath(op.join(tempdir_exp, 'myexperiment.phy/'
                                             'user_settings.py')))

    # User settings.
    assert sm['a'] == 3
    sm._bs._store['a'] = 30
    assert sm['a'] == 30

    # Internal settings.
    sm['c'] = 50
    assert sm['c'] == 50

    # Check persistence.
    sm.save()
    sm = Settings(tempdir)
    sm.on_open(path)
    assert sm['c'] == 50
    assert 'a' not in sm

    assert len(sm.keys()) >= 10
    assert str(sm).startswith('<Settings')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# pylint: disable=W0212

from agate import utils

def rename(self, column_names=None, row_names=None, slug_columns=False, slug_rows=False, **kwargs):
    """
    Create a copy of this table with different column names or row names.

    By enabling :code:`slug_columns` or :code:`slug_rows` and not specifying
    new names you may slugify the table's existing names.

    :code:`kwargs` will be passed through to
    `awesome-slugify's <https://github.com/dimka665/awesome-slugify>`_
    :code:`UniqueSlugify` class.

    :param column_names:
        New column names for the renamed table. May be either an array or
        a dictionary mapping existing column names to new names. If not
        specified, will use this table's existing column names.
    :param row_names:
        New row names for the renamed table. May be either an array or
        a dictionary mapping existing row names to new names. If not
        specified, will use this table's existing row names.
    :param slug_columns:
        If True, column names will be converted to slugs and duplicate names
        will have unique identifiers appended.
    :param slug_rows:
        If True, row names will be converted to slugs and dupicate names will
        have unique identifiers appended.
    """
    from agate.table import Table

    if isinstance(column_names, dict):
        column_names = [column_names[name] if name in column_names else name for name in self._column_names]

    if isinstance(row_names, dict):
        row_names = [row_names[name] if name in row_names else name for name in self._row_names]

    if slug_columns:
        column_names = column_names or self._column_names

        if column_names is not None:
            column_names = utils.slugify(column_names, ensure_unique=True, **kwargs)

    if slug_rows:
        row_names = row_names or self.row_names

        if row_names is not None:
            row_names = utils.slugify(row_names, ensure_unique=True, **kwargs)

    if column_names is not None and column_names != self._column_names:
        if row_names is None:
            row_names = self._row_names

        return Table(self._rows, column_names, self._column_types, row_names=row_names, _is_fork=False)
    else:
        return self._fork(self._rows, column_names, self._column_types, row_names=row_names)
<commit_msg>Remove extra deduplicate call.<commit_after>#!/usr/bin/env python
# pylint: disable=W0212

from agate import utils

def rename(self, column_names=None, row_names=None, slug_columns=False, slug_rows=False, **kwargs):
    """
    Create a copy of this table with different column names or row names.

    By enabling :code:`slug_columns` or :code:`slug_rows` and not specifying
    new names you may slugify the table's existing names.

    :code:`kwargs` will be passed through to
    `awesome-slugify's <https://github.com/dimka665/awesome-slugify>`_
    :code:`UniqueSlugify` class.

    :param column_names:
        New column names for the renamed table. May be either an array or
        a dictionary mapping existing column names to new names. If not
        specified, will use this table's existing column names.
    :param row_names:
        New row names for the renamed table. May be either an array or
        a dictionary mapping existing row names to new names. If not
        specified, will use this table's existing row names.
    :param slug_columns:
        If True, column names will be converted to slugs and duplicate names
        will have unique identifiers appended.
    :param slug_rows:
        If True, row names will be converted to slugs and dupicate names will
        have unique identifiers appended.
    """
    from agate.table import Table

    if isinstance(column_names, dict):
        column_names = [column_names[name] if name in column_names else name for name in self._column_names]

    if isinstance(row_names, dict):
        row_names = [row_names[name] if name in row_names else name for name in self._row_names]

    if slug_columns:
        column_names = column_names or self._column_names

        if column_names is not None:
            column_names = utils.slugify(column_names, ensure_unique=False, **kwargs)

    if slug_rows:
        row_names = row_names or self.row_names

        if row_names is not None:
            row_names = utils.slugify(row_names, ensure_unique=True, **kwargs)

    if column_names is not None and column_names != self._column_names:
        if row_names is None:
            row_names = self._row_names

        return Table(self._rows, column_names, self._column_types, row_names=row_names, _is_fork=False)
    else:
        return self._fork(self._rows, column_names, self._column_types, row_names=row_names)
<|endoftext|>


<commit_before>"""
Django settings for TinhatDyndnsServer project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '^59+5!vq0%fv#+e!j7yn^y+2p0dl8pt3t^-oi4$1rjskax-gqm'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'restapi',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'restapi.error.ErrorMiddleware',
)

ROOT_URLCONF = 'TinhatDyndnsServer.urls'

WSGI_APPLICATION = 'TinhatDyndnsServer.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = False

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'

# Settings for the dyndns service
NSD_CONTROL_PATH = '/usr/local/sbin/nsd-control'
TEMPLATE_DIRS  = (os.path.join(os.path.abspath(os.path.dirname(__name__)), 'templates').replace(os.path.sep, '/'),)
ZONE_TEMPLATE = 'template.zone'
ZONES_DIRECTORY = os.path.join(os.path.abspath(os.path.dirname(__name__)), 'zones')
ZONES_PATTERN = 'dynamic'

GPG_PATH = '"C:\\Program Files (x86)\\GNU\\GnuPG\\gpg.exe"'
KEYS_DIRECTORY = os.path.join(os.path.abspath(os.path.dirname(__name__)), 'keys')<commit_msg>- ignore nsd-control in local settings<commit_after>"""
Django settings for TinhatDyndnsServer project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '^59+5!vq0%fv#+e!j7yn^y+2p0dl8pt3t^-oi4$1rjskax-gqm'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'restapi',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'restapi.error.ErrorMiddleware',
)

ROOT_URLCONF = 'TinhatDyndnsServer.urls'

WSGI_APPLICATION = 'TinhatDyndnsServer.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = False

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'

# Settings for the dyndns service
NSD_CONTROL_PATH = None
TEMPLATE_DIRS  = (os.path.join(os.path.abspath(os.path.dirname(__name__)), 'templates').replace(os.path.sep, '/'),)
ZONE_TEMPLATE = 'template.zone'
ZONES_DIRECTORY = os.path.join(os.path.abspath(os.path.dirname(__name__)), 'zones')
ZONES_PATTERN = 'dynamic'

GPG_PATH = '"C:\\Program Files (x86)\\GNU\\GnuPG\\gpg.exe"'
KEYS_DIRECTORY = os.path.join(os.path.abspath(os.path.dirname(__name__)), 'keys')<|endoftext|>


<commit_before># (C) Datadog, Inc. 2018
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)

class DatadogAgentStub(object):
    def __init__(self):
        self._metadata = {}

    def reset(self):
        self._metadata = {}

    def assert_metadata(self, check_id, data):
        for name, value in data.items():
            assert self._metadata[(check_id, name)] == value

    def assert_metadata_count(self, count):
        assert len(self._metadata) == count

    def get_hostname(self):
        return 'stubbed.hostname'

    def get_config(self, *args, **kwargs):
        return ''

    def get_version(self):
        return '0.0.0'

    def log(self, *args, **kwargs):
        pass

    def set_check_metadata(self, check_id, name, value):
        self._metadata[(check_id, name)] = value

    def set_external_tags(self, *args, **kwargs):
        pass

    def tracemalloc_enabled(self, *args, **kwargs):
        return False

# Use the stub as a singleton
datadog_agent = DatadogAgentStub()
<commit_msg>Better metadata assertion output (#4953)<commit_after># (C) Datadog, Inc. 2018
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)

class DatadogAgentStub(object):
    def __init__(self):
        self._metadata = {}

    def reset(self):
        self._metadata.clear()

    def assert_metadata(self, check_id, data):
        actual = {}
        for name in data:
            key = (check_id, name)
            if key in self._metadata:
                actual[name] = self._metadata[key]
        assert data == actual

    def assert_metadata_count(self, count):
        assert len(self._metadata) == count

    def get_hostname(self):
        return 'stubbed.hostname'

    def get_config(self, *args, **kwargs):
        return ''

    def get_version(self):
        return '0.0.0'

    def log(self, *args, **kwargs):
        pass

    def set_check_metadata(self, check_id, name, value):
        self._metadata[(check_id, name)] = value

    def set_external_tags(self, *args, **kwargs):
        pass

    def tracemalloc_enabled(self, *args, **kwargs):
        return False

# Use the stub as a singleton
datadog_agent = DatadogAgentStub()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix)
    Copyright (C) 2019 Smeulf (original implementation module)
    Operations for changing the playback status

    SPDX-License-Identifier: MIT
    See LICENSES/MIT.md for more information.
"""
from __future__ import absolute_import, division, unicode_literals

import xbmc

import resources.lib.common as common

from .action_manager import ActionManager

class AMPlayback(ActionManager):
    """Operations for changing the playback status"""

    SETTING_ID = 'ResumeManager_enabled'

    def __init__(self):
        super(AMPlayback, self).__init__()
        self.resume_position = None
        self.enabled = True

    def __str__(self):
        return 'enabled={}'.format(self.enabled)

    def initialize(self, data):
        # Due to a bug on Kodi the resume on SRTM files not works correctly, so we force the skip to the resume point
        self.resume_position = data.get('resume_position')

    def on_playback_started(self, player_state):
        if self.resume_position:
            common.info('AMPlayback has forced resume point to {}', self.resume_position)
            xbmc.Player().seekTime(int(self.resume_position))

    def on_tick(self, player_state):
        pass
<commit_msg>Set time limit for paused playback<commit_after># -*- coding: utf-8 -*-
"""
    Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix)
    Copyright (C) 2019 Smeulf (original implementation module)
    Operations for changing the playback status

    SPDX-License-Identifier: MIT
    See LICENSES/MIT.md for more information.
"""
from __future__ import absolute_import, division, unicode_literals

import time

import xbmc

import resources.lib.common as common

from .action_manager import ActionManager

class AMPlayback(ActionManager):
    """Operations for changing the playback status"""

    SETTING_ID = 'ResumeManager_enabled'

    def __init__(self):
        super(AMPlayback, self).__init__()
        self.resume_position = None
        self.enabled = True
        self.start_time = None
        self.is_player_in_pause = False

    def __str__(self):
        return 'enabled={}'.format(self.enabled)

    def initialize(self, data):
        # Due to a bug on Kodi the resume on SRTM files not works correctly, so we force the skip to the resume point
        self.resume_position = data.get('resume_position')

    def on_playback_started(self, player_state):
        if self.resume_position:
            common.info('AMPlayback has forced resume point to {}', self.resume_position)
            xbmc.Player().seekTime(int(self.resume_position))

    def on_tick(self, player_state):
        # Stops playback when paused for more than one hour.
        # Some users leave the playback paused also for more than 12 hours,
        # this complicates things to resume playback, because the manifest data expires and with it also all
        # the streams urls are no longer guaranteed, so we force the stop of the playback.
        if self.is_player_in_pause and (time.time() - self.start_time) > 3600:
            common.info('The playback has been stopped because it has been exceeded 1 hour of pause')
            common.stop_playback()

    def on_playback_pause(self, player_state):
        self.start_time = time.time()
        self.is_player_in_pause = True

    def on_playback_resume(self, player_state):
        self.is_player_in_pause = False
<|endoftext|>


<commit_before>from app import db, models
from time import strptime
from datetime import datetime
from pytz import utc
from calendar import timegm

import thread

IDS = ["stop_id", "route_id", "service_id", "trip_id", "shape_id"]

def object_for_name(name):
    if name == "Agency":
        return models.Agency()
    elif name == "Stop":
        return models.Stop()
    elif name == "Route":
        return models.Route()
    elif name == "Trip":
        return models.Trip()
    elif name == "StopTime":
        return models.StopTime()
    elif name == "Calendar":
        return models.Calendar()
    elif name == "CalendarDate":
        return models.CalendarDate()
    elif name == "Shape":
        return models.Shape()
    else:
        return None

def load_objects(file, name, agency_name = ""):
    objects = []
    try:
        with open(file, 'r') as f:
            clean_first_line = f.readline().strip().replace(' ', '')
            keys = clean_first_line.split(',')
            for line in f:
                obj = object_for_name(name)
                values = line.strip().split(',')
                if len(values) == 0:
                    continue
                for i, key in enumerate(keys):
                    value = values[i].strip()
                    if key in IDS:
                        value = value + " " + agency_name
                    # handle special cases for setting relationships
                    if name == "StopTime" and key == "trip_id":
                        set_trip_for_stop_time(obj, value)
                    elif name == "StopTime" and key == "arrival_time":
                        obj.arrival_time = datetime_from_string(value)
                    elif name == "StopTime" and key == "departure_time":
                        obj.departure_time = datetime_from_string(value)
                    elif name == "StopTime" and key == "stop_id":
                        set_stop_for_stop_time(obj, value)
                    elif name == "Route" and key == "agency_id":
                        set_agency_for_route(obj, value)
                    elif name == "Trip" and key == "route_id":
                        set_route_for_trip(obj, value)
                    else:
                        if hasattr(obj, key):
                            setattr(obj, key, value)
                objects.append(obj)
            f.close()
    except IndexError:
        print "A value is missing from " + file
    return objects

def datetime_from_string(string):
    hr = int(string.split(":")[0])
    if hr >= 24:
        l = list(string)
        l[0] = "0"
        l[1] = str(hr % 24)
        string = "".join(l)
    def to_datetime_from_utc(time_tuple):
        timestamp = timegm(time_tuple)
        if hr >= 24:
            timestamp  = timestamp + 24*60*60
        return datetime.fromtimestamp(timegm(time_tuple), tz = utc)
    return to_datetime_from_utc(strptime(string, "%H:%M:%S"))

# this will always happen before setting the Stop ID for a stop_time
# due to the ordering of trip_id and stop_id in GTFS
def set_trip_for_stop_time(stop_time, trip_id):
    trip = models.Trip.query.filter(models.Trip.trip_id == trip_id).first()
    if not trip is None:
        stop_time.trip = trip

def set_stop_for_stop_time(stop_time, stop_id):
    stop = models.Stop.query.filter(models.Stop.stop_id == stop_id).first()
    if not stop is None:
        stop_time.stop = stop
        stop_time.stop_lat = stop.stop_lat
        stop_time.stop_lon = stop.stop_lon
        # a stop_time's trip must be set before its stop,
        # in order to link trips to stops
        trip = stop_time.trip
        if not trip is None:
            trip.stops.append(stop)

def set_agency_for_route(route, agency_id):
    agency = models.Agency.query.filter(models.Agency.agency_id == agency_id).first()
    if not agency is None:
        route.agency = agency

def set_route_for_trip(trip, route_id):
    route = models.Route.query.filter(models.Route.route_id == route_id).first()
    if not route is None:
        trip.route = route

def commit_objects(objects):
    for obj in objects:
        db.session.add(obj)
    db.session.commit()

def load_agency(gtfs_path):
    try:
        agencies = load_objects(gtfs_path + "agency.txt", "Agency")
        commit_objects(agencies)
        if len(agencies) > 0:
            print "loaded agency:\t" + agencies[0].agency_name
            return agencies[0]
        else:
            print "agency.txt contained no agencies!"
            return None
    except:
        print "Error in loading agency.txt"
        db.session.rollback()
        return None

def load_stops(gtfs_path, agency_name):
    try:
        stops = load_objects(gtfs_path + "stops.txt", "Stop", agency_name)
        commit_objects(stops)
        print "loaded stops:\t" + agency_name
    except:
        print "Error in loading stops.txt"
        db.session.rollback()

def load_routes(gtfs_path, agency_name):
    try:
        routes = load_objects(gtfs_path + "routes.txt", "Route", agency_name)
        commit_objects(routes)
        print "loaded routes:\t" + agency_name
    except:
        print "Error in loading routes.txt"
        db.session.rollback()

def load_trips(gtfs_path, agency_name):
    try:
        trips = load_objects(gtfs_path + "trips.txt", "Trip", agency_name)
        commit_objects(trips)
        print "loaded trips:\t" + agency_name
    except:
        print "Error in loading trips.txt"
        db.session.rollback()

def load_stop_times(gtfs_path, agency_name):
    print "loading stop_times:\t" + agency_name
    try:
        stop_times = load_objects(gtfs_path + "stop_times.txt", "StopTime", agency_name)
        for st in stop_times:
            db.session.add(st)
        commit_objects(stop_times)
        print "loaded stop_times:\t" + agency_name
    except:
        print "Error in loading stop_times.txt"
        db.session.rollback()

def load_calendar(gtfs_path, agency_name):
    try:
        calendar = load_objects(gtfs_path + "calendar.txt", "Calendar", agency_name)
        commit_objects(calendar)
        print "loaded calendar:\t" + agency_name
    except:
        print "Error in loading calendar.txt"
        db.session.rollback()

def load_calendar_dates(gtfs_path, agency_name):
    try:
        calendar_dates = load_objects(gtfs_path + "calendar_dates.txt", "CalendarDate", agency_name)
        commit_objects(calendar_dates)
        print "loaded calendar_dates:\t" + agency_name
    except:
        print "Error in loading calendar_dates.txt"
        db.session.rollback()

def load_shapes(gtfs_path, agency_name):
    print "loading shapes:\t" + agency_name
    try:
        shapes = load_objects(gtfs_path + "shapes.txt", "Shape", agency_name)
        commit_objects(shapes)
        print "loaded shapes:\t" + agency_name
    except:
        print "Error in loading shapes.txt"
        db.session.rollback()

def load_all(gtfs_path):
    # the order is important (necessary for relationships):
    # agencies must be loaded before routes (before everything else actually)
    agency = load_agency(gtfs_path)
    # routes must be loaded before trips and before shapes
    load_routes(gtfs_path, agency.agency_name)
    # trips must be loaded before stop_times
    load_trips(gtfs_path, agency.agency_name)
    # stops must be loaded before stop_times
    load_stops(gtfs_path, agency.agency_name)
    load_calendar(gtfs_path, agency.agency_name)
    load_calendar_dates(gtfs_path, agency.agency_name)
    load_shapes(gtfs_path, agency.agency_name)
    load_stop_times(gtfs_path, agency.agency_name)
    #thread.start_new_thread(load_shapes, (gtfs_path, agency.agency_name, ))
    #thread.start_new_thread(load_stop_times, (gtfs_path, agency.agency_name, ))

<commit_msg>refactored gtfs_parser<commit_after>from app import db, models
from time import strptime
from datetime import datetime
from pytz import utc
from calendar import timegm

import thread

IDS = ["stop_id", "route_id", "service_id", "trip_id", "shape_id"]

def object_for_name(name):
    if name == "Agency":
        return models.Agency()
    elif name == "Stop":
        return models.Stop()
    elif name == "Route":
        return models.Route()
    elif name == "Trip":
        return models.Trip()
    elif name == "StopTime":
        return models.StopTime()
    elif name == "Calendar":
        return models.Calendar()
    elif name == "CalendarDate":
        return models.CalendarDate()
    elif name == "Shape":
        return models.Shape()
    else:
        return None

def load_objects(file, name, agency_name = ""):
    try:
        with open(file, 'r') as f:
            print "Loading " + name + ":\t" + agency_name
            clean_first_line = f.readline().strip().replace(' ', '')
            keys = clean_first_line.split(',')
            for line in f:
                obj = object_for_name(name)
                values = line.strip().split(',')
                if len(values) == 0:
                    continue
                for i, key in enumerate(keys):
                    value = values[i].strip()
                    if key in IDS:
                        value = value + " " + agency_name
                    # handle special cases for setting relationships
                    if name == "Agency" and key == "agency_name":
                        agency_name = value
                    elif name == "StopTime" and key == "trip_id":
                        set_trip_for_stop_time(obj, value)
                    elif name == "StopTime" and key == "arrival_time":
                        obj.arrival_time = datetime_from_string(value)
                    elif name == "StopTime" and key == "departure_time":
                        obj.departure_time = datetime_from_string(value)
                    elif name == "StopTime" and key == "stop_id":
                        set_stop_for_stop_time(obj, value)
                    elif name == "Route" and key == "agency_id":
                        set_agency_for_route(obj, value)
                    elif name == "Trip" and key == "route_id":
                        set_route_for_trip(obj, value)
                    elif hasattr(obj, key):
                        setattr(obj, key, value)
                db.session.add(obj)
            f.close()
            db.session.commit()
            print "Loaded " + name + ":\t" + agency_name
    except IndexError:
        print "A value is missing from " + file
        db.session.rollback()
    except e:
        print "Error in loading " + file + ":\t" + e.description
        db.session.rollback()
    return agency_name

def datetime_from_string(string):
    hr = int(string.split(":")[0])
    if hr >= 24:
        l = list(string)
        l[0] = "0"
        l[1] = str(hr % 24)
        string = "".join(l)
    def to_datetime_from_utc(time_tuple):
        timestamp = timegm(time_tuple)
        if hr >= 24:
            timestamp  = timestamp + 24*60*60
        return datetime.fromtimestamp(timegm(time_tuple), tz = utc)
    return to_datetime_from_utc(strptime(string, "%H:%M:%S"))

# this will always happen before setting the Stop ID for a stop_time
# due to the ordering of trip_id and stop_id in GTFS
def set_trip_for_stop_time(stop_time, trip_id):
    trip = models.Trip.query.filter(models.Trip.trip_id == trip_id).first()
    if not trip is None:
        stop_time.trip = trip

def set_stop_for_stop_time(stop_time, stop_id):
    stop = models.Stop.query.filter(models.Stop.stop_id == stop_id).first()
    if not stop is None:
        stop_time.stop = stop
        stop_time.stop_lat = stop.stop_lat
        stop_time.stop_lon = stop.stop_lon
        # a stop_time's trip must be set before its stop,
        # in order to link trips to stops
        trip = stop_time.trip
        if not trip is None:
            trip.stops.append(stop)

def set_agency_for_route(route, agency_id):
    agency = models.Agency.query.filter(models.Agency.agency_id == agency_id).first()
    if not agency is None:
        route.agency = agency

def set_route_for_trip(trip, route_id):
    route = models.Route.query.filter(models.Route.route_id == route_id).first()
    if not route is None:
        trip.route = route

def load_all(gtfs_path):
    # the order is important (necessary for relationships):
    # agencies must be loaded before routes (before everything else actually)
    agency_name = load_objects(gtfs_path + "agency.txt", "Agency")
    # routes must be loaded before trips and before shapes
    load_objects(gtfs_path + "routes.txt", "Route", agency_name)
    # trips must be loaded before stop_times
    load_objects(gtfs_path + "trips.txt", "Trip", agency_name)
    # stops must be loaded before stop_times
    load_objects(gtfs_path + "stops.txt", "Stop", agency_name)
    load_objects(gtfs_path + "calendar.txt", "Calendar", agency_name)
    load_objects(gtfs_path + "calendar_dates.txt", "CalendarDate", agency_name)
    load_objects(gtfs_path + "shapes.txt", "Shape", agency_name)
    load_objects(gtfs_path + "stop_times.txt", "StopTime", agency_name)
    #thread.start_new_thread(load_shapes, (gtfs_path, agency.agency_name, ))
    #thread.start_new_thread(load_stop_times, (gtfs_path, agency.agency_name, ))

<|endoftext|>


<commit_before># Copyright (C) 2017 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
"""Handlers columns dealing with user roles."""

from sqlalchemy import and_

from ggrc import db
from ggrc.converters import errors
from ggrc.converters.handlers.handlers import UserColumnHandler
from ggrc.login import get_current_user
from ggrc.models import Context
from ggrc.models import Person
from ggrc_basic_permissions.models import Role
from ggrc_basic_permissions.models import UserRole

class ObjectRoleColumnHandler(UserColumnHandler):
  """Basic User role handler"""

  role = -1
  owner_columns = ("program_owner")

  def parse_item(self):
    """Parse new line separated list of emails

    Returns:
      list of User models.
    """
    users = self.get_users_list()
    if not users and self.key in self.owner_columns:
      self.add_warning(errors.OWNER_MISSING, column_name=self.display_name)
      users.append(get_current_user())

    if self.row_converter.is_new and self.mandatory and not users:
      self.add_error(errors.MISSING_VALUE_ERROR,
                     column_name=self.display_name)
    return list(users)

  def set_obj_attr(self):
    pass

  def get_value(self):
    user_role_ids = db.session.query(UserRole.person_id).filter_by(
        role=self.role,
        context_id=self.row_converter.obj.context_id)
    users = Person.query.filter(Person.id.in_(user_role_ids))
    emails = [user.email for user in users]
    return "\n".join(emails)

  def remove_current_roles(self):
    UserRole.query.filter_by(
        role=self.role,
        context_id=self.row_converter.obj.context_id).delete()

  def insert_object(self):
    if self.dry_run or not self.value:
      return
    self.remove_current_roles()
    for owner in self.value:
      user_role = UserRole(
          role=self.role,
          context=self.row_converter.obj.context,
          person=owner
      )
      db.session.add(user_role)
    self.dry_run = True

class ProgramOwnerColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="ProgramOwner").one()
    super(ProgramOwnerColumnHandler, self).__init__(
        row_converter, key, **options)

class ProgramEditorColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="ProgramEditor").one()
    super(ProgramEditorColumnHandler, self).__init__(
        row_converter, key, **options)

class ProgramReaderColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="ProgramReader").one()
    super(ProgramReaderColumnHandler, self).__init__(
        row_converter, key, **options)

class WorkflowOwnerColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="WorkflowOwner").one()
    super(WorkflowOwnerColumnHandler, self).__init__(
        row_converter, key, **options)

class WorkflowMemberColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="WorkflowMember").one()
    super(WorkflowMemberColumnHandler, self).__init__(
        row_converter, key, **options)

class AuditAuditorColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="Auditor").one()
    self.reader = Role.query.filter_by(name="ProgramReader").one()
    super(AuditAuditorColumnHandler, self).__init__(
        row_converter, key, **options)

  def insert_object(self):
    if self.dry_run or not self.value:
      return
    super(AuditAuditorColumnHandler, self).insert_object()
    user_roles = set(map(lambda ur: ur.person_id, self.get_program_roles()))
    context = self.row_converter.obj.program.context
    for auditor in self.value:
      # Check if the role already exists in the database or in the session:
      if auditor.id in user_roles or any(o for o in db.session.new if
                                         isinstance(o, UserRole) and
                                         o.context.id == context.id and
                                         o.person.id == auditor.id):
        continue
      user_role = UserRole(
          role=self.reader,
          context=context,
          person=auditor
      )
      db.session.add(user_role)

  def get_program_roles(self):
    context_id = self.row_converter.obj.program.context.id
    return db.session.query(UserRole).filter(
        UserRole.context_id == context_id).all()

class UserRoleColumnHandler(UserColumnHandler):

  _role_map = {
      "admin": "administrator",
      "ggrc admin": "administrator",
  }

  _allowed_roles = [
      "Reader",
      "Editor",
      "Administrator",
  ]

  def parse_item(self):
    value = self.raw_value.lower()
    name = self._role_map.get(value, value)
    return Role.query.filter_by(name=name).first()

  def set_obj_attr(self):
    pass

  def get_value(self):
    return self.row_converter.obj.system_wide_role

  def remove_current_roles(self):
    allowed_role_ids = db.session.query(Role.id).filter(
        Role.name.in_(self._allowed_roles))
    UserRole.query.filter(and_(
        UserRole.role_id.in_(allowed_role_ids),
        UserRole.person_id == self.row_converter.obj.id)
    ).delete(synchronize_session="fetch")

  def insert_object(self):
    if self.dry_run or not self.value:
      return
    self.remove_current_roles()
    context = None
    if self.value.name == "Administrator":
      context = Context.query.filter_by(name="System Administration").first()
    user_role = UserRole(
        role=self.value,
        person=self.row_converter.obj,
        context=context,
    )
    db.session.add(user_role)
    self.dry_run = True

COLUMN_HANDLERS = {
    "program_editor": ProgramEditorColumnHandler,
    "program_owner": ProgramOwnerColumnHandler,
    "program_reader": ProgramReaderColumnHandler,
    "user_role": UserRoleColumnHandler,
    "user_role:Auditor": AuditAuditorColumnHandler,
    "workflow_member": WorkflowMemberColumnHandler,
    "workflow_owner": WorkflowOwnerColumnHandler,
}
<commit_msg>Add error message for invalid user role import<commit_after># Copyright (C) 2017 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
"""Handlers columns dealing with user roles."""

from sqlalchemy import and_

from ggrc import db
from ggrc.converters import errors
from ggrc.converters.handlers.handlers import UserColumnHandler
from ggrc.login import get_current_user
from ggrc.models import Context
from ggrc.models import Person
from ggrc_basic_permissions.models import Role
from ggrc_basic_permissions.models import UserRole

class ObjectRoleColumnHandler(UserColumnHandler):
  """Basic User role handler"""

  role = -1
  owner_columns = ("program_owner")

  def parse_item(self):
    """Parse new line separated list of emails

    Returns:
      list of User models.
    """
    users = self.get_users_list()
    if not users and self.key in self.owner_columns:
      self.add_warning(errors.OWNER_MISSING, column_name=self.display_name)
      users.append(get_current_user())

    if self.row_converter.is_new and self.mandatory and not users:
      self.add_error(errors.MISSING_VALUE_ERROR,
                     column_name=self.display_name)
    return list(users)

  def set_obj_attr(self):
    pass

  def get_value(self):
    user_role_ids = db.session.query(UserRole.person_id).filter_by(
        role=self.role,
        context_id=self.row_converter.obj.context_id)
    users = Person.query.filter(Person.id.in_(user_role_ids))
    emails = [user.email for user in users]
    return "\n".join(emails)

  def remove_current_roles(self):
    UserRole.query.filter_by(
        role=self.role,
        context_id=self.row_converter.obj.context_id).delete()

  def insert_object(self):
    if self.dry_run or not self.value:
      return
    self.remove_current_roles()
    for owner in self.value:
      user_role = UserRole(
          role=self.role,
          context=self.row_converter.obj.context,
          person=owner
      )
      db.session.add(user_role)
    self.dry_run = True

class ProgramOwnerColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="ProgramOwner").one()
    super(ProgramOwnerColumnHandler, self).__init__(
        row_converter, key, **options)

class ProgramEditorColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="ProgramEditor").one()
    super(ProgramEditorColumnHandler, self).__init__(
        row_converter, key, **options)

class ProgramReaderColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="ProgramReader").one()
    super(ProgramReaderColumnHandler, self).__init__(
        row_converter, key, **options)

class WorkflowOwnerColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="WorkflowOwner").one()
    super(WorkflowOwnerColumnHandler, self).__init__(
        row_converter, key, **options)

class WorkflowMemberColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="WorkflowMember").one()
    super(WorkflowMemberColumnHandler, self).__init__(
        row_converter, key, **options)

class AuditAuditorColumnHandler(ObjectRoleColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.role = Role.query.filter_by(name="Auditor").one()
    self.reader = Role.query.filter_by(name="ProgramReader").one()
    super(AuditAuditorColumnHandler, self).__init__(
        row_converter, key, **options)

  def insert_object(self):
    if self.dry_run or not self.value:
      return
    super(AuditAuditorColumnHandler, self).insert_object()
    user_roles = set(map(lambda ur: ur.person_id, self.get_program_roles()))
    context = self.row_converter.obj.program.context
    for auditor in self.value:
      # Check if the role already exists in the database or in the session:
      if auditor.id in user_roles or any(o for o in db.session.new if
                                         isinstance(o, UserRole) and
                                         o.context.id == context.id and
                                         o.person.id == auditor.id):
        continue
      user_role = UserRole(
          role=self.reader,
          context=context,
          person=auditor
      )
      db.session.add(user_role)

  def get_program_roles(self):
    context_id = self.row_converter.obj.program.context.id
    return db.session.query(UserRole).filter(
        UserRole.context_id == context_id).all()

class UserRoleColumnHandler(UserColumnHandler):

  _role_map = {
      "admin": "administrator",
      "ggrc admin": "administrator",
  }

  _allowed_roles = [
      "Administrator",
      "Creator",
      "Editor",
      "Reader",
      ""
  ]

  def parse_item(self):
    value = self.raw_value.lower()
    if value.title() not in self._allowed_roles:
      self.add_error(errors.WRONG_VALUE, column_name=self.display_name)
    name = self._role_map.get(value, value)
    return Role.query.filter_by(name=name).first()

  def set_obj_attr(self):
    pass

  def get_value(self):
    return self.row_converter.obj.system_wide_role

  def remove_current_roles(self):
    allowed_role_ids = db.session.query(Role.id).filter(
        Role.name.in_(self._allowed_roles))
    UserRole.query.filter(and_(
        UserRole.role_id.in_(allowed_role_ids),
        UserRole.person_id == self.row_converter.obj.id)
    ).delete(synchronize_session="fetch")

  def insert_object(self):
    if self.dry_run or not self.value:
      return
    self.remove_current_roles()
    context = None
    if self.value.name == "Administrator":
      context = Context.query.filter_by(name="System Administration").first()
    user_role = UserRole(
        role=self.value,
        person=self.row_converter.obj,
        context=context,
    )
    db.session.add(user_role)
    self.dry_run = True

COLUMN_HANDLERS = {
    "program_editor": ProgramEditorColumnHandler,
    "program_owner": ProgramOwnerColumnHandler,
    "program_reader": ProgramReaderColumnHandler,
    "user_role": UserRoleColumnHandler,
    "user_role:Auditor": AuditAuditorColumnHandler,
    "workflow_member": WorkflowMemberColumnHandler,
    "workflow_owner": WorkflowOwnerColumnHandler,
}
<|endoftext|>


<commit_before>"""Object hierarchy of property-value annotations in OBO files.
"""

import fastobo

from .utils.impl import set
from .utils.meta import roundrepr, typechecked

class PropertyValue(object):
    """A property-value, which adds annotations to an entity.
    """

    property: str
    __slots__ = ("__weakref__", "property")

@roundrepr
class LiteralPropertyValue(PropertyValue):
    """A property-value which adds a literal annotation to an entity.
    """

    literal: str
    datatype: str
    __slots__ = ("literal", "datatype")

    @typechecked()
    def __init__(self, property: str, literal: str, datatype: str = "xsd:string"):
        """Create a new `LiteralPropertyValue` instance.

        Arguments:
            property (str): The annotation property, as an OBO identifier.
            literal (str): The serialized value of the annotation.
            datatype (str): The datatype of the annotation property value.
                Defaults to `xsd:string`.

        """
        self.property = str(fastobo.id.parse(property))
        self.literal = literal
        self.datatype = str(fastobo.id.parse(datatype))

@roundrepr
class ResourcePropertyValue(PropertyValue):
    """A property-value which adds a resource annotation to an entity.
    """

    resource: str
    __slots__ = ("resource",)

    @typechecked()
    def __init__(self, property: str, resource: str):
        """Create a new `ResourcePropertyValue` instance.

        Arguments:
            property (str): The annotation property, as an OBO identifier.
            resource (str): The annotation entity value, as an OBO identifier.

        """
        self.property = str(fastobo.id.parse(property))
        self.resource = str(fastobo.id.parse(resource))
<commit_msg>Add proper comparison to `PropertyValue` subclasses<commit_after>"""Object hierarchy of property-value annotations in OBO files.
"""

import functools

import fastobo

from .utils.impl import set
from .utils.meta import roundrepr, typechecked

class PropertyValue(object):
    """A property-value, which adds annotations to an entity.
    """

    property: str
    __slots__ = ("__weakref__", "property")

@roundrepr
@functools.total_ordering
class LiteralPropertyValue(PropertyValue):
    """A property-value which adds a literal annotation to an entity.
    """

    literal: str
    datatype: str
    __slots__ = ("literal", "datatype")

    @typechecked()
    def __init__(self, property: str, literal: str, datatype: str = "xsd:string"):
        """Create a new `LiteralPropertyValue` instance.

        Arguments:
            property (str): The annotation property, as an OBO identifier.
            literal (str): The serialized value of the annotation.
            datatype (str): The datatype of the annotation property value.
                Defaults to `xsd:string`.

        """
        self.property = str(fastobo.id.parse(property))
        self.literal = literal
        self.datatype = str(fastobo.id.parse(datatype))

    def __eq__(self, other: object) -> bool:
        if isinstance(other, LiteralPropertyValue):
            return self.property == other.property \
                and self.literal == other.literal \
                and self.datatype == other.datatype
        return False

    def __lt__(self, other: object) -> bool:
        if isinstance(other, LiteralPropertyValue):
            return (self.property, self.literal, self.datatype) \
                < (other.property, other.literal, other.datatype)
        elif isinstance(other, ResourcePropertyValue):
            return self.property < other.property
        else:
            return NotImplemented

    def __hash__(self) -> int:
        return hash((LiteralPropertyValue, self.property, self.literal, self.datatype))

@roundrepr
@functools.total_ordering
class ResourcePropertyValue(PropertyValue):
    """A property-value which adds a resource annotation to an entity.
    """

    resource: str
    __slots__ = ("resource",)

    @typechecked()
    def __init__(self, property: str, resource: str):
        """Create a new `ResourcePropertyValue` instance.

        Arguments:
            property (str): The annotation property, as an OBO identifier.
            resource (str): The annotation entity value, as an OBO identifier.

        """
        self.property = str(fastobo.id.parse(property))
        self.resource = str(fastobo.id.parse(resource))

    def __eq__(self, other: object) -> bool:
        if isinstance(other, ResourcePropertyValue):
            return (self.property, self.resource) == (other.property, other.resource)
        return False

    def __lt__(self, other: object) -> bool:
        if isinstance(other, ResourcePropertyValue):
            return (self.property, self.resource) < (other.property, other.resource)
        elif isinstance(other, LiteralPropertyValue):
            return self.property < other.property
        else:
            return NotImplemented

    def __hash__(self) -> int:
        return hash((LiteralPropertyValue, self.property, self.resource))
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Utility functions."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import StringIO
from matplotlib import pylab as p
# axes3d is being used implictly for visualization.
from mpl_toolkits.mplot3d import axes3d as p3  # pylint:disable=unused-import
import numpy as np
from PIL import Image
from skimage import measure

import tensorflow as tf

def save_image(inp_array, image_file):
  """Function that dumps the image to disk."""
  inp_array = np.clip(inp_array, 0, 255).astype(np.uint8)
  image = Image.fromarray(inp_array)
  buf = StringIO.StringIO()
  image.save(buf, format='JPEG')
  with open(image_file, 'w') as f:
    f.write(buf.getvalue())
  return None

def image_flipud(images):
  """Function that flip (up-down) the np image."""
  quantity = images.get_shape().as_list()[0]
  image_list = []
  for k in xrange(quantity):
    image_list.append(tf.image.flip_up_down(images[k, :, :, :]))
  outputs = tf.stack(image_list)
  return outputs

def resize_image(inp_array, new_height, new_width):
  """Function that resize the np image."""
  inp_array = np.clip(inp_array, 0, 255).astype(np.uint8)
  image = Image.fromarray(inp_array)
  # Reverse order
  image = image.resize((new_width, new_height))
  return np.array(image)

def display_voxel(points, vis_size=128):
  """Function to display 3D voxel."""
  try:
    data = visualize_voxel_spectral(points, vis_size)
  except ValueError:
    data = visualize_voxel_scatter(points, vis_size)
  return data

def visualize_voxel_spectral(points, vis_size=128):
  """Function to visualize voxel (spectral)."""
  points = np.rint(points)
  points = np.swapaxes(points, 0, 2)
  fig = p.figure(figsize=(1, 1), dpi=vis_size)
  verts, faces = measure.marching_cubes(points, 0, spacing=(0.1, 0.1, 0.1))
  ax = fig.add_subplot(111, projection='3d')
  ax.plot_trisurf(
      verts[:, 0], verts[:, 1], faces, verts[:, 2], cmap='Spectral_r', lw=0.1)
  ax.set_axis_off()
  fig.tight_layout(pad=0)
  fig.canvas.draw()
  data = np.fromstring(
      fig.canvas.tostring_rgb(), dtype=np.uint8, sep='').reshape(
          vis_size, vis_size, 3)
  p.close('all')
  return data

def visualize_voxel_scatter(points, vis_size=128):
  """Function to visualize voxel (scatter)."""
  points = np.rint(points)
  points = np.swapaxes(points, 0, 2)
  fig = p.figure(figsize=(1, 1), dpi=vis_size)
  ax = fig.add_subplot(111, projection='3d')
  x = []
  y = []
  z = []
  (x_dimension, y_dimension, z_dimension) = points.shape
  for i in range(x_dimension):
    for j in range(y_dimension):
      for k in range(z_dimension):
        if points[i, j, k]:
          x.append(i)
          y.append(j)
          z.append(k)
  ax.scatter3D(x, y, z)
  ax.set_axis_off()
  fig.tight_layout(pad=0)
  fig.canvas.draw()
  data = np.fromstring(
      fig.canvas.tostring_rgb(), dtype=np.uint8, sep='').reshape(
          vis_size, vis_size, 3)
  p.close('all')
  return data

<commit_msg>enables training the decoder, fixes matplotlib multi-threading issue #2160 (#2174)<commit_after># Copyright 2017 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Utility functions."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import StringIO
import matplotlib
matplotlib.use('Agg')
from matplotlib import pylab as p
# axes3d is being used implictly for visualization.
from mpl_toolkits.mplot3d import axes3d as p3  # pylint:disable=unused-import
import numpy as np
from PIL import Image
from skimage import measure

import tensorflow as tf

def save_image(inp_array, image_file):
  """Function that dumps the image to disk."""
  inp_array = np.clip(inp_array, 0, 255).astype(np.uint8)
  image = Image.fromarray(inp_array)
  buf = StringIO.StringIO()
  image.save(buf, format='JPEG')
  with open(image_file, 'w') as f:
    f.write(buf.getvalue())
  return None

def image_flipud(images):
  """Function that flip (up-down) the np image."""
  quantity = images.get_shape().as_list()[0]
  image_list = []
  for k in xrange(quantity):
    image_list.append(tf.image.flip_up_down(images[k, :, :, :]))
  outputs = tf.stack(image_list)
  return outputs

def resize_image(inp_array, new_height, new_width):
  """Function that resize the np image."""
  inp_array = np.clip(inp_array, 0, 255).astype(np.uint8)
  image = Image.fromarray(inp_array)
  # Reverse order
  image = image.resize((new_width, new_height))
  return np.array(image)

def display_voxel(points, vis_size=128):
  """Function to display 3D voxel."""
  try:
    data = visualize_voxel_spectral(points, vis_size)
  except ValueError:
    data = visualize_voxel_scatter(points, vis_size)
  return data

def visualize_voxel_spectral(points, vis_size=128):
  """Function to visualize voxel (spectral)."""
  points = np.rint(points)
  points = np.swapaxes(points, 0, 2)
  fig = p.figure(figsize=(1, 1), dpi=vis_size)
  verts, faces = measure.marching_cubes(points, 0, spacing=(0.1, 0.1, 0.1))
  ax = fig.add_subplot(111, projection='3d')
  ax.plot_trisurf(
      verts[:, 0], verts[:, 1], faces, verts[:, 2], cmap='Spectral_r', lw=0.1)
  ax.set_axis_off()
  fig.tight_layout(pad=0)
  fig.canvas.draw()
  data = np.fromstring(
      fig.canvas.tostring_rgb(), dtype=np.uint8, sep='').reshape(
          vis_size, vis_size, 3)
  p.close('all')
  return data

def visualize_voxel_scatter(points, vis_size=128):
  """Function to visualize voxel (scatter)."""
  points = np.rint(points)
  points = np.swapaxes(points, 0, 2)
  fig = p.figure(figsize=(1, 1), dpi=vis_size)
  ax = fig.add_subplot(111, projection='3d')
  x = []
  y = []
  z = []
  (x_dimension, y_dimension, z_dimension) = points.shape
  for i in range(x_dimension):
    for j in range(y_dimension):
      for k in range(z_dimension):
        if points[i, j, k]:
          x.append(i)
          y.append(j)
          z.append(k)
  ax.scatter3D(x, y, z)
  ax.set_axis_off()
  fig.tight_layout(pad=0)
  fig.canvas.draw()
  data = np.fromstring(
      fig.canvas.tostring_rgb(), dtype=np.uint8, sep='').reshape(
          vis_size, vis_size, 3)
  p.close('all')
  return data

<|endoftext|>


<commit_before>import threading
import logging
import requests
import argparse
import sys
import string
import random
import os

from ptop import __version__, _log_file
from ptop.statistics import Statistics
from ptop.interfaces import PtopGUI
from ptop.plugins import SENSORS_LIST
from ptop.constants import SUPPORTED_THEMES
from ptop import __version__
from huepy import *

logger = logging.getLogger('ptop.main')

def _update():
    '''
        Try to update ptop at application start after asking the user
    '''
    try:
        CURRENT_VERSION = str(__version__)
        os_name = "{0} {1}".format(platform.system(),
                                   platform.release()
                                   )
        resp = requests.get("http://www.mocky.io/v2/5bec3430330000b92efbc2f7",params={os_name: os_name, version: __version__})
        NEW_VERSION = str(resp.text)
        if NEW_VERSION != CURRENT_VERSION:
            sys.stdout.write(blue("A new version is available, would you like to update (Y/N) ? "))
            sys.stdout.flush()
            user_consent = raw_input()
            if user_consent.lower() == 'y':
                logger.info("main.py :: Updating ptop to version {0}".format(NEW_VERSION))
                # run update instructions
                update_success_status = 0
                source_folder = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
                sys.stdout.write(green("\nCreating a temporary directory /tmp/{0} ...\n".format(source_folder)))
                sys.stdout.flush()
                update_success_status |= os.system('mkdir /tmp/{0}'.format(source_folder))
                sys.stdout.flush()
                update_success_status |= os.system('git clone https://github.com/darxtrix/ptop.git /tmp/{0}'.format(source_folder))
                sys.stdout.write(green("\nInstalling ptop ...\n"))
                sys.stdout.flush()
                update_success_status |= os.system('sudo python /tmp/{0}/ptop/setup.py install'.format(source_folder))
                # if we are not successful in updating status
                if update_success_status != 0:
                    sys.stdout.write(red("\nError occured while updating ptop.\n"))
                    sys.stdout.write(red("Please report the issue at https://github.com/darxtrix/ptop/issues with the terminal output.\n"))
                    sys.stdout.flush()
                    sys.exit(1)

    except Exception as e:
        logger.info("Exception :: main.py :: Exception occured while updating ptop "+str(e))

def main():
    try:
        # app wide global stop flag
        global_stop_event = threading.Event()

        # command line argument parsing
        parser = argparse.ArgumentParser(description='ptop argument parser')
        parser.add_argument('-t',
                            dest='theme',
                            action='store',
                            type=str,
                            required=False,
                            choices=SUPPORTED_THEMES.keys(),
                            help=
                            '''
                                Valid themes are :
                                 elegant
                                 colorful
                                 dark
                                 light
                                 simple
                                 blackonwhite
                            ''')
        parser.add_argument('-v',
                            action='version',
                            version='ptop {}'.format(__version__))

        results = parser.parse_args()
        if results.theme:
            theme = results.theme
        else:
            theme = 'elegant'

        # try to update ptop
        _update()

        # TODO ::  Catch the exception of the child thread and kill the application gracefully
        # https://stackoverflow.com/questions/2829329/catch-a-threads-exception-in-the-caller-thread-in-python
        s = Statistics(SENSORS_LIST,global_stop_event)
        # internally uses a thread Job
        s.generate()
        logger.info('Statistics generating started')
        app = PtopGUI(s.statistics,global_stop_event,theme)
        # blocking call
        logger.info('Starting the GUI application')
        app.run()

    # catch the kill signals here and perform the clean up
    except KeyboardInterrupt:
        global_stop_event.set()
        # clear log file
        # Add code for wait for all the threads before join
        with open(_log_file,'w'):
            pass
        # TODO :Wait for threads to exit before calling systemExist
        raise SystemExit

    except Exception as e:
        global_stop_event.set()
        # don't clear the log file
        logger.info("Exception :: main.py "+str(e))
        raise SystemExit

if __name__ == '__main__':
    main()<commit_msg>Changed telemetry url<commit_after>import threading
import logging
import requests
import argparse
import sys
import string
import random
import os

from ptop import __version__, _log_file
from ptop.statistics import Statistics
from ptop.interfaces import PtopGUI
from ptop.plugins import SENSORS_LIST
from ptop.constants import SUPPORTED_THEMES
from ptop import __version__
from huepy import *

logger = logging.getLogger('ptop.main')

def _update():
    '''
        Try to update ptop at application start after asking the user
    '''
    try:
        CURRENT_VERSION = str(__version__)
        os_name = "{0} {1}".format(platform.system(),
                                   platform.release()
                                   )
        resp = requests.get("http://ptop-telemetry.darxtrix.in",params={os_name: os_name, version: __version__})
        NEW_VERSION = str(resp.text)
        if NEW_VERSION != CURRENT_VERSION:
            sys.stdout.write(blue("A new version is available, would you like to update (Y/N) ? "))
            sys.stdout.flush()
            user_consent = raw_input()
            if user_consent.lower() == 'y':
                logger.info("main.py :: Updating ptop to version {0}".format(NEW_VERSION))
                # run update instructions
                update_success_status = 0
                source_folder = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
                sys.stdout.write(green("\nCreating a temporary directory /tmp/{0} ...\n".format(source_folder)))
                sys.stdout.flush()
                update_success_status |= os.system('mkdir /tmp/{0}'.format(source_folder))
                sys.stdout.flush()
                update_success_status |= os.system('git clone https://github.com/darxtrix/ptop.git /tmp/{0}'.format(source_folder))
                sys.stdout.write(green("\nInstalling ptop ...\n"))
                sys.stdout.flush()
                update_success_status |= os.system('sudo python /tmp/{0}/ptop/setup.py install'.format(source_folder))
                # if we are not successful in updating status
                if update_success_status != 0:
                    sys.stdout.write(red("\nError occured while updating ptop.\n"))
                    sys.stdout.write(red("Please report the issue at https://github.com/darxtrix/ptop/issues with the terminal output.\n"))
                    sys.stdout.flush()
                    sys.exit(1)

    except Exception as e:
        logger.info("Exception :: main.py :: Exception occured while updating ptop "+str(e))

def main():
    try:
        # app wide global stop flag
        global_stop_event = threading.Event()

        # command line argument parsing
        parser = argparse.ArgumentParser(description='ptop argument parser')
        parser.add_argument('-t',
                            dest='theme',
                            action='store',
                            type=str,
                            required=False,
                            choices=SUPPORTED_THEMES.keys(),
                            help=
                            '''
                                Valid themes are :
                                 elegant
                                 colorful
                                 dark
                                 light
                                 simple
                                 blackonwhite
                            ''')
        parser.add_argument('-v',
                            action='version',
                            version='ptop {}'.format(__version__))

        results = parser.parse_args()
        if results.theme:
            theme = results.theme
        else:
            theme = 'elegant'

        # try to update ptop
        _update()

        # TODO ::  Catch the exception of the child thread and kill the application gracefully
        # https://stackoverflow.com/questions/2829329/catch-a-threads-exception-in-the-caller-thread-in-python
        s = Statistics(SENSORS_LIST,global_stop_event)
        # internally uses a thread Job
        s.generate()
        logger.info('Statistics generating started')
        app = PtopGUI(s.statistics,global_stop_event,theme)
        # blocking call
        logger.info('Starting the GUI application')
        app.run()

    # catch the kill signals here and perform the clean up
    except KeyboardInterrupt:
        global_stop_event.set()
        # clear log file
        # Add code for wait for all the threads before join
        with open(_log_file,'w'):
            pass
        # TODO :Wait for threads to exit before calling systemExist
        raise SystemExit

    except Exception as e:
        global_stop_event.set()
        # don't clear the log file
        logger.info("Exception :: main.py "+str(e))
        raise SystemExit

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import configparser

from UM.Settings import SettingsError
from UM.Logger import Logger
from UM.Signal import Signal, SignalEmitter
from UM.SaveFile import SaveFile

class MachineInstance(SignalEmitter):
    MachineInstanceVersion = 1

    def __init__(self, machine_manager, **kwargs):
        super().__init__()

        self._machine_manager = machine_manager

        self._name = kwargs.get("name", "")
        self._machine_definition = kwargs.get("definition", None)
        if self._machine_definition:
            self._machine_definition.loadAll()
        self._machine_setting_overrides = {}

    nameChanged = Signal()

    def getName(self):
        return self._name

    def setName(self, name):
        if name != self._name:
            old_name = self._name
            self._name = name
            self.nameChanged.emit(self, old_name)

    def getMachineDefinition(self):
        return self._machine_definition

    def setMachineDefinition(self, definition):
        if not definition:
            return

        definition.loadAll()
        self._machine_definition = definition

    def setMachineSettingValue(self, setting, value):
        if not self._machine_definition.isMachineSetting(setting):
            Logger.log("w", "Tried to override setting %s that is not a machine setting", setting)
            return

        self._machine_setting_overrides[setting] = value

    def getMachineSettingValue(self, setting):
        if not self._machine_definition.isMachineSetting(setting):
            return

        if setting in self._machine_setting_overrides:
            return self._machine_setting_overrides[setting]

        return self._machine_definition.getSetting(setting).getDefaultValue()

    def getSettingValue(self, key):
        if not self._machine_definition.isSetting(key):
            return None

        if key in self._machine_setting_overrides:
            return self._machine_setting_overrides[key]

        return self._machine_definition.getSetting(key).getDefaultValue()

    def hasMachineSettingValue(self, key):
        return key in self._machine_setting_overrides

    def loadFromFile(self, path):
        config = configparser.ConfigParser(interpolation = None)
        config.read(path, "utf-8")

        if not config.has_section("general"):
            raise SettingsError.InvalidFileError(path)

        if not config.has_option("general", "version"):
            raise SettingsError.InvalidFileError(path)

        if not config.has_option("general", "name") or not config.has_option("general", "type"):
            raise SettingsError.InvalidFileError(path)

        if int(config.get("general", "version")) != self.MachineInstanceVersion:
            raise SettingsError.InvalidVersionError(path)

        type_name = config.get("general", "type")
        variant_name = config.get("general", "variant", fallback = "")

        self._machine_definition = self._machine_manager.findMachineDefinition(type_name, variant_name)
        self._machine_definition.loadAll()

        self._name = config.get("general", "name")

        for key, value in config["machine_settings"].items():
            self._machine_setting_overrides[key] = value

    def saveToFile(self, path):
        config = configparser.ConfigParser(interpolation = None)

        config.add_section("general")
        config["general"]["name"] = self._name
        config["general"]["type"] = self._machine_definition.getId()
        config["general"]["version"] = str(self.MachineInstanceVersion)
        if self._machine_definition.getVariantName():
            config["general"]["variant"] = self._machine_definition.getVariantName()

        config.add_section("machine_settings")
        for key, value in self._machine_setting_overrides:
            config["machine_settings"][key] = str(value)

        try:
            with SaveFile(path, "wt", -1, "utf-8") as f:
                config.write(f)
        except Exception as e:
            Logger.log("e", "Failed to write Instance to %s: %s", path, str(e))
<commit_msg>Properly write machine setting overrides instead of throwing an exception<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import configparser

from UM.Settings import SettingsError
from UM.Logger import Logger
from UM.Signal import Signal, SignalEmitter
from UM.SaveFile import SaveFile

class MachineInstance(SignalEmitter):
    MachineInstanceVersion = 1

    def __init__(self, machine_manager, **kwargs):
        super().__init__()

        self._machine_manager = machine_manager

        self._name = kwargs.get("name", "")
        self._machine_definition = kwargs.get("definition", None)
        if self._machine_definition:
            self._machine_definition.loadAll()
        self._machine_setting_overrides = {}

    nameChanged = Signal()

    def getName(self):
        return self._name

    def setName(self, name):
        if name != self._name:
            old_name = self._name
            self._name = name
            self.nameChanged.emit(self, old_name)

    def getMachineDefinition(self):
        return self._machine_definition

    def setMachineDefinition(self, definition):
        if not definition:
            return

        definition.loadAll()
        self._machine_definition = definition

    def setMachineSettingValue(self, setting, value):
        if not self._machine_definition.isMachineSetting(setting):
            Logger.log("w", "Tried to override setting %s that is not a machine setting", setting)
            return

        self._machine_setting_overrides[setting] = value

    def getMachineSettingValue(self, setting):
        if not self._machine_definition.isMachineSetting(setting):
            return

        if setting in self._machine_setting_overrides:
            return self._machine_setting_overrides[setting]

        return self._machine_definition.getSetting(setting).getDefaultValue()

    def getSettingValue(self, key):
        if not self._machine_definition.isSetting(key):
            return None

        if key in self._machine_setting_overrides:
            return self._machine_setting_overrides[key]

        return self._machine_definition.getSetting(key).getDefaultValue()

    def hasMachineSettingValue(self, key):
        return key in self._machine_setting_overrides

    def loadFromFile(self, path):
        config = configparser.ConfigParser(interpolation = None)
        config.read(path, "utf-8")

        if not config.has_section("general"):
            raise SettingsError.InvalidFileError(path)

        if not config.has_option("general", "version"):
            raise SettingsError.InvalidFileError(path)

        if not config.has_option("general", "name") or not config.has_option("general", "type"):
            raise SettingsError.InvalidFileError(path)

        if int(config.get("general", "version")) != self.MachineInstanceVersion:
            raise SettingsError.InvalidVersionError(path)

        type_name = config.get("general", "type")
        variant_name = config.get("general", "variant", fallback = "")

        self._machine_definition = self._machine_manager.findMachineDefinition(type_name, variant_name)
        self._machine_definition.loadAll()

        self._name = config.get("general", "name")

        for key, value in config["machine_settings"].items():
            self._machine_setting_overrides[key] = value

    def saveToFile(self, path):
        config = configparser.ConfigParser(interpolation = None)

        config.add_section("general")
        config["general"]["name"] = self._name
        config["general"]["type"] = self._machine_definition.getId()
        config["general"]["version"] = str(self.MachineInstanceVersion)
        if self._machine_definition.getVariantName():
            config["general"]["variant"] = self._machine_definition.getVariantName()

        config.add_section("machine_settings")
        for key, value in self._machine_setting_overrides.items():
            config["machine_settings"][key] = str(value)

        try:
            with SaveFile(path, "wt", -1, "utf-8") as f:
                config.write(f)
        except Exception as e:
            Logger.log("e", "Failed to write Instance to %s: %s", path, str(e))
<|endoftext|>


<commit_before># Copyright (c) 2016 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import ast

from UM.Logger import Logger

class IllegalMethodError(Exception):
    pass

##  Encapsulates Python code that provides a simple value calculation function.
#
class SettingFunction:
    ##  Constructor.
    #
    #   \param code The Python code this function should evaluate.
    def __init__(self, code, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self._code = code
        self._settings = set() # Keys of all settings that are referenced to in this function.
        self._compiled = None
        self._valid = False

        try:
            tree = ast.parse(self._code, "eval")
            self._settings = set(_SettingExpressionVisitor().visit(tree))
            self._compiled = compile(self._code, repr(self), "eval")
            self._valid = True
        except (SyntaxError, TypeError) as e:
            Logger.log("e", "Parse error in function ({1}) for setting: {0}".format(str(e), self._code))
        except IllegalMethodError as e:
            Logger.log("e", "Use of illegal method {0} in function ({1}) for setting".format(str(e), self._code))
        except Exception as e:
            Logger.log("e", "Exception in function ({1}) for setting: {1}".format(str(e), self._code))

    ##  Call the actual function to calculate the value.
    def __call__(self, value_provider, *args, **kwargs):
        if not value_provider:
            return None

        if not self._valid:
            return None

        locals = { }
        for name in self._settings:
            locals[name] = value_provider.getValue(name)

        return eval(self._compiled, globals(), locals)

    def __eq__(self, other):
        if not isinstance(other, SettingFunction):
            return False

        return self._code == other._code

    ##  Returns whether the function is ready to be executed.
    #
    #   \return True if the function is valid, or False if it's not.
    def isValid(self):
        return self._valid

    ##  Retrieve a set of the keys of all the settings used in this function.
    #
    #   \return A set of the keys of all the settings used in this functions.
    def getUsedSettings(self):
        return self._settings

    def __str__(self):
        return "SettingFunction({0})".format(self._code)

# Helper class used to analyze a parsed function
class _SettingExpressionVisitor(ast.NodeVisitor):
    def __init__(self):
        super().__init__()
        self.names = []

    def visit(self, node):
        super().visit(node)
        return self.names

    def visit_Name(self, node): # [CodeStyle: ast.NodeVisitor requires this function name]
        if node.id in self._blacklist:
            raise IllegalMethodError(node.id)

        if node.id not in self._knownNames and node.id not in __builtins__:
            self.names.append(node.id)

    _knownNames = [
        "math"
    ]

    _blacklist = [
        "sys",
        "os",
        "import",
        "__import__",
        "eval",
        "exec",
        "subprocess",
    ]
<commit_msg>Pre-import math so it can be used by setting functions<commit_after># Copyright (c) 2016 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import ast
import math # Imported here so it can be used easily by the setting functions.

from UM.Logger import Logger

class IllegalMethodError(Exception):
    pass

##  Encapsulates Python code that provides a simple value calculation function.
#
class SettingFunction:
    ##  Constructor.
    #
    #   \param code The Python code this function should evaluate.
    def __init__(self, code, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self._code = code
        self._settings = set() # Keys of all settings that are referenced to in this function.
        self._compiled = None
        self._valid = False

        try:
            tree = ast.parse(self._code, "eval")
            self._settings = set(_SettingExpressionVisitor().visit(tree))
            self._compiled = compile(self._code, repr(self), "eval")
            self._valid = True
        except (SyntaxError, TypeError) as e:
            Logger.log("e", "Parse error in function ({1}) for setting: {0}".format(str(e), self._code))
        except IllegalMethodError as e:
            Logger.log("e", "Use of illegal method {0} in function ({1}) for setting".format(str(e), self._code))
        except Exception as e:
            Logger.log("e", "Exception in function ({1}) for setting: {1}".format(str(e), self._code))

    ##  Call the actual function to calculate the value.
    def __call__(self, value_provider, *args, **kwargs):
        if not value_provider:
            return None

        if not self._valid:
            return None

        locals = { }
        for name in self._settings:
            locals[name] = value_provider.getValue(name)

        return eval(self._compiled, globals(), locals)

    def __eq__(self, other):
        if not isinstance(other, SettingFunction):
            return False

        return self._code == other._code

    ##  Returns whether the function is ready to be executed.
    #
    #   \return True if the function is valid, or False if it's not.
    def isValid(self):
        return self._valid

    ##  Retrieve a set of the keys of all the settings used in this function.
    #
    #   \return A set of the keys of all the settings used in this functions.
    def getUsedSettings(self):
        return self._settings

    def __str__(self):
        return "SettingFunction({0})".format(self._code)

# Helper class used to analyze a parsed function
class _SettingExpressionVisitor(ast.NodeVisitor):
    def __init__(self):
        super().__init__()
        self.names = []

    def visit(self, node):
        super().visit(node)
        return self.names

    def visit_Name(self, node): # [CodeStyle: ast.NodeVisitor requires this function name]
        if node.id in self._blacklist:
            raise IllegalMethodError(node.id)

        if node.id not in self._knownNames and node.id not in __builtins__:
            self.names.append(node.id)

    _knownNames = [
        "math"
    ]

    _blacklist = [
        "sys",
        "os",
        "import",
        "__import__",
        "eval",
        "exec",
        "subprocess",
    ]
<|endoftext|>


<commit_before># coding=utf-8
import base64
import random
import requests
import json
import logging

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        try:
            r = requests.get('http://101.200.55.192:8000/?type=1&count=20')
            #logging.info('contents:%s' % contents.text)
            data = json.loads(r.text)
            if len(data) > 0:
                proxy = random.choice(data)
                ip = proxy.get('ip')
                port = proxy.get('port')
                address = '%s:%s' % (ip, port)
                #request.meta['proxy'] = "http://%s" % proxy['ip_port']
                request.meta['proxy'] = 'http://%s' % address
                logging.info('********ProxyMiddleware proxy*******:%s' % request.meta['proxy'] )
        except:
            pass<commit_msg>更改解析代理 IP 格式为 json<commit_after># coding=utf-8
import base64
import random
import requests
import json
import logging

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        try:
            r = requests.get('http://101.200.55.192:8000/?type=1&count=20')
            #logging.info('contents:%s' % contents.text)
            data = json.loads(r.text)
            if len(data) > 0:
                proxy = random.choice(data)
                ip = proxy.get('ip')
                port = proxy.get('port')
                address = '%s:%s' % (ip, port)

                request.meta['proxy'] = 'http://%s' % address
                logging.info('********ProxyMiddleware proxy*******:%s' % request.meta['proxy'] )
        except Exception, e:
            logging.warning('ProxyMiddleware Exception:%s' % str(e))
<|endoftext|>


<commit_before>from datetime import datetime
import numpy as np
from keras.models import Sequential
from keras.utils import np_utils
from keras.datasets import mnist, cifar10
from genome_handler import GenomeHandler
from keras.callbacks import EarlyStopping
import random as rand
import math
import csv

# Our genetic algorithm.
# This will contain methods for fitness, crossover, mutation, etc.
class Evolution:

    def __init__(self):
        self.genome_handler = GenomeHandler()
	(self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
	self.process_dataset()
        self.datafile = 'data/' + datetime.now().ctime()  + '.csv'
	print "model and accuracy data stored at", self.datafile
        self.data = []

    def process_dataset(self):
        self.x_train = self.x_train.reshape(self.x_train.shape[0], 1, 28, 28).astype("float32") / 255
        self.x_test = self.x_test.reshape(self.x_test.shape[0], 1, 28, 28).astype('float32') / 255
        self.y_train = np_utils.to_categorical(self.y_train)
        self.y_test = np_utils.to_categorical(self.y_test)

    # Create a population and evolve
    def run(self, num_generations, pop_size):
        # Generate initial random population
        epochs = 5
        members = np.array([self.genome_handler.generate() for _ in range(pop_size)])
        #with open("best_50.csv") as f:
		#members = np.array([line.split() for line in f])
	fit = np.array(self.fitnesses(members, epochs))
        pop = Population(members, fit)
        #epochs -= 1

        # Evolve over generations
        for i in range(1, num_generations):
            members = []
            for i in range(int(pop_size*0.95)): # Crossover
                members.append(self.crossover(pop.select(), pop.select()))
	    members += pop.getBest(pop_size - int(pop_size*0.95))
            for i in range(len(members)): # Mutation
	        members[i] = self.mutate(members[i])
            fit = np.array(self.fitnesses(members, epochs))
	    members = np.array(members)
            pop = Population(members, fit)
            #epochs -= 1
            #epochs = epochs if epochs >= 3 else 3

    def fitnesses(self, genomes, epochs):
	accuracies = [self.evaluate(x, epochs)[1] for x in genomes]
	accuracies -= min(accuracies)
	accuracies /= max(accuracies)
	return map(lambda x: math.exp(x), accuracies)

    # Returns the accuracy for a model as 1 / loss
    def evaluate(self, genome, epochs):
        model = self.genome_handler.decode(genome)
        loss, accuracy = None, None
	model.fit(self.x_train, self.y_train, \
		validation_data=(self.x_test, self.y_test),
		epochs=epochs, batch_size=200, verbose=1,
		callbacks=[EarlyStopping(monitor='val_loss', patience=2, verbose=1)])
	loss, accuracy = model.evaluate(self.x_test, self.y_test, verbose=0)

        # Record the stats
        with open(self.datafile, 'a') as csvfile:
            writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            row = list(genome) + [loss, accuracy]
            writer.writerow(row)

        return loss, accuracy

    def crossover(self, genome1, genome2):
        genome1 = genome1.tolist()
        genome2 = genome2.tolist()
        #swap the genomes split at the crossover index
        crossIndexA = rand.randint(0, len(genome1))
        child = genome1[:crossIndexA] + genome2[crossIndexA:]

        return np.array(child)

    def mutate(self, genome):
        return self.genome_handler.mutate(genome)

class Population:

    def __len__(self):
        return len(self.members)

    def __init__(self, members, fitnesses):
        self.members = members
        self.fitnesses = fitnesses
        self.s_fit = sum(self.fitnesses)

    def getBest(self, n):
	combined = [(self.members[i], self.fitnesses[i]) \
			for i in range(len(self.members))]
	sorted(combined, key=(lambda x: x[1]), reverse=True)
	return map(lambda x: x[0], combined[:n])

    def select(self):
        dart = rand.uniform(0, self.s_fit)
        sum_fits = 0
        for i in range(len(self.members)):
            sum_fits += self.fitnesses[i]
            if sum_fits > dart:
                return self.members[i]
<commit_msg>fixed ability to initialize from csv<commit_after>from datetime import datetime
import numpy as np
from keras.models import Sequential
from keras.utils import np_utils
from keras.datasets import mnist, cifar10
from genome_handler import GenomeHandler
from keras.callbacks import EarlyStopping
import random as rand
import math
import csv

# Our genetic algorithm.
# This will contain methods for fitness, crossover, mutation, etc.
class Evolution:

    def __init__(self):
        self.genome_handler = GenomeHandler()
	(self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
	self.process_dataset()
        self.datafile = 'data/' + datetime.now().ctime()  + '.csv'
	print "model and accuracy data stored at", self.datafile
        self.data = []

    def process_dataset(self):
        self.x_train = self.x_train.reshape(self.x_train.shape[0], 1, 28, 28).astype("float32") / 255
        self.x_test = self.x_test.reshape(self.x_test.shape[0], 1, 28, 28).astype('float32') / 255
        self.y_train = np_utils.to_categorical(self.y_train)
        self.y_test = np_utils.to_categorical(self.y_test)

    def fix_line(self, line):
        split_line = line.strip().split(',')
        for i in range(len(split_line)):
            split_line[i] = int(split_line[i])
        return(split_line)

    # Create a population and evolve
    def run(self, num_generations, pop_size):
        # Generate initial random population
        epochs = 10
        #members = np.array([self.genome_handler.generate() for _ in range(pop_size)])
        with open("best_50.csv") as f:
		members = np.array([self.fix_line(line) for line in f])
	fit = np.array(self.fitnesses(members, epochs))
        pop = Population(members, fit)
        #epochs -= 1

        # Evolve over generations
        for i in range(1, num_generations):
            members = []
            for i in range(int(pop_size*0.95)): # Crossover
                members.append(self.crossover(pop.select(), pop.select()))
	    members += pop.getBest(pop_size - int(pop_size*0.95))
            for i in range(len(members)): # Mutation
	        members[i] = self.mutate(members[i])
            fit = np.array(self.fitnesses(members, epochs))
	    members = np.array(members)
            pop = Population(members, fit)
            #epochs -= 1
            #epochs = epochs if epochs >= 3 else 3

    def fitnesses(self, genomes, epochs):
	accuracies = [self.evaluate(x, epochs)[1] for x in genomes]
	accuracies -= min(accuracies)
	accuracies /= max(accuracies)
	return map(lambda x: self.fitness(x), accuracies)

    def fitness(self, val):
	return (val * 100) ** 4

    # Returns the accuracy for a model as 1 / loss
    def evaluate(self, genome, epochs):
        model = self.genome_handler.decode(genome)
        loss, accuracy = None, None
	model.fit(self.x_train, self.y_train, \
		validation_data=(self.x_test, self.y_test),
		epochs=epochs, batch_size=200, verbose=1,
		callbacks=[EarlyStopping(monitor='val_loss', patience=2, verbose=1)])
	loss, accuracy = model.evaluate(self.x_test, self.y_test, verbose=0)

        # Record the stats
        with open(self.datafile, 'a') as csvfile:
            writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            row = list(genome) + [loss, accuracy]
            writer.writerow(row)

        return loss, accuracy

    def crossover(self, genome1, genome2):
        genome1 = genome1.tolist()
        genome2 = genome2.tolist()
        #swap the genomes split at the crossover index
        crossIndexA = rand.randint(0, len(genome1))
        child = genome1[:crossIndexA] + genome2[crossIndexA:]

        return np.array(child)

    def mutate(self, genome):
        return self.genome_handler.mutate(genome)

class Population:

    def __len__(self):
        return len(self.members)

    def __init__(self, members, fitnesses):
        self.members = members
        self.fitnesses = fitnesses
        self.s_fit = sum(self.fitnesses)

    def getBest(self, n):
	combined = [(self.members[i], self.fitnesses[i]) \
			for i in range(len(self.members))]
	sorted(combined, key=(lambda x: x[1]), reverse=True)
	return map(lambda x: x[0], combined[:n])

    def select(self):
        dart = rand.uniform(0, self.s_fit)
        sum_fits = 0
        for i in range(len(self.members)):
            sum_fits += self.fitnesses[i]
            if sum_fits > dart:
                return self.members[i]
<|endoftext|>