<commit_before>"""Fetch Transitland Feed Registry feeds."""
import argparse
import os
import collections

import mzgtfs.feed

import util
import task
# Temporary:
import tyr

class FeedEaterArtifact(task.FeedEaterTask):
  def run(self):
    # Create GTFS Artifacts
    print "===== Feed: %s ====="%self.feedid
    feed = self.registry.feed(self.feedid)
    filename = self.filename or os.path.join(self.workdir, '%s.zip'%feed.onestop())
    print "Opening: %s"%filename
    gtfsfeed = mzgtfs.feed.Feed(filename, debug=self.debug)

    for stop in gtfsfeed.stops():
      identifier = stop.feedid(self.feedid)
      print "Looking for identifier: %s"%identifier
      found = self.datastore.stops(identifier=identifier)
      if not found:
        print "  No identifier found!"
        stop.set('onestop_id', None)
        stop.set('osm_way_id', None)
        continue
      match = sorted(found, key=lambda x:x.data['updated_at'])[0]
      onestop_id = match.onestop()
      osm_way_id = match.data.get('osm_way_id')
      if not osm_way_id and tyr:
        osm_way_id = tyr.tyr_osm(
          stop,
          apitoken=os.getenv('TYR_AUTH_TOKEN'),
          debug=self.debug
        )
        print "  ... got tyr osm_way_id:", osm_way_id
      print "  onestop_id: %s, osm_way_id: %s"%(onestop_id, osm_way_id)
      stop.set('onestop_id', onestop_id)
      stop.set('osm_way_id', osm_way_id)

    # Write output
    stopstxt = os.path.join(self.workdir, 'stops.txt')
    artifact = os.path.join(self.workdir, '%s.artifact.zip'%feed.onestop())
    if os.path.exists(stopstxt):
      os.unlink(stopstxt)
    if os.path.exists(artifact):
      os.unlink(artifact)
    #
    print "Creating output artifact: %s"%artifact
    gtfsfeed.write(stopstxt, gtfsfeed.stops(), sortkey='stop_id')
    gtfsfeed.make_zip(artifact, files=[stopstxt], clone=filename)
    #
    if os.path.exists(stopstxt):
      os.unlink(stopstxt)

if __name__ == "__main__":
  run()<commit_msg>Bug fix; did not update __main__ to use task<commit_after>"""Fetch Transitland Feed Registry feeds."""
import argparse
import os
import collections

import mzgtfs.feed

import util
import task
# Temporary:
import tyr

class FeedEaterArtifact(task.FeedEaterTask):
  def run(self):
    # Create GTFS Artifacts
    print "===== Feed: %s ====="%self.feedid
    feed = self.registry.feed(self.feedid)
    filename = self.filename or os.path.join(self.workdir, '%s.zip'%feed.onestop())
    print "Opening: %s"%filename
    gtfsfeed = mzgtfs.feed.Feed(filename, debug=self.debug)

    for stop in gtfsfeed.stops():
      identifier = stop.feedid(self.feedid)
      print "Looking for identifier: %s"%identifier
      found = self.datastore.stops(identifier=identifier)
      if not found:
        print "  No identifier found!"
        stop.set('onestop_id', None)
        stop.set('osm_way_id', None)
        continue
      match = sorted(found, key=lambda x:x.data['updated_at'])[0]
      onestop_id = match.onestop()
      osm_way_id = match.data.get('osm_way_id')
      if not osm_way_id and tyr:
        osm_way_id = tyr.tyr_osm(
          stop,
          apitoken=os.getenv('TYR_AUTH_TOKEN'),
          debug=self.debug
        )
        print "  ... got tyr osm_way_id:", osm_way_id
      print "  onestop_id: %s, osm_way_id: %s"%(onestop_id, osm_way_id)
      stop.set('onestop_id', onestop_id)
      stop.set('osm_way_id', osm_way_id)

    # Write output
    stopstxt = os.path.join(self.workdir, 'stops.txt')
    artifact = os.path.join(self.workdir, '%s.artifact.zip'%feed.onestop())
    if os.path.exists(stopstxt):
      os.unlink(stopstxt)
    if os.path.exists(artifact):
      os.unlink(artifact)
    #
    print "Creating output artifact: %s"%artifact
    gtfsfeed.write(stopstxt, gtfsfeed.stops(), sortkey='stop_id')
    gtfsfeed.make_zip(artifact, files=[stopstxt], clone=filename)
    #
    if os.path.exists(stopstxt):
      os.unlink(stopstxt)

if __name__ == "__main__":
  task = FeedEaterArtifact.from_args()
  task.run()
<|endoftext|>


<commit_before>import logging
import re

from pylons import c

from ming.orm import ThreadLocalORMSession

from allura import model as M

from allura.lib import utils
from forgetracker import model as TM
from forgewiki.wiki_main import ForgeWikiApp

log = logging.getLogger(__name__)

def main():
    for a in M.AppConfig.query.find().all():
        if a.tool_name == "tickets":
            is_deleted = True
            while is_deleted:
                is_deleted = False
                for i in range(len(a.acl)):
                    if a.acl[i].permission == "write":
                        del a.acl[i]
                        a.acl.append(M.ACE.allow(a.acl[i].role_id, 'update'))
                        a.acl.append(M.ACE.allow(a.acl[i].role_id, 'create'))
                        is_deleted = True

        ThreadLocalORMSession.flush_all()

if __name__ == '__main__':
    main()
<commit_msg>[#3892] ticket:80 add comment<commit_after>import logging
import re

from pylons import c

from ming.orm import ThreadLocalORMSession

from allura import model as M

from allura.lib import utils
from forgetracker import model as TM
from forgewiki.wiki_main import ForgeWikiApp

log = logging.getLogger(__name__)

# migration script for change write permission to create + update
def main():
    for a in M.AppConfig.query.find().all():
        if a.tool_name == "tickets":
            is_deleted = True
            while is_deleted:
                is_deleted = False
                for i in range(len(a.acl)):
                    if a.acl[i].permission == "write":
                        del a.acl[i]
                        a.acl.append(M.ACE.allow(a.acl[i].role_id, 'update'))
                        a.acl.append(M.ACE.allow(a.acl[i].role_id, 'create'))
                        is_deleted = True

        ThreadLocalORMSession.flush_all()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Django settings for sensible_data_platform project.

import os
import LOCAL_SETTINGS

DEBUG = False
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS
BASE_DIR = LOCAL_SETTINGS.BASE_DIR
ROOT_DIR = LOCAL_SETTINGS.ROOT_DIR
ROOT_URL = LOCAL_SETTINGS.ROOT_URL
DATABASES = LOCAL_SETTINGS.DATABASES
BASE_URL = LOCAL_SETTINGS.BASE_URL

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['.sensible.dtu.dk']

LOGIN_URL = ROOT_URL + 'accounts/login/'
LOGIN_REDIRECT_URL = ROOT_URL

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
	# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ROOT_DIR+'static_root/'

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = ROOT_URL+'static/'

# Additional locations of static files
STATICFILES_DIRS = (
    ROOT_DIR+'static/',
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = LOCAL_SETTINGS.SECRET_KEY

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
        'django.core.context_processors.static',
        'django.contrib.auth.context_processors.auth',
)

ROOT_URLCONF = 'sensible_data_platform.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'sensible_data_platform.wsgi.application'

TEMPLATE_DIRS = (
		ROOT_DIR+'templates',
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'openid_provider',
    'accounts',
    'render',
    'identity_providers',
    'utils',
    'oauth2app',
    'oauth2_authorization_server',
    'uni_form',
    'service_manager',
    'south',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

import hashlib
SESSION_COOKIE_NAME = str(hashlib.sha1(SECRET_KEY).hexdigest())
<commit_msg>changed debug to true for devel<commit_after># Django settings for sensible_data_platform project.

import os
import LOCAL_SETTINGS

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS
BASE_DIR = LOCAL_SETTINGS.BASE_DIR
ROOT_DIR = LOCAL_SETTINGS.ROOT_DIR
ROOT_URL = LOCAL_SETTINGS.ROOT_URL
DATABASES = LOCAL_SETTINGS.DATABASES
BASE_URL = LOCAL_SETTINGS.BASE_URL

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['.sensible.dtu.dk']

LOGIN_URL = ROOT_URL + 'accounts/login/'
LOGIN_REDIRECT_URL = ROOT_URL

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
	# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ROOT_DIR+'static_root/'

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = ROOT_URL+'static/'

# Additional locations of static files
STATICFILES_DIRS = (
    ROOT_DIR+'static/',
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = LOCAL_SETTINGS.SECRET_KEY

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
        'django.core.context_processors.static',
        'django.contrib.auth.context_processors.auth',
)

ROOT_URLCONF = 'sensible_data_platform.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'sensible_data_platform.wsgi.application'

TEMPLATE_DIRS = (
		ROOT_DIR+'templates',
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'openid_provider',
    'accounts',
    'render',
    'identity_providers',
    'utils',
    'oauth2app',
    'oauth2_authorization_server',
    'uni_form',
    'service_manager',
    'south',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

import hashlib
SESSION_COOKIE_NAME = str(hashlib.sha1(SECRET_KEY).hexdigest())
<|endoftext|>


<commit_before># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import subprocess
import sys

import pkg_resources

import click

import guild
import guild.cli
import guild.config
import guild.plugin
import guild.test
import guild.uat
import guild.util

log = logging.getLogger("guild")

CHECK_MODS = [
    "pip",
    "psutil",
    "setuptools",
    "twine",
    "yaml",
    "werkzeug",
    "whoosh",
]

class Check(object):

    def __init__(self, args):
        self.args = args
        self._errors = False

    def error(self):
        self._errors = True

    @property
    def has_error(self):
        return self._errors

def main(args):
    if args.uat:
        _uat_and_exit()
    check = Check(args)
    if not args.no_info:
        _print_info(check)
    if args.all_tests or args.tests:
        _run_tests(check)
    if check.has_error:
        use_verbose = (
            " or rerun check with the --verbose option" if not args.verbose
            else "")
        guild.cli.error(
            "there are problems with your setup\n"
            "Refer to the issues above for more information%s."
            % use_verbose)

def _uat_and_exit():
    guild.uat.run()
    sys.exit(0)

def _run_tests(check):
    if check.args.all_tests:
        if check.args.tests:
            log.warning(
                "running all tests (--all-tests specified) - "
                "ignoring individual tests")
        success = guild.test.run_all(skip=check.args.skip)
    elif check.args.tests:
        if check.args.skip:
            log.warning(
                "running individual tests - ignoring --skip")
        success = guild.test.run(check.args.tests)
    if not success:
        check.error()

def _print_info(check):
    _print_guild_info()
    _print_python_info(check)
    _print_tensorflow_info(check)
    _print_nvidia_tools_info()
    if check.args.verbose:
        _print_mods_info(check)

def _print_guild_info():
    guild.cli.out("guild_version:             %s" % guild.version())
    guild.cli.out("guild_home:                %s" % guild.config.guild_home())
    guild.cli.out("guild_install_location:    %s" % _guild_install_location())
    guild.cli.out("installed_plugins:         %s" % _format_plugins())

def _guild_install_location():
    return pkg_resources.resource_filename("guild", "")

def _format_plugins():
    return ", ".join([
        name
        for name, _ in sorted(guild.plugin.iter_plugins())
    ])

def _print_python_info(check):
    guild.cli.out("python_version:            %s" % _python_version())
    if check.args.verbose:
        guild.cli.out("python_path:           %s" % _python_path())

def _python_version():
    return sys.version.replace("\n", "")

def _python_path():
    return os.path.pathsep.join(sys.path)

def _print_tensorflow_info(check):
    # Run externally to avoid tf logging to our stderr
    cmd = [sys.executable, "-um", "guild.commands.tensorflow_info_main"]
    env = guild.util.safe_osenv()
    env["PYTHONPATH"] = os.path.pathsep.join(sys.path)
    stderr = None if check.args.verbose else open(os.devnull, "w")
    p = subprocess.Popen(cmd, stderr=stderr, env=env)
    exit_status = p.wait()
    if exit_status != 0:
        check.error()

def _print_nvidia_tools_info():
    guild.cli.out("nvidia_smi_version:        %s" % _nvidia_smi_version())

def _nvidia_smi_version():
    cmd = guild.util.which("nvidia-smi")
    if not cmd:
        return "not installed"
    try:
        out = subprocess.check_output([cmd, "--version"])
    except subprocess.CalledProcessError as e:
        return _warn("ERROR: %s" % e.output.strip())
    else:
        return out.strip()

def _print_mods_info(check):
    for mod in CHECK_MODS:
        ver = _try_module_version(mod, check)
        space = " " * (18 - len(mod))
        guild.cli.out("%s_version:%s%s" % (mod, space, ver))

def _try_module_version(name, check):
    try:
        mod = __import__(name)
    except ImportError as e:
        check.error()
        return _warn("NOT INSTALLED (%s)" % e)
    else:
        try:
            ver = mod.__version__
        except AttributeError:
            return _warn("UNKNOWN")
        else:
            return _format_version(ver)

def _format_version(ver):
    if isinstance(ver, tuple):
        return ".".join([str(part) for part in ver])
    else:
        return str(ver)

def _warn(msg):
    return click.style(msg, fg="red", bold=True)
<commit_msg>Fix nvidia-smi version check<commit_after># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re
import subprocess
import sys

import pkg_resources

import click

import guild
import guild.cli
import guild.config
import guild.plugin
import guild.test
import guild.uat
import guild.util

log = logging.getLogger("guild")

CHECK_MODS = [
    "pip",
    "psutil",
    "setuptools",
    "twine",
    "yaml",
    "werkzeug",
    "whoosh",
]

class Check(object):

    def __init__(self, args):
        self.args = args
        self._errors = False

    def error(self):
        self._errors = True

    @property
    def has_error(self):
        return self._errors

def main(args):
    if args.uat:
        _uat_and_exit()
    check = Check(args)
    if not args.no_info:
        _print_info(check)
    if args.all_tests or args.tests:
        _run_tests(check)
    if check.has_error:
        use_verbose = (
            " or rerun check with the --verbose option" if not args.verbose
            else "")
        guild.cli.error(
            "there are problems with your setup\n"
            "Refer to the issues above for more information%s."
            % use_verbose)

def _uat_and_exit():
    guild.uat.run()
    sys.exit(0)

def _run_tests(check):
    if check.args.all_tests:
        if check.args.tests:
            log.warning(
                "running all tests (--all-tests specified) - "
                "ignoring individual tests")
        success = guild.test.run_all(skip=check.args.skip)
    elif check.args.tests:
        if check.args.skip:
            log.warning(
                "running individual tests - ignoring --skip")
        success = guild.test.run(check.args.tests)
    if not success:
        check.error()

def _print_info(check):
    _print_guild_info()
    _print_python_info(check)
    _print_tensorflow_info(check)
    _print_nvidia_tools_info()
    if check.args.verbose:
        _print_mods_info(check)

def _print_guild_info():
    guild.cli.out("guild_version:             %s" % guild.version())
    guild.cli.out("guild_home:                %s" % guild.config.guild_home())
    guild.cli.out("guild_install_location:    %s" % _guild_install_location())
    guild.cli.out("installed_plugins:         %s" % _format_plugins())

def _guild_install_location():
    return pkg_resources.resource_filename("guild", "")

def _format_plugins():
    return ", ".join([
        name
        for name, _ in sorted(guild.plugin.iter_plugins())
    ])

def _print_python_info(check):
    guild.cli.out("python_version:            %s" % _python_version())
    if check.args.verbose:
        guild.cli.out("python_path:           %s" % _python_path())

def _python_version():
    return sys.version.replace("\n", "")

def _python_path():
    return os.path.pathsep.join(sys.path)

def _print_tensorflow_info(check):
    # Run externally to avoid tf logging to our stderr
    cmd = [sys.executable, "-um", "guild.commands.tensorflow_info_main"]
    env = guild.util.safe_osenv()
    env["PYTHONPATH"] = os.path.pathsep.join(sys.path)
    stderr = None if check.args.verbose else open(os.devnull, "w")
    p = subprocess.Popen(cmd, stderr=stderr, env=env)
    exit_status = p.wait()
    if exit_status != 0:
        check.error()

def _print_nvidia_tools_info():
    guild.cli.out("nvidia_smi_version:        %s" % _nvidia_smi_version())

def _nvidia_smi_version():
    cmd = guild.util.which("nvidia-smi")
    if not cmd:
        return "not installed"
    try:
        out = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as e:
        return _warn("ERROR: %s" % e.output.strip())
    else:
        m = re.search(r"NVIDIA-SMI ([0-9\.]+)", out)
        if m:
            return m.group(1)
        else:
            log.debug("Unexpected output from nvidia-smi: %s", out)
            return "unknown"

def _print_mods_info(check):
    for mod in CHECK_MODS:
        ver = _try_module_version(mod, check)
        space = " " * (18 - len(mod))
        guild.cli.out("%s_version:%s%s" % (mod, space, ver))

def _try_module_version(name, check):
    try:
        mod = __import__(name)
    except ImportError as e:
        check.error()
        return _warn("NOT INSTALLED (%s)" % e)
    else:
        try:
            ver = mod.__version__
        except AttributeError:
            return _warn("UNKNOWN")
        else:
            return _format_version(ver)

def _format_version(ver):
    if isinstance(ver, tuple):
        return ".".join([str(part) for part in ver])
    else:
        return str(ver)

def _warn(msg):
    return click.style(msg, fg="red", bold=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# This file is subject to the terms and conditions defined in
# file 'LICENSE.md', which is part of this source code package.
#

import time

from kubernetes.K8sConfig import K8sConfig
from kubernetes.K8sContainer import K8sContainer
from kubernetes.K8sExceptions import BadRequestException
from kubernetes.K8sExceptions import TimedOutException, NotFoundException
from kubernetes.K8sObject import K8sObject
from kubernetes.K8sVolume import K8sVolume
from kubernetes.K8sReplicaSet import K8sReplicaSet
from kubernetes.models.v1beta1.Deployment import Deployment
from kubernetes.models.v1beta1.DeploymentRollback import DeploymentRollback
from kubernetes.models.v1beta1.LabelSelector import LabelSelector

class K8sDeployment(K8sObject):

    SCALE_WAIT_TIMEOUT_SECONDS = 120

    def __init__(self, config=None, name=None, image=None, replicas=0):

        super(K8sDeployment, self).__init__(
            config=config,
            obj_type='Deployment',
            name=name
        )

        self.desired_replicas = replicas

        labels = {'name': name}
        sel = LabelSelector()
        sel.match_labels = labels

        self.selector = sel
        self.labels = labels
        self.pod_labels = labels

        if image is not None:
            container = K8sContainer(name=name, image=image)
            self.add_container(container)

        if self.config.pull_secret is not None:
            self.add_image_pull_secrets(self.config.pull_secret)

    # -------------------------------------------------------------------------------------  override

    def create(self):
        super(K8sDeployment, self).create()
        self.get()
        if self.desired_replicas > 0:
            self._wait_for_desired_replicas()
        return self

    def update(self):
        super(K8sDeployment, self).update()
        self.get()
        if self.desired_replicas > 0:
            self._wait_for_desired_replicas()
        return self

    def list(self, pattern=None):
        ls = super(K8sDeployment, self).list()
        deploys = list(map(lambda x: Deployment(x), ls))
        if pattern is not None:
            deploys = list(filter(lambda x: pattern in x.name, deploys))
        k8s = []
        for x in deploys:
            j = K8sDeployment(config=self.config, name=x.name)
            j.model = x
            k8s.append(j)
        return k8s

    def delete(self, cascade=False):
        if cascade:
            rsets = K8sReplicaSet(
                config=self.config,
                name="yo"
            ).list(pattern=self.name)
            [rset.delete() for rset in rsets]
        super(K8sDeployment, self).delete(cascade)

    # -------------------------------------------------------------------------------------  wait

    def _wait_for_desired_replicas(self):
        start_time = time.time()
        while not self._has_desired_replicas():
            time.sleep(0.5)
            self.get()
            self._check_timeout(start_time)

    def _has_desired_replicas(self):
        if self.updated_replicas == self.desired_replicas \
                and self.current_replicas == self.desired_replicas \
                and self.available_replicas == self.desired_replicas:
            return True
        return False

    def _check_timeout(self, start_time=None):
        elapsed_time = time.time() - start_time
        if elapsed_time >= self.SCALE_WAIT_TIMEOUT_SECONDS:  # timeout
            raise TimedOutException(
                "Timed out scaling Deployment: [ {} ] to replica count: [ {} ]".format(
                    self.name,
                    self.desired_replicas))

    # -------------------------------------------------------------------------------------  add

    def add_container(self, container=None):
        if not isinstance(container, K8sContainer):
            raise SyntaxError('K8sDeployment.add_container() container: [ {0} ] is invalid.'.format(container))
        containers = self.model.spec.template.spec.containers
        if container.model not in containers:
            containers.append(container.model)
        self.model.spec.template.spec.containers = containers
        return self

    def add_image_pull_secrets(self, secret=None):
        self.model.spec.template.spec.add_image_pull_secrets(secret)
        return self

    def add_volume(self, volume=None):
        if not isinstance(volume, K8sVolume):
            raise SyntaxError('K8sDeployment.add_volume() volume: [ {0} ] is invalid.'.format(volume))
        volumes = self.model.spec.template.spec.volumes
        if volume.model not in volumes:
            volumes.append(volume.model)
        self.model.spec.template.spec.volumes = volumes
        return self

    # -------------------------------------------------------------------------------------  get

    def get(self):
        self.model = Deployment(self.get_model())
        return self

    # ------------------------------------------------------------------------------------- namespace

    @property
    def namespace(self):
        return self.model.metadata.namespace

    @namespace.setter
    def namespace(self, nspace=None):
        self.model.metadata.namespace = nspace

    # -------------------------------------------------------------------------------------  pod annotations

    @property
    def pod_annotations(self):
        return self.model.spec.template.metadata.annotations

    @pod_annotations.setter
    def pod_annotations(self, anns=None):
        self.model.spec.template.metadata.annotations = anns

    # -------------------------------------------------------------------------------------  pod labels

    @property
    def pod_labels(self):
        return self.model.spec.template.metadata.labels

    @pod_labels.setter
    def pod_labels(self, labels=None):
        self.model.spec.template.metadata.labels = labels

    # ------------------------------------------------------------------------------------- current replicas

    @property
    def current_replicas(self):
        return self.model.status.replicas

    @current_replicas.setter
    def current_replicas(self, reps=None):
        self.model.status.replicas = reps

    # ------------------------------------------------------------------------------------- desired replicas

    @property
    def desired_replicas(self):
        return self.model.spec.replicas

    @desired_replicas.setter
    def desired_replicas(self, reps=None):
        self.model.spec.replicas = reps

    # ------------------------------------------------------------------------------------- updated replicas

    @property
    def updated_replicas(self):
        return self.model.status.updated_replicas

    @updated_replicas.setter
    def updated_replicas(self, reps=None):
        self.model.status.updated_replicas = reps

    # ------------------------------------------------------------------------------------- available replicas

    @property
    def available_replicas(self):
        return self.model.status.available_replicas

    @available_replicas.setter
    def available_replicas(self, reps=None):
        self.model.status.available_replicas = reps

    # ------------------------------------------------------------------------------------- unavailable replicas

    @property
    def unavailable_replicas(self):
        return self.model.status.unavailable_replicas

    @unavailable_replicas.setter
    def unavailable_replicas(self, reps=None):
        self.model.status.unavailable_replicas = reps

    # -------------------------------------------------------------------------------------  revision

    @property
    def revision(self):
        if 'deployment.kubernetes.io/revision' in self.annotations:
            return int(self.annotations['deployment.kubernetes.io/revision'])
        return None

    @revision.setter
    def revision(self, r=None):
        raise NotImplementedError("K8sDeployment: revision is read-only.")

    # -------------------------------------------------------------------------------------  selector

    @property
    def selector(self):
        return self.model.spec.selector

    @selector.setter
    def selector(self, sel=None):
        self.model.spec.selector = sel

    # -------------------------------------------------------------------------------------  containers

    @property
    def containers(self):
        objs = []
        for c in self.model.spec.template.spec.containers:
            k8scontainer = K8sContainer(name=c.name, image=c.image)
            k8scontainer.model = c
            objs.append(k8scontainer)
        return objs

    @containers.setter
    def containers(self, containers=None):
        models = []
        for obj in containers:
            models.append(obj.model)
        self.model.spec.template.spec.containers = models

    # -------------------------------------------------------------------------------------  container_image

    @property
    def container_image(self, name=None):
        if name is None and len(self.containers) > 1:
            raise SyntaxError("K8sDeployment.container_image() Please specify a container name.")

        if len(self.containers) == 1:
            return self.containers[0].image
        else:
            filtered = list(filter(lambda x: x.name == name, self.containers))
            if filtered:
                return filtered[0].image
            return None

    @container_image.setter
    def container_image(self, tup=None):
        if not isinstance(tup, tuple):
            raise SyntaxError('K8sDeployment.container_image() must be a tuple of the form (name, image)')
        name, image = tup
        found = list(filter(lambda x: x.name == name, self.containers))
        if found:
            new = list(filter(lambda x: x.name != name, self.containers))
            found[0].image = image
            new.append(found[0])
            self.containers = new

    # -------------------------------------------------------------------------------------  get by name

    @staticmethod
    def get_by_name(config=None, name=None):
        """
        Fetches a K8sDeployment by name.

        :param config: A K8sConfig object.
        :param name: The name we want.
        :return: A list of K8sDeployment objects.
        """

        if name is None:
            raise SyntaxError('Deployment: name: [ {0} ] cannot be None.'.format(name))
        if not isinstance(name, str):
            raise SyntaxError('Deployment: name: [ {0} ] must be a string.'.format(name))

        if config is not None and not isinstance(config, K8sConfig):
            raise SyntaxError('Deployment: config: [ {0} ] must be a K8sConfig'.format(config))

        dep_list = list()
        data = {'labelSelector': 'name={0}'.format(name)}
        deps = K8sDeployment(config=config, name=name).get_with_params(data=data)

        for dep in deps:
            try:
                d = Deployment(dep)
                dep_name = d.metadata.name
                dep_list.append(K8sDeployment(config=config, name=dep_name).get())
            except NotFoundException:
                pass

        return dep_list

    # -------------------------------------------------------------------------------------  rollback

    def rollback(self, revision=None, annotations=None):
        """
        Performs a rollback of the Deployment.

        If the 'revision' parameter is omitted, we fetch the Deployment's system-generated
        annotation containing the current revision, and revert to the version immediately
        preceding the current version.

        :param revision: The revision to rollback to.
        :param annotations: Annotations we'd like to update.
        :return: self
        """

        rollback = DeploymentRollback()
        rollback.name = self.name

        # to the specified revision
        if revision is not None:
            rollback.rollback_to.revision = revision
        # to the revision immediately preceding the current revision
        else:
            current_revision = int(self.get_annotation('deployment.kubernetes.io/revision'))
            rev = max(current_revision - 1, 0)
            rollback.rollback_to.revision = rev

        if annotations is not None:
            rollback.updated_annotations.update(annotations)

        url = '{base}/{name}/rollback'.format(base=self.base_url, name=self.name)
        state = self.request(
            method='POST',
            url=url,
            data=rollback.serialize())

        self.get()
        self._wait_for_desired_replicas()

        if not state.get('success'):
            status = state.get('status', '')
            reason = state.get('data', dict()).get('message', None)
            message = 'K8sDeployment: ROLLBACK failed : HTTP {0} : {1}'.format(status, reason)
            raise BadRequestException(message)

        return self

    # -------------------------------------------------------------------------------------  scale

    def scale(self, replicas=None):
        """
        Scales up or down.

        :param replicas: The number of desired replicas.
        :return: self
        """

        self.desired_replicas = replicas
        self.update()
        return self

    # -------------------------------------------------------------------------------------  purge replica sets

    def purge_replica_sets(self, keep=3):
        """
        Builds a list of ReplicaSets, sorted from newest to oldest.
        Slices the array, keeping the most X most recent ReplicaSets.

        :param keep: The number of ReplicaSets to keep.
        :return: None.
        """

        rsets = K8sReplicaSet(
            config=self.config,
            name="yo").list(pattern=self.name, reverse=True)

        to_purge = rsets[keep:]
        for rset in to_purge:
            rset.delete(cascade=True)
<commit_msg>K8sDeployment: fixing updated annotations<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# This file is subject to the terms and conditions defined in
# file 'LICENSE.md', which is part of this source code package.
#

import time

from kubernetes.K8sConfig import K8sConfig
from kubernetes.K8sContainer import K8sContainer
from kubernetes.K8sExceptions import BadRequestException
from kubernetes.K8sExceptions import TimedOutException, NotFoundException
from kubernetes.K8sObject import K8sObject
from kubernetes.K8sVolume import K8sVolume
from kubernetes.K8sReplicaSet import K8sReplicaSet
from kubernetes.models.v1beta1.Deployment import Deployment
from kubernetes.models.v1beta1.DeploymentRollback import DeploymentRollback
from kubernetes.models.v1beta1.LabelSelector import LabelSelector

class K8sDeployment(K8sObject):

    SCALE_WAIT_TIMEOUT_SECONDS = 120

    def __init__(self, config=None, name=None, image=None, replicas=0):

        super(K8sDeployment, self).__init__(
            config=config,
            obj_type='Deployment',
            name=name
        )

        self.desired_replicas = replicas

        labels = {'name': name}
        sel = LabelSelector()
        sel.match_labels = labels

        self.selector = sel
        self.labels = labels
        self.pod_labels = labels

        if image is not None:
            container = K8sContainer(name=name, image=image)
            self.add_container(container)

        if self.config.pull_secret is not None:
            self.add_image_pull_secrets(self.config.pull_secret)

    # -------------------------------------------------------------------------------------  override

    def create(self):
        super(K8sDeployment, self).create()
        self.get()
        if self.desired_replicas > 0:
            self._wait_for_desired_replicas()
        return self

    def update(self):
        super(K8sDeployment, self).update()
        self.get()
        if self.desired_replicas > 0:
            self._wait_for_desired_replicas()
        return self

    def list(self, pattern=None):
        ls = super(K8sDeployment, self).list()
        deploys = list(map(lambda x: Deployment(x), ls))
        if pattern is not None:
            deploys = list(filter(lambda x: pattern in x.name, deploys))
        k8s = []
        for x in deploys:
            j = K8sDeployment(config=self.config, name=x.name)
            j.model = x
            k8s.append(j)
        return k8s

    def delete(self, cascade=False):
        if cascade:
            rsets = K8sReplicaSet(
                config=self.config,
                name="yo"
            ).list(pattern=self.name)
            [rset.delete() for rset in rsets]
        super(K8sDeployment, self).delete(cascade)

    # -------------------------------------------------------------------------------------  wait

    def _wait_for_desired_replicas(self):
        start_time = time.time()
        while not self._has_desired_replicas():
            time.sleep(0.5)
            self.get()
            self._check_timeout(start_time)

    def _has_desired_replicas(self):
        if self.updated_replicas == self.desired_replicas \
                and self.current_replicas == self.desired_replicas \
                and self.available_replicas == self.desired_replicas:
            return True
        return False

    def _check_timeout(self, start_time=None):
        elapsed_time = time.time() - start_time
        if elapsed_time >= self.SCALE_WAIT_TIMEOUT_SECONDS:  # timeout
            raise TimedOutException(
                "Timed out scaling Deployment: [ {} ] to replica count: [ {} ]".format(
                    self.name,
                    self.desired_replicas))

    # -------------------------------------------------------------------------------------  add

    def add_container(self, container=None):
        if not isinstance(container, K8sContainer):
            raise SyntaxError('K8sDeployment.add_container() container: [ {0} ] is invalid.'.format(container))
        containers = self.model.spec.template.spec.containers
        if container.model not in containers:
            containers.append(container.model)
        self.model.spec.template.spec.containers = containers
        return self

    def add_image_pull_secrets(self, secret=None):
        self.model.spec.template.spec.add_image_pull_secrets(secret)
        return self

    def add_volume(self, volume=None):
        if not isinstance(volume, K8sVolume):
            raise SyntaxError('K8sDeployment.add_volume() volume: [ {0} ] is invalid.'.format(volume))
        volumes = self.model.spec.template.spec.volumes
        if volume.model not in volumes:
            volumes.append(volume.model)
        self.model.spec.template.spec.volumes = volumes
        return self

    # -------------------------------------------------------------------------------------  get

    def get(self):
        self.model = Deployment(self.get_model())
        return self

    # ------------------------------------------------------------------------------------- namespace

    @property
    def namespace(self):
        return self.model.metadata.namespace

    @namespace.setter
    def namespace(self, nspace=None):
        self.model.metadata.namespace = nspace

    # -------------------------------------------------------------------------------------  pod annotations

    @property
    def pod_annotations(self):
        return self.model.spec.template.metadata.annotations

    @pod_annotations.setter
    def pod_annotations(self, anns=None):
        self.model.spec.template.metadata.annotations = anns

    # -------------------------------------------------------------------------------------  pod labels

    @property
    def pod_labels(self):
        return self.model.spec.template.metadata.labels

    @pod_labels.setter
    def pod_labels(self, labels=None):
        self.model.spec.template.metadata.labels = labels

    # ------------------------------------------------------------------------------------- current replicas

    @property
    def current_replicas(self):
        return self.model.status.replicas

    @current_replicas.setter
    def current_replicas(self, reps=None):
        self.model.status.replicas = reps

    # ------------------------------------------------------------------------------------- desired replicas

    @property
    def desired_replicas(self):
        return self.model.spec.replicas

    @desired_replicas.setter
    def desired_replicas(self, reps=None):
        self.model.spec.replicas = reps

    # ------------------------------------------------------------------------------------- updated replicas

    @property
    def updated_replicas(self):
        return self.model.status.updated_replicas

    @updated_replicas.setter
    def updated_replicas(self, reps=None):
        self.model.status.updated_replicas = reps

    # ------------------------------------------------------------------------------------- available replicas

    @property
    def available_replicas(self):
        return self.model.status.available_replicas

    @available_replicas.setter
    def available_replicas(self, reps=None):
        self.model.status.available_replicas = reps

    # ------------------------------------------------------------------------------------- unavailable replicas

    @property
    def unavailable_replicas(self):
        return self.model.status.unavailable_replicas

    @unavailable_replicas.setter
    def unavailable_replicas(self, reps=None):
        self.model.status.unavailable_replicas = reps

    # -------------------------------------------------------------------------------------  revision

    @property
    def revision(self):
        if 'deployment.kubernetes.io/revision' in self.annotations:
            return int(self.annotations['deployment.kubernetes.io/revision'])
        return None

    @revision.setter
    def revision(self, r=None):
        raise NotImplementedError("K8sDeployment: revision is read-only.")

    # -------------------------------------------------------------------------------------  selector

    @property
    def selector(self):
        return self.model.spec.selector

    @selector.setter
    def selector(self, sel=None):
        self.model.spec.selector = sel

    # -------------------------------------------------------------------------------------  containers

    @property
    def containers(self):
        objs = []
        for c in self.model.spec.template.spec.containers:
            k8scontainer = K8sContainer(name=c.name, image=c.image)
            k8scontainer.model = c
            objs.append(k8scontainer)
        return objs

    @containers.setter
    def containers(self, containers=None):
        models = []
        for obj in containers:
            models.append(obj.model)
        self.model.spec.template.spec.containers = models

    # -------------------------------------------------------------------------------------  container_image

    @property
    def container_image(self, name=None):
        if name is None and len(self.containers) > 1:
            raise SyntaxError("K8sDeployment.container_image() Please specify a container name.")

        if len(self.containers) == 1:
            return self.containers[0].image
        else:
            filtered = list(filter(lambda x: x.name == name, self.containers))
            if filtered:
                return filtered[0].image
            return None

    @container_image.setter
    def container_image(self, tup=None):
        if not isinstance(tup, tuple):
            raise SyntaxError('K8sDeployment.container_image() must be a tuple of the form (name, image)')
        name, image = tup
        found = list(filter(lambda x: x.name == name, self.containers))
        if found:
            new = list(filter(lambda x: x.name != name, self.containers))
            found[0].image = image
            new.append(found[0])
            self.containers = new

    # -------------------------------------------------------------------------------------  get by name

    @staticmethod
    def get_by_name(config=None, name=None):
        """
        Fetches a K8sDeployment by name.

        :param config: A K8sConfig object.
        :param name: The name we want.
        :return: A list of K8sDeployment objects.
        """

        if name is None:
            raise SyntaxError('Deployment: name: [ {0} ] cannot be None.'.format(name))
        if not isinstance(name, str):
            raise SyntaxError('Deployment: name: [ {0} ] must be a string.'.format(name))

        if config is not None and not isinstance(config, K8sConfig):
            raise SyntaxError('Deployment: config: [ {0} ] must be a K8sConfig'.format(config))

        dep_list = list()
        data = {'labelSelector': 'name={0}'.format(name)}
        deps = K8sDeployment(config=config, name=name).get_with_params(data=data)

        for dep in deps:
            try:
                d = Deployment(dep)
                dep_name = d.metadata.name
                dep_list.append(K8sDeployment(config=config, name=dep_name).get())
            except NotFoundException:
                pass

        return dep_list

    # -------------------------------------------------------------------------------------  rollback

    def rollback(self, revision=None, annotations=None):
        """
        Performs a rollback of the Deployment.

        If the 'revision' parameter is omitted, we fetch the Deployment's system-generated
        annotation containing the current revision, and revert to the version immediately
        preceding the current version.

        :param revision: The revision to rollback to.
        :param annotations: Annotations we'd like to update.
        :return: self
        """

        rollback = DeploymentRollback()
        rollback.name = self.name

        # to the specified revision
        if revision is not None:
            rollback.rollback_to.revision = revision
        # to the revision immediately preceding the current revision
        else:
            current_revision = int(self.get_annotation('deployment.kubernetes.io/revision'))
            rev = max(current_revision - 1, 0)
            rollback.rollback_to.revision = rev

        if annotations is not None:
            rollback.updated_annotations = annotations

        url = '{base}/{name}/rollback'.format(base=self.base_url, name=self.name)
        state = self.request(
            method='POST',
            url=url,
            data=rollback.serialize())

        self.get()
        self._wait_for_desired_replicas()

        if not state.get('success'):
            status = state.get('status', '')
            reason = state.get('data', dict()).get('message', None)
            message = 'K8sDeployment: ROLLBACK failed : HTTP {0} : {1}'.format(status, reason)
            raise BadRequestException(message)

        return self

    # -------------------------------------------------------------------------------------  scale

    def scale(self, replicas=None):
        """
        Scales up or down.

        :param replicas: The number of desired replicas.
        :return: self
        """

        self.desired_replicas = replicas
        self.update()
        return self

    # -------------------------------------------------------------------------------------  purge replica sets

    def purge_replica_sets(self, keep=3):
        """
        Builds a list of ReplicaSets, sorted from newest to oldest.
        Slices the array, keeping the most X most recent ReplicaSets.

        :param keep: The number of ReplicaSets to keep.
        :return: None.
        """

        rsets = K8sReplicaSet(
            config=self.config,
            name="yo").list(pattern=self.name, reverse=True)

        to_purge = rsets[keep:]
        for rset in to_purge:
            rset.delete(cascade=True)
<|endoftext|>


<commit_before>import time

from kunai.log import logger
from kunai.collector import Collector
from kunai.util import to_best_int_float

class Redis(Collector):
    RATE_KEYS = ['used_cpu_sys', 'used_cpu_sys_children', 'used_cpu_user', 'used_cpu_user_children', 'total_commands_processed']
    def __init__(self, config, put_result=None):
        super(Redis, self).__init__(config, put_result)
        self.store = {}
        self.last_launch = 0.0

    def launch(self):
        try:
            import redis
        except ImportError:
            logger.error('Unable to import redis library')
            return False

        now = int(time.time())
        diff = now - self.last_launch
        self.last_launch = now

        conn = redis.Redis()

        start = time.time()
        try:
            info = conn.info()
        except ValueError, e:
            raise
        except Exception, e:
            raise

        latency_ms = round((time.time() - start) * 1000, 2)
        info['connexion_latency_ms'] = latency_ms
        info['available'] = True

        to_add = {}
        for (k,v) in info.iteritems():
            if k in self.RATE_KEYS:
                if k in self.store:
                    to_add['%s/s' % k] = (v - self.store[k]) / diff
                else:
                    to_add['%s/s' % k] = 0
                self.store[k] = info[k]

        for k in self.RATE_KEYS:
            del info[k]
        info.update(to_add)

        return info
<commit_msg>Fix: redis collector when redis is down<commit_after>import time

from kunai.log import logger
from kunai.collector import Collector
from kunai.util import to_best_int_float

class Redis(Collector):
    RATE_KEYS = ['used_cpu_sys', 'used_cpu_sys_children', 'used_cpu_user', 'used_cpu_user_children', 'total_commands_processed']
    def __init__(self, config, put_result=None):
        super(Redis, self).__init__(config, put_result)
        self.store = {}
        self.last_launch = 0.0

    def launch(self):
        try:
            import redis
        except ImportError:
            logger.error('Unable to import redis library')
            return {'available':False}

        now = int(time.time())
        diff = now - self.last_launch
        self.last_launch = now

        conn = redis.Redis()

        start = time.time()
        try:
            info = conn.info()
        except ValueError, e:
            raise
        except redis.ConnectionError, exp:
            logger.info('Redis connexion fail: %s' % exp)
            return {'available':False}
        except Exception, e:
            raise

        latency_ms = round((time.time() - start) * 1000, 2)
        info['connexion_latency_ms'] = latency_ms
        info['available'] = True

        to_add = {}
        for (k,v) in info.iteritems():
            if k in self.RATE_KEYS:
                if k in self.store:
                    to_add['%s/s' % k] = (v - self.store[k]) / diff
                else:
                    to_add['%s/s' % k] = 0
                self.store[k] = info[k]

        for k in self.RATE_KEYS:
            del info[k]
        info.update(to_add)

        return info
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sqlite3 as lite
import os

class Database:

    """
    Handles db connection
    """

    def __init__(self, **kwargs):
        self.connection = None
        path = None

        try:
            path = os.environ.get("SPIFFYDB_PATH")
        except KeyError:
            pass

        if path is None:
            raise RuntimeError("Enviroment variable SPIFFYDB_PATH not found!")

        is_file = os.path.isfile(path)
        is_readable = os.access(path, os.R_OK)

        if not is_file or not is_readable:
            raise RuntimeError("Unable to read DB file: %s" % path)

        self.connection = lite.connect(path)
        self.connection.row_factory = lite.Row

    def get_connection(self):
        return self.connection

    def destroy(self):
        if self.connection is not None:
            self.connection.close()
<commit_msg>Removes Database.destroy (unused)<commit_after># -*- coding: utf-8 -*-
import sqlite3 as lite
import os

class Database:

    """
    Handles db connection
    """

    def __init__(self, **kwargs):
        self.connection = None
        path = None

        try:
            path = os.environ.get("SPIFFYDB_PATH")
        except KeyError:
            pass

        if path is None:
            raise RuntimeError("Enviroment variable SPIFFYDB_PATH not found!")

        is_file = os.path.isfile(path)
        is_readable = os.access(path, os.R_OK)

        if not is_file or not is_readable:
            raise RuntimeError("Unable to read DB file: %s" % path)

        self.connection = lite.connect(path)
        self.connection.row_factory = lite.Row

    def get_connection(self):
        return self.connection
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Jon Surrell
# Copyright (c) 2013 Jon Surrell
#
# License: MIT
#

"""This module exports the Ghc plugin class."""

from SublimeLinter.lint import Linter, util
from os.path import basename

class Ghc(Linter):

    """Provides an interface to ghc."""

    syntax = ('haskell', 'haskell-sublimehaskell', 'literate haskell')
    cmd = ('ghc', '-fno-code', '-Wall', '-Wwarn', '-fno-helpful-errors')
    regex = (
        r'(?i)'
        r'^(?P<filename>.+):'
        r'(?P<line>\d+):(?P<col>\d+):'
        r'\s+(?P<warning>Warning:\s+)?(?P<message>([\n]|.)+?)(?=\n\s*\|)'
    )
    multiline = True

    # No stdin
    tempfile_suffix = {
        'haskell': 'hs',
        'haskell-sublimehaskell': 'hs',
        'literate haskell': 'lhs'
    }

    # ghc writes errors to STDERR
    error_stream = util.STREAM_STDERR

    def split_match(self, match):
        """Override to ignore errors reported in imported files."""
        match, line, col, error, warning, message, near = (
            super().split_match(match)
        )

        match_filename = basename(match.groupdict()['filename'])
        linted_filename = basename(self.filename)

        if match_filename != linted_filename:
            return None, None, None, None, None, '', None

        return match, line, col, error, warning, message, near
<commit_msg>Haskell: Sublime 3.1<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Jon Surrell
# Copyright (c) 2013 Jon Surrell
#
# License: MIT
#

"""This module exports the Ghc plugin class."""

from SublimeLinter.lint import Linter, util
from os.path import basename

class Ghc(Linter):

    """Provides an interface to ghc."""

    syntax = ('haskell', 'haskell-sublimehaskell', 'literate haskell')
    cmd = ('ghc', '-fno-code', '-Wall', '-Wwarn', '-fno-helpful-errors')
    regex = (
        r'(?i)'
        r'^(?P<filename>.+):'
        r'(?P<line>\d+):(?P<col>\d+):'
        r'\s+(?P<warning>Warning:\s+)?(?P<message>([\n]|.)+?)(?=\n\s*\|)'
    )
    multiline = True
    defaults = {
        'selector': 'source.haskell, text.tex.latex.haskell'
    }

    # No stdin
    tempfile_suffix = {
        'haskell': 'hs',
        'haskell-sublimehaskell': 'hs',
        'literate haskell': 'lhs'
    }

    # ghc writes errors to STDERR
    error_stream = util.STREAM_STDERR

    def split_match(self, match):
        """Override to ignore errors reported in imported files."""
        match, line, col, error, warning, message, near = (
            super().split_match(match)
        )

        match_filename = basename(match.groupdict()['filename'])
        linted_filename = basename(self.filename)

        if match_filename != linted_filename:
            return None, None, None, None, None, '', None

        return match, line, col, error, warning, message, near
<|endoftext|>


<commit_before>from django.core.management.base import NoArgsCommand
from ec2spotmanager.models import PoolConfiguration, InstancePool, Instance, INSTANCE_STATE
from django.conf import settings
from ec2spotmanager.management.common import mgmt_lock_required
import warnings
import datetime
import time
import logging
import threading

from django.utils import timezone

from laniakea.laniakea import LaniakeaCommandLine
from laniakea.core.manager import Laniakea
import boto.ec2
import boto.exception

class Command(NoArgsCommand):
    help = "Check the status of all bugs we have"
    @mgmt_lock_required
    def handle_noargs(self, **options):
        while True:
            self.check_instance_pools()
            time.sleep(10)

    def check_instance_pools(self):
        instance_pools = InstancePool.objects.all()

        # Process all instance pools
        for instance_pool in instance_pools:
            config = instance_pool.config.flatten()
            instances = Instance.objects.filter(pool=instance_pool)

            instances_missing = config.size
            running_instances = []

            self.update_pool_instances(instance_pool, instances, config)
            for instance in instances:
                if instance.status_code == INSTANCE_STATE['running'] or instance.status_code == INSTANCE_STATE['pending']:
                    instances_missing -= 1
                    running_instances.append(instance)
                else:
                    # The instance is no longer running, delete it from our database
                    instance.delete()

            # Continue working with the instances we have running
            instances = running_instances

            if (not instance_pool.last_cycled) or instance_pool.last_cycled < timezone.now() - timezone.timedelta(seconds=config.cycle_interval):
                print("[Main] Pool needs to be cycled, terminating all instances...")
                instance_pool.last_cycled = timezone.now()
                self.terminate_pool_instances(instance_pool, instances, config, terminateByPool=True)
                instance_pool.save()
                self.update_pool_instances(instance_pool, instances, config)
                print("[Main] Pool termination complete.")

            # Determine which instances need to be cycled
            #outdated_instances = instances.filter(created__lt = timezone.now() - timezone.timedelta(seconds=config.cycle_interval))

            # Terminate all instances that need cycling
            #for instance in outdated_instances:
            #    self.terminate_instance(instance, config)
            #    instances_missing += 1

            if instances_missing > 0:
                print("[Main] Pool needs %s more instances, starting..." % instances_missing)
                self.start_pool_instances(instance_pool, config, count=instances_missing)
            elif instances_missing < 0:
                # Select the oldest instances we have running and terminate
                # them so we meet the size limitation again.
                print("[Main] Pool has %s instances over limit, terminating..." % -instances_missing)
                instances = Instance.objects.filter(pool=instance_pool).order_by('created')[:-instances_missing]
                self.terminate_pool_instances(instance_pool, instances, config)
            else:
                print("[Main] Pool size ok.")

    def get_best_region_zone(self, config):
        def get_spot_price_per_region(region_name, profile_name, instance_type):
            '''Gets spot prices of the specified region and instance type'''
            now = datetime.datetime.now()
            start = now - datetime.timedelta(hours=6)
            r = boto.ec2.connect_to_region(region_name, profile_name=profile_name).get_spot_price_history(
                                                             start_time=start.isoformat(),
                                                             end_time=now.isoformat(),
                                                             instance_type=instance_type,
                                                             product_description="Linux/UNIX") #TODO: Make configurable
            return r

        def get_price_median(data):
            sdata = sorted(data)
            n = len(sdata)
            if not n % 2:
                return (sdata[n / 2] + sdata[n / 2 - 1]) / 2.0
            return sdata[n / 2]

        from multiprocessing import Pool, cpu_count
        pool = Pool(cpu_count())
        results = []
        for region in config.allowed_regions:
            f = pool.apply_async(get_spot_price_per_region, [region, "laniakea", config.instance_type])
            results.append(f)

        prices = {}
        for result in results:
            #r = result.get()
            for entry in result.get():
                if not entry.region.name in prices:
                    prices[entry.region.name] = {}

                zone = entry.availability_zone

                if not zone in prices[entry.region.name]:
                    prices[entry.region.name][zone] = []

                prices[entry.region.name][zone].append(entry.price)

        # Calculate median values for all availability zones and best zone/price
        best_zone = None
        best_region = None
        best_median = None
        for region in prices:
            for zone in prices[region]:
                # Do not consider a zone/region combination that has a current
                # price higher than the maximum price we are willing to pay,
                # even if the median would end up being lower than our maximum.
                if prices[region][zone][-1] > config.ec2_max_price:
                    continue

                median = get_price_median(prices[region][zone])
                if best_median == None or best_median > median:
                    best_median = median
                    best_zone = zone
                    best_region = region

        return (best_region, best_zone)

    def create_laniakea_images(self, config):
        images = { "default" : {} }

        # These are the configuration keys we want to put into the target configuration
        # without further preprocessing, except for the adjustment of the key name itself.
        keys = [
            'ec2_key_name',
            'ec2_image_name',
            'ec2_instance_type',
            'ec2_security_groups',
        ]

        for key in keys:
            lkey = key.replace("ec2_", "", 1)
            images["default"][lkey] = config[key]

        if config.ec2_raw_config:
            images["default"].update(config.ec2_raw_config)

        return images

    def start_pool_instances(self, pool, config, count=1):
        """ Start an instance with the given configuration """

        images = self.create_laniakea_images(config)
        (region, zone) = self.get_best_region_zone(config)
        instances = []

        # Create all our instances as pending, the async thread will update them once
        # they have been spawned.
        for i in range(0,count):
            instance = Instance()
            instance.ec2_region = region
            instance.status_code = INSTANCE_STATE["pending"]
            instance.pool = pool
            instance.save()
            instances.append(instance)

        # This method will run async to spawn our machines
        def start_instances_async(pool, config, count, images, region, zone, instances):
            userdata = LaniakeaCommandLine.handle_import_tags(config.ec2_userdata)
            userdata = LaniakeaCommandLine.handle_tags(userdata, config.ec2_userdata_macros)
            if not userdata:
                raise RuntimeError("start_instance: Failed to compile userdata")

            images["default"]['user_data'] = userdata
            images["default"]['placement'] = zone
            images["default"]['count'] = count

            cluster = Laniakea(images)
            try:
                cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
            except Exception as msg:
                logging.error("%s: laniakea failure: %s" % ("start_instances_async", msg))
                return

            config.ec2_tags['SpotManager-PoolId'] = str(pool.pk)

            try:
                print("Creating %s instances" % count)
                (boto_instances, boto_pending) = cluster.create_spot(config.ec2_max_price, tags=config.ec2_tags, timeout=20*60)

                assert (len(boto_instances) + len(boto_pending)) == len(instances) == count

                for i in range(0,len(boto_instances)):
                    instances[i].hostname = boto_instances[i].public_dns_name
                    instances[i].ec2_instance_id = boto_instances[i].id
                    instances[i].status_code = boto_instances[i].state_code
                    instances[i].save()

                if boto_pending:
                    for i in range(len(boto_instances),len(boto_pending)):
                        # Delete instances belong to canceled spot requests
                        instances[i].delete()

            except boto.exception.EC2ResponseError as msg:
                logging.error("%s: boto failure: %s" % ("start_instances_async", msg))
                return

        # TODO: We don't get any information back from the async method call here, but should handle failures!
        t = threading.Thread(target=start_instances_async, args = (pool, config, count, images, region, zone, instances))
        t.start()

    def terminate_pool_instances(self, pool, instances, config, terminateByPool=False):
        """ Terminate an instance with the given configuration """
        instance_ids_by_region = self.get_instance_ids_by_region(instances)

        for region in instance_ids_by_region:
            cluster = Laniakea(None)
            try:
                cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
            except Exception as msg:
                logging.error("%s: laniakea failure: %s" % ("terminate_pool_instances", msg))
                return None

            try:
                if terminateByPool:
                    boto_instances = cluster.find(filters={"tag:SpotManager-PoolId" : str(pool.pk)})

                    # Data consistency checks
                    for boto_instance in boto_instances:
                        assert ((boto_instance.id in instance_ids_by_region[region])
                                or (boto_instance.state_code == INSTANCE_STATE['shutting-down']
                                or boto_instance.state_code == INSTANCE_STATE['terminated']))

                    cluster.terminate(boto_instances)
                else:
                    print("Terminating %s instances in region %s" % (len(instance_ids_by_region[region]),region))
                    cluster.terminate(cluster.find(instance_ids=instance_ids_by_region[region]))
            except boto.exception.EC2ResponseError as msg:
                logging.error("%s: boto failure: %s" % ("terminate_pool_instances", msg))
                return 1

    def get_instance_ids_by_region(self, instances):
        instance_ids_by_region = {}

        for instance in instances:
            if not instance.ec2_region in instance_ids_by_region:
                instance_ids_by_region[instance.ec2_region] = []
            instance_ids_by_region[instance.ec2_region].append(instance.ec2_instance_id)

        return instance_ids_by_region

    def get_instances_by_ids(self, instances):
        instances_by_ids = {}
        for instance in instances:
            instances_by_ids[instance.ec2_instance_id] = instance
        return instances_by_ids

    def update_pool_instances(self, pool, instances, config):
        """ Check the state of the instances in a pool and update it in the database """
        instance_ids_by_region = self.get_instance_ids_by_region(instances)
        instances_by_ids = self.get_instances_by_ids(instances)

        for region in instance_ids_by_region:
            cluster = Laniakea(None)
            try:
                cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
            except Exception as msg:
                logging.error("%s: laniakea failure: %s" % ("update_pool_instances", msg))
                return None

            try:
                #cluster.find(instance_ids=instance_ids_by_region[region])
                boto_instances = cluster.find(filters={"tag:SpotManager-PoolId" : str(pool.pk)})

                for boto_instance in boto_instances:
                    # Whenever we see an instance that is not in our instance list for that region,
                    # make sure it's a terminated instance because we should never have running instance
                    #
                    # We must however not perform this check if we still have pending instances.
                    # In this case, the thread that is monitoring the pending instances must first
                    # redeclare them with their proper id in the database before we perform *any*
                    # updates on it. Otherwise, parallel save operations on the instance object
                    # might lead to inconsistent states of the database model
                    if not boto_instance.id in instance_ids_by_region[region]:
                        if not None in instance_ids_by_region:
                            assert (boto_instance.state_code == INSTANCE_STATE['shutting-down']
                                or boto_instance.state_code == INSTANCE_STATE['terminated'])

                        continue

                    instance = instances_by_ids[boto_instance.id]

                    # Check the status code and update if necessary
                    if instance.status_code != boto_instance.state_code:
                        instance.status_code = boto_instance.state_code
                        instance.save()

                    # If for some reason we don't have a hostname yet,
                    # update it accordingly.
                    if not instance.hostname:
                        instance.hostname = boto_instance.public_dns_name
                        instance.save()

            except boto.exception.EC2ResponseError as msg:
                logging.error("%s: boto failure: %s" % ("update_pool_instances", msg))
                return 1

<commit_msg>Delete EBS volumes on termination by default<commit_after>from django.core.management.base import NoArgsCommand
from ec2spotmanager.models import PoolConfiguration, InstancePool, Instance, INSTANCE_STATE
from django.conf import settings
from ec2spotmanager.management.common import mgmt_lock_required
import warnings
import datetime
import time
import logging
import threading

from django.utils import timezone

from laniakea.laniakea import LaniakeaCommandLine
from laniakea.core.manager import Laniakea
import boto.ec2
import boto.exception

class Command(NoArgsCommand):
    help = "Check the status of all bugs we have"
    @mgmt_lock_required
    def handle_noargs(self, **options):
        while True:
            self.check_instance_pools()
            time.sleep(10)

    def check_instance_pools(self):
        instance_pools = InstancePool.objects.all()

        # Process all instance pools
        for instance_pool in instance_pools:
            config = instance_pool.config.flatten()
            instances = Instance.objects.filter(pool=instance_pool)

            instances_missing = config.size
            running_instances = []

            self.update_pool_instances(instance_pool, instances, config)
            for instance in instances:
                if instance.status_code == INSTANCE_STATE['running'] or instance.status_code == INSTANCE_STATE['pending']:
                    instances_missing -= 1
                    running_instances.append(instance)
                else:
                    # The instance is no longer running, delete it from our database
                    instance.delete()

            # Continue working with the instances we have running
            instances = running_instances

            if (not instance_pool.last_cycled) or instance_pool.last_cycled < timezone.now() - timezone.timedelta(seconds=config.cycle_interval):
                print("[Main] Pool needs to be cycled, terminating all instances...")
                instance_pool.last_cycled = timezone.now()
                self.terminate_pool_instances(instance_pool, instances, config, terminateByPool=True)
                instance_pool.save()
                self.update_pool_instances(instance_pool, instances, config)
                print("[Main] Pool termination complete.")

            # Determine which instances need to be cycled
            #outdated_instances = instances.filter(created__lt = timezone.now() - timezone.timedelta(seconds=config.cycle_interval))

            # Terminate all instances that need cycling
            #for instance in outdated_instances:
            #    self.terminate_instance(instance, config)
            #    instances_missing += 1

            if instances_missing > 0:
                print("[Main] Pool needs %s more instances, starting..." % instances_missing)
                self.start_pool_instances(instance_pool, config, count=instances_missing)
            elif instances_missing < 0:
                # Select the oldest instances we have running and terminate
                # them so we meet the size limitation again.
                print("[Main] Pool has %s instances over limit, terminating..." % -instances_missing)
                instances = Instance.objects.filter(pool=instance_pool).order_by('created')[:-instances_missing]
                self.terminate_pool_instances(instance_pool, instances, config)
            else:
                print("[Main] Pool size ok.")

    def get_best_region_zone(self, config):
        def get_spot_price_per_region(region_name, profile_name, instance_type):
            '''Gets spot prices of the specified region and instance type'''
            now = datetime.datetime.now()
            start = now - datetime.timedelta(hours=6)
            r = boto.ec2.connect_to_region(region_name, profile_name=profile_name).get_spot_price_history(
                                                             start_time=start.isoformat(),
                                                             end_time=now.isoformat(),
                                                             instance_type=instance_type,
                                                             product_description="Linux/UNIX") #TODO: Make configurable
            return r

        def get_price_median(data):
            sdata = sorted(data)
            n = len(sdata)
            if not n % 2:
                return (sdata[n / 2] + sdata[n / 2 - 1]) / 2.0
            return sdata[n / 2]

        from multiprocessing import Pool, cpu_count
        pool = Pool(cpu_count())
        results = []
        for region in config.allowed_regions:
            f = pool.apply_async(get_spot_price_per_region, [region, "laniakea", config.instance_type])
            results.append(f)

        prices = {}
        for result in results:
            #r = result.get()
            for entry in result.get():
                if not entry.region.name in prices:
                    prices[entry.region.name] = {}

                zone = entry.availability_zone

                if not zone in prices[entry.region.name]:
                    prices[entry.region.name][zone] = []

                prices[entry.region.name][zone].append(entry.price)

        # Calculate median values for all availability zones and best zone/price
        best_zone = None
        best_region = None
        best_median = None
        for region in prices:
            for zone in prices[region]:
                # Do not consider a zone/region combination that has a current
                # price higher than the maximum price we are willing to pay,
                # even if the median would end up being lower than our maximum.
                if prices[region][zone][-1] > config.ec2_max_price:
                    continue

                median = get_price_median(prices[region][zone])
                if best_median == None or best_median > median:
                    best_median = median
                    best_zone = zone
                    best_region = region

        return (best_region, best_zone)

    def create_laniakea_images(self, config):
        images = { "default" : {} }

        # These are the configuration keys we want to put into the target configuration
        # without further preprocessing, except for the adjustment of the key name itself.
        keys = [
            'ec2_key_name',
            'ec2_image_name',
            'ec2_instance_type',
            'ec2_security_groups',
        ]

        for key in keys:
            lkey = key.replace("ec2_", "", 1)
            images["default"][lkey] = config[key]

        if config.ec2_raw_config:
            images["default"].update(config.ec2_raw_config)

        return images

    def start_pool_instances(self, pool, config, count=1):
        """ Start an instance with the given configuration """

        images = self.create_laniakea_images(config)
        (region, zone) = self.get_best_region_zone(config)
        instances = []

        # Create all our instances as pending, the async thread will update them once
        # they have been spawned.
        for i in range(0,count):
            instance = Instance()
            instance.ec2_region = region
            instance.status_code = INSTANCE_STATE["pending"]
            instance.pool = pool
            instance.save()
            instances.append(instance)

        # This method will run async to spawn our machines
        def start_instances_async(pool, config, count, images, region, zone, instances):
            userdata = LaniakeaCommandLine.handle_import_tags(config.ec2_userdata)
            userdata = LaniakeaCommandLine.handle_tags(userdata, config.ec2_userdata_macros)
            if not userdata:
                raise RuntimeError("start_instance: Failed to compile userdata")

            images["default"]['user_data'] = userdata
            images["default"]['placement'] = zone
            images["default"]['count'] = count

            cluster = Laniakea(images)
            try:
                cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
            except Exception as msg:
                logging.error("%s: laniakea failure: %s" % ("start_instances_async", msg))
                return

            config.ec2_tags['SpotManager-PoolId'] = str(pool.pk)

            try:
                print("Creating %s instances" % count)
                (boto_instances, boto_pending) = cluster.create_spot(config.ec2_max_price, tags=config.ec2_tags, delete_on_termination=True, timeout=20*60)

                assert (len(boto_instances) + len(boto_pending)) == len(instances) == count

                for i in range(0,len(boto_instances)):
                    instances[i].hostname = boto_instances[i].public_dns_name
                    instances[i].ec2_instance_id = boto_instances[i].id
                    instances[i].status_code = boto_instances[i].state_code
                    instances[i].save()

                if boto_pending:
                    for i in range(len(boto_instances),len(boto_pending)):
                        # Delete instances belong to canceled spot requests
                        instances[i].delete()

            except boto.exception.EC2ResponseError as msg:
                logging.error("%s: boto failure: %s" % ("start_instances_async", msg))
                return

        # TODO: We don't get any information back from the async method call here, but should handle failures!
        t = threading.Thread(target=start_instances_async, args = (pool, config, count, images, region, zone, instances))
        t.start()

    def terminate_pool_instances(self, pool, instances, config, terminateByPool=False):
        """ Terminate an instance with the given configuration """
        instance_ids_by_region = self.get_instance_ids_by_region(instances)

        for region in instance_ids_by_region:
            cluster = Laniakea(None)
            try:
                cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
            except Exception as msg:
                logging.error("%s: laniakea failure: %s" % ("terminate_pool_instances", msg))
                return None

            try:
                if terminateByPool:
                    boto_instances = cluster.find(filters={"tag:SpotManager-PoolId" : str(pool.pk)})

                    # Data consistency checks
                    for boto_instance in boto_instances:
                        assert ((boto_instance.id in instance_ids_by_region[region])
                                or (boto_instance.state_code == INSTANCE_STATE['shutting-down']
                                or boto_instance.state_code == INSTANCE_STATE['terminated']))

                    cluster.terminate(boto_instances)
                else:
                    print("Terminating %s instances in region %s" % (len(instance_ids_by_region[region]),region))
                    cluster.terminate(cluster.find(instance_ids=instance_ids_by_region[region]))
            except boto.exception.EC2ResponseError as msg:
                logging.error("%s: boto failure: %s" % ("terminate_pool_instances", msg))
                return 1

    def get_instance_ids_by_region(self, instances):
        instance_ids_by_region = {}

        for instance in instances:
            if not instance.ec2_region in instance_ids_by_region:
                instance_ids_by_region[instance.ec2_region] = []
            instance_ids_by_region[instance.ec2_region].append(instance.ec2_instance_id)

        return instance_ids_by_region

    def get_instances_by_ids(self, instances):
        instances_by_ids = {}
        for instance in instances:
            instances_by_ids[instance.ec2_instance_id] = instance
        return instances_by_ids

    def update_pool_instances(self, pool, instances, config):
        """ Check the state of the instances in a pool and update it in the database """
        instance_ids_by_region = self.get_instance_ids_by_region(instances)
        instances_by_ids = self.get_instances_by_ids(instances)

        for region in instance_ids_by_region:
            cluster = Laniakea(None)
            try:
                cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
            except Exception as msg:
                logging.error("%s: laniakea failure: %s" % ("update_pool_instances", msg))
                return None

            try:
                #cluster.find(instance_ids=instance_ids_by_region[region])
                boto_instances = cluster.find(filters={"tag:SpotManager-PoolId" : str(pool.pk)})

                for boto_instance in boto_instances:
                    # Whenever we see an instance that is not in our instance list for that region,
                    # make sure it's a terminated instance because we should never have running instance
                    #
                    # We must however not perform this check if we still have pending instances.
                    # In this case, the thread that is monitoring the pending instances must first
                    # redeclare them with their proper id in the database before we perform *any*
                    # updates on it. Otherwise, parallel save operations on the instance object
                    # might lead to inconsistent states of the database model
                    if not boto_instance.id in instance_ids_by_region[region]:
                        if not None in instance_ids_by_region:
                            assert (boto_instance.state_code == INSTANCE_STATE['shutting-down']
                                or boto_instance.state_code == INSTANCE_STATE['terminated'])

                        continue

                    instance = instances_by_ids[boto_instance.id]

                    # Check the status code and update if necessary
                    if instance.status_code != boto_instance.state_code:
                        instance.status_code = boto_instance.state_code
                        instance.save()

                    # If for some reason we don't have a hostname yet,
                    # update it accordingly.
                    if not instance.hostname:
                        instance.hostname = boto_instance.public_dns_name
                        instance.save()

            except boto.exception.EC2ResponseError as msg:
                logging.error("%s: boto failure: %s" % ("update_pool_instances", msg))
                return 1

<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8

""" This is my listener for the joy_node. It listens on the topic 'joy' and prints out some information.
    I then switches on motors on my Raspberry when the D-Pad is used on the joystick/gamepad.
    This needs the motor_server to be run on the Raspberry Pi. Like this:

Usage
-----

Raspberry Pi:
1. Run: roscore
2. Run the motor server on the Pi: rosrun minibot motor_server.py

Another Ubuntu machine:
1. export ROS_MASTER_URI=http://hostname-of-your-pi:11311/
   i.E. export ROS_MASTER_URI=http://pi-desktop:11311/
2. Set joystick device if different to js0: rosparam set joy_node/dev "/dev/input/js2"
3. In the next terminal window repeat step 1. (export...)
4. Run the joystick node: rosrun joy joy_node
5. Run this listener: rosrun minibot joy_motor_listener
6. Use the D-Pad buttons on the joystick/gamepad to control the motors.

"""

import rospy
from std_msgs.msg import String
from sensor_msgs.msg import Joy
# name of the package(!).srv
from minibot.srv import *

# set the motor speed (from 0 (off) to 255 (max speed))
startSpeed = 100

#  this will execute the "drive" command
def drive(direction):
    # Service 'motor' from motor_server.py ready?
    rospy.wait_for_service('motor')

    # Send driving direction to motor
    try:
        # Create the handle 'motor_switcher' with the service type 'Motor'.
        # The latter automatically generates the MotorRequest and MotorResponse objects.
        motor_switcher = rospy.ServiceProxy('motor', Motor)

         # the handle can be called like a normal function
        rospy.loginfo("Switching motors to %s @ speed %s.", direction, startSpeed)
        response = motor_switcher(direction, startSpeed)

        # show result
        rospy.loginfo(rospy.get_caller_id() + ' says result is %s.', response.result)

    except rospy.ServiceException, e:
        rospy.logerr("Service call for 'motor' failed: %s", e)

def callback(joy):
    """
    # debug messages
    rospy.loginfo("axis 0: %s", joy.axes[0])
    rospy.loginfo("axis 1: %s", joy.axes[1])
    rospy.loginfo("axis 2: %s", joy.axes[2])
    rospy.loginfo("axis 3: %s", joy.axes[3])
    rospy.loginfo("axis 4: %s", joy.axes[4])
    rospy.loginfo("axis 5: %s", joy.axes[5])
    rospy.loginfo("-------------------")
    """

    # which button was pressed?
    # D-Pad, vertikal up
    if   (joy.axes[5] == 1.0):
      rospy.loginfo("FORWARD button pressed.")
      drive("FORWARD")
    # D-Pad, vertikal down
    elif (joy.axes[5] == -1.0):
      rospy.loginfo("BACKWARD button pressed.")
      drive("BACKWARD")
    # D-Pad, horizontal left
    elif (joy.axes[4] ==  1.0):
      rospy.loginfo("LEFT button pressed.")
      drive("LEFT")
    # D-Pad, horizontal right
    elif (joy.axes[4] == -1.0):
      rospy.loginfo("RIGHT button pressed.")
      drive("RIGHT")

def listener():
    # In ROS, nodes are uniquely named. If two nodes with the same
    # name are launched, the previous one is kicked off. The
    # anonymous=True flag means that rospy will choose a unique
    # name for our 'joy_listener' node so that multiple listeners can
    # run simultaneously.
    rospy.init_node('joy_listener', anonymous=True)

    # subscribe the joy(stick) topic
    rospy.Subscriber('joy', Joy, callback)

    # Ready
    rospy.loginfo("Ready. Press a button on your joystick D-Pad now.")

    # spin() simply keeps python from exiting until this node is stopped
    rospy.spin()

if __name__ == '__main__':
    listener()
<commit_msg>Added 'stop' button code<commit_after>#!/usr/bin/env python
# coding=utf-8

""" This is my listener for the joy_node. It listens on the topic 'joy' and prints out some information.
    I then switches on motors on my Raspberry when the D-Pad is used on the joystick/gamepad.
    This needs the motor_server to be run on the Raspberry Pi. Like this:

Usage
-----

Raspberry Pi:
1. Run: roscore
2. Run the motor server on the Pi: rosrun minibot motor_server.py

Another Ubuntu machine:
1. export ROS_MASTER_URI=http://hostname-of-your-pi:11311/
   i.E. export ROS_MASTER_URI=http://pi-desktop:11311/
2. Set joystick device if different to js0: rosparam set joy_node/dev "/dev/input/js2"
3. In the next terminal window repeat step 1. (export...)
4. Run the joystick node: rosrun joy joy_node
5. Run this listener: rosrun minibot joy_motor_listener
6. Use the D-Pad buttons on the joystick/gamepad to control the motors.

"""

import rospy
from std_msgs.msg import String
from sensor_msgs.msg import Joy
# name of the package(!).srv
from minibot.srv import *

# set the motor speed (from 0 (off) to 255 (max speed))
startSpeed = 100

#  this will execute the "drive" command
def drive(direction):
    # Service 'motor' from motor_server.py ready?
    rospy.wait_for_service('motor')

    # Send driving direction to motor
    try:
        # Create the handle 'motor_switcher' with the service type 'Motor'.
        # The latter automatically generates the MotorRequest and MotorResponse objects.
        motor_switcher = rospy.ServiceProxy('motor', Motor)

         # the handle can be called like a normal function
        rospy.loginfo("Switching motors to %s @ speed %s.", direction, startSpeed)
        response = motor_switcher(direction, startSpeed)

        # show result
        rospy.loginfo(rospy.get_caller_id() + ' says result is %s.', response.result)

    except rospy.ServiceException, e:
        rospy.logerr("Service call for 'motor' failed: %s", e)

def callback(joy):
    """
    # debug messages
    rospy.loginfo("axis 0: %s", joy.axes[0])
    rospy.loginfo("axis 1: %s", joy.axes[1])
    rospy.loginfo("axis 2: %s", joy.axes[2])
    rospy.loginfo("axis 3: %s", joy.axes[3])
    rospy.loginfo("axis 4: %s", joy.axes[4])
    rospy.loginfo("axis 5: %s", joy.axes[5])
    rospy.loginfo("-------------------")
    """

    # which button was pressed?
    # D-Pad, vertikal up
    if   (joy.axes[5] == 1.0):
      rospy.loginfo("FORWARD button pressed.")
      drive("FORWARD")
    # D-Pad, vertikal down
    elif (joy.axes[5] == -1.0):
      rospy.loginfo("BACKWARD button pressed.")
      drive("BACKWARD")
    # D-Pad, horizontal left
    elif (joy.axes[4] ==  1.0):
      rospy.loginfo("LEFT button pressed.")
      drive("LEFT")
    # D-Pad, horizontal right
    elif (joy.axes[4] == -1.0):
      rospy.loginfo("RIGHT button pressed.")
      drive("RIGHT")
    # red button on my gamepad
    elif (joy.buttons[10] == 1.0):
      rospy.loginfo("RED button pressed.")
      drive("STOP")

def listener():
    # In ROS, nodes are uniquely named. If two nodes with the same
    # name are launched, the previous one is kicked off. The
    # anonymous=True flag means that rospy will choose a unique
    # name for our 'joy_listener' node so that multiple listeners can
    # run simultaneously.
    rospy.init_node('joy_listener', anonymous=True)

    # subscribe the joy(stick) topic
    rospy.Subscriber('joy', Joy, callback)

    # Ready
    rospy.loginfo("Ready. Press a button on your joystick D-Pad now.")

    # spin() simply keeps python from exiting until this node is stopped
    rospy.spin()

if __name__ == '__main__':
    listener()
<|endoftext|>