<commit_before>from django.contrib.postgres.fields import JSONField
from django.db import models
from django.utils.functional import cached_property

from db.models.abstract_jobs import AbstractJobStatus, JobMixin
from db.models.plugins import PluginJobBase
from db.models.unique_names import TENSORBOARD_UNIQUE_NAME_FORMAT
from libs.spec_validation import validate_tensorboard_spec_config
from schemas.specifications import TensorboardSpecification

class TensorboardJob(PluginJobBase, JobMixin):
    """A model that represents the configuration for tensorboard job."""
    project = models.ForeignKey(
        'db.Project',
        on_delete=models.CASCADE,
        related_name='tensorboard_jobs')
    config = JSONField(
        help_text='The compiled polyaxonfile for the tensorboard job.',
        validators=[validate_tensorboard_spec_config])
    experiment_group = models.ForeignKey(
        'db.ExperimentGroup',
        on_delete=models.CASCADE,
        related_name='tensorboard_jobs',
        null=True,
        blank=True)
    experiment = models.ForeignKey(
        'db.Experiment',
        on_delete=models.CASCADE,
        related_name='tensorboard_jobs',
        null=True,
        blank=True)
    status = models.OneToOneField(
        'db.TensorboardJobStatus',
        related_name='+',
        blank=True,
        null=True,
        editable=True,
        on_delete=models.SET_NULL)

    class Meta:
        app_label = 'db'

    @cached_property
    def unique_name(self):
        return TENSORBOARD_UNIQUE_NAME_FORMAT.format(
            project_name=self.project.unique_name,
            id=self.id)

    @cached_property
    def specification(self):
        return TensorboardSpecification(values=self.config)

    def set_status(self,  # pylint:disable=arguments-differ
                   status,
                   message=None,
                   traceback=None,
                   details=None):
        return self._set_status(status_model=TensorboardJobStatus,
                                status=status,
                                message=message,
                                traceback=traceback,
                                details=details)

    @cached_property
    def outputs_path(self):
        from libs.paths.experiments import get_experiment_outputs_path

        def get_named_experiment_outputs_path(experiment):
            return '{}:{}'.format(
                self.experiment,
                get_experiment_outputs_path(
                    persistence_outputs=experiment.persistence_outputs,
                    experiment_name=experiment.unique_name,
                    original_name=experiment.original_unique_name,
                    cloning_strategy=experiment.cloning_strategy))

        if self.experiment:
            return get_named_experiment_outputs_path(self.experiment)

        if self.experiment_group:
            return ','.join([get_named_experiment_outputs_path(experiment)
                             for experiment in self.experiment_group.experiments.all()])

        from libs.paths.projects import get_project_outputs_path
        return get_project_outputs_path(persistence_outputs=None,
                                        project_name=self.project.unique_name)

class TensorboardJobStatus(AbstractJobStatus):
    """A model that represents tensorboard job status at certain time."""
    job = models.ForeignKey(
        'db.TensorboardJob',
        on_delete=models.CASCADE,
        related_name='statuses')

    class Meta(AbstractJobStatus.Meta):
        app_label = 'db'
        verbose_name_plural = 'Tensorboard Job Statuses'
<commit_msg>Fix tensorboard path name<commit_after>from django.contrib.postgres.fields import JSONField
from django.db import models
from django.utils.functional import cached_property

from db.models.abstract_jobs import AbstractJobStatus, JobMixin
from db.models.plugins import PluginJobBase
from db.models.unique_names import TENSORBOARD_UNIQUE_NAME_FORMAT
from libs.spec_validation import validate_tensorboard_spec_config
from schemas.specifications import TensorboardSpecification

class TensorboardJob(PluginJobBase, JobMixin):
    """A model that represents the configuration for tensorboard job."""
    project = models.ForeignKey(
        'db.Project',
        on_delete=models.CASCADE,
        related_name='tensorboard_jobs')
    config = JSONField(
        help_text='The compiled polyaxonfile for the tensorboard job.',
        validators=[validate_tensorboard_spec_config])
    experiment_group = models.ForeignKey(
        'db.ExperimentGroup',
        on_delete=models.CASCADE,
        related_name='tensorboard_jobs',
        null=True,
        blank=True)
    experiment = models.ForeignKey(
        'db.Experiment',
        on_delete=models.CASCADE,
        related_name='tensorboard_jobs',
        null=True,
        blank=True)
    status = models.OneToOneField(
        'db.TensorboardJobStatus',
        related_name='+',
        blank=True,
        null=True,
        editable=True,
        on_delete=models.SET_NULL)

    class Meta:
        app_label = 'db'

    @cached_property
    def unique_name(self):
        return TENSORBOARD_UNIQUE_NAME_FORMAT.format(
            project_name=self.project.unique_name,
            id=self.id)

    @cached_property
    def specification(self):
        return TensorboardSpecification(values=self.config)

    def set_status(self,  # pylint:disable=arguments-differ
                   status,
                   message=None,
                   traceback=None,
                   details=None):
        return self._set_status(status_model=TensorboardJobStatus,
                                status=status,
                                message=message,
                                traceback=traceback,
                                details=details)

    @cached_property
    def outputs_path(self):
        from libs.paths.experiments import get_experiment_outputs_path

        def get_named_experiment_outputs_path(experiment):
            return '{}:{}'.format(
                experiment.unique_name,
                get_experiment_outputs_path(
                    persistence_outputs=experiment.persistence_outputs,
                    experiment_name=experiment.unique_name,
                    original_name=experiment.original_unique_name,
                    cloning_strategy=experiment.cloning_strategy))

        if self.experiment:
            return get_named_experiment_outputs_path(self.experiment)

        if self.experiment_group:
            return ','.join([get_named_experiment_outputs_path(experiment)
                             for experiment in self.experiment_group.experiments.all()])

        from libs.paths.projects import get_project_outputs_path
        return get_project_outputs_path(persistence_outputs=None,
                                        project_name=self.project.unique_name)

class TensorboardJobStatus(AbstractJobStatus):
    """A model that represents tensorboard job status at certain time."""
    job = models.ForeignKey(
        'db.TensorboardJob',
        on_delete=models.CASCADE,
        related_name='statuses')

    class Meta(AbstractJobStatus.Meta):
        app_label = 'db'
        verbose_name_plural = 'Tensorboard Job Statuses'
<|endoftext|>


<commit_before>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, cmd, ucmd, ucompile,
                                    upload_file, trigger, ip, mkdir,
                                    rmdir, s_sbranch, s_force, s_trigger,
                                    hash_download_from_master,
                                    property_is_false, clean_unpack)
from polyjit.buildbot.repos import make_cb, codebases
from polyjit.buildbot.master import URL
from buildbot.plugins import util
from buildbot.changes import filter

codebase = make_cb(['benchbuild'])

P = util.Property
BuildFactory = util.BuildFactory

def has_munged(host):
    if "has_munged" in host["properties"]:
        return host["properties"]["has_munged"]
    return False

accepted_builders = slaves.get_hostlist(slaves.infosun, predicate = has_munged)

# yapf: disable
def configure(c):
    llvm_dl = hash_download_from_master("public_html/llvm.tar.gz",
                                        "llvm.tar.gz", "llvm")
    polyjit_dl = hash_download_from_master("public_html/polyjit.tar.gz",
                                        "polyjit.tar.gz", "polyjit")
    steps = [
#        trigger(schedulerNames=['trigger-build-llvm', 'trigger-build-jit']),
        define("scratch", ip("/scratch/pjtest/%(prop:buildnumber)s/"))
    ]
    steps.extend(llvm_dl)
    steps.extend(clean_unpack("llvm.tar.gz", "llvm"))
    steps.extend(polyjit_dl)
    steps.extend(clean_unpack("polyjit.tar.gz", "polyjit"))
    steps.extend([
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('benchbuild', 'develop', codebases, workdir=P("BENCHBUILD_ROOT")),

    ])

    steps.extend([
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('polyjit', ip('%(prop:scratch)s/polyjit')),
        cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env/')),
        cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install', '--upgrade', '.',
            workdir='build/benchbuild'),
        mkdir(P("scratch")),
        cmd("rsync", "-var", "./", P("scratch"))
        cmd(P('benchbuild'), 'bootstrap', env={
            'PATH': '/opt/cmake/bin:/usr/sbin:/sbin:/usr/bin:/bin',
            'BB_ENV_COMPILER_PATH': ip('%(prop:llvm)s/bin'),
            'BB_ENV_COMPILER_LD_LIBRARY_PATH':
                ip('%(prop:llvm)s/lib:'),
            'BB_ENV_LOOKUP_PATH': ip('%(prop:llvm)s/bin:%(prop:polyjit)s/bin'),
            'BB_ENV_LOOKUP_LD_LIBRARY_PATH':
                ip('%(prop:llvm)s/lib:%(prop:polyjit)s/lib'),
            'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
            'BB_LIKWID_PREFIX': '/usr',
            'BB_PAPI_INCLUDE': '/usr/include',
            'BB_PAPI_LIBRARY': '/usr/lib',
            'BB_SRC_DIR': ip('%(prop:scratch)s/benchbuild'),
            'BB_UNIONFS_ENABLE': 'false'
        },
        workdir=P('%(prop:scratch)s'),
    ])

    c['builders'].append(builder("build-slurm-set", None, accepted_builders,
        factory=BuildFactory(steps)))
# yapf: enable

def schedule(c):
    c['schedulers'].extend([
        s_sbranch("build-slurm-set-sched", codebase, ["build-slurm-set"],
                  change_filter=filter.ChangeFilter(branch_re='next|develop'),
                  treeStableTimer=2 * 60),
        s_force("force-build-slurm-set", codebase, ["build-slurm-set"]),
        s_trigger("trigger-slurm-set", codebase, ['build-slurm-set'])
    ])

register(sys.modules[__name__])
<commit_msg>slurm: clean unused steps<commit_after>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, cmd, trigger, ip,
                                    mkdir, s_sbranch, s_force, s_trigger,
                                    hash_download_from_master, clean_unpack)
from polyjit.buildbot.repos import make_cb, codebases
from buildbot.plugins import util
from buildbot.changes import filter

codebase = make_cb(['benchbuild'])

P = util.Property
BuildFactory = util.BuildFactory

def has_munged(host):
    if "has_munged" in host["properties"]:
        return host["properties"]["has_munged"]
    return False

accepted_builders = slaves.get_hostlist(slaves.infosun, predicate=has_munged)

# yapf: disable
def configure(c):
    llvm_dl = hash_download_from_master("public_html/llvm.tar.gz",
                                        "llvm.tar.gz", "llvm")
    polyjit_dl = hash_download_from_master("public_html/polyjit.tar.gz",
                                        "polyjit.tar.gz", "polyjit")
    steps = [
#        trigger(schedulerNames=['trigger-build-llvm', 'trigger-build-jit']),
        define("scratch", ip("/scratch/pjtest/%(prop:buildnumber)s/"))
    ]
    steps.extend(llvm_dl)
    steps.extend(clean_unpack("llvm.tar.gz", "llvm"))
    steps.extend(polyjit_dl)
    steps.extend(clean_unpack("polyjit.tar.gz", "polyjit"))
    steps.extend([
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('benchbuild', 'develop', codebases, workdir=P("BENCHBUILD_ROOT")),

    ])

    steps.extend([
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('polyjit', ip('%(prop:scratch)s/polyjit')),
        cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env/')),
        cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install', '--upgrade', '.',
            workdir='build/benchbuild'),
        mkdir(P("scratch")),
        cmd("rsync", "-var", "./", P("scratch")),
        cmd(P('benchbuild'), 'bootstrap', env={
            'PATH': '/opt/cmake/bin:/usr/sbin:/sbin:/usr/bin:/bin',
            'BB_ENV_COMPILER_PATH': ip('%(prop:llvm)s/bin'),
            'BB_ENV_COMPILER_LD_LIBRARY_PATH':
                ip('%(prop:llvm)s/lib:'),
            'BB_ENV_LOOKUP_PATH': ip('%(prop:llvm)s/bin:%(prop:polyjit)s/bin'),
            'BB_ENV_LOOKUP_LD_LIBRARY_PATH':
                ip('%(prop:llvm)s/lib:%(prop:polyjit)s/lib'),
            'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
            'BB_LIKWID_PREFIX': '/usr',
            'BB_PAPI_INCLUDE': '/usr/include',
            'BB_PAPI_LIBRARY': '/usr/lib',
            'BB_SRC_DIR': ip('%(prop:scratch)s/benchbuild'),
            'BB_UNIONFS_ENABLE': 'false'
        },
        workdir=P('%(prop:scratch)s')),
    ])

    c['builders'].append(builder("build-slurm-set", None, accepted_builders,
        factory=BuildFactory(steps)))
# yapf: enable

def schedule(c):
    c['schedulers'].extend([
        s_sbranch("build-slurm-set-sched", codebase, ["build-slurm-set"],
                  change_filter=filter.ChangeFilter(branch_re='next|develop'),
                  treeStableTimer=2 * 60),
        s_force("force-build-slurm-set", codebase, ["build-slurm-set"]),
        s_trigger("trigger-slurm-set", codebase, ['build-slurm-set'])
    ])

register(sys.modules[__name__])
<|endoftext|>


<commit_before># vim:fileencoding=utf-8:noet
from __future__ import (unicode_literals, division, absolute_import, print_function)

import atexit

from weakref import WeakValueDictionary, ref

import zsh

from powerline.shell import ShellPowerline
from powerline.lib.overrides import parsedotval, parse_override_var
from powerline.lib.unicode import unicode, u
from powerline.lib.encoding import (get_preferred_output_encoding,
                                    get_preferred_environment_encoding)
from powerline.lib.dict import mergeargs

used_powerlines = WeakValueDictionary()

def shutdown():
	for powerline in tuple(used_powerlines.values()):
		powerline.shutdown()

def get_var_config(var):
	try:
		val = zsh.getvalue(var)
		if isinstance(val, dict):
			return mergeargs([parsedotval((u(k), u(v))) for k, v in val.items()])
		elif isinstance(val, (unicode, str, bytes)):
			return mergeargs(parse_override_var(u(val)))
		else:
			return None
	except:
		return None

class Args(object):
	__slots__ = ('last_pipe_status', 'last_exit_code')
	ext = ['shell']
	renderer_module = '.zsh'

	@property
	def config_override(self):
		return get_var_config('POWERLINE_CONFIG_OVERRIDES')

	@property
	def theme_override(self):
		return get_var_config('POWERLINE_THEME_OVERRIDES')

	@property
	def config_path(self):
		try:
			ret = zsh.getvalue('POWERLINE_CONFIG_PATHS')
		except IndexError:
			return None
		else:
			if isinstance(ret, (unicode, str, bytes)):
				return ret.split((b':' if isinstance(ret, bytes) else ':'))
			else:
				return ret

	@property
	def jobnum(self):
		return zsh.getvalue('_POWERLINE_JOBNUM')

def string(s):
	if type(s) is bytes:
		return s.decode(get_preferred_environment_encoding(), 'replace')
	else:
		return str(s)

class Environment(object):
	@staticmethod
	def __getitem__(key):
		try:
			return string(zsh.getvalue(key))
		except IndexError as e:
			raise KeyError(*e.args)

	@staticmethod
	def get(key, default=None):
		try:
			return string(zsh.getvalue(key))
		except IndexError:
			return default

	@staticmethod
	def __contains__(key):
		try:
			zsh.getvalue(key)
			return True
		except IndexError:
			return False

environ = getattr(zsh, 'environ', Environment())

class ZshPowerline(ShellPowerline):
	def init(self, **kwargs):
		super(ZshPowerline, self).init(Args(), **kwargs)

	def precmd(self):
		self.args.last_pipe_status = zsh.pipestatus()
		self.args.last_exit_code = zsh.last_exit_code()

	def do_setup(self, zsh_globals):
		set_prompt(self, 'PS1', 'left', None, above=True)
		set_prompt(self, 'RPS1', 'right', None)
		set_prompt(self, 'PS2', 'left', 'continuation')
		set_prompt(self, 'RPS2', 'right', 'continuation')
		set_prompt(self, 'PS3', 'left', 'select')
		used_powerlines[id(self)] = self
		zsh_globals['_powerline'] = self

class Prompt(object):
	__slots__ = ('powerline', 'side', 'savedpsvar', 'savedps', 'args', 'theme', 'above', '__weakref__')

	def __init__(self, powerline, side, theme, savedpsvar=None, savedps=None, above=False):
		self.powerline = powerline
		self.side = side
		self.above = above
		self.savedpsvar = savedpsvar
		self.savedps = savedps
		self.args = powerline.args
		self.theme = theme

	def __str__(self):
		zsh.eval('_POWERLINE_PARSER_STATE="${(%):-%_}"')
		zsh.eval('_POWERLINE_SHORTENED_PATH="${(%):-%~}"')
		try:
			mode = u(zsh.getvalue('_POWERLINE_MODE'))
		except IndexError:
			mode = None
		segment_info = {
			'args': self.args,
			'environ': environ,
			'client_id': 1,
			'local_theme': self.theme,
			'parser_state': zsh.getvalue('_POWERLINE_PARSER_STATE'),
			'shortened_path': zsh.getvalue('_POWERLINE_SHORTENED_PATH'),
			'mode': mode,
		}
		zsh.setvalue('_POWERLINE_PARSER_STATE', None)
		zsh.setvalue('_POWERLINE_SHORTENED_PATH', None)
		try:
			zle_rprompt_indent = zsh.getvalue('ZLE_RPROMPT_INDENT')
		except IndexError:
			zle_rprompt_indent = 1
		r = ''
		if self.above:
			for line in self.powerline.render_above_lines(
				width=zsh.columns() - zle_rprompt_indent,
				segment_info=segment_info,
			):
				r += line + '\n'
		r += self.powerline.render(
			width=zsh.columns(),
			side=self.side,
			segment_info=segment_info,
			mode=mode,
		)
		if type(r) is not str:
			if type(r) is bytes:
				return r.decode(get_preferred_output_encoding(), 'replace')
			else:
				return r.encode(get_preferred_output_encoding(), 'replace')
		return r

	def __del__(self):
		if self.savedps:
			zsh.setvalue(self.savedpsvar, self.savedps)
		self.powerline.shutdown()

def set_prompt(powerline, psvar, side, theme, above=False):
	try:
		savedps = zsh.getvalue(psvar)
	except IndexError:
		savedps = None
	zpyvar = 'ZPYTHON_POWERLINE_' + psvar
	prompt = Prompt(powerline, side, theme, psvar, savedps, above)
	zsh.eval('unset ' + zpyvar)
	zsh.set_special_string(zpyvar, prompt)
	zsh.setvalue(psvar, '${' + zpyvar + '}')
	return ref(prompt)

def reload():
	for powerline in tuple(used_powerlines.values()):
		powerline.reload()

def reload_config():
	for powerline in used_powerlines.values():
		powerline.create_renderer(load_main=True, load_colors=True, load_colorscheme=True, load_theme=True)

def setup(zsh_globals):
	powerline = ZshPowerline()
	powerline.setup(zsh_globals)
	atexit.register(shutdown)
<commit_msg>Also do not allow empty paths in zpython bindings<commit_after># vim:fileencoding=utf-8:noet
from __future__ import (unicode_literals, division, absolute_import, print_function)

import atexit

from weakref import WeakValueDictionary, ref

import zsh

from powerline.shell import ShellPowerline
from powerline.lib.overrides import parsedotval, parse_override_var
from powerline.lib.unicode import unicode, u
from powerline.lib.encoding import (get_preferred_output_encoding,
                                    get_preferred_environment_encoding)
from powerline.lib.dict import mergeargs

used_powerlines = WeakValueDictionary()

def shutdown():
	for powerline in tuple(used_powerlines.values()):
		powerline.shutdown()

def get_var_config(var):
	try:
		val = zsh.getvalue(var)
		if isinstance(val, dict):
			return mergeargs([parsedotval((u(k), u(v))) for k, v in val.items()])
		elif isinstance(val, (unicode, str, bytes)):
			return mergeargs(parse_override_var(u(val)))
		else:
			return None
	except:
		return None

class Args(object):
	__slots__ = ('last_pipe_status', 'last_exit_code')
	ext = ['shell']
	renderer_module = '.zsh'

	@property
	def config_override(self):
		return get_var_config('POWERLINE_CONFIG_OVERRIDES')

	@property
	def theme_override(self):
		return get_var_config('POWERLINE_THEME_OVERRIDES')

	@property
	def config_path(self):
		try:
			ret = zsh.getvalue('POWERLINE_CONFIG_PATHS')
		except IndexError:
			return None
		else:
			if isinstance(ret, (unicode, str, bytes)):
				return [
					path
					for path in ret.split((b':' if isinstance(ret, bytes) else ':'))
					if path
				]
			else:
				return ret

	@property
	def jobnum(self):
		return zsh.getvalue('_POWERLINE_JOBNUM')

def string(s):
	if type(s) is bytes:
		return s.decode(get_preferred_environment_encoding(), 'replace')
	else:
		return str(s)

class Environment(object):
	@staticmethod
	def __getitem__(key):
		try:
			return string(zsh.getvalue(key))
		except IndexError as e:
			raise KeyError(*e.args)

	@staticmethod
	def get(key, default=None):
		try:
			return string(zsh.getvalue(key))
		except IndexError:
			return default

	@staticmethod
	def __contains__(key):
		try:
			zsh.getvalue(key)
			return True
		except IndexError:
			return False

environ = getattr(zsh, 'environ', Environment())

class ZshPowerline(ShellPowerline):
	def init(self, **kwargs):
		super(ZshPowerline, self).init(Args(), **kwargs)

	def precmd(self):
		self.args.last_pipe_status = zsh.pipestatus()
		self.args.last_exit_code = zsh.last_exit_code()

	def do_setup(self, zsh_globals):
		set_prompt(self, 'PS1', 'left', None, above=True)
		set_prompt(self, 'RPS1', 'right', None)
		set_prompt(self, 'PS2', 'left', 'continuation')
		set_prompt(self, 'RPS2', 'right', 'continuation')
		set_prompt(self, 'PS3', 'left', 'select')
		used_powerlines[id(self)] = self
		zsh_globals['_powerline'] = self

class Prompt(object):
	__slots__ = ('powerline', 'side', 'savedpsvar', 'savedps', 'args', 'theme', 'above', '__weakref__')

	def __init__(self, powerline, side, theme, savedpsvar=None, savedps=None, above=False):
		self.powerline = powerline
		self.side = side
		self.above = above
		self.savedpsvar = savedpsvar
		self.savedps = savedps
		self.args = powerline.args
		self.theme = theme

	def __str__(self):
		zsh.eval('_POWERLINE_PARSER_STATE="${(%):-%_}"')
		zsh.eval('_POWERLINE_SHORTENED_PATH="${(%):-%~}"')
		try:
			mode = u(zsh.getvalue('_POWERLINE_MODE'))
		except IndexError:
			mode = None
		segment_info = {
			'args': self.args,
			'environ': environ,
			'client_id': 1,
			'local_theme': self.theme,
			'parser_state': zsh.getvalue('_POWERLINE_PARSER_STATE'),
			'shortened_path': zsh.getvalue('_POWERLINE_SHORTENED_PATH'),
			'mode': mode,
		}
		zsh.setvalue('_POWERLINE_PARSER_STATE', None)
		zsh.setvalue('_POWERLINE_SHORTENED_PATH', None)
		try:
			zle_rprompt_indent = zsh.getvalue('ZLE_RPROMPT_INDENT')
		except IndexError:
			zle_rprompt_indent = 1
		r = ''
		if self.above:
			for line in self.powerline.render_above_lines(
				width=zsh.columns() - zle_rprompt_indent,
				segment_info=segment_info,
			):
				r += line + '\n'
		r += self.powerline.render(
			width=zsh.columns(),
			side=self.side,
			segment_info=segment_info,
			mode=mode,
		)
		if type(r) is not str:
			if type(r) is bytes:
				return r.decode(get_preferred_output_encoding(), 'replace')
			else:
				return r.encode(get_preferred_output_encoding(), 'replace')
		return r

	def __del__(self):
		if self.savedps:
			zsh.setvalue(self.savedpsvar, self.savedps)
		self.powerline.shutdown()

def set_prompt(powerline, psvar, side, theme, above=False):
	try:
		savedps = zsh.getvalue(psvar)
	except IndexError:
		savedps = None
	zpyvar = 'ZPYTHON_POWERLINE_' + psvar
	prompt = Prompt(powerline, side, theme, psvar, savedps, above)
	zsh.eval('unset ' + zpyvar)
	zsh.set_special_string(zpyvar, prompt)
	zsh.setvalue(psvar, '${' + zpyvar + '}')
	return ref(prompt)

def reload():
	for powerline in tuple(used_powerlines.values()):
		powerline.reload()

def reload_config():
	for powerline in used_powerlines.values():
		powerline.create_renderer(load_main=True, load_colors=True, load_colorscheme=True, load_theme=True)

def setup(zsh_globals):
	powerline = ZshPowerline()
	powerline.setup(zsh_globals)
	atexit.register(shutdown)
<|endoftext|>


<commit_before>import logging
from django.core.management.base import BaseCommand
from optparse import make_option

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    help = "Starts a pq worker"
    args = "<queue queue ...>"

    option_list = BaseCommand.option_list + (
        make_option('--burst', '-b', action='store_true', dest='burst',
            default=False, help='Run in burst mode (quit after all work is done)'),
        make_option('--name', '-n', default=None, dest='name',
            help='Specify a different name'),
        make_option('--sentry-dsn', action='store', default=None, metavar='URL',
                    help='Report exceptions to this Sentry DSN'),
    )

    def handle(self, *args, **options):
        """
        The actual logic of the command. Subclasses must implement
        this method.

        """
        from django.conf import settings
        from pq.queue import Queue
        from pq.worker import Worker

        sentry_dsn = options.get('sentry_dsn')
        if not sentry_dsn:
            sentry_dsn = settings.SENTRY_DSN if hasattr(settings, 'SENTRY_DSN') else None

        verbosity = int(options.get('verbosity'))

        queues = map(Queue.create, args)
        w = Worker.create(queues, name=options.get('name'))

        # Should we configure Sentry?
        if sentry_dsn:
            from raven import Client
            from pq.contrib.sentry import register_sentry
            client = Client(args.sentry_dsn)
            register_sentry(client, w)

        w.work(burst=options['burst'])

<commit_msg>A connection to pqworker<commit_after>import logging
from django.core.management.base import BaseCommand
from optparse import make_option

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    help = "Starts a pq worker"
    args = "<queue queue ...>"

    option_list = BaseCommand.option_list + (
        make_option('--burst', '-b', action='store_true', dest='burst',
            default=False, help='Run in burst mode (quit after all work is done)'),
        make_option('--name', '-n', default=None, dest='name',
            help='Specify a different name'),
        make_option('--connection', '-c', action='store', default='default',
                    help='Report exceptions to this Sentry DSN'),
        make_option('--sentry-dsn', action='store', default=None, metavar='URL',
                    help='Report exceptions to this Sentry DSN'),
    )

    def handle(self, *args, **options):
        """
        The actual logic of the command. Subclasses must implement
        this method.

        """
        from django.conf import settings
        from pq.queue import Queue
        from pq.worker import Worker

        sentry_dsn = options.get('sentry_dsn')
        if not sentry_dsn:
            sentry_dsn = settings.SENTRY_DSN if hasattr(settings, 'SENTRY_DSN') else None

        verbosity = int(options.get('verbosity'))

        queues = map(Queue.create, args)
        w = Worker.create(queues, name=options.get('name'), connection=options['connection'])

        # Should we configure Sentry?
        if sentry_dsn:
            from raven import Client
            from pq.contrib.sentry import register_sentry
            client = Client(args.sentry_dsn)
            register_sentry(client, w)

        w.work(burst=options['burst'])

<|endoftext|>


<commit_before># Copyright (c) 2012, GPy authors (see AUTHORS.txt).
# Licensed under the BSD 3-clause license (see LICENSE.txt)

from kernpart import kernpart
import numpy as np
import hashlib
#from scipy import integrate # This may not be necessary (Nicolas, 20th Feb)

class product_orthogonal(kernpart):
    """
    Computes the product of 2 kernels

    :param k1, k2: the kernels to multiply
    :type k1, k2: kernpart
    :rtype: kernel object

    """
    def __init__(self,k1,k2):
        self.D = k1.D + k2.D
        self.Nparam = k1.Nparam + k2.Nparam
        self.name = k1.name + '<times>' + k2.name
        self.k1 = k1
        self.k2 = k2
        self._set_params(np.hstack((k1._get_params(),k2._get_params())))

    def _get_params(self):
        """return the value of the parameters."""
        return self.params

    def _set_params(self,x):
        """set the value of the parameters."""
        self.k1._set_params(x[:self.k1.Nparam])
        self.k2._set_params(x[self.k1.Nparam:])
        self.params = x

    def _get_param_names(self):
        """return parameter names."""
        return [self.k1.name + '_' + param_name for param_name in self.k1._get_param_names()] + [self.k2.name + '_' + param_name for param_name in self.k2._get_param_names()]

    def K(self,X,X2,target):
        """Compute the covariance matrix between X and X2."""
        if X2 is None: X2 = X
        target1 = np.zeros((X.shape[0],X2.shape[0]))
        target2 = np.zeros((X.shape[0],X2.shape[0]))
        self.k1.K(X[:,0:self.k1.D],X2[:,0:self.k1.D],target1)
        self.k2.K(X[:,self.k1.D:],X2[:,self.k1.D:],target2)
        target += target1 * target2

    def dK_dtheta(self,partial,X,X2,target):
        """derivative of the covariance matrix with respect to the parameters."""
        if X2 is None: X2 = X
        K1 = np.zeros((X.shape[0],X2.shape[0]))
        K2 = np.zeros((X.shape[0],X2.shape[0]))
        self.k1.K(X[:,0:self.k1.D],X2[:,0:self.k1.D],K1)
        self.k2.K(X[:,self.k1.D:],X2[:,self.k1.D:],K2)

        k1_target = np.zeros(self.k1.Nparam)
        k2_target = np.zeros(self.k2.Nparam)
        self.k1.dK_dtheta(partial*K2, X[:,:self.k1.D], X2[:,:self.k1.D], k1_target)
        self.k2.dK_dtheta(partial*K1, X[:,self.k1.D:], X2[:,self.k1.D:], k2_target)

        target[:self.k1.Nparam] += k1_target
        target[self.k1.Nparam:] += k2_target

    def Kdiag(self,X,target):
        """Compute the diagonal of the covariance matrix associated to X."""
        target1 = np.zeros((X.shape[0],))
        target2 = np.zeros((X.shape[0],))
        self.k1.Kdiag(X[:,:self.k1.D],target1)
        self.k2.Kdiag(X[:,self.k1.D:],target2)
        target += target1 * target2

    def dKdiag_dtheta(self,partial,X,target):
        K1 = np.zeros(X.shape[0])
        K2 = np.zeros(X.shape[0])
        self.k1.Kdiag(X[:,:self.k1.D],K1)
        self.k2.Kdiag(X[:,self.k1.D:],K2)
        self.k1.dKdiag_dtheta(partial*K2,X[:,:self.k1.D],target[:self.k1.Nparam])
        self.k2.dKdiag_dtheta(partial*K1,X[:,self.k1.D:],target[self.k1.Nparam:])

    def dK_dX(self,partial,X,X2,target):
        """derivative of the covariance matrix with respect to X."""
        if X2 is None: X2 = X
        K1 = np.zeros((X.shape[0],X2.shape[0]))
        K2 = np.zeros((X.shape[0],X2.shape[0]))
        self.k1.K(X[:,0:self.k1.D],X2[:,0:self.k1.D],K1)
        self.k2.K(X[:,self.k1.D:],X2[:,self.k1.D:],K2)

        self.k1.dK_dX(partial*K2, X[:,:self.k1.D], X2[:,:self.k1.D], target)
        self.k2.dK_dX(partial*K1, X[:,self.k1.D:], X2[:,self.k1.D:], target)

    def dKdiag_dX(self,X,target):
        pass
<commit_msg>some changes to product_orthogonal<commit_after># Copyright (c) 2012, GPy authors (see AUTHORS.txt).
# Licensed under the BSD 3-clause license (see LICENSE.txt)

from kernpart import kernpart
import numpy as np
import hashlib
#from scipy import integrate # This may not be necessary (Nicolas, 20th Feb)

class product_orthogonal(kernpart):
    """
    Computes the product of 2 kernels

    :param k1, k2: the kernels to multiply
    :type k1, k2: kernpart
    :rtype: kernel object

    """
    def __init__(self,k1,k2):
        self.D = k1.D + k2.D
        self.Nparam = k1.Nparam + k2.Nparam
        self.name = k1.name + '<times>' + k2.name
        self.k1 = k1
        self.k2 = k2
        self._set_params(np.hstack((k1._get_params(),k2._get_params())))

    def _get_params(self):
        """return the value of the parameters."""
        return self.params

    def _set_params(self,x):
        """set the value of the parameters."""
        self.k1._set_params(x[:self.k1.Nparam])
        self.k2._set_params(x[self.k1.Nparam:])
        self.params = x

    def _get_param_names(self):
        """return parameter names."""
        return [self.k1.name + '_' + param_name for param_name in self.k1._get_param_names()] + [self.k2.name + '_' + param_name for param_name in self.k2._get_param_names()]

    def K(self,X,X2,target):
        """Compute the covariance matrix between X and X2."""
        if X2 is None: X2 = X
        target1 = np.zeros((X.shape[0],X2.shape[0]))
        target2 = np.zeros((X.shape[0],X2.shape[0]))
        self.k1.K(X[:,:self.k1.D],X2[:,:self.k1.D],target1)
        self.k2.K(X[:,self.k1.D:],X2[:,self.k1.D:],target2)
        target += target1 * target2

    def dK_dtheta(self,partial,X,X2,target):
        """derivative of the covariance matrix with respect to the parameters."""
        if X2 is None: X2 = X
        K1 = np.zeros((X.shape[0],X2.shape[0]))
        K2 = np.zeros((X.shape[0],X2.shape[0]))
        self.k1.K(X[:,:self.k1.D],X2[:,:self.k1.D],K1)
        self.k2.K(X[:,self.k1.D:],X2[:,self.k1.D:],K2)

        self.k1.dK_dtheta(partial*K2, X[:,:self.k1.D], X2[:,:self.k1.D], target[:self.k1.Nparam])
        self.k2.dK_dtheta(partial*K1, X[:,self.k1.D:], X2[:,self.k1.D:], target[self.k1.Nparam:])

    def Kdiag(self,X,target):
        """Compute the diagonal of the covariance matrix associated to X."""
        target1 = np.zeros(X.shape[0])
        target2 = np.zeros(X.shape[0])
        self.k1.Kdiag(X[:,:self.k1.D],target1)
        self.k2.Kdiag(X[:,self.k1.D:],target2)
        target += target1 * target2

    def dKdiag_dtheta(self,partial,X,target):
        K1 = np.zeros(X.shape[0])
        K2 = np.zeros(X.shape[0])
        self.k1.Kdiag(X[:,:self.k1.D],K1)
        self.k2.Kdiag(X[:,self.k1.D:],K2)
        self.k1.dKdiag_dtheta(partial*K2,X[:,:self.k1.D],target[:self.k1.Nparam])
        self.k2.dKdiag_dtheta(partial*K1,X[:,self.k1.D:],target[self.k1.Nparam:])

    def dK_dX(self,partial,X,X2,target):
        """derivative of the covariance matrix with respect to X."""
        if X2 is None: X2 = X
        K1 = np.zeros((X.shape[0],X2.shape[0]))
        K2 = np.zeros((X.shape[0],X2.shape[0]))
        self.k1.K(X[:,0:self.k1.D],X2[:,0:self.k1.D],K1)
        self.k2.K(X[:,self.k1.D:],X2[:,self.k1.D:],K2)

        self.k1.dK_dX(partial*K2, X[:,:self.k1.D], X2[:,:self.k1.D], target)
        self.k2.dK_dX(partial*K1, X[:,self.k1.D:], X2[:,self.k1.D:], target)

    def dKdiag_dX(self, partial, X, target):
        K1 = np.zeros(X.shape[0])
        K2 = np.zeros(X.shape[0])
        self.k1.Kdiag(X[:,0:self.k1.D],K1)
        self.k2.Kdiag(X[:,self.k1.D:],K2)

        self.k1.dK_dX(partial*K2, X[:,:self.k1.D], target)
        self.k2.dK_dX(partial*K1, X[:,self.k1.D:], target)

<|endoftext|>


<commit_before># Copyright (c) 2014 The GPy authors (see AUTHORS.txt)
# Licensed under the BSD 3-clause license (see LICENSE.txt)

try:
    import sympy as sym
    sympy_available=True
    from sympy.utilities.lambdify import lambdify
    from GPy.util.symbolic import normcdfln, normcdf
except ImportError:
    sympy_available=False

import numpy as np
from ..util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
from GPy.util.functions import clip_exp
import link_functions
from symbolic import Symbolic
from scipy import stats

class Skew_normal(Symbolic):
    """
    Skew Normal distribution.

    .. math::

    .. Note::
    Y takes real values.
    link function is identity

    .. See also::
    symbolic.py, for the parent class
    """
    def __init__(self, gp_link=None, shape=1.0, scale=1.0):
        parameters = {'scale': scale, 'shape':shape}
        if gp_link is None:
            gp_link = link_functions.Identity()

        # # this likelihood has severe problems with likelihoods saturating exponentials, so clip_exp is used in place of the true exp as a solution for dealing with the numerics.
        # func_modules = [{'exp':clip_exp}]
        func_modules = []

        scale = sym.Symbol('scale', positive=True, real=True)
        shape = sym.Symbol('shape', real=True)
        y_0 = sym.Symbol('y_0', real=True)
        f_0 = sym.Symbol('f_0', real=True)
        log_pdf=-sym.log(scale)-1./2*sym.log(2*sym.pi)-1./2*((y_0-f_0)/scale)**2 + sym.log(2) + normcdfln(shape*(y_0-f_0)/scale)
        super(Skew_normal, self).__init__(log_pdf=log_pdf, parameters=parameters, gp_link=gp_link, name='Skew_normal', func_modules=func_modules)

        self.log_concave = True

<commit_msg>removed more imports<commit_after># Copyright (c) 2014 The GPy authors (see AUTHORS.txt)
# Licensed under the BSD 3-clause license (see LICENSE.txt)

try:
    import sympy as sym
    sympy_available=True
    from sympy.utilities.lambdify import lambdify
    from GPy.util.symbolic import normcdfln, normcdf
except ImportError:
    sympy_available=False

import numpy as np
from GPy.util.functions import clip_exp
import link_functions
from symbolic import Symbolic
from scipy import stats

class Skew_normal(Symbolic):
    """
    Skew Normal distribution.

    .. math::

    .. Note::
    Y takes real values.
    link function is identity

    .. See also::
    symbolic.py, for the parent class
    """
    def __init__(self, gp_link=None, shape=1.0, scale=1.0):
        parameters = {'scale': scale, 'shape':shape}
        if gp_link is None:
            gp_link = link_functions.Identity()

        # # this likelihood has severe problems with likelihoods saturating exponentials, so clip_exp is used in place of the true exp as a solution for dealing with the numerics.
        # func_modules = [{'exp':clip_exp}]
        func_modules = []

        scale = sym.Symbol('scale', positive=True, real=True)
        shape = sym.Symbol('shape', real=True)
        y_0 = sym.Symbol('y_0', real=True)
        f_0 = sym.Symbol('f_0', real=True)
        log_pdf=-sym.log(scale)-1./2*sym.log(2*sym.pi)-1./2*((y_0-f_0)/scale)**2 + sym.log(2) + normcdfln(shape*(y_0-f_0)/scale)
        super(Skew_normal, self).__init__(log_pdf=log_pdf, parameters=parameters, gp_link=gp_link, name='Skew_normal', func_modules=func_modules)

        self.log_concave = True

<|endoftext|>


<commit_before>import os
import sys

from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_factory import PluginFactory
from pandaharvester.harvestercore.communicator_pool import CommunicatorPool
from pandaharvester.harvestercore.job_spec import JobSpec

queueName = sys.argv[1]
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)

pluginFactory = PluginFactory()

# get job
com = CommunicatorPool()
jobs, errStr = com.get_jobs(queueConfig.queueName, 'nodeName', queueConfig.prodSourceLabel,
                            'computingElement', 1)
if len(jobs) == 0:
    print "Failed to get jobs at {0} due to {1}".format(queueConfig.queueName, errStr)
    sys.exit(0)

jobSpec = JobSpec()
jobSpec.convert_job_json(jobs[0])

# set input file paths
inFiles = jobSpec.get_input_file_attributes()
for inLFN, inFile in inFiles.iteritems():
    inFile['path'] = '{0}/{1}'.format(os.getcwd(), inLFN)
jobSpec.set_input_file_paths(inFiles)

maker = pluginFactory.get_plugin(queueConfig.workerMaker)
workSpec = maker.make_worker([jobSpec], queueConfig)
workSpec.accessPoint = os.getcwd()
workSpec.mapType = queueConfig.mapType

# set job to worker if not job-level late binding
if not queueConfig.useJobLateBinding:
    workSpec.hasJob = 1
    workSpec.set_jobspec_list([jobSpec])

# get plugin for messenger
messenger = pluginFactory.get_plugin(queueConfig.messenger)
messenger.feed_jobs(workSpec, [jobSpec])

if 'eventService' in jobSpec.jobParams:
    workSpec.eventsRequest = WorkSpec.EV_useEvents

if workSpec.hasJob == 1 and workSpec.eventsRequest == WorkSpec.EV_useEvents:
    workSpec.eventsRequest = WorkSpec.EV_requestEvents
    eventsRequestParams = dict()
    eventsRequestParams[jobSpec.PandaID] = {'pandaID': jobSpec.PandaID,
                                            'taskID': jobSpec.taskID,
                                            'jobsetID': jobSpec.jobParams['jobsetID'],
                                            'nRanges': jobSpec.jobParams['coreCount'],
                                            }
    workSpec.eventsRequestParams = eventsRequestParams

    tmpStat, events = com.get_event_ranges(workSpec.eventsRequestParams)
    # failed
    if tmpStat is False:
        print('failed to get events with {0}'.format(events))
        sys.exit(0)
    tmpStat = messenger.feed_events(workSpec, events)

# get submitter plugin
submitterCore = pluginFactory.get_plugin(queueConfig.submitter)
print "testing submission with plugin={0}".format(submitterCore.__class__.__name__)
tmpRetList = submitterCore.submit_workers([workSpec])
tmpStat, tmpOut = tmpRetList[0]
if tmpStat:
    print " OK batchID={0}".format(workSpec.batchID)
else:
    print " NG {0}".format(tmpOut)
    sys.exit(1)

print

# get monitoring plug-in
monCore = pluginFactory.get_plugin(queueConfig.monitor)
print "testing monitoring for batchID={0} with plugin={1}".format(workSpec.batchID,
                                                                  monCore.__class__.__name__)
tmpStat, tmpOut = monCore.check_workers([workSpec])
tmpOut = tmpOut[0]
if tmpStat:
    print " OK workerStatus={0}".format(tmpOut[0])
else:
    print " NG {0}".format(tmpOut[1])
    sys.exit(1)
<commit_msg>tests with NoJob<commit_after>import os
import sys

from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_factory import PluginFactory
from pandaharvester.harvestercore.communicator_pool import CommunicatorPool
from pandaharvester.harvestercore.job_spec import JobSpec

queueName = sys.argv[1]
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)

pluginFactory = PluginFactory()

com = CommunicatorPool()

# get job
jobSpecList = []
if queueConfig.mapType != WorkSpec.MT_NoJob:
    jobs, errStr = com.get_jobs(queueConfig.queueName, 'nodeName', queueConfig.prodSourceLabel,
                                'computingElement', 1)
    if len(jobs) == 0:
        print "Failed to get jobs at {0} due to {1}".format(queueConfig.queueName, errStr)
        sys.exit(0)

    jobSpec = JobSpec()
    jobSpec.convert_job_json(jobs[0])

    # set input file paths
    inFiles = jobSpec.get_input_file_attributes()
    for inLFN, inFile in inFiles.iteritems():
        inFile['path'] = '{0}/{1}'.format(os.getcwd(), inLFN)
    jobSpec.set_input_file_paths(inFiles)
    jobSpecList.append(jobSpec)

maker = pluginFactory.get_plugin(queueConfig.workerMaker)
workSpec = maker.make_worker(jobSpecList, queueConfig)
workSpec.accessPoint = os.getcwd()
workSpec.mapType = queueConfig.mapType

# set job to worker if not job-level late binding
if not queueConfig.useJobLateBinding:
    workSpec.hasJob = 1
    workSpec.set_jobspec_list(jobSpecList)

# get plugin for messenger
if queueConfig.mapType != WorkSpec.MT_NoJob:
    messenger = pluginFactory.get_plugin(queueConfig.messenger)
    messenger.feed_jobs(workSpec, jobSpecList)

    jobSpec = jobSpecList[0]
    if 'eventService' in jobSpec.jobParams:
        workSpec.eventsRequest = WorkSpec.EV_useEvents

    if workSpec.hasJob == 1 and workSpec.eventsRequest == WorkSpec.EV_useEvents:
        workSpec.eventsRequest = WorkSpec.EV_requestEvents
        eventsRequestParams = dict()
        eventsRequestParams[jobSpec.PandaID] = {'pandaID': jobSpec.PandaID,
                                                'taskID': jobSpec.taskID,
                                                'jobsetID': jobSpec.jobParams['jobsetID'],
                                                'nRanges': jobSpec.jobParams['coreCount'],
                                                }
        workSpec.eventsRequestParams = eventsRequestParams

        tmpStat, events = com.get_event_ranges(workSpec.eventsRequestParams)
        # failed
        if tmpStat is False:
            print('failed to get events with {0}'.format(events))
            sys.exit(0)
        tmpStat = messenger.feed_events(workSpec, events)
        if tmpStat is False:
            print('failed to feed events with {0}'.format(events))
            sys.exit(0)

# get submitter plugin
submitterCore = pluginFactory.get_plugin(queueConfig.submitter)
print "testing submission with plugin={0}".format(submitterCore.__class__.__name__)
tmpRetList = submitterCore.submit_workers([workSpec])
tmpStat, tmpOut = tmpRetList[0]
if tmpStat:
    print " OK batchID={0}".format(workSpec.batchID)
else:
    print " NG {0}".format(tmpOut)
    sys.exit(1)

print

# get monitoring plug-in
monCore = pluginFactory.get_plugin(queueConfig.monitor)
print "testing monitoring for batchID={0} with plugin={1}".format(workSpec.batchID,
                                                                  monCore.__class__.__name__)
tmpStat, tmpOut = monCore.check_workers([workSpec])
tmpOut = tmpOut[0]
if tmpStat:
    print " OK workerStatus={0}".format(tmpOut[0])
else:
    print " NG {0}".format(tmpOut[1])
    sys.exit(1)
<|endoftext|>


<commit_before># Program to perform breadth first traversal in a graph
from collections import defaultdict, deque

class Graph:
    def __init__(self, directed=False):
        self.graph = defaultdict(list)
        self.directed = directed

    def addEdge(self, frm, to):
        self.graph[frm].append(to)

        if self.directed is False:
            self.graph[to].append(frm)

    def bfsUtil(self, s, visited):
        queue = deque([])
        queue.append(s)
        visited[s] = True

        while queue:
            vertex = queue.popleft()
            print(vertex, end=' ')

            # traverse vertices adjacent to vertex
            for i in self.graph[vertex]:
                if not visited[i]:
                    visited[i] = True
                    queue.append(i)
        print()

    def bfs(self, s=None):
        visited = {i: False for i in self.graph}

        # do bfs from the node specified
        if s is not None:
            self.bfsUtil(s, visited)

        for v in self.graph:
            if not visited[v]:
                self.bfsUtil(v, visited)

if __name__ == '__main__':
    graph = Graph()

    # component 1 of the graph
    graph.addEdge(0, 1)
    graph.addEdge(0, 2)
    graph.addEdge(1, 2)
    graph.addEdge(2, 3)
    graph.addEdge(3, 3)
    graph.addEdge(1, 4)
    graph.addEdge(1, 5)
    graph.addEdge(3, 6)

    # component 2 of the graph
    graph.addEdge(7, 8)
    graph.addEdge(8, 9)
    graph.addEdge(7, 10)

    # call bfs from 2 vertex
    print("Breadth First Traversal:")
    graph.bfs(2)

<commit_msg>updated documentation<commit_after># Program to perform breadth first traversal in a graph
from collections import defaultdict, deque

class Graph:
    def __init__(self, directed=False):
        self.graph = defaultdict(list)
        self.directed = directed

    def addEdge(self, frm, to):
        self.graph[frm].append(to)

        if self.directed is False:
            self.graph[to].append(frm)

    def bfsUtil(self, s, visited):
        queue = deque([])
        queue.append(s)
        visited[s] = True

        while queue:
            vertex = queue.popleft()
            print(vertex, end=' ')

            # traverse vertices adjacent to vertex
            for i in self.graph[vertex]:
                if not visited[i]:
                    visited[i] = True
                    queue.append(i)
        print()

    def bfs(self, s=None):
        visited = {i: False for i in self.graph}

        # do bfs from the node specified
        if s is not None:
            self.bfsUtil(s, visited)
        # traverse for all the vertices in other components of graph
        for v in self.graph:
            if not visited[v]:
                self.bfsUtil(v, visited)

if __name__ == '__main__':
    graph = Graph()

    # component 1 of the graph
    graph.addEdge(0, 1)
    graph.addEdge(0, 2)
    graph.addEdge(1, 2)
    graph.addEdge(2, 3)
    graph.addEdge(3, 3)
    graph.addEdge(1, 4)
    graph.addEdge(1, 5)
    graph.addEdge(3, 6)

    # component 2 of the graph
    graph.addEdge(7, 8)
    graph.addEdge(8, 9)
    graph.addEdge(7, 10)

    # call bfs from 2 vertex
    print("Breadth First Traversal:")
    graph.bfs(2)

<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Class to represent a GCE Virtual Machine object.

Zones:
run 'gcutil listzones'
Machine Types:
run 'gcutil listmachinetypes'
Images:
run 'gcutil listimages'

All VM specifics are self-contained and the class provides methods to
operate on the VM: boot, shutdown, etc.
"""

import json
import re
import tempfile

from perfkitbenchmarker import disk
from perfkitbenchmarker import flags
from perfkitbenchmarker import package_managers
from perfkitbenchmarker import virtual_machine
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.gcp import gce_disk
from perfkitbenchmarker.gcp import util

flags.DEFINE_integer('gce_num_local_ssds', 0,
                     'The number of ssds that should be added to the VM. Note '
                     'that this is currently only supported in certain zones '
                     '(see https://cloud.google.com/compute/docs/local-ssd).')
flags.DEFINE_string('gcloud_scopes', None, 'If set, space-separated list of '
                    'scopes to apply to every created machine')

FLAGS = flags.FLAGS

SET_INTERRUPTS_SH = 'set-interrupts.sh'
BOOT_DISK_SIZE_GB = 10
BOOT_DISK_TYPE = 'pd-standard'
DRIVE_START_LETTER = 'b'
NVME = 'nvme'
SCSI = 'SCSI'

class GceVirtualMachine(virtual_machine.BaseVirtualMachine):
  """Object representing a Google Compute Engine Virtual Machine."""

  instance_counter = 0

  def __init__(self, vm_spec):
    """Initialize a GCE virtual machine.

    Args:
      vm_spec: virtual_machine.BaseVirtualMachineSpec object of the vm.
    """
    super(GceVirtualMachine, self).__init__(vm_spec)
    self.name = 'perfkit-%s-%s' % (FLAGS.run_uri, self.instance_counter)
    GceVirtualMachine.instance_counter += 1
    disk_spec = disk.BaseDiskSpec(BOOT_DISK_SIZE_GB, BOOT_DISK_TYPE, None)
    self.boot_disk = gce_disk.GceDisk(
        disk_spec, self.name, self.zone, self.project, self.image)
    self.num_ssds = FLAGS.gce_num_local_ssds

  def _CreateDependencies(self):
    """Create VM dependencies."""
    self.boot_disk.Create()

  def _DeleteDependencies(self):
    """Delete VM dependencies."""
    self.boot_disk.Delete()

  def _Create(self):
    """Create a GCE VM instance."""
    super(GceVirtualMachine, self)._Create()
    with open(self.ssh_public_key) as f:
      public_key = f.read().rstrip('\n')
    with tempfile.NamedTemporaryFile(dir=vm_util.GetTempDir(),
                                     prefix='key-metadata') as tf:
      tf.write('%s:%s\n' % (self.user_name, public_key))
      tf.flush()
      create_cmd = [FLAGS.gcloud_path,
                    'compute',
                    'instances',
                    'create', self.name,
                    '--disk',
                    'name=%s' % self.boot_disk.name,
                    'boot=yes',
                    'mode=rw',
                    '--machine-type', self.machine_type,
                    '--tags=perfkitbenchmarker',
                    '--maintenance-policy', 'TERMINATE',
                    '--metadata-from-file',
                    'sshKeys=%s' % tf.name,
                    '--metadata',
                    'owner=%s' % FLAGS.owner]
      ssd_interface_option = NVME if NVME in self.image else SCSI
      for _ in range(self.num_ssds):
        create_cmd.append('--local-ssd')
        create_cmd.append('interface=%s' % ssd_interface_option)
      if FLAGS.gcloud_scopes:
        create_cmd.extend(['--scopes'] +
                          re.split(r'[,; ]', FLAGS.gcloud_scopes))
      create_cmd.extend(util.GetDefaultGcloudFlags(self))
      vm_util.IssueCommand(create_cmd)

  @vm_util.Retry()
  def _PostCreate(self):
    """Get the instance's data."""
    getinstance_cmd = [FLAGS.gcloud_path,
                       'compute',
                       'instances',
                       'describe', self.name]
    getinstance_cmd.extend(util.GetDefaultGcloudFlags(self))
    stdout, _, _ = vm_util.IssueCommand(getinstance_cmd)
    response = json.loads(stdout)
    network_interface = response['networkInterfaces'][0]
    self.internal_ip = network_interface['networkIP']
    self.ip_address = network_interface['accessConfigs'][0]['natIP']

  def _Delete(self):
    """Delete a GCE VM instance."""
    delete_cmd = [FLAGS.gcloud_path,
                  'compute',
                  'instances',
                  'delete', self.name]
    delete_cmd.extend(util.GetDefaultGcloudFlags(self))
    vm_util.IssueCommand(delete_cmd)

  def _Exists(self):
    """Returns true if the VM exists."""
    getinstance_cmd = [FLAGS.gcloud_path,
                       'compute',
                       'instances',
                       'describe', self.name]
    getinstance_cmd.extend(util.GetDefaultGcloudFlags(self))
    stdout, _, _ = vm_util.IssueCommand(getinstance_cmd, suppress_warning=True)
    try:
      json.loads(stdout)
    except ValueError:
      return False
    return True

  def CreateScratchDisk(self, disk_spec):
    """Create a VM's scratch disk.

    Args:
      disk_spec: virtual_machine.BaseDiskSpec object of the disk.
    """
    name = '%s-scratch-%s' % (self.name, len(self.scratch_disks))
    scratch_disk = gce_disk.GceDisk(disk_spec, name, self.zone, self.project)
    self.scratch_disks.append(scratch_disk)

    scratch_disk.Create()
    scratch_disk.Attach(self)

    self.FormatDisk(scratch_disk.GetDevicePath())
    self.MountDisk(scratch_disk.GetDevicePath(), disk_spec.mount_point)

  def GetName(self):
    """Get a GCE VM's unique name."""
    return self.name

  def GetLocalDrives(self):
    """Returns a list of local drives on the VM.

    Returns:
      A list of strings, where each string is the absolute path to the local
          drives on the VM (e.g. '/dev/sdb').
    """
    return ['/dev/disk/by-id/google-local-ssd-%d' % i
            for i in range(self.num_ssds)]

  def SetupLocalDrives(self, mount_path=virtual_machine.LOCAL_MOUNT_PATH):
    """Set up any local drives that exist.

    Sets up drives as usual, then performs GCE specific setup
    (runs set-interrupts.sh).

    Args:
      mount_path: The path where the local drives should be mounted. If this
          is None, then the device won't be formatted or mounted.

    Returns:
      A boolean indicating whether the setup occured.
    """
    ret = super(GceVirtualMachine, self).SetupLocalDrives(mount_path=mount_path)
    if ret:
      self.PushDataFile(SET_INTERRUPTS_SH)
      self.RemoteCommand(
          'chmod +rx set-interrupts.sh; sudo ./set-interrupts.sh')
    return ret

  def AddMetadata(self, **kwargs):
    """Adds metadata to the VM via 'gcloud compute instances add-metadata'."""
    if not kwargs:
      return
    cmd = [FLAGS.gcloud_path, 'compute', 'instances', 'add-metadata',
           '--zone', self.zone, self.name, '--metadata']
    for key, value in kwargs.iteritems():
      cmd.append('{0}={1}'.format(key, value))
    vm_util.IssueCommand(cmd)

class DebianBasedGceVirtualMachine(GceVirtualMachine,
                                   package_managers.AptMixin):
  pass

class RhelBasedGceVirtualMachine(GceVirtualMachine,
                                 package_managers.YumMixin):
  pass
<commit_msg>Fixes small bug wth GCP metadata<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Class to represent a GCE Virtual Machine object.

Zones:
run 'gcutil listzones'
Machine Types:
run 'gcutil listmachinetypes'
Images:
run 'gcutil listimages'

All VM specifics are self-contained and the class provides methods to
operate on the VM: boot, shutdown, etc.
"""

import json
import re
import tempfile

from perfkitbenchmarker import disk
from perfkitbenchmarker import flags
from perfkitbenchmarker import package_managers
from perfkitbenchmarker import virtual_machine
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.gcp import gce_disk
from perfkitbenchmarker.gcp import util

flags.DEFINE_integer('gce_num_local_ssds', 0,
                     'The number of ssds that should be added to the VM. Note '
                     'that this is currently only supported in certain zones '
                     '(see https://cloud.google.com/compute/docs/local-ssd).')
flags.DEFINE_string('gcloud_scopes', None, 'If set, space-separated list of '
                    'scopes to apply to every created machine')

FLAGS = flags.FLAGS

SET_INTERRUPTS_SH = 'set-interrupts.sh'
BOOT_DISK_SIZE_GB = 10
BOOT_DISK_TYPE = 'pd-standard'
DRIVE_START_LETTER = 'b'
NVME = 'nvme'
SCSI = 'SCSI'

class GceVirtualMachine(virtual_machine.BaseVirtualMachine):
  """Object representing a Google Compute Engine Virtual Machine."""

  instance_counter = 0

  def __init__(self, vm_spec):
    """Initialize a GCE virtual machine.

    Args:
      vm_spec: virtual_machine.BaseVirtualMachineSpec object of the vm.
    """
    super(GceVirtualMachine, self).__init__(vm_spec)
    self.name = 'perfkit-%s-%s' % (FLAGS.run_uri, self.instance_counter)
    GceVirtualMachine.instance_counter += 1
    disk_spec = disk.BaseDiskSpec(BOOT_DISK_SIZE_GB, BOOT_DISK_TYPE, None)
    self.boot_disk = gce_disk.GceDisk(
        disk_spec, self.name, self.zone, self.project, self.image)
    self.num_ssds = FLAGS.gce_num_local_ssds

  def _CreateDependencies(self):
    """Create VM dependencies."""
    self.boot_disk.Create()

  def _DeleteDependencies(self):
    """Delete VM dependencies."""
    self.boot_disk.Delete()

  def _Create(self):
    """Create a GCE VM instance."""
    super(GceVirtualMachine, self)._Create()
    with open(self.ssh_public_key) as f:
      public_key = f.read().rstrip('\n')
    with tempfile.NamedTemporaryFile(dir=vm_util.GetTempDir(),
                                     prefix='key-metadata') as tf:
      tf.write('%s:%s\n' % (self.user_name, public_key))
      tf.flush()
      create_cmd = [FLAGS.gcloud_path,
                    'compute',
                    'instances',
                    'create', self.name,
                    '--disk',
                    'name=%s' % self.boot_disk.name,
                    'boot=yes',
                    'mode=rw',
                    '--machine-type', self.machine_type,
                    '--tags=perfkitbenchmarker',
                    '--maintenance-policy', 'TERMINATE',
                    '--metadata-from-file',
                    'sshKeys=%s' % tf.name,
                    '--metadata',
                    'owner=%s' % FLAGS.owner]
      ssd_interface_option = NVME if NVME in self.image else SCSI
      for _ in range(self.num_ssds):
        create_cmd.append('--local-ssd')
        create_cmd.append('interface=%s' % ssd_interface_option)
      if FLAGS.gcloud_scopes:
        create_cmd.extend(['--scopes'] +
                          re.split(r'[,; ]', FLAGS.gcloud_scopes))
      create_cmd.extend(util.GetDefaultGcloudFlags(self))
      vm_util.IssueCommand(create_cmd)

  @vm_util.Retry()
  def _PostCreate(self):
    """Get the instance's data."""
    getinstance_cmd = [FLAGS.gcloud_path,
                       'compute',
                       'instances',
                       'describe', self.name]
    getinstance_cmd.extend(util.GetDefaultGcloudFlags(self))
    stdout, _, _ = vm_util.IssueCommand(getinstance_cmd)
    response = json.loads(stdout)
    network_interface = response['networkInterfaces'][0]
    self.internal_ip = network_interface['networkIP']
    self.ip_address = network_interface['accessConfigs'][0]['natIP']

  def _Delete(self):
    """Delete a GCE VM instance."""
    delete_cmd = [FLAGS.gcloud_path,
                  'compute',
                  'instances',
                  'delete', self.name]
    delete_cmd.extend(util.GetDefaultGcloudFlags(self))
    vm_util.IssueCommand(delete_cmd)

  def _Exists(self):
    """Returns true if the VM exists."""
    getinstance_cmd = [FLAGS.gcloud_path,
                       'compute',
                       'instances',
                       'describe', self.name]
    getinstance_cmd.extend(util.GetDefaultGcloudFlags(self))
    stdout, _, _ = vm_util.IssueCommand(getinstance_cmd, suppress_warning=True)
    try:
      json.loads(stdout)
    except ValueError:
      return False
    return True

  def CreateScratchDisk(self, disk_spec):
    """Create a VM's scratch disk.

    Args:
      disk_spec: virtual_machine.BaseDiskSpec object of the disk.
    """
    name = '%s-scratch-%s' % (self.name, len(self.scratch_disks))
    scratch_disk = gce_disk.GceDisk(disk_spec, name, self.zone, self.project)
    self.scratch_disks.append(scratch_disk)

    scratch_disk.Create()
    scratch_disk.Attach(self)

    self.FormatDisk(scratch_disk.GetDevicePath())
    self.MountDisk(scratch_disk.GetDevicePath(), disk_spec.mount_point)

  def GetName(self):
    """Get a GCE VM's unique name."""
    return self.name

  def GetLocalDrives(self):
    """Returns a list of local drives on the VM.

    Returns:
      A list of strings, where each string is the absolute path to the local
          drives on the VM (e.g. '/dev/sdb').
    """
    return ['/dev/disk/by-id/google-local-ssd-%d' % i
            for i in range(self.num_ssds)]

  def SetupLocalDrives(self, mount_path=virtual_machine.LOCAL_MOUNT_PATH):
    """Set up any local drives that exist.

    Sets up drives as usual, then performs GCE specific setup
    (runs set-interrupts.sh).

    Args:
      mount_path: The path where the local drives should be mounted. If this
          is None, then the device won't be formatted or mounted.

    Returns:
      A boolean indicating whether the setup occured.
    """
    ret = super(GceVirtualMachine, self).SetupLocalDrives(mount_path=mount_path)
    if ret:
      self.PushDataFile(SET_INTERRUPTS_SH)
      self.RemoteCommand(
          'chmod +rx set-interrupts.sh; sudo ./set-interrupts.sh')
    return ret

  def AddMetadata(self, **kwargs):
    """Adds metadata to the VM via 'gcloud compute instances add-metadata'."""
    if not kwargs:
      return
    cmd = [FLAGS.gcloud_path, 'compute', 'instances', 'add-metadata',
           '--zone', self.zone, self.name, '--metadata']
    for key, value in kwargs.iteritems():
      cmd.append('{0}={1}'.format(key, value))
    cmd.extend(util.GetDefaultGcloudFlags(self))
    vm_util.IssueCommand(cmd)

class DebianBasedGceVirtualMachine(GceVirtualMachine,
                                   package_managers.AptMixin):
  pass

class RhelBasedGceVirtualMachine(GceVirtualMachine,
                                 package_managers.YumMixin):
  pass
<|endoftext|>


<commit_before>"""
test_frost_number_bmi.py
  tests of the frost_number component of permamodel using bmi API
"""

from permamodel.components import frost_number_Geo
from permamodel.components import bmi_frost_number_Geo
from permamodel.components import perma_base
from dateutil.relativedelta import relativedelta
import os
import numpy as np
from .. import examples_directory
from nose.tools import (assert_is_instance, assert_greater_equal,
                        assert_less_equal, assert_almost_equal,
                        assert_greater, assert_in, assert_true,
                        assert_false, assert_equal, assert_raises,
                        assert_not_equal)
from numpy.testing import assert_array_equal
import datetime

# Set the file names for the example cfg files
config_filename = \
        os.path.join(examples_directory,
                     'FrostnumberGeo_Default.cfg')

# List of files to be removed after testing is complete
# use files_to_remove.append(<filename>) to add to it
files_to_remove = []

def setup_module():
    """ Standard fixture called before any tests in this file are performed """
    pass

def teardown_module():
    """ Standard fixture called after all tests in this file are performed """
    # If need to remove files that are created:
    #for f in files_to_remove:
    #    if os.path.exists(f):
    #        os.remove(f)
    pass

# ---------------------------------------------------
# Tests that ensure we have bmi functionality
# Note: the netcdf functions seem to lead to errors if the netcdf
# files are not closed.  This is done in the finalize() routine
# which should therefore be called after every test
# ---------------------------------------------------
def test_frost_number_Geo_has_initialize():
    # Can we call an initialize function?
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize(cfg_file=config_filename)
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_initialize_sets_year():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize(cfg_file=config_filename)

    # Assert the values from the cfg file
    assert_equal(fng._model._date_current.year, 1901)
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_initialize_sets_air_min_and_max():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize(cfg_file=config_filename)

    # The temperature arrays are initialized to all NaNs
    nan_array = np.zeros((3, 2), dtype=np.float32)
    nan_array.fill(np.nan)
    assert_array_equal(fng._model.T_air_min, nan_array)
    assert_array_equal(fng._model.T_air_max, nan_array)
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_update_increments_time():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    fng.update()

    assert_equal(fng._model._date_current,
                 fng._model._start_date + \
                 relativedelta(years=fng._model._timestep_duration))
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_update_changes_air_frost_number():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    fng.update()
    afn0 = fng._model.air_frost_number_Geo
    fng.update_until(1.0)
    afn1 = fng._model.air_frost_number_Geo
    try:
        # If this is none, then the first array was all NaNs
        assert_true(assert_array_equal(afn0, afn1) is None)
    except AssertionError:
        # If this is raised, then the arrays are different, which is nood
        pass
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_get_current_time_returns_scalar():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    current_time = fng.get_current_time()
    assert_true(isinstance(current_time, float) \
                or isinstance(current_time, int))
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_get_end_time_returns_scalar():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    end_time = fng.get_end_time()
    assert(isinstance(end_time, float) \
           or isinstance(end_time, int))
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_implements_update_until():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    fng.update_until(fng.get_end_time())
    assert_true(fng._model._date_current, fng._model._end_date)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_computes_default_values():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    for n in range(5):
        if (fng._model.T_air_min[0, 0] + fng._model.T_air_max[0, 0]) == 0:
            assert_equal(fng._values['frostnumber__air'][0, 0], 0.5)
        if (fng._model.T_air_min[0, 0] <= 0.0) and \
           (fng._model.T_air_max[0, 0] <= 0.0):
            assert_equal(fng._values['frostnumber__air'][0, 0], 1.0)
        if (fng._model.T_air_min[0, 0] > 0.0) and \
           (fng._model.T_air_max[0, 0] > 0.0):
            assert_equal(fng._values['frostnumber__air'][0, 0], 0.0)
        #print("FNGeo frostnumber__air: %d" % n)
        #print(fng._model.T_air_min)
        #print(fng._model.T_air_max)
        #print(fng._values['frostnumber__air'])
        fng.update()
    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_attribute():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    # Check an attribute that exists
    this_att = fng.get_attribute('time_units')
    assert_equal(this_att, 'years')

    # Check an attribute that doesn't exist
    assert_raises(KeyError, fng.get_attribute, 'not_an_attribute')

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_input_var_names():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    assert_in('atmosphere_bottom_air__temperature', fng.get_input_var_names())

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_output_var_names():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    assert_in('frostnumber__air', fng.get_output_var_names())

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_set_value():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtempval = fng.get_value('atmosphere_bottom_air__temperature')
    airtempnew = 123 * np.ones_like(airtempval)
    fng.set_value('atmosphere_bottom_air__temperature', airtempnew)
    assert_raises(AssertionError, assert_array_equal, airtempval, airtempnew)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_grid_shape():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    assert_equal((3, 2), fng.get_grid_shape(airtemp_gridid))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_grid_size():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    assert_equal(6, fng.get_grid_size(airtemp_gridid))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_grid_spacing():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    assert_array_equal(
        np.array([1.0, 1.0]),
        fng.get_grid_spacing(airtemp_gridid))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_jan_and_jul_temperatures_are_grids():
    """ Test that FNGeo BMI has input variables for jan and jul data """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    jan_airtemp_gridid = \
            fng.get_var_grid('atmosphere_bottom_air__temperature_mean_jan')
    assert_true(jan_airtemp_gridid is not None)
    jul_airtemp_gridid = \
            fng.get_var_grid('atmosphere_bottom_air__temperature_mean_jul')
    assert_true(jul_airtemp_gridid is not None)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_can_set_current_and_jan_temperatures():
    """ Test that FNGeo BMI can set jan temperature field """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_values = fng.get_value('atmosphere_bottom_air__temperature')
    airtemp_values = np.ones_like(airtemp_values)
    fng.set_value('atmosphere_bottom_air__temperature', airtemp_values)

    jan_airtemp_values = \
        fng.get_value('atmosphere_bottom_air__temperature_mean_jan')
    jan_airtemp_values = np.ones_like(jan_airtemp_values)
    fng.set_value('atmosphere_bottom_air__temperature_mean_jan',
                  jan_airtemp_values)
    assert_array_equal(
        fng.get_value('atmosphere_bottom_air__temperature'),
        fng.get_value('atmosphere_bottom_air__temperature_mean_jan'))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_update_zero_fraction_does_not_change_time():
    """ Test that running update_frac(0) does not change the time """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    init_time = fng.get_current_time()
    fng.update_frac(0)
    plus_zero_time = fng.get_current_time()
    assert_equal(init_time, plus_zero_time)
    fng.update()
    one_update_time = fng.get_current_time()
    assert_not_equal(init_time, one_update_time)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_simulate_WMT_run():
    """ Test that we can set values as if running in WMT """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    wmt_cfg_file = os.path.join(examples_directory, "FNGeo_WMT_testing.cfg")
    fng.initialize(wmt_cfg_file)
    assert_equal(fng._name, "Permamodel FrostnumberGeo Component")

    # Until set, e.g. by WMT with cru values, all vals are NaN
    # Note: these are actually setting references to the underlying
    #       model arrays!
    airtemp_values = fng.get_value('atmosphere_bottom_air__temperature')
    jan_airtemp_values = \
        fng.get_value('atmosphere_bottom_air__temperature_mean_jan')
    jul_airtemp_values = \
        fng.get_value('atmosphere_bottom_air__temperature_mean_jul')

    # In WMT mode, must set monthly temperature values, then run update_frac(0)
    #   to get valid values in frost number array
    # use January as 'coldest'
    # use July as 'warmest'
    airtemps_of_one = np.ones_like(airtemp_values)
    jan_airtemp_values[:, :] = -10.0 * airtemps_of_one[:, :]
    jul_airtemp_values[:, :] = 10.0 * airtemps_of_one[:, :]
    fng.set_value('atmosphere_bottom_air__temperature_mean_jan',
                  jan_airtemp_values)
    fng.set_value('atmosphere_bottom_air__temperature_mean_jul',
                  jul_airtemp_values)
    fng.update_frac(0)
    airfn_values = fng.get_value('frostnumber__air')

    assert_array_equal(0.5 * airtemps_of_one, airfn_values)

    fng.finalize()  # Must have this or get IOError later
<commit_msg>Air temperatures are returned as flattened arrays.<commit_after>"""
test_frost_number_bmi.py
  tests of the frost_number component of permamodel using bmi API
"""

from permamodel.components import frost_number_Geo
from permamodel.components import bmi_frost_number_Geo
from permamodel.components import perma_base
from dateutil.relativedelta import relativedelta
import os
import numpy as np
from .. import examples_directory
from nose.tools import (assert_is_instance, assert_greater_equal,
                        assert_less_equal, assert_almost_equal,
                        assert_greater, assert_in, assert_true,
                        assert_false, assert_equal, assert_raises,
                        assert_not_equal)
from numpy.testing import assert_array_equal
import datetime

# Set the file names for the example cfg files
config_filename = \
        os.path.join(examples_directory,
                     'FrostnumberGeo_Default.cfg')

# List of files to be removed after testing is complete
# use files_to_remove.append(<filename>) to add to it
files_to_remove = []

def setup_module():
    """ Standard fixture called before any tests in this file are performed """
    pass

def teardown_module():
    """ Standard fixture called after all tests in this file are performed """
    # If need to remove files that are created:
    #for f in files_to_remove:
    #    if os.path.exists(f):
    #        os.remove(f)
    pass

# ---------------------------------------------------
# Tests that ensure we have bmi functionality
# Note: the netcdf functions seem to lead to errors if the netcdf
# files are not closed.  This is done in the finalize() routine
# which should therefore be called after every test
# ---------------------------------------------------
def test_frost_number_Geo_has_initialize():
    # Can we call an initialize function?
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize(cfg_file=config_filename)
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_initialize_sets_year():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize(cfg_file=config_filename)

    # Assert the values from the cfg file
    assert_equal(fng._model._date_current.year, 1901)
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_initialize_sets_air_min_and_max():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize(cfg_file=config_filename)

    # The temperature arrays are initialized to all NaNs
    nan_array = np.zeros((3, 2), dtype=np.float32)
    nan_array.fill(np.nan)
    assert_array_equal(fng._model.T_air_min, nan_array)
    assert_array_equal(fng._model.T_air_max, nan_array)
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_update_increments_time():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    fng.update()

    assert_equal(fng._model._date_current,
                 fng._model._start_date + \
                 relativedelta(years=fng._model._timestep_duration))
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_update_changes_air_frost_number():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    fng.update()
    afn0 = fng._model.air_frost_number_Geo
    fng.update_until(1.0)
    afn1 = fng._model.air_frost_number_Geo
    try:
        # If this is none, then the first array was all NaNs
        assert_true(assert_array_equal(afn0, afn1) is None)
    except AssertionError:
        # If this is raised, then the arrays are different, which is nood
        pass
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_get_current_time_returns_scalar():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    current_time = fng.get_current_time()
    assert_true(isinstance(current_time, float) \
                or isinstance(current_time, int))
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_get_end_time_returns_scalar():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    end_time = fng.get_end_time()
    assert(isinstance(end_time, float) \
           or isinstance(end_time, int))
    fng.finalize()  # Must have this or get IOError later

def test_frost_number_implements_update_until():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    fng.update_until(fng.get_end_time())
    assert_true(fng._model._date_current, fng._model._end_date)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_computes_default_values():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    for n in range(5):
        if (fng._model.T_air_min[0, 0] + fng._model.T_air_max[0, 0]) == 0:
            assert_equal(fng._values['frostnumber__air'][0, 0], 0.5)
        if (fng._model.T_air_min[0, 0] <= 0.0) and \
           (fng._model.T_air_max[0, 0] <= 0.0):
            assert_equal(fng._values['frostnumber__air'][0, 0], 1.0)
        if (fng._model.T_air_min[0, 0] > 0.0) and \
           (fng._model.T_air_max[0, 0] > 0.0):
            assert_equal(fng._values['frostnumber__air'][0, 0], 0.0)
        #print("FNGeo frostnumber__air: %d" % n)
        #print(fng._model.T_air_min)
        #print(fng._model.T_air_max)
        #print(fng._values['frostnumber__air'])
        fng.update()
    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_attribute():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    # Check an attribute that exists
    this_att = fng.get_attribute('time_units')
    assert_equal(this_att, 'years')

    # Check an attribute that doesn't exist
    assert_raises(KeyError, fng.get_attribute, 'not_an_attribute')

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_input_var_names():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    assert_in('atmosphere_bottom_air__temperature', fng.get_input_var_names())

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_output_var_names():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    assert_in('frostnumber__air', fng.get_output_var_names())

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_set_value():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtempval = fng.get_value('atmosphere_bottom_air__temperature')
    airtempnew = 123 * np.ones_like(airtempval)
    fng.set_value('atmosphere_bottom_air__temperature', airtempnew)
    assert_raises(AssertionError, assert_array_equal, airtempval, airtempnew)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_grid_shape():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    assert_equal((3, 2), fng.get_grid_shape(airtemp_gridid))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_grid_size():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    assert_equal(6, fng.get_grid_size(airtemp_gridid))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_get_grid_spacing():
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    assert_array_equal(
        np.array([1.0, 1.0]),
        fng.get_grid_spacing(airtemp_gridid))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_jan_and_jul_temperatures_are_grids():
    """ Test that FNGeo BMI has input variables for jan and jul data """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_gridid = fng.get_var_grid('atmosphere_bottom_air__temperature')
    jan_airtemp_gridid = \
            fng.get_var_grid('atmosphere_bottom_air__temperature_mean_jan')
    assert_true(jan_airtemp_gridid is not None)
    jul_airtemp_gridid = \
            fng.get_var_grid('atmosphere_bottom_air__temperature_mean_jul')
    assert_true(jul_airtemp_gridid is not None)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_can_set_current_and_jan_temperatures():
    """ Test that FNGeo BMI can set jan temperature field """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()

    airtemp_values = fng.get_value('atmosphere_bottom_air__temperature')
    airtemp_values = np.ones_like(airtemp_values)
    fng.set_value('atmosphere_bottom_air__temperature', airtemp_values)

    jan_airtemp_values = \
        fng.get_value('atmosphere_bottom_air__temperature_mean_jan')
    jan_airtemp_values = np.ones_like(jan_airtemp_values)
    fng.set_value('atmosphere_bottom_air__temperature_mean_jan',
                  jan_airtemp_values)
    assert_array_equal(
        fng.get_value('atmosphere_bottom_air__temperature'),
        fng.get_value('atmosphere_bottom_air__temperature_mean_jan'))

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_update_zero_fraction_does_not_change_time():
    """ Test that running update_frac(0) does not change the time """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    fng.initialize()
    init_time = fng.get_current_time()
    fng.update_frac(0)
    plus_zero_time = fng.get_current_time()
    assert_equal(init_time, plus_zero_time)
    fng.update()
    one_update_time = fng.get_current_time()
    assert_not_equal(init_time, one_update_time)

    fng.finalize()  # Must have this or get IOError later

def test_FNGeo_simulate_WMT_run():
    """ Test that we can set values as if running in WMT """
    fng = bmi_frost_number_Geo.BmiFrostnumberGeoMethod()
    wmt_cfg_file = os.path.join(examples_directory, "FNGeo_WMT_testing.cfg")
    fng.initialize(wmt_cfg_file)
    assert_equal(fng._name, "Permamodel FrostnumberGeo Component")

    # Until set, e.g. by WMT with cru values, all vals are NaN
    # Note: these are actually setting references to the underlying
    #       model arrays!
    airtemp_values = fng.get_value('atmosphere_bottom_air__temperature')
    jan_airtemp_values = \
        fng.get_value('atmosphere_bottom_air__temperature_mean_jan')
    jul_airtemp_values = \
        fng.get_value('atmosphere_bottom_air__temperature_mean_jul')

    # In WMT mode, must set monthly temperature values, then run update_frac(0)
    #   to get valid values in frost number array
    # use January as 'coldest'
    # use July as 'warmest'
    airtemps_of_one = np.ones_like(airtemp_values)
    jan_airtemp_values[:] = -10.0 * airtemps_of_one
    jul_airtemp_values[:] = 10.0 * airtemps_of_one
    fng.set_value('atmosphere_bottom_air__temperature_mean_jan',
                  jan_airtemp_values)
    fng.set_value('atmosphere_bottom_air__temperature_mean_jul',
                  jul_airtemp_values)
    fng.update_frac(0)
    airfn_values = fng.get_value('frostnumber__air')

    assert_array_equal(0.5 * airtemps_of_one, airfn_values)

    fng.finalize()  # Must have this or get IOError later
<|endoftext|>