<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Contains various utility functions for Emacs Lisp rules.

These functions are internal and subject to change without notice."""

load("@bazel_skylib//lib:paths.bzl", "paths")
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")

def check_relative_filename(filename):
    """Returns `filename`, checking whether it is relative.

    The file name must be relative and represent either the current directory
    or an entry within the current directory or any of its subdirectories.  In
    other words, it may not point above the current directory.  To specify the
    current directory, pass an empty string or a single dot (`.`).  This
    function also checks whether the filename contains special characters.  If
    the filename is invalid in any way (absolute, containing special
    characters, or pointing above the current directory), this function calls
    `fail` with a descriptive error message.  Otherwise, it returns the
    normalized version of `filename`, using purely lexical simplifications (not
    resolving symbolic links).

    Args:
      filename (string): the filename to check

    Returns:
      the normalized version of the `filename` argument
    """
    if paths.is_absolute(filename):
        fail("filename {} is absolute".format(filename))
    filename = paths.normalize(filename)
    if filename != "." and not filename[0].isalpha():
        fail("filename {} has to start with a letter".format(filename))
    for char in filename.elems():
        if not char.isalnum() and char not in "-_./":
            fail("invalid character {} in filename {}".format(char, filename))
    return filename

def runfile_location(ctx, file):
    """Return the filename of the given file relative to the runfiles root.

    Args:
      ctx (ctx): the current rule context
      file (File): any file that’s included in the runfiles

    Returns:
      a string representing the filename of the file relative to the runfiles
      root
    """

    # It might seem surprising that we can use “ctx.workspace_name”
    # unconditionally.  However, for files in external workspaces, “short_path”
    # will start with “../〈workspace〉/…”, so the logic here is correct.
    # “check_relative_filename” not only ensures that the filename is relative,
    # but also canonicalizes it.
    return check_relative_filename(
        paths.join(ctx.workspace_name, file.short_path),
    )

def configure_cc_toolchain(ctx):
    """
    Configures the C/C++ toolchain for use with the current rule.

    Args:
      ctx (ctx): the current rule context

    Returns:
      a pair (toolchain, config) of a `CcToolchainInfo` provider and a
      `FeatureConfiguration` object appropriate for C and C++ compilation
      actions
    """
    toolchain = find_cpp_toolchain(ctx)
    config = cc_common.configure_features(
        ctx = ctx,
        cc_toolchain = toolchain,
        requested_features = ctx.features,
        unsupported_features = ctx.disabled_features,
    )
    return toolchain, config

def cc_wrapper(ctx, cc_toolchain, feature_configuration, driver):
    """Builds a wrapper executable that starts Emacs.

    You can use `configure_cc_toolchain` to construct appropriate values for
    `cc_toolchain` and `feature_configuration`.

    Args:
      ctx (ctx): rule context
      cc_toolchain (Provider): the C++ toolchain to use to compile the wrapper
      feature_configuration (FeatureConfiguration): the features to use to
          compile the wrapper
      driver (File): C++ driver file to compile

    Returns:
      a File representing the executable that starts Emacs
    """
    exec = ctx.attr._exec[CcInfo]
    _, objs = cc_common.compile(
        name = ctx.label.name,
        actions = ctx.actions,
        feature_configuration = feature_configuration,
        cc_toolchain = cc_toolchain,
        srcs = [driver],
        compilation_contexts = [exec.compilation_context],
        user_compile_flags = COPTS,
    )
    bin = cc_common.link(
        name = ctx.label.name,
        actions = ctx.actions,
        feature_configuration = feature_configuration,
        cc_toolchain = cc_toolchain,
        compilation_outputs = objs,
        linking_contexts = [exec.linking_context],
    )
    return bin.executable

def cpp_strings(strings):
    """Formats the given string list as C++ initializer list.

    This function makes an effort to support strings with special characters.

    Args:
      strings (list of string): strings to be formatted

    Returns:
      a string containing C++ code representing the given string list
    """
    return ", ".join([cpp_string(s) for s in strings])

def cpp_string(string):
    """Formats the given string as C++ string literal.

    This function makes an effort to support strings with special characters.

    Args:
      string: any string

    Returns:
      a string containing a properly escaped C++ string literal
    """

    # Use raw strings and choose a delimiter that’s extremely unlikely to occur
    # in real-world code.
    delim = "#*?&"
    open = 'R"' + delim + "("
    close = ")" + delim + '"'
    if close in string:
        fail("String {} can’t be transferred to C++".format(string))
    return open + string + close

def cpp_ints(ints):
    """Formats the given integer list as C++ initializer list.

    Args:
      ints (list of int): numbers to be formatted

    Returns:
      a string containing C++ code representing the given number list
    """
    return ", ".join([cpp_int(i) for i in ints])

def cpp_int(int):
    """Format the given integer as C++ decimal literal.

    Args:
      int: an integer

    Returns:
      a string containing a C++ decimal literal
    """

    # See https://stackoverflow.com/a/1819236 for the guarantees on the C++ int
    # type range.
    if int < -32767 or int > 32767:
        fail("integer {} out of range".format(int))
    return str(int)

# Shared C++ compilation options.
COPTS = [
    "-fno-exceptions",
    "-Werror",
    "-Wall",
    "-Wextra",
    "-Wconversion",
    "-Wsign-conversion",
    "-pedantic-errors",
    # GCC appears to treat some moves as redundant that are in fact necessary.
    "-Wno-redundant-move",
    "-D_GNU_SOURCE",
]
<commit_msg>Allow a few more harmless characters in filenames<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Contains various utility functions for Emacs Lisp rules.

These functions are internal and subject to change without notice."""

load("@bazel_skylib//lib:paths.bzl", "paths")
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")

def check_relative_filename(filename):
    """Returns `filename`, checking whether it is relative.

    The file name must be relative and represent either the current directory
    or an entry within the current directory or any of its subdirectories.  In
    other words, it may not point above the current directory.  To specify the
    current directory, pass an empty string or a single dot (`.`).  This
    function also checks whether the filename contains special characters.  If
    the filename is invalid in any way (absolute, containing special
    characters, or pointing above the current directory), this function calls
    `fail` with a descriptive error message.  Otherwise, it returns the
    normalized version of `filename`, using purely lexical simplifications (not
    resolving symbolic links).

    Args:
      filename (string): the filename to check

    Returns:
      the normalized version of the `filename` argument
    """
    if paths.is_absolute(filename):
        fail("filename {} is absolute".format(filename))
    filename = paths.normalize(filename)
    if filename != "." and not filename[0].isalpha():
        fail("filename {} has to start with a letter".format(filename))
    for char in filename.elems():
        if not char.isalnum() and char not in "-_./+$":
            fail("invalid character {} in filename {}".format(char, filename))
    return filename

def runfile_location(ctx, file):
    """Return the filename of the given file relative to the runfiles root.

    Args:
      ctx (ctx): the current rule context
      file (File): any file that’s included in the runfiles

    Returns:
      a string representing the filename of the file relative to the runfiles
      root
    """

    # It might seem surprising that we can use “ctx.workspace_name”
    # unconditionally.  However, for files in external workspaces, “short_path”
    # will start with “../〈workspace〉/…”, so the logic here is correct.
    # “check_relative_filename” not only ensures that the filename is relative,
    # but also canonicalizes it.
    return check_relative_filename(
        paths.join(ctx.workspace_name, file.short_path),
    )

def configure_cc_toolchain(ctx):
    """
    Configures the C/C++ toolchain for use with the current rule.

    Args:
      ctx (ctx): the current rule context

    Returns:
      a pair (toolchain, config) of a `CcToolchainInfo` provider and a
      `FeatureConfiguration` object appropriate for C and C++ compilation
      actions
    """
    toolchain = find_cpp_toolchain(ctx)
    config = cc_common.configure_features(
        ctx = ctx,
        cc_toolchain = toolchain,
        requested_features = ctx.features,
        unsupported_features = ctx.disabled_features,
    )
    return toolchain, config

def cc_wrapper(ctx, cc_toolchain, feature_configuration, driver):
    """Builds a wrapper executable that starts Emacs.

    You can use `configure_cc_toolchain` to construct appropriate values for
    `cc_toolchain` and `feature_configuration`.

    Args:
      ctx (ctx): rule context
      cc_toolchain (Provider): the C++ toolchain to use to compile the wrapper
      feature_configuration (FeatureConfiguration): the features to use to
          compile the wrapper
      driver (File): C++ driver file to compile

    Returns:
      a File representing the executable that starts Emacs
    """
    exec = ctx.attr._exec[CcInfo]
    _, objs = cc_common.compile(
        name = ctx.label.name,
        actions = ctx.actions,
        feature_configuration = feature_configuration,
        cc_toolchain = cc_toolchain,
        srcs = [driver],
        compilation_contexts = [exec.compilation_context],
        user_compile_flags = COPTS,
    )
    bin = cc_common.link(
        name = ctx.label.name,
        actions = ctx.actions,
        feature_configuration = feature_configuration,
        cc_toolchain = cc_toolchain,
        compilation_outputs = objs,
        linking_contexts = [exec.linking_context],
    )
    return bin.executable

def cpp_strings(strings):
    """Formats the given string list as C++ initializer list.

    This function makes an effort to support strings with special characters.

    Args:
      strings (list of string): strings to be formatted

    Returns:
      a string containing C++ code representing the given string list
    """
    return ", ".join([cpp_string(s) for s in strings])

def cpp_string(string):
    """Formats the given string as C++ string literal.

    This function makes an effort to support strings with special characters.

    Args:
      string: any string

    Returns:
      a string containing a properly escaped C++ string literal
    """

    # Use raw strings and choose a delimiter that’s extremely unlikely to occur
    # in real-world code.
    delim = "#*?&"
    open = 'R"' + delim + "("
    close = ")" + delim + '"'
    if close in string:
        fail("String {} can’t be transferred to C++".format(string))
    return open + string + close

def cpp_ints(ints):
    """Formats the given integer list as C++ initializer list.

    Args:
      ints (list of int): numbers to be formatted

    Returns:
      a string containing C++ code representing the given number list
    """
    return ", ".join([cpp_int(i) for i in ints])

def cpp_int(int):
    """Format the given integer as C++ decimal literal.

    Args:
      int: an integer

    Returns:
      a string containing a C++ decimal literal
    """

    # See https://stackoverflow.com/a/1819236 for the guarantees on the C++ int
    # type range.
    if int < -32767 or int > 32767:
        fail("integer {} out of range".format(int))
    return str(int)

# Shared C++ compilation options.
COPTS = [
    "-fno-exceptions",
    "-Werror",
    "-Wall",
    "-Wextra",
    "-Wconversion",
    "-Wsign-conversion",
    "-pedantic-errors",
    # GCC appears to treat some moves as redundant that are in fact necessary.
    "-Wno-redundant-move",
    "-D_GNU_SOURCE",
]
<|endoftext|>


<commit_before>"""
Django settings for sampleproj project.

Generated by 'django-admin startproject' using Django 1.8.19.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.staticfiles',
    # 'rest_framework',
    # 'mdot_rest',
    'django_user_agents',
    'mdot',
    'compressor',
    'uw_saml'
)

MIDDLEWARE = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django_user_agents.middleware.UserAgentMiddleware',
    'htmlmin.middleware.HtmlMinifyMiddleware',
    'htmlmin.middleware.MarkRequestMiddleware',
    'django.contrib.auth.middleware.PersistentRemoteUserMiddleware',
)

AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.RemoteUserBackend',
    # use for mock saml
    'django.contrib.auth.backends.ModelBackend',
]

from django.core.urlresolvers import reverse_lazy
LOGIN_URL  = reverse_lazy('saml_login')

ROOT_URLCONF = 'sampleproj.urls'

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/
STATIC_URL = '/static/'

STATICFILES_FINDERS = (
    "django.contrib.staticfiles.finders.FileSystemFinder",
    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
    'compressor.finders.CompressorFinder',
)

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
        ],
        'APP_DIRS': True,
    },
]

# django_compressor

COMPRESS_ROOT = "compress_files"
<commit_msg>added uw saml dict to settings<commit_after>"""
Django settings for sampleproj project.

Generated by 'django-admin startproject' using Django 1.8.19.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.staticfiles',
    # 'rest_framework',
    # 'mdot_rest',
    'django_user_agents',
    'mdot',
    'compressor',
    'uw_saml'
)

MIDDLEWARE = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django_user_agents.middleware.UserAgentMiddleware',
    'htmlmin.middleware.HtmlMinifyMiddleware',
    'htmlmin.middleware.MarkRequestMiddleware',
    'django.contrib.auth.middleware.PersistentRemoteUserMiddleware',
)

AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.RemoteUserBackend',
    # use for mock saml
    'django.contrib.auth.backends.ModelBackend',
]

from django.core.urlresolvers import reverse_lazy
LOGIN_URL  = reverse_lazy('saml_login')

# uw saml dict
UW_SAML = {
    # SET TO TRUE FOR PRODUCTION
    'strict': False,
    'debug': True,
    'sp': {
        'entityId': 'https://example.uw.edu/saml2',
        'assertionConsumerService': {
            'url': 'https://example.uw.edu/saml/sso',
            'binding': 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST'
        },
        'singleLogoutService': {
            'url': 'https://example.uw.edu/saml/logout',
            'binding': 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect'
        },
        'NameIDFormat': 'urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified',
        'x509cert': '',
        # for encrypted saml assertions uncomment and add the private key
        # 'privateKey': '',
    },
    'idp': {
        'entityId': 'urn:mace:incommon:washington.edu',
        'singleSignOnService': {
            'url': 'https://idp.u.washington.edu/idp/profile/SAML2/Redirect/SSO',
            'binding': 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect'
        },
        'singleLogoutService': {
            'url': 'https://idp.u.washington.edu/idp/logout',
            'binding': 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect'
        },
        'x509cert': '',
    },
    'security': {
        # for encrypted saml assertions
        # 'wantAssertionsEncrypted': True,
        # for 2FA uncomment this line
        # 'requestedAuthnContext':  ['urn:oasis:names:tc:SAML:2.0:ac:classes:TimeSyncToken']
    }
}

ROOT_URLCONF = 'sampleproj.urls'

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/
STATIC_URL = '/static/'

STATICFILES_FINDERS = (
    "django.contrib.staticfiles.finders.FileSystemFinder",
    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
    'compressor.finders.CompressorFinder',
)

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
        ],
        'APP_DIRS': True,
    },
]

# django_compressor

COMPRESS_ROOT = "compress_files"
<|endoftext|>


<commit_before>from django.conf import settings
from persistent_messages.models import Message
from persistent_messages.constants import PERSISTENT_MESSAGE_LEVELS
from django.contrib.auth.models import AnonymousUser
from django.contrib.messages.storage.fallback import FallbackStorage
from django.db.models import Q
import datetime

def get_user(request):
    if hasattr(request, 'user') and request.user.__class__ != AnonymousUser:
        return request.user
    else:
        return AnonymousUser()

"""
Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
Hence, they need to be stored when being added. You can disable this, but then you'll only be able to
close a message when it is displayed for the second time.
"""
STORE_WHEN_ADDING = getattr(settings, 'MESSAGES_STORE_WHEN_ADDING', True)

#@TODO USE FALLBACK
class PersistentMessageStorage(FallbackStorage):
    def __init__(self, *args, **kwargs):
        super(PersistentMessageStorage, self).__init__(*args, **kwargs)
        self.non_persistent_messages = []
        self.is_anonymous = not get_user(self.request).is_authenticated()

    def _message_queryset(self, exclude_unread=True):
        qs = Message.objects.filter(user=get_user(self.request)).filter(
            Q(expires=None) | Q(expires__gt=datetime.datetime.now()))
        if exclude_unread:
            qs = qs.exclude(read=True)
        return qs

    def _get(self, *args, **kwargs):
        """
        Retrieves a list of stored messages. Returns a tuple of the messages
        and a flag indicating whether or not all the messages originally
        intended to be stored in this storage were, in fact, stored and
        retrieved; e.g., ``(messages, all_retrieved)``.
        """
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._get(*args, **kwargs)
        messages = []
        for message in self._message_queryset():
            if not message.is_persistent():
                self.non_persistent_messages.append(message)
            messages.append(message)
        return (messages, True)

    def get_persistent(self):
        return self._message_queryset(exclude_unread=False).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def get_persistent_unread(self):
        return self._message_queryset(exclude_unread=True).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def count_unread(self):
        return self._message_queryset(exclude_unread=True).count()

    def count_persistent_unread(self):
        return self.get_persistent_unread().count()

    def _delete_non_persistent(self):
        for message in self.non_persistent_messages:
            message.delete()
        self.non_persistent_messages = []

    def __iter__(self):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).__iter__()
        self.used = True
        messages = []
        messages.extend(self._loaded_messages)
        if self._queued_messages:
            messages.extend(self._queued_messages)
        return iter(messages)

    def _prepare_messages(self, messages):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._prepare_messages(messages)
        """
        Obsolete method since model takes care of this.
        """
        pass

    def _store(self, messages, response, *args, **kwargs):
        """
        Stores a list of messages, returning a list of any messages which could
        not be stored.

        If STORE_WHEN_ADDING is True, messages are already stored at this time and won't be
        saved again.
        """

        messages = [message for message in messages if not message.is_sticky()]

        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._store(messages, response, *args, **kwargs)
        for message in messages:
            if not self.used or message.is_persistent():
                if not message.pk:
                    message.save()
        return []

    def update(self, response):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).update(response)
        """
        Deletes all non-persistent, read messages. Saves all unstored messages.
        """
        if self.used:
            self._delete_non_persistent()
        return super(PersistentMessageStorage, self).update(response)

    def add(self, level, message, extra_tags='', subject='', user=None, from_user=None, expires=None,
            close_timeout=None):
        """
        Queues a message to be stored.

        The message is only queued if it contained something and its level is
        not less than the recording level (``self.level``).
        """
        to_user = user or get_user(self.request)
        if not to_user.is_authenticated():
            if Message(level=level).is_persistent():
                raise NotImplementedError('Persistent message levels cannot be used for anonymous users.')
            else:
                return super(PersistentMessageStorage, self).add(level, message, extra_tags)
        if not message:
            return
            # Check that the message level is not less than the recording level.
        level = int(level)
        if level < self.level:
            return
            # Add the message.
        message = Message(user=to_user, level=level, message=message, extra_tags=extra_tags, subject=subject,
            from_user=from_user, expires=expires, close_timeout=close_timeout)
        # Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
        # Hence, save it now instead of adding it to queue:
        if STORE_WHEN_ADDING:
            message.save()
        else:
            self.added_new = True
            self._queued_messages.append(message)
<commit_msg>Fix sticky messages always was saved<commit_after>from django.conf import settings
from persistent_messages.models import Message
from persistent_messages.constants import PERSISTENT_MESSAGE_LEVELS
from django.contrib.auth.models import AnonymousUser
from django.contrib.messages.storage.fallback import FallbackStorage
from django.db.models import Q
import datetime

def get_user(request):
    if hasattr(request, 'user') and request.user.__class__ != AnonymousUser:
        return request.user
    else:
        return AnonymousUser()

"""
Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
Hence, they need to be stored when being added. You can disable this, but then you'll only be able to
close a message when it is displayed for the second time.
"""
STORE_WHEN_ADDING = getattr(settings, 'MESSAGES_STORE_WHEN_ADDING', True)

#@TODO USE FALLBACK
class PersistentMessageStorage(FallbackStorage):
    def __init__(self, *args, **kwargs):
        super(PersistentMessageStorage, self).__init__(*args, **kwargs)
        self.non_persistent_messages = []
        self.is_anonymous = not get_user(self.request).is_authenticated()

    def _message_queryset(self, exclude_unread=True):
        qs = Message.objects.filter(user=get_user(self.request)).filter(
            Q(expires=None) | Q(expires__gt=datetime.datetime.now()))
        if exclude_unread:
            qs = qs.exclude(read=True)
        return qs

    def _get(self, *args, **kwargs):
        """
        Retrieves a list of stored messages. Returns a tuple of the messages
        and a flag indicating whether or not all the messages originally
        intended to be stored in this storage were, in fact, stored and
        retrieved; e.g., ``(messages, all_retrieved)``.
        """
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._get(*args, **kwargs)
        messages = []
        for message in self._message_queryset():
            if not message.is_persistent():
                self.non_persistent_messages.append(message)
            messages.append(message)
        return (messages, True)

    def get_persistent(self):
        return self._message_queryset(exclude_unread=False).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def get_persistent_unread(self):
        return self._message_queryset(exclude_unread=True).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def count_unread(self):
        return self._message_queryset(exclude_unread=True).count()

    def count_persistent_unread(self):
        return self.get_persistent_unread().count()

    def _delete_non_persistent(self):
        for message in self.non_persistent_messages:
            message.delete()
        self.non_persistent_messages = []
        self.non_persistent_messages = []

    def __iter__(self):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).__iter__()
        self.used = True
        messages = []
        messages.extend(self._loaded_messages)
        if self._queued_messages:
            messages.extend(self._queued_messages)
        return iter(messages)

    def _prepare_messages(self, messages):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._prepare_messages(messages)
        """
        Obsolete method since model takes care of this.
        """
        pass

    def _store(self, messages, response, *args, **kwargs):
        """
        Stores a list of messages, returning a list of any messages which could
        not be stored.

        If STORE_WHEN_ADDING is True, messages are already stored at this time and won't be
        saved again.
        """

        messages = [message for message in messages if not message.is_sticky()]

        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._store(messages, response, *args, **kwargs)
        for message in messages:
            if not self.used or message.is_persistent():
                if not message.pk:
                    message.save()
        return []

    def update(self, response):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).update(response)
        """
        Deletes all non-persistent, read messages. Saves all unstored messages.
        """
        if self.used:
            self._delete_non_persistent()
        return super(PersistentMessageStorage, self).update(response)

    def add(self, level, message, extra_tags='', subject='', user=None, from_user=None, expires=None,
            close_timeout=None):
        """
        Queues a message to be stored.

        The message is only queued if it contained something and its level is
        not less than the recording level (``self.level``).
        """
        to_user = user or get_user(self.request)
        if not to_user.is_authenticated() or Message(
            level=level).is_sticky():# si es sticky también dejar que lo procese la session
            if Message(level=level).is_persistent():
                raise NotImplementedError('Persistent message levels cannot be used for anonymous users.')
            else:
                return super(PersistentMessageStorage, self).add(level, message, extra_tags)
        if not message:
            return
            # Check that the message level is not less than the recording level.
        level = int(level)
        if level < self.level:
            return
            # Add the message.
        message = Message(user=to_user, level=level, message=message, extra_tags=extra_tags, subject=subject,
            from_user=from_user, expires=expires, close_timeout=close_timeout)
        # Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
        # Hence, save it now instead of adding it to queue:
        if STORE_WHEN_ADDING:
            message.save()
        else:
            self.added_new = True
            self._queued_messages.append(message)
<|endoftext|>


<commit_before>from pathlib import Path
from ansicolor import Colors, colorize

DEFAULT_FORMAT = '{file_path}:{line_number}:{column_number}: {description} (see {reference})'

FORMAT_COLOR_MAP = {
    'file_path': Colors.Red,
    'file_name': Colors.Red,
    'line_number': Colors.White,
    'column_number': Colors.White,
    'severity': Colors.Red,
    'description': Colors.White,
    'policy_name': Colors.White,
    'reference': Colors.White,
}

class Formatter(object):
    def __init__(self, config_dict):
        if 'cmdargs' in config_dict:
            cmdargs = config_dict['cmdargs']
        else:
            cmdargs = {}

        if 'format' in cmdargs:
            self._format = cmdargs['format']
        else:
            self._format = DEFAULT_FORMAT

        if 'color' in cmdargs:
            self._should_be_colorized = cmdargs['color']
        else:
            self._should_be_colorized = False

    def format_violations(self, violations):
        line_number = lambda violation: violation['position']['line']
        sorted_violations = sorted(violations, key=line_number)

        formatted_lines = map(self.format_violation, sorted_violations)

        return '\n'.join(formatted_lines)

    def format_violation(self, violation):
        if self._should_be_colorized:
            formatter_map = self._get_colorize_formatter_map(violation)
        else:
            formatter_map = self._get_formatter_map(violation)

        formatted_line = self._format.format(**formatter_map)
        return formatted_line

    def _get_colorize_formatter_map(self, violation):
        formatter_map = self._get_formatter_map(violation)
        colorized_formatter_map = {}

        for key, value in formatter_map.items():
            if key in FORMAT_COLOR_MAP:
                Color = FORMAT_COLOR_MAP[key]
                colorized_formatter_map[key] = colorize(str(value), Color())
            else:
                colorized_formatter_map[key] = value

        return colorized_formatter_map

    def _get_formatter_map(self, violation):
        file_path = Path(violation['position']['path'])

        return {
            'file_path': file_path,
            'file_name': file_path.name,
            'line_number': violation['position']['line'],
            'column_number': violation['position']['column'],
            'severity': violation['level'].name.lower(),
            'description': violation['description'],
            'policy_name': violation['name'],
            'reference': violation['reference'],
        }
<commit_msg>Sort by file name<commit_after>from pathlib import Path
from ansicolor import Colors, colorize
from operator import attrgetter

DEFAULT_FORMAT = '{file_path}:{line_number}:{column_number}: {description} (see {reference})'

FORMAT_COLOR_MAP = {
    'file_path': Colors.Red,
    'file_name': Colors.Red,
    'line_number': Colors.White,
    'column_number': Colors.White,
    'severity': Colors.Red,
    'description': Colors.White,
    'policy_name': Colors.White,
    'reference': Colors.White,
}

class Formatter(object):
    def __init__(self, config_dict):
        if 'cmdargs' in config_dict:
            cmdargs = config_dict['cmdargs']
        else:
            cmdargs = {}

        if 'format' in cmdargs:
            self._format = cmdargs['format']
        else:
            self._format = DEFAULT_FORMAT

        if 'color' in cmdargs:
            self._should_be_colorized = cmdargs['color']
        else:
            self._should_be_colorized = False

    def _sort_violations(self, violations):
        return sorted(violations, key=lambda violation: (violation['position']['path'],
                                                         violation['position']['line']))

    def format_violations(self, violations):
        sorted_violations = self._sort_violations(violations)
        formatted_lines = map(self.format_violation, sorted_violations)

        return '\n'.join(formatted_lines)

    def format_violation(self, violation):
        if self._should_be_colorized:
            formatter_map = self._get_colorize_formatter_map(violation)
        else:
            formatter_map = self._get_formatter_map(violation)

        formatted_line = self._format.format(**formatter_map)
        return formatted_line

    def _get_colorize_formatter_map(self, violation):
        formatter_map = self._get_formatter_map(violation)
        colorized_formatter_map = {}

        for key, value in formatter_map.items():
            if key in FORMAT_COLOR_MAP:
                Color = FORMAT_COLOR_MAP[key]
                colorized_formatter_map[key] = colorize(str(value), Color())
            else:
                colorized_formatter_map[key] = value

        return colorized_formatter_map

    def _get_formatter_map(self, violation):
        file_path = Path(violation['position']['path'])

        return {
            'file_path': file_path,
            'file_name': file_path.name,
            'line_number': violation['position']['line'],
            'column_number': violation['position']['column'],
            'severity': violation['level'].name.lower(),
            'description': violation['description'],
            'policy_name': violation['name'],
            'reference': violation['reference'],
        }
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8
"""
    em
    ~~

    em program is a terminal tool that prints FILE(s), or standard input
    to standard output and highlights the expressions that are matched
    the PATTERN.

    The expression will be highlighted iff the terminal is ANSI-compatible.

    :copyright: (c) 2014, Igor Kalnitsky
    :license: BSD, see LICENSE for details
"""
from __future__ import print_function, unicode_literals

import re
import os
import sys
import gettext
import argparse

__version__ = '0.2-dev'

BUF_SIZE = 32768

#: FileNotFoundError has been introduced in Python 3.3, and replaced
#: more abstract IOError.
if not hasattr(__builtins__, 'FileNotFoundError'):
    FileNotFoundError = IOError

def get_ansi_color(color):
    """
    Returns an ANSI-escape code for a given color.
    """
    colors = {
        # attributes
        'reset':        '\033[0m',
        'bold':         '\033[1m',
        'underline':    '\033[4m',

        # colors
        'grey':         '\033[30m',
        'red':          '\033[31m',
        'green':        '\033[32m',
        'yellow':       '\033[33m',
        'blue':         '\033[34m',
        'magenta':      '\033[35m',
        'cyan':         '\033[36m',
        'white':        '\033[37m',
    }
    return colors.get(color.lower())

def show_error_and_exit(status_code, message):
    """
    Print a given error message to standard error and terminate the program.
    """
    print(message, file=sys.stderr)
    sys.exit(status_code)

def emphasize(stream, patterns):
    """
    Emphasize a given ``patterns`` in a given ``stream`` and print the
    result to ``stdout``. The ``patterns`` argument must be represent as
    a dictionay where the key is a pattern, and the value is a format
    lexem (e.g. RED, BLUE, UNDERLINE).
    """
    # compile patterns for quick execution
    def re_compile(k, v):
        return re.compile('(%s)' % k, re.I if v['ignore_case'] else 0)
    patterns = {re_compile(k, v): v for k, v in patterns.items()}

    # don't use stream.read() here as it can possibly block until the block of
    # the requested size is read fully, while os.read() returns immediately
    # after something has been read (actual for python 2.x)
    for buf in iter(lambda: os.read(stream.fileno(), BUF_SIZE), b''):
        # colorize matched patterns with ANSI-escapes
        for line in buf.decode().split('\n'):
            for pattern, style in patterns.items():
                line = pattern.sub(r'{style}\1{reset}'.format(
                    style=get_ansi_color(style['format']),
                    reset=get_ansi_color('reset')
                ), line)
            sys.stdout.write(line + '\n')

def get_arguments():
    """
    Create and parse command line interface.
    """
    parser = argparse.ArgumentParser(
        description=_(
            '%(prog)s program is a terminal tool that prints FILE(s), '
            'or standard input to standard output and highlights the '
            'expressions that are matched the PATTERN.'),
        epilog=_(
            'With no FILE, or when FILE is -, read standard input.'
            '  '
            'The FORMAT option must be one of: BOLD, UNDERLINE, GREY, RED, '
            'GREEN, YELLOW, BLUE, MAGENTA, CYAN or WHITE.')
    )

    arg = parser.add_argument('pattern', metavar='PATTERN')
    arg.help = _('a pattern to highlight')

    arg = parser.add_argument('format', metavar='FORMAT')
    arg.help = _('a color to highlight matched expressions')

    arg = parser.add_argument('files', metavar='FILE', nargs='*', default=['-'])
    arg.help = _('search for pattern in these file(s)')

    arg = parser.add_argument('-i', '--ignore-case', action='store_true')
    arg.help = _('ignore case distinctions')

    arg = parser.add_argument('-s', '--safe-output', action='store_true')
    arg.help = _('print ansi sequences only on tty')

    parser.add_argument(
        '--version', action='version', version='%(prog)s ' + __version__)

    # TODO: add option to highlight entire line
    # TODO: add option to load pattern/format settings from the file

    return parser.parse_args()

def validate_arguments(arguments):
    """
    Validate arguments from the command line.
    """
    if not get_ansi_color(arguments.format):
        show_error_and_exit(2, '{0}: {1}: {2}'.format(
            __name__, arguments.format,
            _('Unknown format value')
        ))

def main():
    PY2 = sys.version_info[0] == 2

    # initialize localization subsystem
    localedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'locale')
    if PY2:
        gettext.install('em', localedir, unicode=True)
    else:
        gettext.install('em', localedir)

    # parse command line arguments and validate it.
    # terminate the program if error was occured.
    arguments = get_arguments()
    validate_arguments(arguments)
    patterns = {
        arguments.pattern: {
            'format': arguments.format,
            'ignore_case': arguments.ignore_case,
        }
    }

    # iterate over the files and highlight the given patterns
    for filename in arguments.files:
        try:
            stream = sys.stdin if filename == '-' else open(filename, 'r')

            if arguments.safe_output and not sys.stdout.isatty():
                sys.stdout.write(stream.read())
            else:
                emphasize(stream, patterns)

        except FileNotFoundError as e:
            show_error_and_exit(2, '{0}: {1}: {2}'.format(
                __name__, e.filename, e.strerror
            ))
        finally:
            if filename != '-':
                stream.close()

if __name__ == '__main__':
    main()
<commit_msg>Fix handling of UTF-8 args and files in Py 2.x<commit_after>#!/usr/bin/env python
# coding: utf-8
"""
    em
    ~~

    em program is a terminal tool that prints FILE(s), or standard input
    to standard output and highlights the expressions that are matched
    the PATTERN.

    The expression will be highlighted iff the terminal is ANSI-compatible.

    :copyright: (c) 2014, Igor Kalnitsky
    :license: BSD, see LICENSE for details
"""
from __future__ import print_function, unicode_literals

import re
import os
import sys
import gettext
import argparse

__version__ = '0.2-dev'

BUF_SIZE = 32768
PY2 = sys.version_info[0] == 2

#: FileNotFoundError has been introduced in Python 3.3, and replaced
#: more abstract IOError.
if not hasattr(__builtins__, 'FileNotFoundError'):
    FileNotFoundError = IOError

def get_ansi_color(color):
    """
    Returns an ANSI-escape code for a given color.
    """
    colors = {
        # attributes
        'reset':        '\033[0m',
        'bold':         '\033[1m',
        'underline':    '\033[4m',

        # colors
        'grey':         '\033[30m',
        'red':          '\033[31m',
        'green':        '\033[32m',
        'yellow':       '\033[33m',
        'blue':         '\033[34m',
        'magenta':      '\033[35m',
        'cyan':         '\033[36m',
        'white':        '\033[37m',
    }
    return colors.get(color.lower())

def show_error_and_exit(status_code, message):
    """
    Print a given error message to standard error and terminate the program.
    """
    print(message, file=sys.stderr)
    sys.exit(status_code)

def emphasize(stream, patterns):
    """
    Emphasize a given ``patterns`` in a given ``stream`` and print the
    result to ``stdout``. The ``patterns`` argument must be represent as
    a dictionay where the key is a pattern, and the value is a format
    lexem (e.g. RED, BLUE, UNDERLINE).
    """
    # compile patterns for quick execution
    def re_compile(k, v):
        return re.compile('(%s)' % k, re.I if v['ignore_case'] else 0)
    patterns = {re_compile(k, v): v for k, v in patterns.items()}

    # don't use stream.read() here as it can possibly block until the block of
    # the requested size is read fully, while os.read() returns immediately
    # after something has been read (actual for python 2.x)
    for buf in iter(lambda: os.read(stream.fileno(), BUF_SIZE), b''):
        # colorize matched patterns with ANSI-escapes
        for line in buf.decode(sys.getfilesystemencoding()).split('\n'):
            for pattern, style in patterns.items():
                line = pattern.sub(r'{style}\1{reset}'.format(
                    style=get_ansi_color(style['format']),
                    reset=get_ansi_color('reset')
                ), line)
            sys.stdout.write(line + '\n')

def get_arguments():
    """
    Create and parse command line interface.
    """
    parser = argparse.ArgumentParser(
        description=_(
            '%(prog)s program is a terminal tool that prints FILE(s), '
            'or standard input to standard output and highlights the '
            'expressions that are matched the PATTERN.'),
        epilog=_(
            'With no FILE, or when FILE is -, read standard input.'
            '  '
            'The FORMAT option must be one of: BOLD, UNDERLINE, GREY, RED, '
            'GREEN, YELLOW, BLUE, MAGENTA, CYAN or WHITE.')
    )

    arg = parser.add_argument(
        'pattern', metavar='PATTERN',
        type=lambda v: v.decode(sys.getfilesystemencoding()) if PY2 else v
    )
    arg.help = _('a pattern to highlight')

    arg = parser.add_argument('format', metavar='FORMAT')
    arg.help = _('a color to highlight matched expressions')

    arg = parser.add_argument('files', metavar='FILE', nargs='*', default=['-'])
    arg.help = _('search for pattern in these file(s)')

    arg = parser.add_argument('-i', '--ignore-case', action='store_true')
    arg.help = _('ignore case distinctions')

    arg = parser.add_argument('-s', '--safe-output', action='store_true')
    arg.help = _('print ansi sequences only on tty')

    parser.add_argument(
        '--version', action='version', version='%(prog)s ' + __version__)

    # TODO: add option to highlight entire line
    # TODO: add option to load pattern/format settings from the file

    return parser.parse_args()

def validate_arguments(arguments):
    """
    Validate arguments from the command line.
    """
    if not get_ansi_color(arguments.format):
        show_error_and_exit(2, '{0}: {1}: {2}'.format(
            __name__, arguments.format,
            _('Unknown format value')
        ))

def main():
    # initialize localization subsystem
    localedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'locale')
    if PY2:
        gettext.install('em', localedir, unicode=True)
    else:
        gettext.install('em', localedir)

    # parse command line arguments and validate it.
    # terminate the program if error was occured.
    arguments = get_arguments()
    validate_arguments(arguments)
    patterns = {
        arguments.pattern: {
            'format': arguments.format,
            'ignore_case': arguments.ignore_case,
        }
    }

    # iterate over the files and highlight the given patterns
    for filename in arguments.files:
        try:
            stream = sys.stdin if filename == '-' else open(filename, 'r')

            if arguments.safe_output and not sys.stdout.isatty():
                sys.stdout.write(stream.read())
            else:
                emphasize(stream, patterns)

        except FileNotFoundError as e:
            show_error_and_exit(2, '{0}: {1}: {2}'.format(
                __name__, e.filename, e.strerror
            ))
        finally:
            if filename != '-':
                stream.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Error handelers of errors originating from the Submission systems.
"""

from __future__ import division

__author__ = "Michiel van Setten"
__copyright__ = " "
__version__ = "0.9"
__maintainer__ = "Michiel van Setten"
__email__ = "mjvansetten@gmail.com"
__date__ = "Mar 24, 2014"

import os
import os.path
import abc
import re

from abc import ABCMeta

from custodian.custodian import Custodian, ErrorHandler

class SchedulerErrorHandler(ErrorHandler):
    """
    Custodian error handler for scheduler related errors
      scheduler takes the schduler, currently: slurm is supported
    """
    def __init__(self, scheduler, err_file='queue.err', out_file='queue.out', run_err_file='run.err', batch_err_file='batch.err'):
        self.scheduler = scheduler
        self.err_file = err_file
        self.out_file = out_file
        self.run_err_file = run_err_file
        self.batch_err_file = batch_err_file
        self.errors = []

    def check(self):
        parser = AbstractErrorParser.factory('slurm', err_file=self.err_file, out_file=self.out_file, run_err_file=self.run_err_file, batch_err_file=self.batch_err_file)
        parser.parse()
        self.errors = parser.errors
        if len(self.errors) == 0:
            return False
        else:
            return True

    def correct(self):
        pass

class AbstractError():
    """
    Error base class
    """
    __metaclass__ = ABCMeta

    def __init__(self, errmsg, meta_data):
        self.errmsg = errmsg
        self.meta_data = meta_data
        self._message = str

    def __str__(self):
        self._message = '%s  %s\n' \
                        '  error message : %s \n' \
                        '  meta data     : %s' % (self.name, self.__doc__, self.errmsg, str(self.meta_data))
        return self._message

    @property
    def name(self):
        return self.__class__.__name__

class SubmitError(AbstractError):
    """
    Errors occurring at submission. The limits on the cluster may have changed.
    """
    def __init__(self, errmsg, meta_data):
        super(SubmitError, self).__init__(errmsg, meta_data)

class FullQueueError(AbstractError):
    """
    Errors occurring at submission. To many jobs in the queue / total cpus / .. .
    """
    def __init__(self, errmsg, meta_data):
        super(FullQueueError, self).__init__(errmsg, meta_data)

class DiskError(AbstractError):
    """
    Errors involving problems writing to disk.
    """
    def __init__(self, errmsg, meta_data):
        super(DiskError, self).__init__(errmsg, meta_data)

class TimeCancelError(AbstractError):
    """
    Error due to exceeding the time limit for the job.
      .limit will return the limit that was broken, None if it could not be determined.
    """
    def __init__(self, errmsg, meta_data):
        super(TimeCancelError, self).__init__(errmsg, meta_data)
        self.limit = None
        if meta_data is not None and 'broken_limit' in meta_data.keys():
                self.limit = meta_data['broken_limit']

class MemoryCancelError(AbstractError):
    """
    Error due to exceeding the memory limit for the job.
      .limit will return the limit that was broken, None if it could not be determined.
    """
    def __init__(self, errmsg, meta_data):
        super(MemoryCancelError, self).__init__(errmsg, meta_data)
        self.limit = None
        if meta_data is not None and 'broken_limit' in meta_data.keys():
                self.limit = meta_data['broken_limit']

class NodeFailureError(AbstractError):
    """
    Error due the hardware failure of a specific node.
     .node will return the problematic node, None if it could not be determined.
    """
    def __init__(self, errmsg, meta_data):
        super(NodeFailureError, self).__init__(errmsg, meta_data)
        self.node = None
        if len(meta_data) > None and 'node' in meta_data.keys():
                self.node = meta_data['node']

class AbstractErrorParser():
    """
    Abstract class for parsing errors originating from the scheduler system and error that are not reported by the
    program itself, i.e. segmentation faults.

    A concrete implementation of this class for a specific sceduler needs a class attribute ERRORS for containing a
    dictionary specifying error:

    ERRORS = {ErrorClass: {'filespecifier' : {'string': "the string to be lookedfor",
                                              'metafilter': "string specifing the regular exprecion to obtian the meta data"}}

    """
    __metaclass__ = ABCMeta

    def __init__(self, err_file, out_file, run_err_file, batch_err_file):
        self.files = {'err': err_file, 'out': out_file, 'run_err': run_err_file, 'batch_err': batch_err_file}
        self.errors = []
        return

    def factory(scheduler, *kwargs):
        if scheduler == "slurm":
            return SlurmErrorParser(*kwargs)
        elif scheduler == "pbs":
            return PBSErrorParse(*kwargs)
        assert 0, "undefined scheduler " + scheduler
    factory = staticmethod(factory)

    @staticmethod
    def extract_metadata(message, metafilter):
        meta_dict = {}
        for key in metafilter.keys():
            print key
            print metafilter[key]
            print metafilter[key][0]
            print " !!!!!!!  next line"
            print message
            match = re.match(metafilter[key][0], message)
            if match is not None:
                print ' !!! match found'
                print match
                meta_dict.update({key: re.match(metafilter[key][0], message).group(metafilter[key][1])})
        return meta_dict

    def parse_single(self, errmsg):
        """
        Parse the provided files for the corresponding strings.
        """
        found = False
        message = None
        metadata = None
        for k in errmsg.keys():
            print 'parsing ', self.files[k], ' for ', errmsg[k]['string']
            try:
                with open(self.files[k], mode='r') as f:
                    lines = f.read().split('\n')
                for line in lines:
                    if errmsg[k]['string'] in line:
                        found = True
                        message = line.strip()
                        metadata = self.extract_metadata(message, errmsg[k]['metafilter'])
            except (IOError, OSError):
                print self.files[k], not found

        return found, message, metadata

    def parse(self):
        """
        Parse for the occurce of all errors defined in ERRORS
        """
        for my_error in self.ERRORS:
            result = self.parse_single(self.ERRORS[my_error])
            if result[0]:
                self.errors.append(my_error(result[1], result[2]))

class SlurmErrorParser(AbstractErrorParser):
    """
    Implementation for the Slurm scheduler
    """
    def __init__(self, err_file, out_file, run_err_file, batch_err_file):
        super(SlurmErrorParser, self).__init__(err_file, out_file, run_err_file, batch_err_file)

    ERRORS = {SubmitError: {'batch_err': {'string': "sbatch: error: Batch job submission failed:",
                                          'metafilter': ""}},
              FullQueueError: {'batch_err': {'string': "sbatch: error: Batch job submission failed: Job violates accounting/QOS policy",
                                             'metafilter': ""}},
              MemoryCancelError: {'err': {'string': "Exceeded job memory limit",
                                          'metafilter': ""}},
              NodeFailureError: {'run_err': {'string': "can't open /dev/ipath, network down",
                                      'metafilter': {'node': [r"node(\d+)\.(\d+)can't open (\S*), network down \(err=26\)", 1]}}},
              AbstractError: {'out': {'string': "a string to be found",
                                      'metafilter': ""}},
              }

class PBSErrorParse(AbstractErrorParser):
    """
    Implementation for the PBS scheduler
    """

if __name__ == "__main__":
    my_parser = SlurmErrorParser(err_file='queue.err', out_file='queue.out', run_err_file='run.err', batch_err_file='batch.err')
    my_parser.parse()
    print 'parser.errors', my_parser.errors
    for error in my_parser.errors:
        print error
        print error.node<commit_msg>scheduler errors<commit_after>"""
Error handelers of errors originating from the Submission systems.
"""

from __future__ import division

__author__ = "Michiel van Setten"
__copyright__ = " "
__version__ = "0.9"
__maintainer__ = "Michiel van Setten"
__email__ = "mjvansetten@gmail.com"
__date__ = "Mar 24, 2014"

import os
import os.path
import abc
import re

from abc import ABCMeta

from custodian.custodian import Custodian, ErrorHandler

class SchedulerErrorHandler(ErrorHandler):
    """
    Custodian error handler for scheduler related errors
      scheduler takes the schduler, currently: slurm is supported
    """
    def __init__(self, scheduler, err_file='queue.err', out_file='queue.out', run_err_file='run.err', batch_err_file='batch.err'):
        self.scheduler = scheduler
        self.err_file = err_file
        self.out_file = out_file
        self.run_err_file = run_err_file
        self.batch_err_file = batch_err_file
        self.errors = []

    def check(self):
        parser = AbstractErrorParser.factory('slurm', err_file=self.err_file, out_file=self.out_file, run_err_file=self.run_err_file, batch_err_file=self.batch_err_file)
        parser.parse()
        self.errors = parser.errors
        if len(self.errors) == 0:
            return False
        else:
            return True

    def correct(self):
        pass

class AbstractError():
    """
    Error base class
    """
    __metaclass__ = ABCMeta

    def __init__(self, errmsg, meta_data):
        self.errmsg = errmsg
        self.meta_data = meta_data
        self._message = str

    def __str__(self):
        self._message = '%s  %s\n' \
                        '  error message : %s \n' \
                        '  meta data     : %s' % (self.name, self.__doc__, self.errmsg, str(self.meta_data))
        return self._message

    @property
    def name(self):
        return self.__class__.__name__

class SubmitError(AbstractError):
    """
    Errors occurring at submission. The limits on the cluster may have changed.
    """
    def __init__(self, errmsg, meta_data):
        super(SubmitError, self).__init__(errmsg, meta_data)

class FullQueueError(AbstractError):
    """
    Errors occurring at submission. To many jobs in the queue / total cpus / .. .
    """
    def __init__(self, errmsg, meta_data):
        super(FullQueueError, self).__init__(errmsg, meta_data)

class DiskError(AbstractError):
    """
    Errors involving problems writing to disk.
    """
    def __init__(self, errmsg, meta_data):
        super(DiskError, self).__init__(errmsg, meta_data)

class TimeCancelError(AbstractError):
    """
    Error due to exceeding the time limit for the job.
      .limit will return the limit that was broken, None if it could not be determined.
    """
    def __init__(self, errmsg, meta_data):
        super(TimeCancelError, self).__init__(errmsg, meta_data)
        self.limit = None
        if meta_data is not None and 'broken_limit' in meta_data.keys():
                self.limit = meta_data['broken_limit']

class MemoryCancelError(AbstractError):
    """
    Error due to exceeding the memory limit for the job.
      .limit will return the limit that was broken, None if it could not be determined.
    """
    def __init__(self, errmsg, meta_data):
        super(MemoryCancelError, self).__init__(errmsg, meta_data)
        self.limit = None
        if meta_data is not None and 'broken_limit' in meta_data.keys():
                self.limit = meta_data['broken_limit']

class NodeFailureError(AbstractError):
    """
    Error due the hardware failure of a specific node.
     .node will return the problematic node, None if it could not be determined.
    """
    def __init__(self, errmsg, meta_data):
        super(NodeFailureError, self).__init__(errmsg, meta_data)
        self.node = None
        if len(meta_data) > None and 'node' in meta_data.keys():
                self.node = meta_data['node']

class AbstractErrorParser():
    """
    Abstract class for parsing errors originating from the scheduler system and error that are not reported by the
    program itself, i.e. segmentation faults.

    A concrete implementation of this class for a specific sceduler needs a class attribute ERRORS for containing a
    dictionary specifying error:

    ERRORS = {ErrorClass: {'filespecifier' : {'string': "the string to be lookedfor",
                                              'metafilter': "string specifing the regular exprecion to obtian the meta data"}}

    """
    __metaclass__ = ABCMeta

    def __init__(self, err_file, out_file, run_err_file, batch_err_file):
        self.files = {'err': err_file, 'out': out_file, 'run_err': run_err_file, 'batch_err': batch_err_file}
        self.errors = []
        return

    def factory(scheduler, *kwargs):
        if scheduler == "slurm":
            return SlurmErrorParser(*kwargs)
        elif scheduler == "pbs":
            return PBSErrorParse(*kwargs)
        assert 0, "undefined scheduler " + scheduler
    factory = staticmethod(factory)

    @staticmethod
    def extract_metadata(message, metafilter):
        meta_dict = {}
        for key in metafilter.keys():
            match = re.match(metafilter[key][0], message)
            if match is not None:
                meta_dict.update({key: re.match(metafilter[key][0], message).group(metafilter[key][1])})
        return meta_dict

    def parse_single(self, errmsg):
        """
        Parse the provided files for the corresponding strings.
        """
        found = False
        message = None
        metadata = None
        for k in errmsg.keys():
            print 'parsing ', self.files[k], ' for ', errmsg[k]['string']
            try:
                with open(self.files[k], mode='r') as f:
                    lines = f.read().split('\n')
                for line in lines:
                    if errmsg[k]['string'] in line:
                        found = True
                        message = line.strip()
                        metadata = self.extract_metadata(message, errmsg[k]['metafilter'])
            except (IOError, OSError):
                print self.files[k], not found

        return found, message, metadata

    def parse(self):
        """
        Parse for the occurce of all errors defined in ERRORS
        """
        for my_error in self.ERRORS:
            result = self.parse_single(self.ERRORS[my_error])
            if result[0]:
                self.errors.append(my_error(result[1], result[2]))

class SlurmErrorParser(AbstractErrorParser):
    """
    Implementation for the Slurm scheduler
    """
    def __init__(self, err_file, out_file, run_err_file, batch_err_file):
        super(SlurmErrorParser, self).__init__(err_file, out_file, run_err_file, batch_err_file)

    ERRORS = {SubmitError: {'batch_err': {'string': "sbatch: error: Batch job submission failed:",
                                          'metafilter': ""}},
              FullQueueError: {'batch_err': {'string': "sbatch: error: Batch job submission failed: Job violates accounting/QOS policy",
                                             'metafilter': ""}},
              MemoryCancelError: {'err': {'string': "Exceeded job memory limit",
                                          'metafilter': ""}},
              NodeFailureError: {'run_err': {'string': "can't open /dev/ipath, network down",
                                      'metafilter': {'node': [r"node(\d+)\.(\d+)can't open (\S*), network down \(err=26\)", 1]}}},
              AbstractError: {'out': {'string': "a string to be found",
                                      'metafilter': ""}},
              }

class PBSErrorParse(AbstractErrorParser):
    """
    Implementation for the PBS scheduler
    """

if __name__ == "__main__":
    my_parser = SlurmErrorParser(err_file='queue.err', out_file='queue.out', run_err_file='run.err', batch_err_file='batch.err')
    my_parser.parse()
    print 'parser.errors', my_parser.errors
    for error in my_parser.errors:
        print error
        print error.node<|endoftext|>


<commit_before>import requests

def download_file(url, fileName):
    """A procedural download, to allow for interruptions and large file sizes."""
    url_request = requests.get(url, stream=True)
    with open(fileName,'wb') as fileOpen:
        for chunk in url_request.iter_content(chunk_size=1024):
            if chunk:
                fileOpen.write(chunk)
                fileOpen.flush()

download_file('https://raw.githubusercontent.com/bpampuch/pdfmake/master/build/pdfmake.min.js', '.doccu/static/js/pdfmake.min.js')
download_file('https://raw.githubusercontent.com/bpampuch/pdfmake/master/build/vfs_fonts.js', '.doccu/static/js/vfs_fonts.js')

import os

for dirname, dirnames, filenames in os.walk('.'):
    # print path to all subdirectories first.
    for subdirname in dirnames:
        print(os.path.join(dirname, subdirname))

    # print path to all filenames.
    for filename in filenames:
        print(os.path.join(dirname, filename))

    # Advanced usage:
    # editing the 'dirnames' list will stop os.walk() from recursing into there.
    if '.git' in dirnames:
        # don't go into any .git directories.
        dirnames.remove('.git')
<commit_msg>Troubleshooting env name for Travis<commit_after>import requests

def download_file(url, fileName):
    """A procedural download, to allow for interruptions and large file sizes."""
    url_request = requests.get(url, stream=True)
    with open(fileName,'wb') as fileOpen:
        for chunk in url_request.iter_content(chunk_size=1024):
            if chunk:
                fileOpen.write(chunk)
                fileOpen.flush()

download_file('https://raw.githubusercontent.com/bpampuch/pdfmake/master/build/pdfmake.min.js', '.doccu/static/js/pdfmake.min.js')
download_file('https://raw.githubusercontent.com/bpampuch/pdfmake/master/build/vfs_fonts.js', '.doccu/static/js/vfs_fonts.js')

import os

print(os.getenv('Travis'))

for dirname, dirnames, filenames in os.walk('.'):
    # print path to all subdirectories first.
    for subdirname in dirnames:
        print(os.path.join(dirname, subdirname))

    # print path to all filenames.
    for filename in filenames:
        print(os.path.join(dirname, filename))

    # Advanced usage:
    # editing the 'dirnames' list will stop os.walk() from recursing into there.
    if '.git' in dirnames:
        # don't go into any .git directories.
        dirnames.remove('.git')
<|endoftext|>


<commit_before>from bokeh.io import push_notebook, show, output_notebook
import matplotlib.pyplot as plt
import numpy as np
from bokeh.io import push_notebook, show, output_notebook
from bokeh.plotting import figure

from pextant.lib.geoshapely import GeoPolygon

class TriExpandViz(object):
    def __init__(self, env_model, start_point, end_point, counter_interval=10):
        self.mesh = env_model.dataset.mesh
        self.env_model = env_model
        self.points = GeoPolygon([start_point, end_point])
        self.y, self.x = self.mesh.vertices[:, :2].transpose()
        self.zfaces = np.zeros(env_model.size)
        self.counter = 0
        self.counter_interval = counter_interval

    def draw(self):
        px,py = self.points.to(self.env_model.ROW_COL)
        plt.tripcolor(self.x, self.y, self.mesh.faces, facecolors=self.zfaces, edgecolors='k')
        plt.plot(px, py, 'o')
        plt.axis('equal')
        plt.show()

    def addcount(self):
        self.counter += 1

        if self.counter % self.counter_interval == 0:
            print self.counter

        if self.counter % self.counter_interval == 0:
            self.draw()

    def add(self, state, cost):
        self.zfaces[state] = cost

class ExpandViz(object):
    def __init__(self, env_model, counter_interval=1000):
        self.env_model = env_model
        self.expandedgrid = np.zeros((env_model.y_size, env_model.x_size))
        self.counter = 0
        self.counter_interval = counter_interval
        self.expanded = []

    #cmap = 'viridis'
    def draw(self):
        expanded = np.array(self.expanded).transpose()
        gp_expanded = GeoPolygon(self.env_model.ROW_COL,*expanded)
        upper_left, lower_right = gp_expanded.geoEnvelope()
        upper_row, left_col = upper_left.to(self.env_model.ROW_COL)
        lower_row, right_col = lower_right.to(self.env_model.ROW_COL)
        plt.matshow(self.expandedgrid[upper_row:lower_row+1,left_col:right_col+1])
        print((upper_row, lower_row), (left_col,right_col))
        #print(waypoints.to(env_model.COL_ROW))
        #plt.scatter(*waypoints.to(env_model.COL_ROW), c='r')
        plt.show()

    def drawsolution(self, rawpoints):
        np_rawpoints = GeoPolygon(self.env_model.ROW_COL, *np.array(rawpoints).transpose())
        plt.matshow(self.env_model.dataset)
        #plt.scatter(*waypoints.to(env_model.COL_ROW), c='r')
        plt.scatter(*np_rawpoints.to(self.env_model.COL_ROW), c='b')
        plt.show()

    def addcount(self):
        self.counter += 1

        if self.counter % 1000 == 0:
            print self.counter

        if self.counter % self.counter_interval == 0:
            self.draw()

    def add(self, state, cost):
        self.expanded.append(np.array(state))
        self.expandedgrid[state] = cost

class MeshViz:
    def __init__(self, notebook=False):
        self.notebook = notebook
        if notebook:
            output_notebook()

    def viz(self, mesh, x=None, y=None, palette="Spectral11", viz=True, type="line"):
        dh, dw = mesh.shape
        size = max(dh, dw)
        self.mesh = mesh
        self.dh = dh
        self.dw = dw
        self.p = figure(webgl=True, title="MD2", x_axis_label='x', y_axis_label='y', x_range=[0, size], y_range=[0, size])
        self.p.image(image=[mesh[::-1, :]], x=0, y=0, dw=dw, dh=dh, palette=palette)
        if not x is None:
            if type=="line":
                self.p.line(x, self.dh - np.array(y), line_color="green", line_width=3)
            else:
                self.p.circle(x, self.dh - np.array(y), fill_color="yellow", line_color="black", size=10)
        if self.notebook and viz:
            self.t = show(self.p, notebook_handle = self.notebook)
        else:
            #self.t = show(self.p)
            pass

    def show(self):
        self.t = show(self.p, notebook_handle = self.notebook)

    def vizpoints(self, x, y):
        print(x)
        self.p.circle(y, self.dh - np.array(x), fill_color="yellow", size=10)
        push_notebook(handle=self.t)

class MeshVizM:
    def __init__(self):
        pass

    def viz(self, mesh, x=None, y=None):
        plt.matshow(mesh)
        plt.show()

if __name__ == '__main__':
    MeshViz().viz(np.zeros([4,4]))<commit_msg>visualization tweaks<commit_after>from bokeh.io import push_notebook, show, output_notebook
import matplotlib.pyplot as plt
import numpy as np
from bokeh.io import push_notebook, show, output_notebook
from bokeh.plotting import figure

from pextant.lib.geoshapely import GeoPolygon

class TriExpandViz(object):
    def __init__(self, env_model, start_point, end_point, counter_interval=10):
        self.mesh = env_model.dataset.mesh
        self.env_model = env_model
        #self.points = GeoPolygon([start_point, end_point])
        self.y, self.x = self.mesh.vertices[:, :2].transpose()
        self.zfaces = self.mesh.triangles_center[:, 2]
        self.counter = 0
        self.counter_interval = counter_interval

    def draw(self, x = (), y=()):
        #px,py = self.points.to(self.env_model.ROW_COL)
        plt.tripcolor(self.x, self.y, self.mesh.faces, facecolors=self.zfaces, edgecolors='k')
        if len(x) != 0:
            plt.plot(x, y)
        plt.axis('equal')
        plt.show()

    def addcount(self):
        self.counter += 1

        if self.counter % self.counter_interval == 0:
            print self.counter

        if self.counter % self.counter_interval == 0:
            self.draw()

    def add(self, state, cost):
        self.zfaces[state] = cost

class ExpandViz(object):
    def __init__(self, env_model, counter_interval=1000):
        self.env_model = env_model
        self.expandedgrid = np.zeros((env_model.y_size, env_model.x_size))
        self.counter = 0
        self.counter_interval = counter_interval
        self.expanded = []

    #cmap = 'viridis'
    def draw(self):
        expanded = np.array(self.expanded).transpose()
        gp_expanded = GeoPolygon(self.env_model.ROW_COL,*expanded)
        upper_left, lower_right = gp_expanded.geoEnvelope()
        upper_row, left_col = upper_left.to(self.env_model.ROW_COL)
        lower_row, right_col = lower_right.to(self.env_model.ROW_COL)
        plt.matshow(self.expandedgrid[upper_row:lower_row+1,left_col:right_col+1])
        print((upper_row, lower_row), (left_col,right_col))
        #print(waypoints.to(env_model.COL_ROW))
        #plt.scatter(*waypoints.to(env_model.COL_ROW), c='r')
        plt.show()

    def drawsolution(self, rawpoints):
        np_rawpoints = GeoPolygon(self.env_model.ROW_COL, *np.array(rawpoints).transpose())
        plt.matshow(self.env_model.dataset)
        #plt.scatter(*waypoints.to(env_model.COL_ROW), c='r')
        plt.scatter(*np_rawpoints.to(self.env_model.COL_ROW), c='b')
        plt.show()

    def addcount(self):
        self.counter += 1

        if self.counter % 1000 == 0:
            print self.counter

        if self.counter % self.counter_interval == 0:
            self.draw()

    def add(self, state, cost):
        self.expanded.append(np.array(state))
        self.expandedgrid[state] = cost

class MeshViz:
    def __init__(self, notebook=False):
        self.notebook = notebook
        if notebook:
            output_notebook()

    def viz(self, mesh, x=None, y=None, palette="Spectral11", viz=True, type="line"):
        dh, dw = mesh.shape
        size = max(dh, dw)
        self.mesh = mesh
        self.dh = dh
        self.dw = dw
        self.p = figure(webgl=True, title="MD2", x_axis_label='x', y_axis_label='y', x_range=[0, size], y_range=[0, size])
        self.p.image(image=[mesh[::-1, :]], x=0, y=0, dw=dw, dh=dh, palette=palette)
        if not x is None:
            if type=="line":
                self.p.line(x, self.dh - np.array(y), line_color="green", line_width=3)
            else:
                self.p.circle(x, self.dh - np.array(y), fill_color="yellow", line_color="black", size=10)
        if self.notebook and viz:
            self.t = show(self.p, notebook_handle = self.notebook)
        else:
            #self.t = show(self.p)
            pass

    def show(self):
        self.t = show(self.p, notebook_handle = self.notebook)

    def vizpoints(self, x, y):
        print(x)
        self.p.circle(y, self.dh - np.array(x), fill_color="yellow", size=10)
        push_notebook(handle=self.t)

class MeshVizM:
    def __init__(self):
        pass

    def viz(self, mesh, x=None, y=None):
        plt.matshow(mesh)
        plt.show()

if __name__ == '__main__':
    MeshViz().viz(np.zeros([4,4]))<|endoftext|>


<commit_before>"""
Wextracto is a library for extracting data from web resources.

:copyright: (c) 2012-2014

"""

__version__ = '0.1.0'  # pragma: no cover
<commit_msg>Revision bump<commit_after>"""
Wextracto is a library for extracting data from web resources.

:copyright: (c) 2012-2014

"""

__version__ = '0.1.1'  # pragma: no cover
<|endoftext|>


<commit_before>import warnings
import sys
sys.path.append('/home/MDEJONG1/pythonplayground/pymatgen/pymatgen_repo/') # (If one does not want to change $PYTHONPATH)
import unittest
import pymatgen
from pymatgen.io.vaspio import Poscar
from pymatgen.io.vaspio import Poscar
from pymatgen.io.vaspio import Vasprun
from pymatgen.io.cifio import CifWriter
from pymatgen.io.cifio import CifParser
from pymatgen.core.lattice import Lattice
from pymatgen.core.structure import Structure
from pymatgen.transformations.standard_transformations import *
from pymatgen.core.structure_modifier import StructureEditor
import numpy as np
import random
from pymatgen.phonons.strain import Strain
from pymatgen.phonons.strain import IndependentStrain

for i in range(0, 24):

    path = '/home/MDEJONG1/Re/elasticity/Cij_redo/Re_Ta/M' + str(i) + '/vasprun.xml'
    A = Vasprun(path)
    sigma =  A.ionic_steps[-1]['stress']
    print sigma[0,0], sigma[0,1], sigma[0,2]
    print sigma[1,0], sigma[1,1], sigma[1,2]
    print sigma[2,0], sigma[2,1], sigma[2,2]

<commit_msg>expanded test fot fit_elas further<commit_after>import warnings
import sys
sys.path.append('/home/MDEJONG1/pythonplayground/pymatgen/pymatgen_repo/') # (If one does not want to change $PYTHONPATH)
import unittest
from StringIO import StringIO as sio
import pymatgen
from pymatgen.io.vaspio import Poscar
from pymatgen.io.vaspio import Poscar
from pymatgen.io.vaspio import Vasprun
from pymatgen.io.cifio import CifWriter
from pymatgen.io.cifio import CifParser
from pymatgen.core.lattice import Lattice
from pymatgen.core.structure import Structure
from pymatgen.transformations.standard_transformations import *
from pymatgen.core.structure_modifier import StructureEditor
import numpy as np
import random
from pymatgen.phonons.strain import Strain
from pymatgen.phonons.strain import IndependentStrain
from pymatgen.phonons.genstrain import DeformGeometry

if __name__ == "__main__":

    f = open('stresses', 'r')
    lines = f.readlines()
    f.close()

    lines = [line.strip() for line in lines]
    sig1 = np.matrix(np.loadtxt(sio('\n'.join(lines[0:3]))))

    stress_dict = {}

    struct = CifParser('/home/MDEJONG1/pythonplayground/pymatgen/pymatgen_repo/pymatgen/phonons/aluminum.cif').get_structures()[0]
    D = DeformGeometry(struct)
    D = D.deform()

#   stress_dict: {IndependentStrain:stress matrix}

    strain_array = []
    count = 0

    for c in D:
#        stress_dict[c] = np.matrix(np.loadtxt(sio('\n'.join(lines[3*count:3*count+3]))))
        if c.i==0 and c.j==0:
            strain_array.append(c.strain[c.i, c.j])

    count = 0
    strain_array = sorted(strain_array)

    for n in strain_array:
        for c in D:
            if c.i==0 and c.j==0 and np.abs(c.strain[c.i, c.j] - n) < 0.00001:
                stress_dict[c] = np.matrix(np.loadtxt(sio('\n'.join(lines[3*count:3*count+3]))))
                count+=1

#    print stress_dict

#    print strain_array

    for mode in stress_dict:

#        print stress_dict[mode]
        print mode._dfm

#       if mode.i==0 and mode.j==0:

#           print mode.strain

#        print mode.i, mode.j, mode.strain[mode.i, mode.j]
#        print i
#        print stress_dict[i]

<|endoftext|>


<commit_before>import os
from os import path
from subprocess import Popen
import time
import re

#-----------------------------------------------------------------------------
class TimeoutError(Exception):
  def __init__(self, value):
    self.value = value
  def __str__(self):
    return repr(self.value)

#
# needs from cherrypy:
#   log
#   - config. use:
#      if 'demo.extra_path' in cherrypy.config:
#        set_extra_path(cherrypy.config['demo.extra_path'])
#   TimeoutError
#   get_MATLAB_path()
#

#-------------------------------------------------------------------------------
class RunDemoBase:

  # default timeout to 1 minute
  default_timeout = 60

  #-----------------------------------------------------------------------------
  def __init__(self,base_dir, work_dir):
    self.base_dir    = base_dir
    self.work_dir    = work_dir
    self.pytools_dir = os.path.join(self.base_dir,'../../PythonTools/')
    self.bin_dir     = os.path.join(self.base_dir,'bin/')
    self.scripts_dir = os.path.join(self.base_dir,'scripts/')
    self.python_dir  = os.path.join(self.base_dir,'python/')
    self.dl_dir      = os.path.join(self.base_dir,'dl/')
    self.log_file    = os.path.join(self.base_dir, "build.log"   )
    self.logger      = None
    self.MATLAB_path = None
    self.demo_id     = None

  #-----------------------------------------------------------------------------
  # set the running commands as a dictionnary (usually read from JSON file)
  def set_commands(self,commands):
    self.commands=commands

  #-----------------------------------------------------------------------------
  def set_demo_id(self,id):
    self.demo_id = id

  #-----------------------------------------------------------------------------
  def get_demo_id(self):
    return self.demo_id

  #-----------------------------------------------------------------------------
  # set the algorihtm parameters as a  dictionnary
  def set_algo_params(self,algo_params):
    self.algo_params=algo_params

  #-----------------------------------------------------------------------------
  def set_extra_path(self,p):
    self.extra_path = p

  #-----------------------------------------------------------------------------
  def get_extra_path(self):
    try:
      return self.extra_path
    except:
      return None

  #-----------------------------------------------------------------------------
  def set_MATLAB_path(self,p):
    self.MATH_path = p

  #-----------------------------------------------------------------------------
  def get_MATLAB_path(self):
    return self.MATLAB_path

  #-----------------------------------------------------------------------------
  def set_logger(self,logger):
    self.logger = logger

  #-----------------------------------------------------------------------------
  def log(self,*args, **kwargs):
    if self.logger!=None:
      return self.logger(*args,**kwargs)

  #-----------------------------------------------------------------------------
  def run_algo(self, timeout=False):
    """
    the core algo runner
    could also be called by a batch processor
    """

    # convert parameters to variables
    for k in self.algo_params:
      exec("{0} = {1}".format(k,repr(self.algo_params[k])))

    # if run several commands, is it in series?
    # TODO: deal with timeout for each command

    for cmd in self.commands:
      print "cmd = ", cmd
      args = cmd.split()
      stdout_file = None
      stderr_file = open(self.work_dir+"stderr.txt", 'w')
      # replace variables
      for i,p in enumerate(args):
        print p
        # 1 replace simple variables
        v = re.search(r'\$\w+',p)
        # more complicate, search all variables and replace them
        if v!=None:
          while v!=None:
            p = p[:v.start()]+str(eval(v.group()[1:]))+p[v.end():]
            print "argument number ",i," evaluated to: ",p
            v = re.search(r'\$\w+',p)
          args[i] = p

        # 2 replace more complex expressions, of type ${expression},
        # where expression does not contain '{' or '}' characters
        v = re.search(r'\$\{[^\{\}]*\}',p)
        # more complicate, search all variables and replace them
        if v!=None:
          while v!=None:
            p = p[:v.start()]+str(eval(v.group()[2:-1]))+p[v.end():]
            print "argument number ",i," evaluated to: ",p
            v = re.search(r'\$\{[^\{\}]*\}',p)
          args[i] = p

        # output file >filename finishes also the build command
        if p[0]=='>':
          try:
            print "opening ", self.work_dir+p[1:]
            stdout_file = open(self.work_dir+p[1:], 'w')
          except:
            print "failed"
            stdout_file = None
          del args[i:]
        # redirect errors to stdout
        if p=='2>&1':
          stderr_file = stdout_file

      try:
        self.log("running %s" % repr(args),
                  context='SETUP/%s' % self.get_demo_id(),
                  traceback=False)
        p = self.run_proc(args, stdout=stdout_file, stderr=stderr_file)
        self.wait_proc(p)
      except ValueError as e:
        self.log("Error %s" % e,
                  context='SETUP/%s' % self.get_demo_id(),
                  traceback=False)
      except RuntimeError:
        print "**** run_algo: RuntimeError "
        with open(self.work_dir+"stderr.txt", "r") as errfile:
          errors=errfile.read()
        print errors
        print "***"
        raise RuntimeError(errors)

      # the files should close automatically with their scope ...
      # but we do it anyway just in case
      if stderr_file!=None and stderr_file!=stdout_file:
        stderr_file.close()
      if stdout_file!=None:
        stdout_file.close()

  #-----------------------------------------------------------------------------
  #
  # SUBPROCESS
  #
  def run_proc(self, args, stdin=None, stdout=None, stderr=None, env=None):
    """
    execute a sub-process from the 'tmp' folder
    """

    if env is None:
        env = {}
    # update the environment
    newenv = os.environ.copy()
    # add local environment settings

    newenv.update(env)

    # TODO clear the PATH, hard-rewrite the exec arg0
    # TODO use shell-string execution

    # Add PATH in configuration
    path = self.bin_dir
    # add also the scripts dir
    path = path + ":" + self.scripts_dir
    # add also the python dir
    path = path + ":" + self.python_dir
    # add also the python tools dir
    path = path + ":" + self.pytools_dir

    # Check if there are extra paths
    if self.get_extra_path()!=None:
      path = path + ":" + self.get_extra_path()

    #
    p = self.get_MATLAB_path()
    if p is None:
      self.log("warning: MATLAB path directory %s does not exist" % p,
                context='SETUP/%s' % self.get_demo_id(),
                traceback=False)
    else:
      path = path + ":" + p

    newenv.update({'PATH' : path})

    # check for pipelines in arguments??
    # TODO: allow pipelines: more tricky since we need to measure the
    # processing time ...
    #if args.count('|')>1:
      #self.log("error: only one pipe is allowed in running command line" % p,
                #context='SETUP/%s' % self.get_demo_id(),
                #traceback=False)

    #if args.count('|')==1:
      #pipe_pos = args.index('|')
      #args1 = args[:pipe_pos]
      #args2 = args[pipe_pos+1:]
      #p1 = Popen(args1, stdin=stdin,      stdout=PIPE)
      #p2 = Popen(args2, stdin=p1.stdout,  stdout=PIPE)
      #p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
      #output = p2.communicate()[0]
    #else:

    # run
    return Popen(args,  stdin=stdin, stdout=stdout, stderr=stderr,
                        env=newenv, cwd=self.work_dir)

  #-----------------------------------------------------------------------------
  @staticmethod
  def wait_proc(process, timeout=default_timeout):
    """
    wait for the end of a process execution with an optional timeout
    timeout: False (no timeout) or a numeric value (seconds)
    process: a process or a process list, tuple, ...
    """

    ## If production and timeout is not set, assign a security value

    if isinstance(process, Popen):
      # require a list
      process_list = [process]
    else:
      # duck typing, suppose we have an iterable
      process_list = process

    # http://stackoverflow.com/questions/1191374/
    # wainting for better : http://bugs.python.org/issue5673
    start_time = time.time()
    run_time = 0
    while True:
      if all([p.poll() is not None for p in process_list]):
        # all processes have terminated
        break
      run_time = time.time() - start_time
      if run_time > timeout:
        for p in process_list:
          try:
            p.terminate()
          except OSError:
            # could not stop the process
            # probably self-terminated
            pass
        raise TimeoutError()
      time.sleep(0.1)

    if any([0 != p.returncode for p in process_list]):
      #with open(self.work_dir+"stderr.txt", "r") as errfile:
        #errors=errfile.read()
      #raise RuntimeError(errors)
      #print "**** wait_proc: raising RuntimeError"
      raise RuntimeError
    return

<commit_msg>[lib] added standard paths (/bin,/usr/bin,/usr/local/bin) to run demo commands<commit_after>import os
from os import path
from subprocess import Popen
import time
import re

#-----------------------------------------------------------------------------
class TimeoutError(Exception):
  def __init__(self, value):
    self.value = value
  def __str__(self):
    return repr(self.value)

#
# needs from cherrypy:
#   log
#   - config. use:
#      if 'demo.extra_path' in cherrypy.config:
#        set_extra_path(cherrypy.config['demo.extra_path'])
#   TimeoutError
#   get_MATLAB_path()
#

#-------------------------------------------------------------------------------
class RunDemoBase:

  # default timeout to 1 minute
  default_timeout = 60

  #-----------------------------------------------------------------------------
  def __init__(self,base_dir, work_dir):
    self.base_dir    = base_dir
    self.work_dir    = work_dir
    self.pytools_dir = os.path.join(self.base_dir,'../../PythonTools/')
    self.bin_dir     = os.path.join(self.base_dir,'bin/')
    self.scripts_dir = os.path.join(self.base_dir,'scripts/')
    self.python_dir  = os.path.join(self.base_dir,'python/')
    self.dl_dir      = os.path.join(self.base_dir,'dl/')
    self.log_file    = os.path.join(self.base_dir, "build.log"   )
    self.logger      = None
    self.MATLAB_path = None
    self.demo_id     = None

  #-----------------------------------------------------------------------------
  # set the running commands as a dictionnary (usually read from JSON file)
  def set_commands(self,commands):
    self.commands=commands

  #-----------------------------------------------------------------------------
  def set_demo_id(self,id):
    self.demo_id = id

  #-----------------------------------------------------------------------------
  def get_demo_id(self):
    return self.demo_id

  #-----------------------------------------------------------------------------
  # set the algorihtm parameters as a  dictionnary
  def set_algo_params(self,algo_params):
    self.algo_params=algo_params

  #-----------------------------------------------------------------------------
  def set_extra_path(self,p):
    self.extra_path = p

  #-----------------------------------------------------------------------------
  def get_extra_path(self):
    try:
      return self.extra_path
    except:
      return None

  #-----------------------------------------------------------------------------
  def set_MATLAB_path(self,p):
    self.MATH_path = p

  #-----------------------------------------------------------------------------
  def get_MATLAB_path(self):
    return self.MATLAB_path

  #-----------------------------------------------------------------------------
  def set_logger(self,logger):
    self.logger = logger

  #-----------------------------------------------------------------------------
  def log(self,*args, **kwargs):
    if self.logger!=None:
      return self.logger(*args,**kwargs)

  #-----------------------------------------------------------------------------
  def run_algo(self, timeout=False):
    """
    the core algo runner
    could also be called by a batch processor
    """

    # convert parameters to variables
    for k in self.algo_params:
      exec("{0} = {1}".format(k,repr(self.algo_params[k])))

    # if run several commands, is it in series?
    # TODO: deal with timeout for each command

    for cmd in self.commands:
      print "cmd = ", cmd
      args = cmd.split()
      stdout_file = None
      stderr_file = open(self.work_dir+"stderr.txt", 'w')
      # replace variables
      for i,p in enumerate(args):
        print p
        # 1 replace simple variables
        v = re.search(r'\$\w+',p)
        # more complicate, search all variables and replace them
        if v!=None:
          while v!=None:
            p = p[:v.start()]+str(eval(v.group()[1:]))+p[v.end():]
            print "argument number ",i," evaluated to: ",p
            v = re.search(r'\$\w+',p)
          args[i] = p

        # 2 replace more complex expressions, of type ${expression},
        # where expression does not contain '{' or '}' characters
        v = re.search(r'\$\{[^\{\}]*\}',p)
        # more complicate, search all variables and replace them
        if v!=None:
          while v!=None:
            p = p[:v.start()]+str(eval(v.group()[2:-1]))+p[v.end():]
            print "argument number ",i," evaluated to: ",p
            v = re.search(r'\$\{[^\{\}]*\}',p)
          args[i] = p

        # output file >filename finishes also the build command
        if p[0]=='>':
          try:
            print "opening ", self.work_dir+p[1:]
            stdout_file = open(self.work_dir+p[1:], 'w')
          except:
            print "failed"
            stdout_file = None
          del args[i:]
        # redirect errors to stdout
        if p=='2>&1':
          stderr_file = stdout_file

      try:
        self.log("running %s" % repr(args),
                  context='SETUP/%s' % self.get_demo_id(),
                  traceback=False)
        p = self.run_proc(args, stdout=stdout_file, stderr=stderr_file)
        self.wait_proc(p)
      except ValueError as e:
        self.log("Error %s" % e,
                  context='SETUP/%s' % self.get_demo_id(),
                  traceback=False)
      except RuntimeError:
        print "**** run_algo: RuntimeError "
        with open(self.work_dir+"stderr.txt", "r") as errfile:
          errors=errfile.read()
        print errors
        print "***"
        raise RuntimeError(errors)

      # the files should close automatically with their scope ...
      # but we do it anyway just in case
      if stderr_file!=None and stderr_file!=stdout_file:
        stderr_file.close()
      if stdout_file!=None:
        stdout_file.close()

  #-----------------------------------------------------------------------------
  #
  # SUBPROCESS
  #
  def run_proc(self, args, stdin=None, stdout=None, stderr=None, env=None):
    """
    execute a sub-process from the 'tmp' folder
    """

    if env is None:
        env = {}
    # update the environment
    newenv = os.environ.copy()
    # add local environment settings

    newenv.update(env)

    # TODO clear the PATH, hard-rewrite the exec arg0
    # TODO use shell-string execution

    # Add PATH in configuration
    path = self.bin_dir
    # add also the scripts dir
    path = path + ":" + self.scripts_dir
    # add also the python dir
    path = path + ":" + self.python_dir
    # add also the python tools dir
    path = path + ":" + self.pytools_dir

    path=path+":/bin:/usr/bin:/usr/local/bin"

    # Check if there are extra paths
    if self.get_extra_path()!=None:
      path = path + ":" + self.get_extra_path()

    #
    p = self.get_MATLAB_path()
    if p is None:
      self.log("warning: MATLAB path directory %s does not exist" % p,
                context='SETUP/%s' % self.get_demo_id(),
                traceback=False)
    else:
      path = path + ":" + p

    newenv.update({'PATH' : path})

    # check for pipelines in arguments??
    # TODO: allow pipelines: more tricky since we need to measure the
    # processing time ...
    #if args.count('|')>1:
      #self.log("error: only one pipe is allowed in running command line" % p,
                #context='SETUP/%s' % self.get_demo_id(),
                #traceback=False)

    #if args.count('|')==1:
      #pipe_pos = args.index('|')
      #args1 = args[:pipe_pos]
      #args2 = args[pipe_pos+1:]
      #p1 = Popen(args1, stdin=stdin,      stdout=PIPE)
      #p2 = Popen(args2, stdin=p1.stdout,  stdout=PIPE)
      #p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
      #output = p2.communicate()[0]
    #else:

    # run
    return Popen(args,  stdin=stdin, stdout=stdout, stderr=stderr,
                        env=newenv, cwd=self.work_dir)

  #-----------------------------------------------------------------------------
  @staticmethod
  def wait_proc(process, timeout=default_timeout):
    """
    wait for the end of a process execution with an optional timeout
    timeout: False (no timeout) or a numeric value (seconds)
    process: a process or a process list, tuple, ...
    """

    ## If production and timeout is not set, assign a security value

    if isinstance(process, Popen):
      # require a list
      process_list = [process]
    else:
      # duck typing, suppose we have an iterable
      process_list = process

    # http://stackoverflow.com/questions/1191374/
    # wainting for better : http://bugs.python.org/issue5673
    start_time = time.time()
    run_time = 0
    while True:
      if all([p.poll() is not None for p in process_list]):
        # all processes have terminated
        break
      run_time = time.time() - start_time
      if run_time > timeout:
        for p in process_list:
          try:
            p.terminate()
          except OSError:
            # could not stop the process
            # probably self-terminated
            pass
        raise TimeoutError()
      time.sleep(0.1)

    if any([0 != p.returncode for p in process_list]):
      #with open(self.work_dir+"stderr.txt", "r") as errfile:
        #errors=errfile.read()
      #raise RuntimeError(errors)
      #print "**** wait_proc: raising RuntimeError"
      raise RuntimeError
    return

<|endoftext|>