<commit_before>from __future__ import absolute_import

import collections
import re

from .schema import SolrError

class LuceneQuery(object):
    default_term_re = re.compile(r'^\w+$')
    range_query_templates = {
        "lt": "* TO %s",
        "gt": "%s TO *",
        "ra": "%s TO %s",
    }
    def __init__(self, schema):
        self.schema = schema
        self.terms = collections.defaultdict(set)
        self.phrases = collections.defaultdict(set)
        self.ranges = []

    # Below, we sort all our value_sets - this is for predictability when testing.
    def serialize_term_queries(self):
        s = []
        for name, value_set in sorted(self.terms.items()):
            if name:
                field = self.schema.fields[name]
            else:
                field = self.schema.default_field
            if field.type is unicode:
                serializer = lambda v: self.__lqs_escape(v)
            else:
                serializer = lambda v: field.serialize(v)
            if name:
                s += [u'%s:%s' % (name, serializer(value))
                      for value in sorted(value_set)]
            else:
                s += [serializer(value) for value in sorted(value_set)]
        return ' '.join(s)

    # I'm very much not sure we're doing the right thing here:
    lucene_special_chars = re.compile(r'([+\-&|!\(\){}\[\]\^\"~\*\?:\\])')
    def __lqs_escape(self, s):
        return self.lucene_special_chars.sub(r'\\\1', s)

    def serialize_phrase_queries(self):
        s = []
        for name, value_set in sorted(self.phrases.items()):
            if name:
                field = self.schema.fields[name]
            else:
                field = self.schema.default_field
            if field.type is unicode:
                serializer = self.__phrase_escape
            else:
                serializer = field.serialize
            if name:
                s += [u'%s:"%s"' % (name, serializer(value))
                      for value in sorted(value_set)]
            else:
                s += ['"%s"' % serializer(value) for value in sorted(value_set)]
        return ' '.join(s)

    phrase_special_chars = re.compile(r'"')
    def __phrase_escape(self, s):
        return self.phrase_special_chars.sub(r'\\\1', s)

    def serialize_range_queries(self):
        s = []
        for name, rel, value in sorted(self.ranges):
            if rel in ('lte', 'gte', 'range'):
                left, right = "[", "]"
            else:
                left, right = "{", "}"
            if isinstance(value, tuple):
                value = tuple(self.schema.serialize_value(name, v) for v in value)
            else:
                value = self.schema.serialize_value(name, value)
            range = self.range_query_templates[rel[:2]] % value
            s.append("%(name)s:%(left)s%(range)s%(right)s" % vars())
        return ' '.join(s)

    def __unicode__(self):
        u = [self.serialize_term_queries(),
             self.serialize_phrase_queries(),
             self.serialize_range_queries()]
        return ' '.join(s for s in u if s)

    def __nonzero__(self):
        return bool(self.terms) or bool(self.phrases) or bool(self.ranges)

    def add(self, args, kwargs, terms_or_phrases=None):
        try:
            terms_or_phrases = kwargs.pop("__terms_or_phrases")
        except KeyError:
            terms_or_phrases = None
        for value in args:
            self.add_exact(None, value, terms_or_phrases)
        for k, v in kwargs.items():
            try:
                field_name, rel = k.split("__")
            except ValueError:
                field_name, rel = k, 'eq'
            if field_name not in self.schema.fields:
                raise ValueError("%s is not a valid field name" % k)
            field_type = self.schema.fields[field_name].type
            if rel == 'eq':
                self.add_exact(field_name, v, terms_or_phrases)
            else:
                self.add_range(field_name, rel, v)

    def add_exact(self, field_name, value, term_or_phrase):
        if field_name:
            field_type = self.schema.fields[field_name].type
        else:
            field_type = self.schema.default_field.type
        if field_type is unicode:
            term_or_phrase = term_or_phrase or self.term_or_phrase(value)
        else:
            try:
                value = field_type(value)
            except TypeError:
                raise SolrError("Invalid value %s for field %s"
                                % (value, field_name))
            term_or_phrase = "terms"
        getattr(self, term_or_phrase)[field_name].add(value)

    def add_range(self, field_name, rel, value):
        field_type = self.schema.fields[field_name].type
        if field_type is bool:
            raise ValueError("Cannot do a '%s' query on a bool field" % rel)
        if rel.startswith('range'):
            try:
                assert len(value) == 2
            except (AssertionError, TypeError):
                raise ValueError("'%s__%s' argument must be a length-2 iterable"
                                 % (field_name, rel))
        try:
            if rel.startswith('range'):
                value = tuple(sorted(field_type(v) for v in value))
            else:
                value = field_type(value)
        except (ValueError, TypeError):
                raise ValueError("'%s__%s' arguments of the wrong type"
                                 % (field_name, rel))
        self.ranges.append((field_name, rel, value))

    def term_or_phrase(self, arg, force=None):
        return 'terms' if self.default_term_re.match(arg) else 'phrases'

class SolrSearch(object):
    def __init__(self, interface):
        self.interface = interface
        self.schema = interface.schema
        self.query_obj = LuceneQuery(self.schema)
        self.filter_obj = LuceneQuery(self.schema)
        self.options = {}

    def query_by_term(self, *args, **kwargs):
        return self.query(__terms_or_phrases="terms", *args, **kwargs)

    def query_by_phrase(self, *args, **kwargs):
        return self.query(__terms_or_phrases="phrases", *args, **kwargs)

    def filter_by_term(self, *args, **kwargs):
        return self.filter(__terms_or_phrases="terms", *args, **kwargs)

    def filter_by_phrase(self, *args, **kwargs):
        return self.filter(__terms_or_phrases="phrases", *args, **kwargs)

    def query(self, *args, **kwargs):
        self.query_obj.add(args, kwargs)
        return self

    def filter(self, *args, **kwargs):
        self.filter_obj.add(args, kwargs)
        return self

    def _check_fields(self, fields):
        if isinstance(fields, basestring):
            fields = [fields]
        for field in fields:
            if field not in self.schema.fields:
                raise ValueError("Field '%s' not defined in schema" % field)
        return fields

    def facet_by(self, field, limit=None, mincount=None):
        self._check_fields(field)
        self.options.update({"facet":"true",
                             "facet.field":field})
        if limit:
            self.options["f.%s.facet.limit" % field] = limit
        if mincount:
            self.options["f.%s.facet.mincount" % field] = mincount
        return self

    def highlight(self, fields=None, snippets=None, fragsize=None):
        self.options["hl"] = "true"
        if fields:
            fields = self._check_fields(fields)
            self.options["hl.fl"] = ','.join(fields)
            # what if fields has a comma in it?
        if snippets is not None:
            for field in fields:
                self.options["f.%s.hl.snippets" % field] = snippets
        if fragsize is not None:
            for field in fields:
                self.options["f.%s.hl.fragsize" % field] = fragsize
        return self

    def mlt(self, fields, query_fields=None, **kwargs):
        self.options["mlt"] = "true"
        fields = self._check_fields(fields)
        self.options["mlt.fl"] = ",".join(fields)
        if query_fields is not None:
            qf_arg = []
            for k, v in query_fields.items():
                if k not in fields:
                    raise SolrError("'%s' specified in query_fields but not fields")
                if v is None:
                    qf_arg.append(k)
                else:
                    try:
                        v = float(v)
                    except ValueError:
                        raise SolrError("'%s' has non-numerical boost value")
                    qf_arg.append("%s^%s" % (k, v))
            self.options["mlt.qf"] = " ".join(qf_arg)
        mlt_options = MoreLikeThisOptions(self.schema)
        self.options.update(mlt_options(**kwargs))
        return self

    def paginate(self, start=1, rows=10):
        self.options["start"] = start
        self.options["rows"] = rows
        return self

    def execute(self):
        q = unicode(self.query_obj)
        if q:
            self.options["q"] = q
        fq = unicode(self.filter_obj)
        if fq:
            self.options["fq"] = fq
        return self.interface.search(**self.options)

class MoreLikeThisOptions(object):
    opts = {"count":int,
            "mintf":float,
            "mindf":float,
            "minwl":int,
            "maxwl":int,
            "maxqt":int,
            "maxntp":int,
            "boost":bool,
            }
    def __init__(self, schema):
        self.schema = schema

    def __call__(self, **kwargs):
        options = {}
        for opt_name, opt_value in kwargs.items():
            try:
                opt_type = self.opts[opt_name]
            except IndexError:
                raise SolrError("Invalid MLT option %s" % opt_name)
            try:
                options["mlt.%s" % opt_name] = opt_type(opt_value)
            except (ValueError, TypeError):
                raise SolrError("'mlt.%s' should be an '%s'"%
                                (opt_name, opt_type.__name__))
        return options
<commit_msg>Use field normalizer when adding data to search query<commit_after>from __future__ import absolute_import

import collections
import re

from .schema import SolrError

class LuceneQuery(object):
    default_term_re = re.compile(r'^\w+$')
    range_query_templates = {
        "lt": "* TO %s",
        "gt": "%s TO *",
        "ra": "%s TO %s",
    }
    def __init__(self, schema):
        self.schema = schema
        self.terms = collections.defaultdict(set)
        self.phrases = collections.defaultdict(set)
        self.ranges = []

    # Below, we sort all our value_sets - this is for predictability when testing.
    def serialize_term_queries(self):
        s = []
        for name, value_set in sorted(self.terms.items()):
            if name:
                field = self.schema.fields[name]
            else:
                field = self.schema.default_field
            if field.type is unicode:
                serializer = lambda v: self.__lqs_escape(v)
            else:
                serializer = lambda v: field.serialize(v)
            if name:
                s += [u'%s:%s' % (name, serializer(value))
                      for value in sorted(value_set)]
            else:
                s += [serializer(value) for value in sorted(value_set)]
        return ' '.join(s)

    # I'm very much not sure we're doing the right thing here:
    lucene_special_chars = re.compile(r'([+\-&|!\(\){}\[\]\^\"~\*\?:\\])')
    def __lqs_escape(self, s):
        return self.lucene_special_chars.sub(r'\\\1', s)

    def serialize_phrase_queries(self):
        s = []
        for name, value_set in sorted(self.phrases.items()):
            if name:
                field = self.schema.fields[name]
            else:
                field = self.schema.default_field
            if field.type is unicode:
                serializer = self.__phrase_escape
            else:
                serializer = field.serialize
            if name:
                s += [u'%s:"%s"' % (name, serializer(value))
                      for value in sorted(value_set)]
            else:
                s += ['"%s"' % serializer(value) for value in sorted(value_set)]
        return ' '.join(s)

    phrase_special_chars = re.compile(r'"')
    def __phrase_escape(self, s):
        return self.phrase_special_chars.sub(r'\\\1', s)

    def serialize_range_queries(self):
        s = []
        for name, rel, value in sorted(self.ranges):
            if rel in ('lte', 'gte', 'range'):
                left, right = "[", "]"
            else:
                left, right = "{", "}"
            if isinstance(value, tuple):
                value = tuple(self.schema.serialize_value(name, v) for v in value)
            else:
                value = self.schema.serialize_value(name, value)
            range = self.range_query_templates[rel[:2]] % value
            s.append("%(name)s:%(left)s%(range)s%(right)s" % vars())
        return ' '.join(s)

    def __unicode__(self):
        u = [self.serialize_term_queries(),
             self.serialize_phrase_queries(),
             self.serialize_range_queries()]
        return ' '.join(s for s in u if s)

    def __nonzero__(self):
        return bool(self.terms) or bool(self.phrases) or bool(self.ranges)

    def add(self, args, kwargs, terms_or_phrases=None):
        try:
            terms_or_phrases = kwargs.pop("__terms_or_phrases")
        except KeyError:
            terms_or_phrases = None
        for value in args:
            self.add_exact(None, value, terms_or_phrases)
        for k, v in kwargs.items():
            try:
                field_name, rel = k.split("__")
            except ValueError:
                field_name, rel = k, 'eq'
            if field_name not in self.schema.fields:
                raise ValueError("%s is not a valid field name" % k)
            field_type = self.schema.fields[field_name].type
            if rel == 'eq':
                self.add_exact(field_name, v, terms_or_phrases)
            else:
                self.add_range(field_name, rel, v)

    def add_exact(self, field_name, value, term_or_phrase):
        if field_name:
            field = self.schema.fields[field_name]
        else:
            field = self.schema.default_field
        if field.type is unicode:
            term_or_phrase = term_or_phrase or self.term_or_phrase(value)
        else:
            value = field.normalize(value)
            term_or_phrase = "terms"
        getattr(self, term_or_phrase)[field_name].add(value)

    def add_range(self, field_name, rel, value):
        field_type = self.schema.fields[field_name].type
        if field_type is bool:
            raise ValueError("Cannot do a '%s' query on a bool field" % rel)
        if rel.startswith('range'):
            try:
                assert len(value) == 2
            except (AssertionError, TypeError):
                raise ValueError("'%s__%s' argument must be a length-2 iterable"
                                 % (field_name, rel))
        try:
            if rel.startswith('range'):
                value = tuple(sorted(field_type(v) for v in value))
            else:
                value = field_type(value)
        except (ValueError, TypeError):
                raise ValueError("'%s__%s' arguments of the wrong type"
                                 % (field_name, rel))
        self.ranges.append((field_name, rel, value))

    def term_or_phrase(self, arg, force=None):
        return 'terms' if self.default_term_re.match(arg) else 'phrases'

class SolrSearch(object):
    def __init__(self, interface):
        self.interface = interface
        self.schema = interface.schema
        self.query_obj = LuceneQuery(self.schema)
        self.filter_obj = LuceneQuery(self.schema)
        self.options = {}

    def query_by_term(self, *args, **kwargs):
        return self.query(__terms_or_phrases="terms", *args, **kwargs)

    def query_by_phrase(self, *args, **kwargs):
        return self.query(__terms_or_phrases="phrases", *args, **kwargs)

    def filter_by_term(self, *args, **kwargs):
        return self.filter(__terms_or_phrases="terms", *args, **kwargs)

    def filter_by_phrase(self, *args, **kwargs):
        return self.filter(__terms_or_phrases="phrases", *args, **kwargs)

    def query(self, *args, **kwargs):
        self.query_obj.add(args, kwargs)
        return self

    def filter(self, *args, **kwargs):
        self.filter_obj.add(args, kwargs)
        return self

    def _check_fields(self, fields):
        if isinstance(fields, basestring):
            fields = [fields]
        for field in fields:
            if field not in self.schema.fields:
                raise ValueError("Field '%s' not defined in schema" % field)
        return fields

    def facet_by(self, field, limit=None, mincount=None):
        self._check_fields(field)
        self.options.update({"facet":"true",
                             "facet.field":field})
        if limit:
            self.options["f.%s.facet.limit" % field] = limit
        if mincount:
            self.options["f.%s.facet.mincount" % field] = mincount
        return self

    def highlight(self, fields=None, snippets=None, fragsize=None):
        self.options["hl"] = "true"
        if fields:
            fields = self._check_fields(fields)
            self.options["hl.fl"] = ','.join(fields)
            # what if fields has a comma in it?
        if snippets is not None:
            for field in fields:
                self.options["f.%s.hl.snippets" % field] = snippets
        if fragsize is not None:
            for field in fields:
                self.options["f.%s.hl.fragsize" % field] = fragsize
        return self

    def mlt(self, fields, query_fields=None, **kwargs):
        self.options["mlt"] = "true"
        fields = self._check_fields(fields)
        self.options["mlt.fl"] = ",".join(fields)
        if query_fields is not None:
            qf_arg = []
            for k, v in query_fields.items():
                if k not in fields:
                    raise SolrError("'%s' specified in query_fields but not fields")
                if v is None:
                    qf_arg.append(k)
                else:
                    try:
                        v = float(v)
                    except ValueError:
                        raise SolrError("'%s' has non-numerical boost value")
                    qf_arg.append("%s^%s" % (k, v))
            self.options["mlt.qf"] = " ".join(qf_arg)
        mlt_options = MoreLikeThisOptions(self.schema)
        self.options.update(mlt_options(**kwargs))
        return self

    def paginate(self, start=1, rows=10):
        self.options["start"] = start
        self.options["rows"] = rows
        return self

    def execute(self):
        q = unicode(self.query_obj)
        if q:
            self.options["q"] = q
        fq = unicode(self.filter_obj)
        if fq:
            self.options["fq"] = fq
        return self.interface.search(**self.options)

class MoreLikeThisOptions(object):
    opts = {"count":int,
            "mintf":float,
            "mindf":float,
            "minwl":int,
            "maxwl":int,
            "maxqt":int,
            "maxntp":int,
            "boost":bool,
            }
    def __init__(self, schema):
        self.schema = schema

    def __call__(self, **kwargs):
        options = {}
        for opt_name, opt_value in kwargs.items():
            try:
                opt_type = self.opts[opt_name]
            except IndexError:
                raise SolrError("Invalid MLT option %s" % opt_name)
            try:
                options["mlt.%s" % opt_name] = opt_type(opt_value)
            except (ValueError, TypeError):
                raise SolrError("'mlt.%s' should be an '%s'"%
                                (opt_name, opt_type.__name__))
        return options
<|endoftext|>


<commit_before># Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

import re
import sys
import unicodedata

import six

from savannaclient.openstack.common.gettextutils import _

# Used for looking up extensions of text
# to their 'multiplied' byte amount
BYTE_MULTIPLIERS = {
    '': 1,
    't': 1024 ** 4,
    'g': 1024 ** 3,
    'm': 1024 ** 2,
    'k': 1024,
}
BYTE_REGEX = re.compile(r'(^-?\d+)(\D*)')

TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')

SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")

def int_from_bool_as_string(subject):
    """Interpret a string as a boolean and return either 1 or 0.

    Any string value in:

        ('True', 'true', 'On', 'on', '1')

    is interpreted as a boolean True.

    Useful for JSON-decoded stuff and config file parsing
    """
    return bool_from_string(subject) and 1 or 0

def bool_from_string(subject, strict=False, default=False):
    """Interpret a string as a boolean.

    A case-insensitive match is performed such that strings matching 't',
    'true', 'on', 'y', 'yes', or '1' are considered True and, when
    `strict=False`, anything else returns the value specified by 'default'.

    Useful for JSON-decoded stuff and config file parsing.

    If `strict=True`, unrecognized values, including None, will raise a
    ValueError which is useful when parsing values passed in from an API call.
    Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
    """
    if not isinstance(subject, six.string_types):
        subject = str(subject)

    lowered = subject.strip().lower()

    if lowered in TRUE_STRINGS:
        return True
    elif lowered in FALSE_STRINGS:
        return False
    elif strict:
        acceptable = ', '.join(
            "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
        msg = _("Unrecognized value '%(val)s', acceptable values are:"
                " %(acceptable)s") % {'val': subject,
                                      'acceptable': acceptable}
        raise ValueError(msg)
    else:
        return default

def safe_decode(text, incoming=None, errors='strict'):
    """Decodes incoming str using `incoming` if they're not already unicode.

    :param incoming: Text's current encoding
    :param errors: Errors handling policy. See here for valid
        values http://docs.python.org/2/library/codecs.html
    :returns: text or a unicode `incoming` encoded
                representation of it.
    :raises TypeError: If text is not an instance of str
    """
    if not isinstance(text, six.string_types):
        raise TypeError("%s can't be decoded" % type(text))

    if isinstance(text, six.text_type):
        return text

    if not incoming:
        incoming = (sys.stdin.encoding or
                    sys.getdefaultencoding())

    try:
        return text.decode(incoming, errors)
    except UnicodeDecodeError:
        # Note(flaper87) If we get here, it means that
        # sys.stdin.encoding / sys.getdefaultencoding
        # didn't return a suitable encoding to decode
        # text. This happens mostly when global LANG
        # var is not set correctly and there's no
        # default encoding. In this case, most likely
        # python will use ASCII or ANSI encoders as
        # default encodings but they won't be capable
        # of decoding non-ASCII characters.
        #
        # Also, UTF-8 is being used since it's an ASCII
        # extension.
        return text.decode('utf-8', errors)

def safe_encode(text, incoming=None,
                encoding='utf-8', errors='strict'):
    """Encodes incoming str/unicode using `encoding`.

    If incoming is not specified, text is expected to be encoded with
    current python's default encoding. (`sys.getdefaultencoding`)

    :param incoming: Text's current encoding
    :param encoding: Expected encoding for text (Default UTF-8)
    :param errors: Errors handling policy. See here for valid
        values http://docs.python.org/2/library/codecs.html
    :returns: text or a bytestring `encoding` encoded
                representation of it.
    :raises TypeError: If text is not an instance of str
    """
    if not isinstance(text, six.string_types):
        raise TypeError("%s can't be encoded" % type(text))

    if not incoming:
        incoming = (sys.stdin.encoding or
                    sys.getdefaultencoding())

    if isinstance(text, six.text_type):
        if six.PY3:
            return text.encode(encoding, errors).decode(incoming)
        else:
            return text.encode(encoding, errors)
    elif text and encoding != incoming:
        # Decode text before encoding it with `encoding`
        text = safe_decode(text, incoming, errors)
        if six.PY3:
            return text.encode(encoding, errors).decode(incoming)
        else:
            return text.encode(encoding, errors)

    return text

def to_bytes(text, default=0):
    """Converts a string into an integer of bytes.

    Looks at the last characters of the text to determine
    what conversion is needed to turn the input text into a byte number.
    Supports "B, K(B), M(B), G(B), and T(B)". (case insensitive)

    :param text: String input for bytes size conversion.
    :param default: Default return value when text is blank.

    """
    match = BYTE_REGEX.search(text)
    if match:
        magnitude = int(match.group(1))
        mult_key_org = match.group(2)
        if not mult_key_org:
            return magnitude
    elif text:
        msg = _('Invalid string format: %s') % text
        raise TypeError(msg)
    else:
        return default
    mult_key = mult_key_org.lower().replace('b', '', 1)
    multiplier = BYTE_MULTIPLIERS.get(mult_key)
    if multiplier is None:
        msg = _('Unknown byte multiplier: %s') % mult_key_org
        raise TypeError(msg)
    return magnitude * multiplier

def to_slug(value, incoming=None, errors="strict"):
    """Normalize string.

    Convert to lowercase, remove non-word characters, and convert spaces
    to hyphens.

    Inspired by Django's `slugify` filter.

    :param value: Text to slugify
    :param incoming: Text's current encoding
    :param errors: Errors handling policy. See here for valid
        values http://docs.python.org/2/library/codecs.html
    :returns: slugified unicode representation of `value`
    :raises TypeError: If text is not an instance of str
    """
    value = safe_decode(value, incoming, errors)
    # NOTE(aababilov): no need to use safe_(encode|decode) here:
    # encodings are always "ascii", error handling is always "ignore"
    # and types are always known (first: unicode; second: str)
    value = unicodedata.normalize("NFKD", value).encode(
        "ascii", "ignore").decode("ascii")
    value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
    return SLUGIFY_HYPHENATE_RE.sub("-", value)
<commit_msg>update oslo-incubator strutils module<commit_after># Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

import math
import re
import sys
import unicodedata

import six

from savannaclient.openstack.common.gettextutils import _

UNIT_PREFIX_EXPONENT = {
    'k': 1,
    'K': 1,
    'Ki': 1,
    'M': 2,
    'Mi': 2,
    'G': 3,
    'Gi': 3,
    'T': 4,
    'Ti': 4,
}
UNIT_SYSTEM_INFO = {
    'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
    'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
}

TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')

SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")

def int_from_bool_as_string(subject):
    """Interpret a string as a boolean and return either 1 or 0.

    Any string value in:

        ('True', 'true', 'On', 'on', '1')

    is interpreted as a boolean True.

    Useful for JSON-decoded stuff and config file parsing
    """
    return bool_from_string(subject) and 1 or 0

def bool_from_string(subject, strict=False, default=False):
    """Interpret a string as a boolean.

    A case-insensitive match is performed such that strings matching 't',
    'true', 'on', 'y', 'yes', or '1' are considered True and, when
    `strict=False`, anything else returns the value specified by 'default'.

    Useful for JSON-decoded stuff and config file parsing.

    If `strict=True`, unrecognized values, including None, will raise a
    ValueError which is useful when parsing values passed in from an API call.
    Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
    """
    if not isinstance(subject, six.string_types):
        subject = str(subject)

    lowered = subject.strip().lower()

    if lowered in TRUE_STRINGS:
        return True
    elif lowered in FALSE_STRINGS:
        return False
    elif strict:
        acceptable = ', '.join(
            "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
        msg = _("Unrecognized value '%(val)s', acceptable values are:"
                " %(acceptable)s") % {'val': subject,
                                      'acceptable': acceptable}
        raise ValueError(msg)
    else:
        return default

def safe_decode(text, incoming=None, errors='strict'):
    """Decodes incoming str using `incoming` if they're not already unicode.

    :param incoming: Text's current encoding
    :param errors: Errors handling policy. See here for valid
        values http://docs.python.org/2/library/codecs.html
    :returns: text or a unicode `incoming` encoded
                representation of it.
    :raises TypeError: If text is not an instance of str
    """
    if not isinstance(text, six.string_types):
        raise TypeError("%s can't be decoded" % type(text))

    if isinstance(text, six.text_type):
        return text

    if not incoming:
        incoming = (sys.stdin.encoding or
                    sys.getdefaultencoding())

    try:
        return text.decode(incoming, errors)
    except UnicodeDecodeError:
        # Note(flaper87) If we get here, it means that
        # sys.stdin.encoding / sys.getdefaultencoding
        # didn't return a suitable encoding to decode
        # text. This happens mostly when global LANG
        # var is not set correctly and there's no
        # default encoding. In this case, most likely
        # python will use ASCII or ANSI encoders as
        # default encodings but they won't be capable
        # of decoding non-ASCII characters.
        #
        # Also, UTF-8 is being used since it's an ASCII
        # extension.
        return text.decode('utf-8', errors)

def safe_encode(text, incoming=None,
                encoding='utf-8', errors='strict'):
    """Encodes incoming str/unicode using `encoding`.

    If incoming is not specified, text is expected to be encoded with
    current python's default encoding. (`sys.getdefaultencoding`)

    :param incoming: Text's current encoding
    :param encoding: Expected encoding for text (Default UTF-8)
    :param errors: Errors handling policy. See here for valid
        values http://docs.python.org/2/library/codecs.html
    :returns: text or a bytestring `encoding` encoded
                representation of it.
    :raises TypeError: If text is not an instance of str
    """
    if not isinstance(text, six.string_types):
        raise TypeError("%s can't be encoded" % type(text))

    if not incoming:
        incoming = (sys.stdin.encoding or
                    sys.getdefaultencoding())

    if isinstance(text, six.text_type):
        if six.PY3:
            return text.encode(encoding, errors).decode(incoming)
        else:
            return text.encode(encoding, errors)
    elif text and encoding != incoming:
        # Decode text before encoding it with `encoding`
        text = safe_decode(text, incoming, errors)
        if six.PY3:
            return text.encode(encoding, errors).decode(incoming)
        else:
            return text.encode(encoding, errors)

    return text

def string_to_bytes(text, unit_system='IEC', return_int=False):
    """Converts a string into an float representation of bytes.

    The units supported for IEC ::

        Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
        KB, KiB, MB, MiB, GB, GiB, TB, TiB

    The units supported for SI ::

        kb(it), Mb(it), Gb(it), Tb(it)
        kB, MB, GB, TB

    Note that the SI unit system does not support capital letter 'K'

    :param text: String input for bytes size conversion.
    :param unit_system: Unit system for byte size conversion.
    :param return_int: If True, returns integer representation of text
                       in bytes. (default: decimal)
    :returns: Numerical representation of text in bytes.
    :raises ValueError: If text has an invalid value.

    """
    try:
        base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
    except KeyError:
        msg = _('Invalid unit system: "%s"') % unit_system
        raise ValueError(msg)
    match = reg_ex.match(text)
    if match:
        magnitude = float(match.group(1))
        unit_prefix = match.group(2)
        if match.group(3) in ['b', 'bit']:
            magnitude /= 8
    else:
        msg = _('Invalid string format: %s') % text
        raise ValueError(msg)
    if not unit_prefix:
        res = magnitude
    else:
        res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
    if return_int:
        return int(math.ceil(res))
    return res

def to_slug(value, incoming=None, errors="strict"):
    """Normalize string.

    Convert to lowercase, remove non-word characters, and convert spaces
    to hyphens.

    Inspired by Django's `slugify` filter.

    :param value: Text to slugify
    :param incoming: Text's current encoding
    :param errors: Errors handling policy. See here for valid
        values http://docs.python.org/2/library/codecs.html
    :returns: slugified unicode representation of `value`
    :raises TypeError: If text is not an instance of str
    """
    value = safe_decode(value, incoming, errors)
    # NOTE(aababilov): no need to use safe_(encode|decode) here:
    # encodings are always "ascii", error handling is always "ignore"
    # and types are always known (first: unicode; second: str)
    value = unicodedata.normalize("NFKD", value).encode(
        "ascii", "ignore").decode("ascii")
    value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
    return SLUGIFY_HYPHENATE_RE.sub("-", value)
<|endoftext|>


<commit_before>import re
from datetime import datetime
from datetime import timedelta

__all__ = ['find_time', 'extract_time', 'parse_time', 'is_time', 'day_of_year', 'break_time', 'get_day']

# Mapping of time format codes to regular expressions.
REGEX = {
    '%Y': '(?P<year>\d{4})',
    '%m': '(?P<month>\d{1,2})',
    '%d': '(?P<day>\d{1,2})',
    '%H': '(?P<hour>\d{1,2})',
    '%M': '(?P<minute>\d{1,2})',
    '%S': '(?P<second>\d{1,2})',
    '%f': '(?P<microsecond>\d+)',
    '%b': '(?P<month_str>[a-zA-Z]+)',
}

TIME_FORMAT_LIST = [
    "%Y-%m-%dT%H:%M:%S.%f",    # Example 2007-05-04T21:08:12.999999
    "%Y/%m/%dT%H:%M:%S.%f",    # Example 2007/05/04T21:08:12.999999
    "%Y-%m-%dT%H:%M:%S.%fZ",   # Example 2007-05-04T21:08:12.999Z
    "%Y-%m-%dT%H:%M:%S",       # Example 2007-05-04T21:08:12
    "%Y/%m/%dT%H:%M:%S",       # Example 2007/05/04T21:08:12
    "%Y%m%dT%H%M%S.%f",        # Example 20070504T210812.999999
    "%Y%m%dT%H%M%S",           # Example 20070504T210812
    "%Y/%m/%d %H:%M:%S",       # Example 2007/05/04 21:08:12
    "%Y/%m/%d %H:%M",          # Example 2007/05/04 21:08
    "%Y/%m/%d %H:%M:%S.%f",    # Example 2007/05/04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S.%f",    # Example 2007-05-04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S",       # Example 2007-05-04 21:08:12
    "%Y-%m-%d %H:%M",          # Example 2007-05-04 21:08
    "%Y-%b-%d %H:%M:%S",       # Example 2007-May-04 21:08:12
    "%Y-%b-%d %H:%M",          # Example 2007-May-04 21:08
    "%Y-%b-%d",                # Example 2007-May-04
    "%Y-%m-%d",                # Example 2007-05-04
    "%Y/%m/%d",                # Example 2007/05/04
    "%d-%b-%Y",                # Example 04-May-2007
    "%Y%m%d_%H%M%S",           # Example 20070504_210812
]

def _group_or_none(match, group, fun):
    try:
        ret = match.group(group)
    except IndexError:
        return None
    else:
        return fun(ret)

def _n_or_eq(a, b):
    return a is None or a == b

def _regex_parse_time(inp, format):
    # Parser for finding out the minute value so we can adjust the string
    # from 24:00:00 to 00:00:00 the next day because strptime does not
    # understand the former.
    for key, value in REGEX.iteritems():
        format = format.replace(key, value)
    match = re.match(format, inp)
    if match is None:
        return None, None
    try:
        hour = match.group("hour")
    except IndexError:
        return inp, timedelta(days=0)
    if match.group("hour") == "24":
        if not all(_n_or_eq(_group_or_none(match, g, int), 00)
            for g in ["minute", "second", "microsecond"]
        ):
            raise ValueError
        from_, to = match.span("hour")
        return inp[:from_] + "00" + inp[to:], timedelta(days=1)
    return inp, timedelta(days=0)

def find_time(string, format):
    """ Return iterator of occurences of date formatted with format
    in string. Currently supported format codes: """
    re_format = format
    for key, value in REGEX.iteritems():
        re_format = re_format.replace(key, value)
    matches = re.finditer(re_format, string)
    for match in matches:
        try:
            matchstr = string[slice(*match.span())]
            dt = datetime.strptime(matchstr, format)
        except ValueError:
            continue
        else:
            yield dt

find_time.__doc__ += ', '.join(REGEX.keys())

def _iter_empty(iter):
    try:
        iter.next()
    except StopIteration:
        return True
    return False

def extract_time(string):
    """ Find subset of string that corresponds to a datetime and return
    its value as a a datetime. If more than one or none is found, raise
    ValueError. """
    matched = None
    bestmatch = None
    for time_format in TIME_FORMAT_LIST:
        found = find_time(string, time_format)
        try:
            match = found.next()
        except StopIteration:
            continue
        else:
            if matched is not None:
                if time_format.startswith(matched):
                    # Already matched is a substring of the one just matched.
                    matched = time_format
                    bestmatch = match
                elif not matched.startswith(time_format):
                    # If just matched is substring of time_format, just ignore
                    # just matched.
                    raise ValueError("Ambiguous string")
            else:
                matched = time_format
                bestmatch = match
            if not _iter_empty(found):
                raise ValueError("Ambiguous string")
    if not matched:
        raise ValueError("Time not found")
    return bestmatch

def parse_time(time_string):
    """Given a time string will parse and return a datetime object.
    Similar to the anytim function in IDL.

    Parameters
    ----------
    time_string : string
        Datestring to parse

    Returns
    -------
    out : datetime
        DateTime corresponding to input date string

    Examples
    --------
    >>> sunpy.time.parse_time('2012/08/01')
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')

    .. todo::
        add ability to parse tai (International Atomic Time seconds since
        Jan 1, 1958)
    """
    if isinstance(time_string, datetime):
        return time_string
    elif isinstance(time_string, tuple):
        return datetime(*time_string)
    elif isinstance(time_string, int) or isinstance(time_string, float):
        return datetime(1979, 1, 1) + timedelta(0, time_string)
    else:
        # remove trailing zeros and the final dot to allow any
        # number of zeros. This solves issue #289
        if '.' in time_string:
            time_string = time_string.rstrip("0").rstrip(".")
        for time_format in TIME_FORMAT_LIST:
            try:
                try:
                    ts, time_delta = _regex_parse_time(time_string, time_format)
                except TypeError:
                    break
                if ts is None:
                    continue
                return datetime.strptime(ts, time_format) + time_delta
            except ValueError:
                pass

        raise ValueError("%s is not a valid time string!" % time_string)

def is_time(time_string):
    """Returns true if the input is a valid date/time representation

    Parameters
    ----------
    time_string : string
        Datestring to parse

    Returns
    -------
    out : bool
        True if can be parsed by parse_time

    Examples
    --------
    >>> sunpy.time.parse_time('2012/08/01')
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')

    .. todo:: add ability to parse tai (International Atomic Time seconds since
    Jan 1, 1958)"""
    if None:
        return False
    elif isinstance(time_string, datetime):
        return True

    try:
        parse_time(time_string)
    except ValueError:
        return False
    else:
        return True

def day_of_year(time_string):
    """Returns the (fractional) day of year.

    Parameters
    ----------
    time_string : string
        A parse_time compatible string

    Returns
    -------
    out : float
        The fractional day of year (where Jan 1st is 1).

    Examples
    --------
    >>> sunpy.time.day_of_year('2012/01/01')
    1.00
    >>> sunpy.time.day_of_year('2012/08/01')
    214.00
    >>> sunpy.time.day_of_year('2005-08-04T00:18:02.000Z')
    216.01252314814815

    """
    SECONDS_IN_DAY = 60 * 60 * 24.0
    time = parse_time(time_string)
    time_diff = time - datetime(time.year, 1, 1, 0, 0, 0)
    return time_diff.days + time_diff.seconds / SECONDS_IN_DAY + 1

def break_time(t=None):
    """Given a time returns a string. Useful for naming files."""
    #TODO: should be able to handle a time range
    return parse_time(t).strftime("%Y%m%d_%H%M%S")

def get_day(dt):
    """ Return datetime for the beginning of the day of given datetime. """
    return datetime(dt.year, dt.month, dt.day)

<commit_msg>Added support for IRIS timeline date formats<commit_after>import re
from datetime import datetime
from datetime import timedelta

__all__ = ['find_time', 'extract_time', 'parse_time', 'is_time', 'day_of_year', 'break_time', 'get_day']

# Mapping of time format codes to regular expressions.
REGEX = {
    '%Y': '(?P<year>\d{4})',
    '%j': '(?P<dayofyear>\d{3})',
    '%m': '(?P<month>\d{1,2})',
    '%d': '(?P<day>\d{1,2})',
    '%H': '(?P<hour>\d{1,2})',
    '%M': '(?P<minute>\d{1,2})',
    '%S': '(?P<second>\d{1,2})',
    '%f': '(?P<microsecond>\d+)',
    '%b': '(?P<month_str>[a-zA-Z]+)',
}

TIME_FORMAT_LIST = [
    "%Y-%m-%dT%H:%M:%S.%f",    # Example 2007-05-04T21:08:12.999999
    "%Y/%m/%dT%H:%M:%S.%f",    # Example 2007/05/04T21:08:12.999999
    "%Y-%m-%dT%H:%M:%S.%fZ",   # Example 2007-05-04T21:08:12.999Z
    "%Y-%m-%dT%H:%M:%S",       # Example 2007-05-04T21:08:12
    "%Y/%m/%dT%H:%M:%S",       # Example 2007/05/04T21:08:12
    "%Y%m%dT%H%M%S.%f",        # Example 20070504T210812.999999
    "%Y%m%dT%H%M%S",           # Example 20070504T210812
    "%Y/%m/%d %H:%M:%S",       # Example 2007/05/04 21:08:12
    "%Y/%m/%d %H:%M",          # Example 2007/05/04 21:08
    "%Y/%m/%d %H:%M:%S.%f",    # Example 2007/05/04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S.%f",    # Example 2007-05-04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S",       # Example 2007-05-04 21:08:12
    "%Y-%m-%d %H:%M",          # Example 2007-05-04 21:08
    "%Y-%b-%d %H:%M:%S",       # Example 2007-May-04 21:08:12
    "%Y-%b-%d %H:%M",          # Example 2007-May-04 21:08
    "%Y-%b-%d",                # Example 2007-May-04
    "%Y-%m-%d",                # Example 2007-05-04
    "%Y/%m/%d",                # Example 2007/05/04
    "%d-%b-%Y",                # Example 04-May-2007
    "%Y%m%d_%H%M%S",           # Example 20070504_210812
    "%Y:%j:%H:%M:%S",          # Example 2012:124:21:08:12
    "%Y:%j:%H:%M:%S.%f",       # Example 2012:124:21:08:12.999999
]

def _group_or_none(match, group, fun):
    try:
        ret = match.group(group)
    except IndexError:
        return None
    else:
        return fun(ret)

def _n_or_eq(a, b):
    return a is None or a == b

def _regex_parse_time(inp, format):
    # Parser for finding out the minute value so we can adjust the string
    # from 24:00:00 to 00:00:00 the next day because strptime does not
    # understand the former.
    for key, value in REGEX.iteritems():
        format = format.replace(key, value)
    match = re.match(format, inp)
    if match is None:
        return None, None
    try:
        hour = match.group("hour")
    except IndexError:
        return inp, timedelta(days=0)
    if match.group("hour") == "24":
        if not all(_n_or_eq(_group_or_none(match, g, int), 00)
            for g in ["minute", "second", "microsecond"]
        ):
            raise ValueError
        from_, to = match.span("hour")
        return inp[:from_] + "00" + inp[to:], timedelta(days=1)
    return inp, timedelta(days=0)

def find_time(string, format):
    """ Return iterator of occurences of date formatted with format
    in string. Currently supported format codes: """
    re_format = format
    for key, value in REGEX.iteritems():
        re_format = re_format.replace(key, value)
    matches = re.finditer(re_format, string)
    for match in matches:
        try:
            matchstr = string[slice(*match.span())]
            dt = datetime.strptime(matchstr, format)
        except ValueError:
            continue
        else:
            yield dt

find_time.__doc__ += ', '.join(REGEX.keys())

def _iter_empty(iter):
    try:
        iter.next()
    except StopIteration:
        return True
    return False

def extract_time(string):
    """ Find subset of string that corresponds to a datetime and return
    its value as a a datetime. If more than one or none is found, raise
    ValueError. """
    matched = None
    bestmatch = None
    for time_format in TIME_FORMAT_LIST:
        found = find_time(string, time_format)
        try:
            match = found.next()
        except StopIteration:
            continue
        else:
            if matched is not None:
                if time_format.startswith(matched):
                    # Already matched is a substring of the one just matched.
                    matched = time_format
                    bestmatch = match
                elif not matched.startswith(time_format):
                    # If just matched is substring of time_format, just ignore
                    # just matched.
                    raise ValueError("Ambiguous string")
            else:
                matched = time_format
                bestmatch = match
            if not _iter_empty(found):
                raise ValueError("Ambiguous string")
    if not matched:
        raise ValueError("Time not found")
    return bestmatch

def parse_time(time_string):
    """Given a time string will parse and return a datetime object.
    Similar to the anytim function in IDL.

    Parameters
    ----------
    time_string : string
        Datestring to parse

    Returns
    -------
    out : datetime
        DateTime corresponding to input date string

    Examples
    --------
    >>> sunpy.time.parse_time('2012/08/01')
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')

    .. todo::
        add ability to parse tai (International Atomic Time seconds since
        Jan 1, 1958)
    """
    if isinstance(time_string, datetime):
        return time_string
    elif isinstance(time_string, tuple):
        return datetime(*time_string)
    elif isinstance(time_string, int) or isinstance(time_string, float):
        return datetime(1979, 1, 1) + timedelta(0, time_string)
    else:
        # remove trailing zeros and the final dot to allow any
        # number of zeros. This solves issue #289
        if '.' in time_string:
            time_string = time_string.rstrip("0").rstrip(".")
        for time_format in TIME_FORMAT_LIST:
            try:
                try:
                    ts, time_delta = _regex_parse_time(time_string, time_format)
                except TypeError:
                    break
                if ts is None:
                    continue
                return datetime.strptime(ts, time_format) + time_delta
            except ValueError:
                pass

        raise ValueError("%s is not a valid time string!" % time_string)

def is_time(time_string):
    """Returns true if the input is a valid date/time representation

    Parameters
    ----------
    time_string : string
        Datestring to parse

    Returns
    -------
    out : bool
        True if can be parsed by parse_time

    Examples
    --------
    >>> sunpy.time.parse_time('2012/08/01')
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')

    .. todo:: add ability to parse tai (International Atomic Time seconds since
    Jan 1, 1958)"""
    if None:
        return False
    elif isinstance(time_string, datetime):
        return True

    try:
        parse_time(time_string)
    except ValueError:
        return False
    else:
        return True

def day_of_year(time_string):
    """Returns the (fractional) day of year.

    Parameters
    ----------
    time_string : string
        A parse_time compatible string

    Returns
    -------
    out : float
        The fractional day of year (where Jan 1st is 1).

    Examples
    --------
    >>> sunpy.time.day_of_year('2012/01/01')
    1.00
    >>> sunpy.time.day_of_year('2012/08/01')
    214.00
    >>> sunpy.time.day_of_year('2005-08-04T00:18:02.000Z')
    216.01252314814815

    """
    SECONDS_IN_DAY = 60 * 60 * 24.0
    time = parse_time(time_string)
    time_diff = time - datetime(time.year, 1, 1, 0, 0, 0)
    return time_diff.days + time_diff.seconds / SECONDS_IN_DAY + 1

def break_time(t=None):
    """Given a time returns a string. Useful for naming files."""
    #TODO: should be able to handle a time range
    return parse_time(t).strftime("%Y%m%d_%H%M%S")

def get_day(dt):
    """ Return datetime for the beginning of the day of given datetime. """
    return datetime(dt.year, dt.month, dt.day)

<|endoftext|>


<commit_before>from django.test import TestCase

# Create your tests here.

import datetime

from django.shortcuts import render, get_object_or_404
from django.template import loader
from django.core.urlresolvers import reverse
from django.views import generic
from django.contrib.auth import views, logout
from django.contrib.auth.models import User, AnonymousUser
from django.contrib.auth.forms import AuthenticationForm
from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseServerError
import logging
from django.utils import timezone
from .forms import *
from .models import *
from xcom40k.settings import BUILD_NAME, BUILD_VERSION
from . import models,views,forms,urls

class QuestionViewTests(TestCase):
	# a following sample IS TO BE FOLLOWED STRICTLY:
	# 					#####
	#def test_<some verbose name> (self):
	#''' COMPONENTS: <name of components being tested>
	#	 ABOUT:
	#	 EXPECTED: <an expected result>
	#'''
	#
	# 					#####
	# The following example should be self-explanatory:
    def test_try_to_ping(self):
        """ COMPONENT: app:*
        	ABOUT: Tries to ping up the server.
        	EXPECTED: A '200 OK' response should be received from the server on pages:
        			* app:index,
        			* app:login,
        			* app:missions,
        			* app:train.
        		A '302 REDIRECT' should be received on pages:
        			* app:profile,
        			* app:stash.
        """
        self.assertEqual(self.client.get(reverse('app:index')).status_code,    200)
        self.assertEqual(self.client.get(reverse('app:login')).status_code,    200)
        self.assertEqual(self.client.get(reverse('app:missions')).status_code, 200)
        self.assertEqual(self.client.get(reverse('app:train')).status_code,    200)

        self.assertEqual(self.client.get(reverse('app:profile')).status_code,  302)
        self.assertEqual(self.client.get(reverse('app:stash')).status_code,    302)

    def test_root_is_present(self):
        """ COMPONENT: *
            ABOUT: Asserts the root user exists.
            EXPECTED: A user with name 'root' with staff status = True should be found.
        """
        root_set = User.objects.filter(username = 'root')
        self.assertEqual(len(root_set), 1)
        self.assertEqual(root_set[0].is_staff, True)
        self.assertEqual(root_set[0].is_superuser, True)
<commit_msg>rewrite sample test<commit_after>from django.test import TestCase

# Create your tests here.

import datetime

from django.shortcuts import render, get_object_or_404
from django.template import loader
from django.core.urlresolvers import reverse
from django.views import generic
from django.contrib.auth import views, logout
from django.contrib.auth.models import User, AnonymousUser
from django.contrib.auth.forms import AuthenticationForm
from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseServerError
import logging
from django.utils import timezone
from .forms import *
from .models import *
from xcom40k.settings import BUILD_NAME, BUILD_VERSION
from . import models,views,forms,urls

class QuestionViewTests(TestCase):
	# a following sample IS TO BE FOLLOWED STRICTLY:
	# 					#####
	#def test_<some verbose name> (self):
	#''' COMPONENTS: <name of components being tested>
	#	 ABOUT:
	#	 EXPECTED: <an expected result>
	#'''
	#
	# 					#####
	# The following example should be self-explanatory:
    def test_try_to_ping(self):
        """ COMPONENT: app:*
        	ABOUT: Tries to ping up the server.
        	EXPECTED: A '200 OK' response should be received from the server
        """
        self.assertEqual(self.client.get(reverse('app:index')).status_code,    200)
<|endoftext|>


<commit_before>"""
Django settings for task_burndown project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = ')dt$nnw@#1*l$p5gioc^j_=%p!=@4$%*qtbvl+kg7wzh^#5*(('

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    #django
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    #third party
    'rest_framework',

    #apps
    'tasks',

)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'task_burndown.urls'

WSGI_APPLICATION = 'task_burndown.wsgi.application'

# Database
# https://docs.djangoproject.com/en/dev/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/dev/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

TEMPLATE_DIRS = (os.path.join(BASE_DIR,  'templates'),)
STATICFILES_DIRS = (BASE_DIR + '/assets',)
STATIC_URL = '/static/'

try:
    from local_settings import *
except ImportError as exp:
    pass
<commit_msg>added django rest settings<commit_after>"""
Django settings for task_burndown project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = ')dt$nnw@#1*l$p5gioc^j_=%p!=@4$%*qtbvl+kg7wzh^#5*(('

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    #django
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    #third party
    'rest_framework',

    #apps
    'tasks',

)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'task_burndown.urls'

WSGI_APPLICATION = 'task_burndown.wsgi.application'

# Database
# https://docs.djangoproject.com/en/dev/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

REST_FRAMEWORK = {
    # Use Django's standard `django.contrib.auth` permissions,
    # or allow read-only access for unauthenticated users.
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
    ]
}

# Internationalization
# https://docs.djangoproject.com/en/dev/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

TEMPLATE_DIRS = (os.path.join(BASE_DIR,  'templates'),)
STATICFILES_DIRS = (BASE_DIR + '/assets',)
STATIC_URL = '/static/'

try:
    from local_settings import *
except ImportError as exp:
    pass
<|endoftext|>


<commit_before>import argparse
import random
import json
import sys
import generator_tools

def make_turing_machine_rules(n_states, n_symbols):
    the_rules = [   [   (random.randrange(n_symbols), random.randrange(n_states), random.choice('LNR'))
                        for symbol in range(n_symbols)]
                    for state in range(n_states)]
    return the_rules

def encode_turing_machine_rules(rules, starting_state=None, story=None):
    if story is None:
        story = generator_tools.Story()
    graph = story.graph
    if starting_state is None:
        starting_state = random.choice(len(rules))
    the_edges = [(cstate, read, write, nstate, direc)
                    for (cstate, stuff) in enumerate(rules)
                    for (read, (write, nstate, direc)) in enumerate(stuff)]
    random.shuffle(the_edges)
    for cstate, read, write, nstate, direc in the_edges:
        source = graph.make_unique('state_{}'.format(cstate))
        dest = graph.make_unique('state_{}'.format(nstate))
        edge_type = "rule_{}_{}_{}".format(read,write,direc)
        source[edge_type] = dest
        story.add_line("rule {} {} {} {} {}".format(source.type, read, write, dest.type, direc))
    head = graph.make_unique('head')

    head.state = graph.make_unique('state_{}'.format(starting_state))
    story.add_line("start {}".format(head.state.type))
    return story

def encode_turing_machine_process(rules, starting_state, iptlist, process_len, head_index=0, story=None, update_state=False):
    if story is None:
        story = generator_tools.Story()
    graph = story.graph
    last_input = None
    cells = []
    for i,symbol in enumerate(iptlist):
        cell = graph.make('cell')
        cell.left = last_input
        cell.value = graph.make_unique('symbol_{}'.format(symbol))
        cells.append(cell)
        last_input = cell
        if head_index == i:
            head = graph.make_unique('head')
            head.cell = cell
            story.add_line("input {} head".format(cell.value.type))
        else:
            story.add_line("input {}".format(cell.value.type))

    cstate = starting_state
    cell_values = iptlist[:]
    for _ in range(process_len):
        cell = cells[head_index]
        read = cell_values[head_index]
        write, nstate, direc = rules[cstate][read]
        cell.value = graph.make_unique('symbol_{}'.format(write))
        cstate = nstate
        if update_state:
            head.state = graph.make_unique('state_{}'.format(nstate))

        if direc == "L":
            if head_index == 0:
                newcell = graph.make('cell')
                cells.insert(0, newcell)
                cells[1].left = newcell
                newcell.value = graph.make_unique('symbol_{}'.format(0))
                cell_values.insert(0, 0)
                head_index += 1
            head_index -= 1
            head.cell = cells[head_index]
        elif direc == "R":
            if head_index == len(cells)-1:
                newcell = graph.make('cell')
                cells.append(newcell)
                newcell.left = cells[-2]
                newcell.value = graph.make_unique('symbol_{}'.format(0))
                cell_values.append(0)
            head_index += 1
            head.cell = cells[head_index]
        story.add_line('[RUN]')
    story.no_query()
    return story

def generate_universal(num_seqs, num_states, num_symbols, input_len, run_len):
    result = []
    for _ in range(num_seqs):
        rules = make_turing_machine_rules(num_states, num_symbols)
        start_state = random.randrange(num_states)
        input_list = [random.choice(range(num_symbols)) for _ in range(input_len)]
        head_index = random.randrange(input_len)
        story = encode_turing_machine_rules(rules, start_state)
        story = encode_turing_machine_process(rules, start_state, input_list, run_len, head_index, story, True)
        result.extend(story.lines)
    return "\n".join(result)+"\n"

def main(num_seqs, num_states, num_symbols, input_len, run_len, file):
    generated = generate_universal(num_seqs, num_states, num_symbols, input_len, run_len)
    file.write(generated)

parser = argparse.ArgumentParser(description='Generate a universal turing machine task')
parser.add_argument("file", nargs="?", default=sys.stdout, type=argparse.FileType('w'), help="Output file")
parser.add_argument("--num-states", type=int, default=4, help="Number of states")
parser.add_argument("--num-symbols", type=int, default=4, help="Number of symbols")
parser.add_argument("--input-len", type=int, default=5, help="Length of input")
parser.add_argument("--run-len", type=int, default=20, help="How many steps to simulate")
parser.add_argument("--num-seqs", type=int, default=1, help="Number of sequences to generate")

if __name__ == '__main__':
    args = vars(parser.parse_args())
    main(**args)

<commit_msg>Change turing machine default run length<commit_after>import argparse
import random
import json
import sys
import generator_tools

def make_turing_machine_rules(n_states, n_symbols):
    the_rules = [   [   (random.randrange(n_symbols), random.randrange(n_states), random.choice('LNR'))
                        for symbol in range(n_symbols)]
                    for state in range(n_states)]
    return the_rules

def encode_turing_machine_rules(rules, starting_state=None, story=None):
    if story is None:
        story = generator_tools.Story()
    graph = story.graph
    if starting_state is None:
        starting_state = random.choice(len(rules))
    the_edges = [(cstate, read, write, nstate, direc)
                    for (cstate, stuff) in enumerate(rules)
                    for (read, (write, nstate, direc)) in enumerate(stuff)]
    random.shuffle(the_edges)
    for cstate, read, write, nstate, direc in the_edges:
        source = graph.make_unique('state_{}'.format(cstate))
        dest = graph.make_unique('state_{}'.format(nstate))
        edge_type = "rule_{}_{}_{}".format(read,write,direc)
        source[edge_type] = dest
        story.add_line("rule {} {} {} {} {}".format(source.type, read, write, dest.type, direc))
    head = graph.make_unique('head')

    head.state = graph.make_unique('state_{}'.format(starting_state))
    story.add_line("start {}".format(head.state.type))
    return story

def encode_turing_machine_process(rules, starting_state, iptlist, process_len, head_index=0, story=None, update_state=False):
    if story is None:
        story = generator_tools.Story()
    graph = story.graph
    last_input = None
    cells = []
    for i,symbol in enumerate(iptlist):
        cell = graph.make('cell')
        cell.left = last_input
        cell.value = graph.make_unique('symbol_{}'.format(symbol))
        cells.append(cell)
        last_input = cell
        if head_index == i:
            head = graph.make_unique('head')
            head.cell = cell
            story.add_line("input {} head".format(cell.value.type))
        else:
            story.add_line("input {}".format(cell.value.type))

    cstate = starting_state
    cell_values = iptlist[:]
    for _ in range(process_len):
        cell = cells[head_index]
        read = cell_values[head_index]
        write, nstate, direc = rules[cstate][read]
        cell.value = graph.make_unique('symbol_{}'.format(write))
        cstate = nstate
        if update_state:
            head.state = graph.make_unique('state_{}'.format(nstate))

        if direc == "L":
            if head_index == 0:
                newcell = graph.make('cell')
                cells.insert(0, newcell)
                cells[1].left = newcell
                newcell.value = graph.make_unique('symbol_{}'.format(0))
                cell_values.insert(0, 0)
                head_index += 1
            head_index -= 1
            head.cell = cells[head_index]
        elif direc == "R":
            if head_index == len(cells)-1:
                newcell = graph.make('cell')
                cells.append(newcell)
                newcell.left = cells[-2]
                newcell.value = graph.make_unique('symbol_{}'.format(0))
                cell_values.append(0)
            head_index += 1
            head.cell = cells[head_index]
        story.add_line('[RUN]')
    story.no_query()
    return story

def generate_universal(num_seqs, num_states, num_symbols, input_len, run_len):
    result = []
    for _ in range(num_seqs):
        rules = make_turing_machine_rules(num_states, num_symbols)
        start_state = random.randrange(num_states)
        input_list = [random.choice(range(num_symbols)) for _ in range(input_len)]
        head_index = random.randrange(input_len)
        story = encode_turing_machine_rules(rules, start_state)
        story = encode_turing_machine_process(rules, start_state, input_list, run_len, head_index, story, True)
        result.extend(story.lines)
    return "\n".join(result)+"\n"

def main(num_seqs, num_states, num_symbols, input_len, run_len, file):
    generated = generate_universal(num_seqs, num_states, num_symbols, input_len, run_len)
    file.write(generated)

parser = argparse.ArgumentParser(description='Generate a universal turing machine task')
parser.add_argument("file", nargs="?", default=sys.stdout, type=argparse.FileType('w'), help="Output file")
parser.add_argument("--num-states", type=int, default=4, help="Number of states")
parser.add_argument("--num-symbols", type=int, default=4, help="Number of symbols")
parser.add_argument("--input-len", type=int, default=5, help="Length of input")
parser.add_argument("--run-len", type=int, default=10, help="How many steps to simulate")
parser.add_argument("--num-seqs", type=int, default=1, help="Number of sequences to generate")

if __name__ == '__main__':
    args = vars(parser.parse_args())
    main(**args)

<|endoftext|>


<commit_before>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

from django.test import TestCase

class xgds_notes2Test(TestCase):
    """
    Tests for xgds_notes2
    """
    def test_xgds_notes2(self):
        pass<commit_msg>replaced TestCase with TransactionTestCase<commit_after>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

from django.test import TransactionTestCase

class xgds_notes2Test(TransactionTestCase):
    """
    Tests for xgds_notes2
    """
    def test_xgds_notes2(self):
        pass<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Keep the Zoe scheduler loaded with BOINC clients.

Usage: boinc_load.py <zapp_json_file>
"""

import csv
import json
import os
import sys
import time
import random

from zoe_lib.statistics import ZoeStatisticsAPI
from zoe_lib.executions import ZoeExecutionsAPI

TARGET_QUEUE_LENGTH = 10
NUMBER_OF_HOSTS = 10
ZAPP_PER_HOST = 8
TOTAL_JOBS = NUMBER_OF_HOSTS * ZAPP_PER_HOST
# TOTAL_JOBS = 1000
MAX_TO_START_PER_LOOP = 10

def zoe_url():
    """Gets the API URL."""
    return os.environ['ZOE_URL']

def zoe_user():
    """Gets the API user name."""
    return os.environ['ZOE_USER']

def zoe_pass():
    """Gets the API password."""
    return os.environ['ZOE_PASS']

def check_queue_length():
    """Checks how many zapps are in the scheduler queue."""
    stats_api = ZoeStatisticsAPI(zoe_url(), zoe_user(), zoe_pass())
    sched = stats_api.scheduler()
#    print('Scheduler queue length: {}'.format(sched['queue_length']))
    return sched['queue_length']

def load_zapp(filename):
    """Loads and parses the ZApp json file."""
    return json.load(open(filename, 'r'))

def submit_zapp(zapp):
    """Submits one ZApp for execution."""
    exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
    ret = exec_api.start('boinc-loader', zapp)
    return ret

def count_jobs(all=False):
    """Count how many zapps have already been submitted."""
    exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
    execs = exec_api.list()
    count = 0
    for e_id in execs:
        e = exec_api.get(e_id)
        if not all and e['name'] != 'boinc-loader':
            continue
        if e['status'] != 'terminated':
            count += 1
    return count

def delete_finished():
    """Delete finished executions from Zoe."""
    exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
    execs = exec_api.list()
    for e_id in execs:
        e = exec_api.get(e_id)
        if e is not None and e['name'] == 'boinc-loader' and e['status'] == 'terminated':
            print('Execution {} has finished, deleting...'.format(e_id))
            exec_api.delete(e['id'])

def start_batches(zapp, log):
    zapps_to_start = TOTAL_JOBS - count_jobs()
    print('I need to start {} zapps'.format(zapps_to_start))
    while zapps_to_start > 0:
        queue_length = check_queue_length()
        if queue_length >= TARGET_QUEUE_LENGTH:
            to_sleep = random.randint(1, 60)
            print("Target scheduler queue length reached, sleeping for {} seconds".format(to_sleep))
            time.sleep(to_sleep)
            continue
        to_start_now = random.randint(1, MAX_TO_START_PER_LOOP)
        print('Will submit {} new zapps'.format(to_start_now))
        for i in range(to_start_now):
            zapp_id = submit_zapp(zapp)
            zapps_to_start -= 1
            print("ZApp submitted with ID {}, queue length {}, {} zapps to go".format(zapp_id, queue_length, zapps_to_start))
            log.writerow({'zapp_id': zapp_id, 'queue_length': queue_length})

def start_continuous(zapp, log):
    zapps_to_start = TOTAL_JOBS - count_jobs()
    print('I need to start {} zapps'.format(zapps_to_start))
    while zapps_to_start > 0:
        zapp_id = submit_zapp(zapp)
        zapps_to_start -= 1
        time_to_sleep = random.uniform(0, 1)
        queue_length = check_queue_length()
        print("ZApp submitted with ID {}, queue length {}, will sleep for {}, {} zapps to go".format(zapp_id, queue_length, time_to_sleep, zapps_to_start))
        log.writerow({'zapp_id': zapp_id, 'queue_length': queue_length})
        time.sleep(time_to_sleep)

def keep_some_running(zapp):
    while True:
        running_zapps = count_jobs(all=True)
        zapps_to_start = TOTAL_JOBS - running_zapps
        print('I need to start {} zapps ({} running)'.format(zapps_to_start, running_zapps))
        if zapps_to_start > 0:
            for i in range(zapps_to_start):
                queue_length = check_queue_length()
                zapp_id = submit_zapp(zapp)
                print("ZApp submitted with ID {}, queue length {}, {} zapps to go".format(zapp_id, queue_length, zapps_to_start))
        try:
            print('Sleeping')
            time.sleep(60)
        except KeyboardInterrupt:
            print('Exiting infinite loop')
            break
        delete_finished()

def main():
    """Main."""
    log_fieldnames = ['zapp_id', 'queue_length']
    log = csv.DictWriter(open('run.csv', 'w'), fieldnames=log_fieldnames)
    log.writeheader()
    zapp = load_zapp(sys.argv[1])
    # start_batches(zapp, log)
    # start_continuous(zapp, log)
    keep_some_running(zapp)

    print('All Zapps submitted, my work is done.')

if __name__ == "__main__":
    main()
<commit_msg>Fix bug in boinc script<commit_after>#!/usr/bin/env python3

"""
Keep the Zoe scheduler loaded with BOINC clients.

Usage: boinc_load.py <zapp_json_file>
"""

import csv
import json
import os
import sys
import time
import random

from zoe_lib.statistics import ZoeStatisticsAPI
from zoe_lib.executions import ZoeExecutionsAPI

TARGET_QUEUE_LENGTH = 10
NUMBER_OF_HOSTS = 10
ZAPP_PER_HOST = 8
TOTAL_JOBS = NUMBER_OF_HOSTS * ZAPP_PER_HOST
# TOTAL_JOBS = 1000
MAX_TO_START_PER_LOOP = 10

def zoe_url():
    """Gets the API URL."""
    return os.environ['ZOE_URL']

def zoe_user():
    """Gets the API user name."""
    return os.environ['ZOE_USER']

def zoe_pass():
    """Gets the API password."""
    return os.environ['ZOE_PASS']

def check_queue_length():
    """Checks how many zapps are in the scheduler queue."""
    stats_api = ZoeStatisticsAPI(zoe_url(), zoe_user(), zoe_pass())
    sched = stats_api.scheduler()
#    print('Scheduler queue length: {}'.format(sched['queue_length']))
    return sched['queue_length']

def load_zapp(filename):
    """Loads and parses the ZApp json file."""
    return json.load(open(filename, 'r'))

def submit_zapp(zapp):
    """Submits one ZApp for execution."""
    exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
    ret = exec_api.start('boinc-loader', zapp)
    return ret

def count_jobs(all=False):
    """Count how many zapps have already been submitted."""
    exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
    execs = exec_api.list()
    count = 0
    for e_id in execs:
        e = exec_api.get(e_id)
        if e is None:
            continue
        if not all and e['name'] != 'boinc-loader':
            continue
        if e['status'] != 'terminated':
            count += 1
    return count

def delete_finished():
    """Delete finished executions from Zoe."""
    exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
    execs = exec_api.list()
    for e_id in execs:
        e = exec_api.get(e_id)
        if e is None:
            continue
        if e['name'] == 'boinc-loader' and e['status'] == 'terminated':
            print('Execution {} has finished, deleting...'.format(e_id))
            exec_api.delete(e['id'])

def start_batches(zapp, log):
    zapps_to_start = TOTAL_JOBS - count_jobs()
    print('I need to start {} zapps'.format(zapps_to_start))
    while zapps_to_start > 0:
        queue_length = check_queue_length()
        if queue_length >= TARGET_QUEUE_LENGTH:
            to_sleep = random.randint(1, 60)
            print("Target scheduler queue length reached, sleeping for {} seconds".format(to_sleep))
            time.sleep(to_sleep)
            continue
        to_start_now = random.randint(1, MAX_TO_START_PER_LOOP)
        print('Will submit {} new zapps'.format(to_start_now))
        for i in range(to_start_now):
            zapp_id = submit_zapp(zapp)
            zapps_to_start -= 1
            print("ZApp submitted with ID {}, queue length {}, {} zapps to go".format(zapp_id, queue_length, zapps_to_start))
            log.writerow({'zapp_id': zapp_id, 'queue_length': queue_length})

def start_continuous(zapp, log):
    zapps_to_start = TOTAL_JOBS - count_jobs()
    print('I need to start {} zapps'.format(zapps_to_start))
    while zapps_to_start > 0:
        zapp_id = submit_zapp(zapp)
        zapps_to_start -= 1
        time_to_sleep = random.uniform(0, 1)
        queue_length = check_queue_length()
        print("ZApp submitted with ID {}, queue length {}, will sleep for {}, {} zapps to go".format(zapp_id, queue_length, time_to_sleep, zapps_to_start))
        log.writerow({'zapp_id': zapp_id, 'queue_length': queue_length})
        time.sleep(time_to_sleep)

def keep_some_running(zapp):
    while True:
        running_zapps = count_jobs(all=True)
        zapps_to_start = TOTAL_JOBS - running_zapps
        print('I need to start {} zapps ({} running)'.format(zapps_to_start, running_zapps))
        if zapps_to_start > 0:
            for i in range(zapps_to_start):
                queue_length = check_queue_length()
                zapp_id = submit_zapp(zapp)
                print("ZApp submitted with ID {}, queue length {}, {} zapps to go".format(zapp_id, queue_length, zapps_to_start))
        try:
            print('Sleeping')
            time.sleep(60)
        except KeyboardInterrupt:
            print('Exiting infinite loop')
            break
        delete_finished()

def main():
    """Main."""
    log_fieldnames = ['zapp_id', 'queue_length']
    log = csv.DictWriter(open('run.csv', 'w'), fieldnames=log_fieldnames)
    log.writeheader()
    zapp = load_zapp(sys.argv[1])
    # start_batches(zapp, log)
    # start_continuous(zapp, log)
    keep_some_running(zapp)

    print('All Zapps submitted, my work is done.')

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright (c) 2020, Michael Boyle
# See LICENSE file for details: <https://github.com/moble/spherical_functions/blob/master/LICENSE>

import warnings
import os
import numpy as np
import quaternion
from quaternion.numba_wrapper import jit, xrange
import spherical_functions as sf
from ... import WaveformModes, DataType, DataNames

def save(w, file_name=None, L2norm_fractional_tolerance=1e-10, log_frame=None, compress=True):
    import tempfile
    import contextlib
    import pathlib
    import json
    import numpy
    import scipy
    import h5py
    import sxs
    import scri
    from ...utilities import xor_timeseries

    # Make sure that we can understand the file_name and create the directory
    if file_name is None:
        # We'll just be creating a temp directory below, to check
        warnings.warn(
            'Input `file_name` is None.  Running in temporary directory.\n'
            'Note that this option is mostly for debugging purposes.'
        )
    else:
        h5_path = pathlib.Path(file_name).expanduser().resolve().with_suffix('.h5')
        if not h5_path.parent.exists():
            h5_path.parent.mkdir(parents=True)

    compression_options = {
        'compression': 'gzip',
        'compression_opts': 9,
        'shuffle': True,
        'fletcher32': True,
    } if compress else {}

    if L2norm_fractional_tolerance == 0.0:
        log_frame = quaternion.as_float_array(np.log(w.frame))[:, 1:]
    else:
        # We need this storage anyway, so let's just make a copy and work in-place
        w = w.copy()
        if log_frame is not None:
            log_frame = log_frame.copy()

        # Ensure waveform is in corotating frame
        if w.frameType == scri.Inertial:
            try:
                initial_time = w.t[0]
                relaxation_time = w.metadata.relaxation_time
                max_norm_time = w.max_norm_time()
                z_alignment_region = ((relaxation_time - initial_time) / (max_norm_time - initial_time), 0.95)
            except:
                z_alignment_region = (0.1, 0.95)
            w, log_frame = w.to_corotating_frame(
                tolerance=1e-10, z_alignment_region=z_alignment_region, truncate_log_frame=True
            )
            log_frame = log_frame[:, 1:]
        if w.frameType != scri.Corotating:
            raise ValueError("Frame type of input waveform must be 'Corotating' or 'Inertial'; "
                             f"it is {w.frame_type_string}")

        # Convert mode structure to conjugate pairs
        w.convert_to_conjugate_pairs()

        # Set bits below the desired significance level to 0
        w.truncate(tol=L2norm_fractional_tolerance)

        # Compute log(frame)
        if log_frame is None:
            log_frame = quaternion.as_float_array(np.log(w.frame))[:, 1:]
            power_of_2 = 2 ** (-np.floor(np.log2(L2norm_fractional_tolerance/10))).astype('int')
            log_frame = np.round(log_frame * power_of_2) / power_of_2

        # Change -0.0 to 0.0 (~.5% compression for non-precessing systems)
        w.t += 0.0
        w.data += 0.0
        log_frame += 0.0

        # XOR successive instants in time
        xor_timeseries(w.t)
        xor_timeseries(w.data)
        xor_timeseries(log_frame)

    # Make sure we have a place to keep all this
    with contextlib.ExitStack() as context:
        if file_name is None:
            temp_dir = context.enter_context(tempfile.TemporaryDirectory())
            h5_path = pathlib.Path(f'{temp_dir}') / 'test.h5'
        else:
            print(f'Saving H5 to "{h5_path}"')

        # Write the H5 file
        with h5py.File(h5_path, 'w') as f:
            f.attrs['sxs_format'] = 'corotating_paired_xor'
            warnings.warn('sxs_format is being set to "corotating_paired_xor"')
            f.create_dataset('time', data=w.t.view(np.uint64), **compression_options)
            f.create_dataset('modes', data=w.data.view(np.uint64), chunks=(w.n_times, 1), **compression_options)
            f['modes'].attrs['ell_min'] = w.ell_min
            f['modes'].attrs['ell_max'] = w.ell_max
            if log_frame.size > 1:
                f.create_dataset('log_frame', data=log_frame.view(np.uint64), chunks=(w.n_times, 1), **compression_options)

        h5_size = os.stat(h5_path).st_size
        if file_name is None:
            print(f'Output H5 file size: {h5_size:_} B')

        # Write the corresponding JSON file
        json_path = h5_path.with_suffix('.json')
        json_data = {
            'sxs_format': 'corotating_paired_xor',
            'data_info': {
                'data_type': w.data_type_string,
                'spin_weight': int(w.spin_weight),
                'ell_min': int(w.ell_min),
                'ell_max': int(w.ell_max),
            },
            'transformations': {
                'truncation': L2norm_fractional_tolerance,
                # see below for 'boost_velocity'
                # see below for 'space_translation'
            },
            'version_info': {
                # see below 'spec_version_hist'
                'numpy': numpy.__version__,
                'scipy': scipy.__version__,
                'h5py': h5py.__version__,
                'quaternion': quaternion.__version__,
                'spherical_functions': sf.__version__,
                'scri': scri.__version__,
                'sxs': sxs.__version__,
            },
            'validation': {
                'h5_file_size': h5_size,
                'n_times': w.n_times,
                # 'fletcher32': {'time': [], 'modes': [], 'log_frame': []}
            }
        }
        if hasattr(w, 'boost_velocity'):
            json_data['transformations']['boost_velocity'] = w.boost_velocity.tolist()
        if hasattr(w, 'space_translation'):
            json_data['transformations']['space_translation'] = w.space_translation.tolist()
        if hasattr(w, 'version_hist'):
            json_data['version_info']['spec_version_history'] = w.version_hist
        print(f'Saving JSON to "{json_path}"')
        with json_path.open('w') as f:
            json.dump(json_data, f, indent=2, separators=(',', ': '), ensure_ascii=True)

    return w

def load(file_name):
    import pathlib
    import json
    import h5py
    import scri
    from ...utilities import xor_timeseries_reverse

    h5_path = pathlib.Path(file_name).expanduser().resolve().with_suffix('.h5')
    json_path = h5_path.with_suffix('.json')

    with open(json_path, 'r') as f:
        json_data = json.load(f)
        dataType = scri.DataType[scri.DataNames.index(json_data['data_info']['data_type'])]

    with h5py.File(h5_path, 'r') as f:
        sxs_format = f.attrs['sxs_format']
        assert sxs_format in ['corotating_paired_xor',]
        time = f['time'][:].view(np.float64)
        modes = f['modes'][:].view(np.complex128)
        ell_min = f['modes'].attrs['ell_min']
        ell_max = f['modes'].attrs['ell_max']
        if 'log_frame' in f:
            log_frame = f['log_frame'][:].view(np.float64)
        else:
            log_frame = np.empty((0, 3), dtype=np.float64)

    xor_timeseries_reverse(time)
    xor_timeseries_reverse(modes)
    xor_timeseries_reverse(log_frame)
    frame = np.exp(quaternion.as_quat_array(np.insert(log_frame, 0, 0.0, axis=1)))

    w = WaveformModes(
        t=time, frame=frame, data=modes,
        frameType=scri.Corotating, dataType=dataType,
        m_is_scaled_out=True, r_is_scaled_out=True,
        ell_min=ell_min, ell_max=ell_max
    )
    w.convert_from_conjugate_pairs()
    w.json_data = json_data

    return w
<commit_msg>Forgot to chunk the time data; saves a few kB<commit_after># Copyright (c) 2020, Michael Boyle
# See LICENSE file for details: <https://github.com/moble/spherical_functions/blob/master/LICENSE>

import warnings
import os
import numpy as np
import quaternion
from quaternion.numba_wrapper import jit, xrange
import spherical_functions as sf
from ... import WaveformModes, DataType, DataNames

def save(w, file_name=None, L2norm_fractional_tolerance=1e-10, log_frame=None, compress=True):
    import tempfile
    import contextlib
    import pathlib
    import json
    import numpy
    import scipy
    import h5py
    import sxs
    import scri
    from ...utilities import xor_timeseries

    # Make sure that we can understand the file_name and create the directory
    if file_name is None:
        # We'll just be creating a temp directory below, to check
        warnings.warn(
            'Input `file_name` is None.  Running in temporary directory.\n'
            'Note that this option is mostly for debugging purposes.'
        )
    else:
        h5_path = pathlib.Path(file_name).expanduser().resolve().with_suffix('.h5')
        if not h5_path.parent.exists():
            h5_path.parent.mkdir(parents=True)

    compression_options = {
        'compression': 'gzip',
        'compression_opts': 9,
        'shuffle': True,
        'fletcher32': True,
    } if compress else {}

    if L2norm_fractional_tolerance == 0.0:
        log_frame = quaternion.as_float_array(np.log(w.frame))[:, 1:]
    else:
        # We need this storage anyway, so let's just make a copy and work in-place
        w = w.copy()
        if log_frame is not None:
            log_frame = log_frame.copy()

        # Ensure waveform is in corotating frame
        if w.frameType == scri.Inertial:
            try:
                initial_time = w.t[0]
                relaxation_time = w.metadata.relaxation_time
                max_norm_time = w.max_norm_time()
                z_alignment_region = ((relaxation_time - initial_time) / (max_norm_time - initial_time), 0.95)
            except:
                z_alignment_region = (0.1, 0.95)
            w, log_frame = w.to_corotating_frame(
                tolerance=1e-10, z_alignment_region=z_alignment_region, truncate_log_frame=True
            )
            log_frame = log_frame[:, 1:]
        if w.frameType != scri.Corotating:
            raise ValueError("Frame type of input waveform must be 'Corotating' or 'Inertial'; "
                             f"it is {w.frame_type_string}")

        # Convert mode structure to conjugate pairs
        w.convert_to_conjugate_pairs()

        # Set bits below the desired significance level to 0
        w.truncate(tol=L2norm_fractional_tolerance)

        # Compute log(frame)
        if log_frame is None:
            log_frame = quaternion.as_float_array(np.log(w.frame))[:, 1:]
            power_of_2 = 2 ** (-np.floor(np.log2(L2norm_fractional_tolerance/10))).astype('int')
            log_frame = np.round(log_frame * power_of_2) / power_of_2

        # Change -0.0 to 0.0 (~.5% compression for non-precessing systems)
        w.t += 0.0
        w.data += 0.0
        log_frame += 0.0

        # XOR successive instants in time
        xor_timeseries(w.t)
        xor_timeseries(w.data)
        xor_timeseries(log_frame)

    # Make sure we have a place to keep all this
    with contextlib.ExitStack() as context:
        if file_name is None:
            temp_dir = context.enter_context(tempfile.TemporaryDirectory())
            h5_path = pathlib.Path(f'{temp_dir}') / 'test.h5'
        else:
            print(f'Saving H5 to "{h5_path}"')

        # Write the H5 file
        with h5py.File(h5_path, 'w') as f:
            f.attrs['sxs_format'] = 'corotating_paired_xor'
            warnings.warn('sxs_format is being set to "corotating_paired_xor"')
            f.create_dataset('time', data=w.t.view(np.uint64), chunks=(w.n_times,), **compression_options)
            f.create_dataset('modes', data=w.data.view(np.uint64), chunks=(w.n_times, 1), **compression_options)
            f['modes'].attrs['ell_min'] = w.ell_min
            f['modes'].attrs['ell_max'] = w.ell_max
            if log_frame.size > 1:
                f.create_dataset('log_frame', data=log_frame.view(np.uint64), chunks=(w.n_times, 1), **compression_options)

        h5_size = os.stat(h5_path).st_size
        if file_name is None:
            print(f'Output H5 file size: {h5_size:_} B')

        # Write the corresponding JSON file
        json_path = h5_path.with_suffix('.json')
        json_data = {
            'sxs_format': 'corotating_paired_xor',
            'data_info': {
                'data_type': w.data_type_string,
                'spin_weight': int(w.spin_weight),
                'ell_min': int(w.ell_min),
                'ell_max': int(w.ell_max),
            },
            'transformations': {
                'truncation': L2norm_fractional_tolerance,
                # see below for 'boost_velocity'
                # see below for 'space_translation'
            },
            'version_info': {
                'numpy': numpy.__version__,
                'scipy': scipy.__version__,
                'h5py': h5py.__version__,
                'quaternion': quaternion.__version__,
                'spherical_functions': sf.__version__,
                'scri': scri.__version__,
                'sxs': sxs.__version__,
                # see below 'spec_version_hist'
            },
            'validation': {
                'h5_file_size': h5_size,
                'n_times': w.n_times,
                # 'fletcher32': {'time': [], 'modes': [], 'log_frame': []}
            }
        }
        if hasattr(w, 'boost_velocity'):
            json_data['transformations']['boost_velocity'] = w.boost_velocity.tolist()
        if hasattr(w, 'space_translation'):
            json_data['transformations']['space_translation'] = w.space_translation.tolist()
        if hasattr(w, 'version_hist'):
            json_data['version_info']['spec_version_history'] = w.version_hist
        if file_name is not None:
            print(f'Saving JSON to "{json_path}"')
        with json_path.open('w') as f:
            json.dump(json_data, f, indent=2, separators=(',', ': '), ensure_ascii=True)

    return w

def load(file_name):
    import pathlib
    import json
    import h5py
    import scri
    from ...utilities import xor_timeseries_reverse

    h5_path = pathlib.Path(file_name).expanduser().resolve().with_suffix('.h5')
    json_path = h5_path.with_suffix('.json')

    with open(json_path, 'r') as f:
        json_data = json.load(f)
        dataType = scri.DataType[scri.DataNames.index(json_data['data_info']['data_type'])]

    with h5py.File(h5_path, 'r') as f:
        sxs_format = f.attrs['sxs_format']
        assert sxs_format in ['corotating_paired_xor',]
        time = f['time'][:].view(np.float64)
        modes = f['modes'][:].view(np.complex128)
        ell_min = f['modes'].attrs['ell_min']
        ell_max = f['modes'].attrs['ell_max']
        if 'log_frame' in f:
            log_frame = f['log_frame'][:].view(np.float64)
        else:
            log_frame = np.empty((0, 3), dtype=np.float64)

    xor_timeseries_reverse(time)
    xor_timeseries_reverse(modes)
    xor_timeseries_reverse(log_frame)
    frame = np.exp(quaternion.as_quat_array(np.insert(log_frame, 0, 0.0, axis=1)))

    w = WaveformModes(
        t=time, frame=frame, data=modes,
        frameType=scri.Corotating, dataType=dataType,
        m_is_scaled_out=True, r_is_scaled_out=True,
        ell_min=ell_min, ell_max=ell_max
    )
    w.convert_from_conjugate_pairs()
    w.json_data = json_data

    return w
<|endoftext|>