<commit_before>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#
# pylint: disable=invalid-name,missing-docstring
"""
Find highly-connected k-mers and output them in a .stoptags file, for use
in partitioning.

% python scripts/find-knots.py <base>
"""

import argparse
import glob
import os
import textwrap
import khmer
import sys
from khmer.kfile import check_file_status, check_space
from khmer.khmer_args import info

# counting hash parameters.
DEFAULT_COUNTING_HT_SIZE = 3e6                # number of bytes
DEFAULT_COUNTING_HT_N = 4                     # number of counting hash tables

# Lump removal parameters.  Probably shouldn't be changed, but who knows?
#
# explanation:
#
# We will walk EXCURSION_DISTANCE out from each tag; if we find more than
# EXCURSION_KMER_THRESHOLD kmers within that range, this will be a "big"
# excursion and we will track all k-mers visited.  If we find that any
# k-mer has been visited more than EXCURSION_KMER_COUNT_THRESHOLD times,
# we will mark it as BAD and make it a stop tag for traversal.

# don't change these!
EXCURSION_DISTANCE = 40
EXCURSION_KMER_THRESHOLD = 200
EXCURSION_KMER_COUNT_THRESHOLD = 2
# EXCURSION_KMER_COUNT_THRESHOLD=5 # -- works ok for non-diginormed data

def get_parser():
    epilog = """
    Load an k-mer presence table/tagset pair created by load-graph, and a set
    of pmap files created by partition-graph. Go through each pmap file,
    select the largest partition in each, and do the same kind of traversal as
    in :program:`make-initial-stoptags.py` from each of the waypoints in that
    partition; this should identify all of the HCKs in that partition. These
    HCKs are output to <graphbase>.stoptags after each pmap file.

    Parameter choice is reasonably important. See the pipeline in
    :doc:`partitioning-big-data` for an example run.

    This script is not very scalable and may blow up memory and die horribly.
    You should be able to use the intermediate stoptags to restart the
    process, and if you eliminate the already-processed pmap files, you can
    continue where you left off.
    """
    parser = argparse.ArgumentParser(
        description="Find all highly connected k-mers.",
        epilog=textwrap.dedent(epilog),
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--n_tables', '-N', type=int,
                        default=DEFAULT_COUNTING_HT_N,
                        help='number of k-mer counting tables to use')
    parser.add_argument('--min-tablesize', '-x', type=float,
                        default=DEFAULT_COUNTING_HT_SIZE, help='lower bound on'
                        ' the size of the k-mer counting table(s)')
    parser.add_argument('graphbase', help='Basename for the input and output '
                        'files.')
    parser.add_argument('--version', action='version', version='%(prog)s '
                        + khmer.__version__)
    return parser

def main():
    info('find-knots.py', ['graph'])
    args = get_parser().parse_args()

    graphbase = args.graphbase

    # @RamRS: This might need some more work
    infiles = [graphbase + '.pt', graphbase + '.tagset']
    if os.path.exists(graphbase + '.stoptags'):
        infiles.append(graphbase + '.stoptags')
    for _ in infiles:
        check_file_status(_)

    check_space(infiles)

    print >>sys.stderr, 'loading k-mer presence table %s.pt' % graphbase
    htable = khmer.load_hashbits(graphbase + '.pt')

    print >>sys.stderr, 'loading tagset %s.tagset...' % graphbase
    htable.load_tagset(graphbase + '.tagset')

    initial_stoptags = False    # @CTB regularize with make-initial
    if os.path.exists(graphbase + '.stoptags'):
        print >>sys.stderr, 'loading stoptags %s.stoptags' % graphbase
        htable.load_stop_tags(graphbase + '.stoptags')
        initial_stoptags = True

    pmap_files = glob.glob(args.graphbase + '.subset.*.pmap')

    print >>sys.stderr, 'loading %d pmap files (first one: %s)' % \
        (len(pmap_files), pmap_files[0])
    print >>sys.stderr, '---'
    print >>sys.stderr, 'output stoptags will be in', graphbase + '.stoptags'
    if initial_stoptags:
        print >>sys.stderr, \
            '(these output stoptags will include the already-loaded set)'
    print >>sys.stderr, '---'

    # create counting hash
    ksize = htable.ksize()
    counting = khmer.new_counting_hash(ksize, args.min_tablesize,
                                       args.n_tables)

    # load & merge
    for index, subset_file in enumerate(pmap_files):
        print >>sys.stderr, '<-', subset_file
        subset = htable.load_subset_partitionmap(subset_file)

        print >>sys.stderr, '** repartitioning subset... %s' % subset_file
        htable.repartition_largest_partition(subset, counting,
                                             EXCURSION_DISTANCE,
                                             EXCURSION_KMER_THRESHOLD,
                                             EXCURSION_KMER_COUNT_THRESHOLD)

        print >>sys.stderr, '** merging subset... %s' % subset_file
        htable.merge_subset(subset)

        print >>sys.stderr, '** repartitioning, round 2... %s' % subset_file
        size = htable.repartition_largest_partition(
            None, counting, EXCURSION_DISTANCE, EXCURSION_KMER_THRESHOLD,
            EXCURSION_KMER_COUNT_THRESHOLD)

        print >>sys.stderr, '** repartitioned size:', size

        print >>sys.stderr, 'saving stoptags binary'
        htable.save_stop_tags(graphbase + '.stoptags')
        os.rename(subset_file, subset_file + '.processed')
        print >>sys.stderr, '(%d of %d)\n' % (index, len(pmap_files))

    print >>sys.stderr, 'done!'

if __name__ == '__main__':
    main()
<commit_msg>Fixed check_file_status error<commit_after>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#
# pylint: disable=invalid-name,missing-docstring
"""
Find highly-connected k-mers and output them in a .stoptags file, for use
in partitioning.

% python scripts/find-knots.py <base>
"""

import argparse
import glob
import os
import textwrap
import khmer
import sys
from khmer.kfile import check_file_status, check_space
from khmer.khmer_args import info

# counting hash parameters.
DEFAULT_COUNTING_HT_SIZE = 3e6                # number of bytes
DEFAULT_COUNTING_HT_N = 4                     # number of counting hash tables

# Lump removal parameters.  Probably shouldn't be changed, but who knows?
#
# explanation:
#
# We will walk EXCURSION_DISTANCE out from each tag; if we find more than
# EXCURSION_KMER_THRESHOLD kmers within that range, this will be a "big"
# excursion and we will track all k-mers visited.  If we find that any
# k-mer has been visited more than EXCURSION_KMER_COUNT_THRESHOLD times,
# we will mark it as BAD and make it a stop tag for traversal.

# don't change these!
EXCURSION_DISTANCE = 40
EXCURSION_KMER_THRESHOLD = 200
EXCURSION_KMER_COUNT_THRESHOLD = 2
# EXCURSION_KMER_COUNT_THRESHOLD=5 # -- works ok for non-diginormed data

def get_parser():
    epilog = """
    Load an k-mer presence table/tagset pair created by load-graph, and a set
    of pmap files created by partition-graph. Go through each pmap file,
    select the largest partition in each, and do the same kind of traversal as
    in :program:`make-initial-stoptags.py` from each of the waypoints in that
    partition; this should identify all of the HCKs in that partition. These
    HCKs are output to <graphbase>.stoptags after each pmap file.

    Parameter choice is reasonably important. See the pipeline in
    :doc:`partitioning-big-data` for an example run.

    This script is not very scalable and may blow up memory and die horribly.
    You should be able to use the intermediate stoptags to restart the
    process, and if you eliminate the already-processed pmap files, you can
    continue where you left off.
    """
    parser = argparse.ArgumentParser(
        description="Find all highly connected k-mers.",
        epilog=textwrap.dedent(epilog),
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--n_tables', '-N', type=int,
                        default=DEFAULT_COUNTING_HT_N,
                        help='number of k-mer counting tables to use')
    parser.add_argument('--min-tablesize', '-x', type=float,
                        default=DEFAULT_COUNTING_HT_SIZE, help='lower bound on'
                        ' the size of the k-mer counting table(s)')
    parser.add_argument('graphbase', help='Basename for the input and output '
                        'files.')
    parser.add_argument('--version', action='version', version='%(prog)s '
                        + khmer.__version__)
    return parser

def main():
    info('find-knots.py', ['graph'])
    args = get_parser().parse_args()

    graphbase = args.graphbase

    # @RamRS: This might need some more work
    infiles = [graphbase + '.pt', graphbase + '.tagset']
    if os.path.exists(graphbase + '.stoptags'):
        infiles.append(graphbase + '.stoptags')
    for _ in infiles:
        check_file_status(_, False)

    check_space(infiles)

    print >>sys.stderr, 'loading k-mer presence table %s.pt' % graphbase
    htable = khmer.load_hashbits(graphbase + '.pt')

    print >>sys.stderr, 'loading tagset %s.tagset...' % graphbase
    htable.load_tagset(graphbase + '.tagset')

    initial_stoptags = False    # @CTB regularize with make-initial
    if os.path.exists(graphbase + '.stoptags'):
        print >>sys.stderr, 'loading stoptags %s.stoptags' % graphbase
        htable.load_stop_tags(graphbase + '.stoptags')
        initial_stoptags = True

    pmap_files = glob.glob(args.graphbase + '.subset.*.pmap')

    print >>sys.stderr, 'loading %d pmap files (first one: %s)' % \
        (len(pmap_files), pmap_files[0])
    print >>sys.stderr, '---'
    print >>sys.stderr, 'output stoptags will be in', graphbase + '.stoptags'
    if initial_stoptags:
        print >>sys.stderr, \
            '(these output stoptags will include the already-loaded set)'
    print >>sys.stderr, '---'

    # create counting hash
    ksize = htable.ksize()
    counting = khmer.new_counting_hash(ksize, args.min_tablesize,
                                       args.n_tables)

    # load & merge
    for index, subset_file in enumerate(pmap_files):
        print >>sys.stderr, '<-', subset_file
        subset = htable.load_subset_partitionmap(subset_file)

        print >>sys.stderr, '** repartitioning subset... %s' % subset_file
        htable.repartition_largest_partition(subset, counting,
                                             EXCURSION_DISTANCE,
                                             EXCURSION_KMER_THRESHOLD,
                                             EXCURSION_KMER_COUNT_THRESHOLD)

        print >>sys.stderr, '** merging subset... %s' % subset_file
        htable.merge_subset(subset)

        print >>sys.stderr, '** repartitioning, round 2... %s' % subset_file
        size = htable.repartition_largest_partition(
            None, counting, EXCURSION_DISTANCE, EXCURSION_KMER_THRESHOLD,
            EXCURSION_KMER_COUNT_THRESHOLD)

        print >>sys.stderr, '** repartitioned size:', size

        print >>sys.stderr, 'saving stoptags binary'
        htable.save_stop_tags(graphbase + '.stoptags')
        os.rename(subset_file, subset_file + '.processed')
        print >>sys.stderr, '(%d of %d)\n' % (index, len(pmap_files))

    print >>sys.stderr, 'done!'

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
This module implements a parser for language tags, according to the RFC 5646
(BCP 47) standard.

Here, we're only concerned with the syntax of the language tag. Looking up
what they actually mean in a data file is a separate step.

For a full description of the syntax of a language tag, see page 3 of
    http://tools.ietf.org/html/bcp47

>>> parse_tag('en')
[('language', 'en')]

>>> parse_tag('en_US')
[('language', 'en'), ('territory', 'US')]

>>> parse_tag('en-Latn')
[('language', 'en'), ('script', 'Latn')]

>>> parse_tag('es-419')
[('language', 'es'), ('territory', '419')]

>>> parse_tag('zh-hant-tw')
[('language', 'zh'), ('script', 'Hant'), ('territory', 'TW')]

>>> parse_tag('zh-tw-hant')
Traceback (most recent call last):
    ...
langcodes.tag_parser.LanguageTagError: This script subtag, 'hant', is out of place. Expected variant, extension, or end of string.

>>> parse_tag('de-DE-1901')
[('language', 'de'), ('territory', 'DE'), ('variant', '1901')]

>>> parse_tag('ja-latn-hepburn')
[('language', 'ja'), ('script', 'Latn'), ('variant', 'hepburn')]

>>> parse_tag('ja-hepburn-latn')
Traceback (most recent call last):
    ...
langcodes.tag_parser.LanguageTagError: This script subtag, 'latn', is out of place. Expected variant, extension, or end of string.

>>> parse_tag('zh-yue')
[('language', 'zh'), ('extlang', 'yue')]

>>> parse_tag('zh-yue-Hant')
[('language', 'zh'), ('extlang', 'yue'), ('script', 'Hant')]

>>> parse_tag('zh-min-nan')
[('grandfathered', 'zh-min-nan')]

>>> parse_tag('x-dothraki')
[('language', 'x-dothraki')]

>>> parse_tag('en-u-co-backwards-x-pig-latin')
[('language', 'en'), ('extension', 'u-co-backwards'), ('private', 'x-pig-latin')]

>>> parse_tag('en-x-pig-latin-u-co-backwards')
[('language', 'en'), ('private', 'x-pig-latin-u-co-backwards')]

>>> parse_tag('u-co-backwards')
Traceback (most recent call last):
    ...
langcodes.tag_parser.LanguageTagError: Expected a language code, got 'u'
"""
from __future__ import print_function, unicode_literals

# These tags should not be parsed by the usual parser; they're grandfathered
# in from RFC 3066. The 'irregular' ones don't fit the syntax at all; the
# 'regular' ones do, but would give meaningless results when parsed.
#
# These are all lowercased so they can be matched case-insensitively, as the
# standard requires.
EXCEPTIONS = {
    # Irregular exceptions
    "en-gb-oed", "i-ami", "i-bnn", "i-default", "i-enochian", "i-hak",
    "i-klingon", "i-lux", "i-mingo", "i-navajo", "i-pwn", "i-tao", "i-tay",
    "i-tsu", "sgn-be-fr", "sgn-be-nl", "sgn-ch-de",

    # Regular exceptions
    "art-lojban", "cel-gaulish", "no-bok", "no-nyn", "zh-guoyu", "zh-hakka",
    "zh-min", "zh-min-nan", "zh-xiang"
}

# Define the order of subtags as integer constants, but also give them names
# so we can describe them in error messages
EXTLANG, SCRIPT, TERRITORY, VARIANT, EXTENSION = range(5)
SUBTAG_TYPES = ['extlang', 'script', 'territory', 'variant', 'extension',
                'end of string']

def normalize_characters(tag):
    """
    BCP 47 is case-insensitive, and considers underscores equivalent to
    hyphens. So here we smash tags into lowercase with hyphens, so we can
    make exact comparisons.

    >>> normalize_characters('en_US')
    'en-us'
    >>> normalize_characters('zh-Hant_TW')
    'zh-hant-tw'
    """
    return tag.lower().replace('_', '-')

def parse_tag(tag):
    """
    Parse the syntax of a language tag, without looking up anything in the
    registry, yet. Returns a list of (type, value) tuples indicating what
    information will need to be looked up.
    """
    tag = normalize_characters(tag)
    if tag in EXCEPTIONS:
        return [('grandfathered', tag)]
    else:
        # The first subtag is always either the language code, or 'x' to mark
        # the entire tag as private-use. Other subtags are distinguished
        # by their length and format, but the language code is distinguished
        # entirely by the fact that it is required to come first.
        subtags = tag.split('-')
        if subtags[0] == 'x':
            if len(subtags) == 1:
                raise LanguageTagError("'x' is not a language tag on its own")
            else:
                # the entire language tag is private use, but we know that,
                # whatever it is, it fills the "language" slot
                return [('language', tag)]
        elif len(subtags[0]) >= 2:
            return [('language', subtags[0])] + parse_subtags(subtags[1:])
        else:
            subtag_error(subtags[0], 'a language code')

def parse_subtags(subtags, expect=EXTLANG):
    """
    Parse everything that comes after the language tag: scripts, territories,
    variants, and assorted extensions.
    """
    # We parse the parts of a language code recursively: each step of
    # language code parsing handles one component of the code, recurses
    # to handle the rest of the code, and adds what it found onto the
    # list of things that were in the rest of the code.
    #
    # This could just as well have been iterative, but the loops would have
    # been convoluted.
    #
    # So here's the base case.
    if not subtags:
        return []

    # There's a subtag that comes next. We need to find out what it is.
    #
    # The primary thing that distinguishes different types of subtags is
    # length, but the subtags also come in a specified order. The 'expect'
    # parameter keeps track of where we are in that order. expect=TERRITORY,
    # for example, means we're expecting a territory code, or anything later
    # (because everything but the language is optional).
    subtag = subtags[0]
    tag_length = len(subtag)

    # In the usual case, our goal is to recognize what kind of tag this is,
    # and set it in 'tagtype' -- as an integer, so we can compare where it
    # should go in order. You can see the enumerated list of tagtypes above,
    # where the SUBTAG_TYPES global is defined.
    tagtype = None

    if tag_length == 0 or tag_length > 8:
        # Unless you're inside a private use tag or something -- in which case,
        # you're not in this function at the moment -- every component needs to
        # be between 1 and 8 characters.
        subtag_error(subtag, '1-8 characters')

    elif tag_length == 1:
        # A one-character subtag introduces an extension, which can itself have
        # sub-subtags, so we dispatch to a different function at this point.
        #
        # We don't need to check anything about the order, because extensions
        # necessarily come last.
        return parse_extension(subtags)

    elif tag_length == 2:
        if subtag.isalpha():
            # Two-letter alphabetic subtags are territories. These are the only
            # two-character subtags after the language.
            tagtype = TERRITORY

    elif tag_length == 3:
        if subtag.isalpha():
            # Three-letter alphabetic subtags are 'extended languages'.
            # It's allowed for there to be up to three of them in a row, so we
            # need another function to enforce that. Before we dispatch to that
            # function, though, we need to check whether we're in the right
            # place in order.
            if expect <= EXTLANG:
                return parse_extlang(subtags)
            else:
                order_error(subtag, EXTLANG, expect)
        elif subtag.isdigit():
            # Three-digit subtags are territories representing broad regions,
            # such as Latin America (419).
            tagtype = TERRITORY

    elif tag_length == 4:
        if subtag.isalpha():
            # Four-letter alphabetic subtags are scripts.
            tagtype = SCRIPT
        elif subtag[0].isdigit():
            # Four-character subtags that start with a digit are variants.
            tagtype = VARIANT

    else:
        # Tags of length 5-8 are variants.
        tagtype = VARIANT

    # That's the end of the big elif block for figuring out what kind of
    # subtag we have based on its length. Now we should do something with that
    # kind of subtag.

    if tagtype is None:
        # We haven't recognized a type of tag. This subtag just doesn't fit the
        # standard.
        subtag_error(subtag)

    elif tagtype < expect:
        # We got a tag type that was supposed to appear earlier in the order.
        order_error(subtag, tagtype, expect)

    else:
        # We've recognized a subtag of a particular type. If it's a territory or
        # script, we expect the next subtag to be a strictly later type, because
        # there can be at most one territory and one script. Otherwise, we expect
        # the next subtag to be the type we got or later.

        if tagtype in (SCRIPT, TERRITORY):
            expect = tagtype + 1
        else:
            expect = tagtype

        # Get the name of this subtag type instead of its integer value.
        typename = SUBTAG_TYPES[tagtype]

        # Some subtags are conventionally written with capitalization. Apply
        # those conventions.
        if tagtype == SCRIPT:
            subtag = subtag.title()
        elif tagtype == TERRITORY:
            subtag = subtag.upper()

        # Recurse on the remaining subtags.
        return [(typename, subtag)] + parse_subtags(subtags[1:], expect)

def parse_extlang(subtags):
    """
    Parse an 'extended language' tag, which consists of 1 to 3 three-letter
    language codes.

    Extended languages are used for distinguishing dialects/sublanguages
    (depending on your view) of macrolanguages such as Arabic, Bahasa Malay,
    and Chinese.

    It's supposed to also be acceptable to just use the sublanguage as the
    primary language code, and your code should know what's a macrolanguage of
    what. For example, 'zh-yue' and 'yue' are the same language (Cantonese),
    and differ only in whether they explicitly spell out that Cantonese is a
    kind of Chinese.
    """
    index = 0
    parsed = []
    while index < len(subtags) and len(subtags[index]) == 3 and index < 3:
        parsed.append(('extlang', subtags[index]))
        index += 1
    return parsed + parse_subtags(subtags[index:], SCRIPT)

def parse_extension(subtags):
    """
    An extension tag consists of a 'singleton' -- a one-character subtag --
    followed by other subtags. Extension tags are in the BCP 47 syntax, but
    their meaning is outside the scope of the standard.

    For example, there's the u- extension, which is used for setting Unicode
    properties in some context I'm not aware of.

    If the singleton is 'x', it's a private use extension, and consumes the
    rest of the tag. Otherwise, it stops at the next singleton.
    """
    subtag = subtags[0]
    if len(subtags) == 1:
        raise LanguageTagError(
            "The subtag %r must be followed by something" % subtag
        )

    if subtag == 'x':
        # Private use. Everything after this is arbitrary codes that we
        # can't look up.
        return [('private', '-'.join(subtags))]

    else:
        # Look for the next singleton, if there is one.
        boundary = 1
        while boundary < len(subtags) and len(subtags[boundary]) != 1:
            boundary += 1

        # We've parsed a complete extension subtag. Return to the main
        # parse_subtags function, but expect to find nothing but more
        # extensions at this point.
        return ([('extension', '-'.join(subtags[:boundary]))]
                + parse_subtags(subtags[boundary:], EXTENSION))

class LanguageTagError(ValueError):
    pass

def order_error(subtag, got, expected):
    """
    Output an error indicating that tags were out of order.
    """
    options = SUBTAG_TYPES[expected:]
    if len(options) == 1:
        expect_str = options[0]
    elif len(options) == 2:
        expect_str = '%s or %s' % (options[0], options[1])
    else:
        expect_str = '%s, or %s' % (', '.join(options[:-1]), options[-1])
    got_str = SUBTAG_TYPES[got]
    raise LanguageTagError("This %s subtag, %r, is out of place. "
                           "Expected %s." % (got_str, subtag, expect_str))

def subtag_error(subtag, expected='a valid subtag'):
    """
    Try to output a reasonably helpful error message based on our state of
    parsing. Most of this code is about how to list, in English, the kinds
    of things we were expecting to find.
    """
    raise LanguageTagError("Expected %s, got %r" % (expected, subtag))

<commit_msg>don't parse language subtags that are longer than 4 characters<commit_after>"""
This module implements a parser for language tags, according to the RFC 5646
(BCP 47) standard.

Here, we're only concerned with the syntax of the language tag. Looking up
what they actually mean in a data file is a separate step.

For a full description of the syntax of a language tag, see page 3 of
    http://tools.ietf.org/html/bcp47

>>> parse_tag('en')
[('language', 'en')]

>>> parse_tag('en_US')
[('language', 'en'), ('territory', 'US')]

>>> parse_tag('en-Latn')
[('language', 'en'), ('script', 'Latn')]

>>> parse_tag('es-419')
[('language', 'es'), ('territory', '419')]

>>> parse_tag('zh-hant-tw')
[('language', 'zh'), ('script', 'Hant'), ('territory', 'TW')]

>>> parse_tag('zh-tw-hant')
Traceback (most recent call last):
    ...
langcodes.tag_parser.LanguageTagError: This script subtag, 'hant', is out of place. Expected variant, extension, or end of string.

>>> parse_tag('de-DE-1901')
[('language', 'de'), ('territory', 'DE'), ('variant', '1901')]

>>> parse_tag('ja-latn-hepburn')
[('language', 'ja'), ('script', 'Latn'), ('variant', 'hepburn')]

>>> parse_tag('ja-hepburn-latn')
Traceback (most recent call last):
    ...
langcodes.tag_parser.LanguageTagError: This script subtag, 'latn', is out of place. Expected variant, extension, or end of string.

>>> parse_tag('zh-yue')
[('language', 'zh'), ('extlang', 'yue')]

>>> parse_tag('zh-yue-Hant')
[('language', 'zh'), ('extlang', 'yue'), ('script', 'Hant')]

>>> parse_tag('zh-min-nan')
[('grandfathered', 'zh-min-nan')]

>>> parse_tag('x-dothraki')
[('language', 'x-dothraki')]

>>> parse_tag('en-u-co-backwards-x-pig-latin')
[('language', 'en'), ('extension', 'u-co-backwards'), ('private', 'x-pig-latin')]

>>> parse_tag('en-x-pig-latin-u-co-backwards')
[('language', 'en'), ('private', 'x-pig-latin-u-co-backwards')]

>>> parse_tag('u-co-backwards')
Traceback (most recent call last):
    ...
langcodes.tag_parser.LanguageTagError: Expected a language code, got 'u'
"""
from __future__ import print_function, unicode_literals

# These tags should not be parsed by the usual parser; they're grandfathered
# in from RFC 3066. The 'irregular' ones don't fit the syntax at all; the
# 'regular' ones do, but would give meaningless results when parsed.
#
# These are all lowercased so they can be matched case-insensitively, as the
# standard requires.
EXCEPTIONS = {
    # Irregular exceptions
    "en-gb-oed", "i-ami", "i-bnn", "i-default", "i-enochian", "i-hak",
    "i-klingon", "i-lux", "i-mingo", "i-navajo", "i-pwn", "i-tao", "i-tay",
    "i-tsu", "sgn-be-fr", "sgn-be-nl", "sgn-ch-de",

    # Regular exceptions
    "art-lojban", "cel-gaulish", "no-bok", "no-nyn", "zh-guoyu", "zh-hakka",
    "zh-min", "zh-min-nan", "zh-xiang"
}

# Define the order of subtags as integer constants, but also give them names
# so we can describe them in error messages
EXTLANG, SCRIPT, TERRITORY, VARIANT, EXTENSION = range(5)
SUBTAG_TYPES = ['extlang', 'script', 'territory', 'variant', 'extension',
                'end of string']

def normalize_characters(tag):
    """
    BCP 47 is case-insensitive, and considers underscores equivalent to
    hyphens. So here we smash tags into lowercase with hyphens, so we can
    make exact comparisons.

    >>> normalize_characters('en_US')
    'en-us'
    >>> normalize_characters('zh-Hant_TW')
    'zh-hant-tw'
    """
    return tag.lower().replace('_', '-')

def parse_tag(tag):
    """
    Parse the syntax of a language tag, without looking up anything in the
    registry, yet. Returns a list of (type, value) tuples indicating what
    information will need to be looked up.
    """
    tag = normalize_characters(tag)
    if tag in EXCEPTIONS:
        return [('grandfathered', tag)]
    else:
        # The first subtag is always either the language code, or 'x' to mark
        # the entire tag as private-use. Other subtags are distinguished
        # by their length and format, but the language code is distinguished
        # by the fact that it is required to come first.
        subtags = tag.split('-')
        if subtags[0] == 'x':
            if len(subtags) == 1:
                raise LanguageTagError("'x' is not a language tag on its own")
            else:
                # the entire language tag is private use, but we know that,
                # whatever it is, it fills the "language" slot
                return [('language', tag)]
        elif 2 <= len(subtags[0]) <= 4:
            # Language codes should be 2 or 3 letters, but 4-letter codes
            # are allowed to parse for legacy Unicode reasons
            return [('language', subtags[0])] + parse_subtags(subtags[1:])
        else:
            subtag_error(subtags[0], 'a language code')

def parse_subtags(subtags, expect=EXTLANG):
    """
    Parse everything that comes after the language tag: scripts, territories,
    variants, and assorted extensions.
    """
    # We parse the parts of a language code recursively: each step of
    # language code parsing handles one component of the code, recurses
    # to handle the rest of the code, and adds what it found onto the
    # list of things that were in the rest of the code.
    #
    # This could just as well have been iterative, but the loops would have
    # been convoluted.
    #
    # So here's the base case.
    if not subtags:
        return []

    # There's a subtag that comes next. We need to find out what it is.
    #
    # The primary thing that distinguishes different types of subtags is
    # length, but the subtags also come in a specified order. The 'expect'
    # parameter keeps track of where we are in that order. expect=TERRITORY,
    # for example, means we're expecting a territory code, or anything later
    # (because everything but the language is optional).
    subtag = subtags[0]
    tag_length = len(subtag)

    # In the usual case, our goal is to recognize what kind of tag this is,
    # and set it in 'tagtype' -- as an integer, so we can compare where it
    # should go in order. You can see the enumerated list of tagtypes above,
    # where the SUBTAG_TYPES global is defined.
    tagtype = None

    if tag_length == 0 or tag_length > 8:
        # Unless you're inside a private use tag or something -- in which case,
        # you're not in this function at the moment -- every component needs to
        # be between 1 and 8 characters.
        subtag_error(subtag, '1-8 characters')

    elif tag_length == 1:
        # A one-character subtag introduces an extension, which can itself have
        # sub-subtags, so we dispatch to a different function at this point.
        #
        # We don't need to check anything about the order, because extensions
        # necessarily come last.
        return parse_extension(subtags)

    elif tag_length == 2:
        if subtag.isalpha():
            # Two-letter alphabetic subtags are territories. These are the only
            # two-character subtags after the language.
            tagtype = TERRITORY

    elif tag_length == 3:
        if subtag.isalpha():
            # Three-letter alphabetic subtags are 'extended languages'.
            # It's allowed for there to be up to three of them in a row, so we
            # need another function to enforce that. Before we dispatch to that
            # function, though, we need to check whether we're in the right
            # place in order.
            if expect <= EXTLANG:
                return parse_extlang(subtags)
            else:
                order_error(subtag, EXTLANG, expect)
        elif subtag.isdigit():
            # Three-digit subtags are territories representing broad regions,
            # such as Latin America (419).
            tagtype = TERRITORY

    elif tag_length == 4:
        if subtag.isalpha():
            # Four-letter alphabetic subtags are scripts.
            tagtype = SCRIPT
        elif subtag[0].isdigit():
            # Four-character subtags that start with a digit are variants.
            tagtype = VARIANT

    else:
        # Tags of length 5-8 are variants.
        tagtype = VARIANT

    # That's the end of the big elif block for figuring out what kind of
    # subtag we have based on its length. Now we should do something with that
    # kind of subtag.

    if tagtype is None:
        # We haven't recognized a type of tag. This subtag just doesn't fit the
        # standard.
        subtag_error(subtag)

    elif tagtype < expect:
        # We got a tag type that was supposed to appear earlier in the order.
        order_error(subtag, tagtype, expect)

    else:
        # We've recognized a subtag of a particular type. If it's a territory or
        # script, we expect the next subtag to be a strictly later type, because
        # there can be at most one territory and one script. Otherwise, we expect
        # the next subtag to be the type we got or later.

        if tagtype in (SCRIPT, TERRITORY):
            expect = tagtype + 1
        else:
            expect = tagtype

        # Get the name of this subtag type instead of its integer value.
        typename = SUBTAG_TYPES[tagtype]

        # Some subtags are conventionally written with capitalization. Apply
        # those conventions.
        if tagtype == SCRIPT:
            subtag = subtag.title()
        elif tagtype == TERRITORY:
            subtag = subtag.upper()

        # Recurse on the remaining subtags.
        return [(typename, subtag)] + parse_subtags(subtags[1:], expect)

def parse_extlang(subtags):
    """
    Parse an 'extended language' tag, which consists of 1 to 3 three-letter
    language codes.

    Extended languages are used for distinguishing dialects/sublanguages
    (depending on your view) of macrolanguages such as Arabic, Bahasa Malay,
    and Chinese.

    It's supposed to also be acceptable to just use the sublanguage as the
    primary language code, and your code should know what's a macrolanguage of
    what. For example, 'zh-yue' and 'yue' are the same language (Cantonese),
    and differ only in whether they explicitly spell out that Cantonese is a
    kind of Chinese.
    """
    index = 0
    parsed = []
    while index < len(subtags) and len(subtags[index]) == 3 and index < 3:
        parsed.append(('extlang', subtags[index]))
        index += 1
    return parsed + parse_subtags(subtags[index:], SCRIPT)

def parse_extension(subtags):
    """
    An extension tag consists of a 'singleton' -- a one-character subtag --
    followed by other subtags. Extension tags are in the BCP 47 syntax, but
    their meaning is outside the scope of the standard.

    For example, there's the u- extension, which is used for setting Unicode
    properties in some context I'm not aware of.

    If the singleton is 'x', it's a private use extension, and consumes the
    rest of the tag. Otherwise, it stops at the next singleton.
    """
    subtag = subtags[0]
    if len(subtags) == 1:
        raise LanguageTagError(
            "The subtag %r must be followed by something" % subtag
        )

    if subtag == 'x':
        # Private use. Everything after this is arbitrary codes that we
        # can't look up.
        return [('private', '-'.join(subtags))]

    else:
        # Look for the next singleton, if there is one.
        boundary = 1
        while boundary < len(subtags) and len(subtags[boundary]) != 1:
            boundary += 1

        # We've parsed a complete extension subtag. Return to the main
        # parse_subtags function, but expect to find nothing but more
        # extensions at this point.
        return ([('extension', '-'.join(subtags[:boundary]))]
                + parse_subtags(subtags[boundary:], EXTENSION))

class LanguageTagError(ValueError):
    pass

def order_error(subtag, got, expected):
    """
    Output an error indicating that tags were out of order.
    """
    options = SUBTAG_TYPES[expected:]
    if len(options) == 1:
        expect_str = options[0]
    elif len(options) == 2:
        expect_str = '%s or %s' % (options[0], options[1])
    else:
        expect_str = '%s, or %s' % (', '.join(options[:-1]), options[-1])
    got_str = SUBTAG_TYPES[got]
    raise LanguageTagError("This %s subtag, %r, is out of place. "
                           "Expected %s." % (got_str, subtag, expect_str))

def subtag_error(subtag, expected='a valid subtag'):
    """
    Try to output a reasonably helpful error message based on our state of
    parsing. Most of this code is about how to list, in English, the kinds
    of things we were expecting to find.
    """
    raise LanguageTagError("Expected %s, got %r" % (expected, subtag))

<|endoftext|>


<commit_before><commit_msg>OPT: removed the split data option<commit_after><|endoftext|>


<commit_before>"""
Django settings for lap project.

Generated by 'django-admin startproject' using Django 1.10.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

DEBUG = os.environ.get('DEBUG', 'on') == 'on'
SECRET_KEY = os.environ.get('SECRET_KEY', os.urandom(32))

ALLOWED_HOSTS = ['0.0.0.0']

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = BASE_DIR + '/ihc-static/'

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    BASE_DIR + '/static/',
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder'
)

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
    'analytics.apps.AnalyticsConfig',
    'authenticate',
    'django_filters',
    'rest_framework_swagger'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'lap.urls'
LOGIN_URL = '/login/'
SESSION_EXPIRE_AT_BROWSER_CLOSE = False
SESSION_SAVE_EVERY_REQUEST = True
# Session timeout age in seconds, important to set above 15 minutes as the client-side
# Angular code is set to automatically timeout after 15 minutes, but will use the heartbeat
# API endpoint to update on any mouse or keyboard activity once a minute
SESSION_COOKIE_AGE = 1200

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            BASE_DIR + '/templates',
            BASE_DIR + '/authenticate/templates'
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'lap.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

REST_FRAMEWORK = {
    'PAGINATE_BY': None,
    'PAGINATE_BY_PARAM': 'paginate_by',
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.SessionAuthentication',
        'rest_framework.authentication.TokenAuthentication',
    ),
    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',)
}

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

MEDIA_ROOT = BASE_DIR + '/media/'
MEDIA_URL = '/media/'
<commit_msg>added django_extensions to example Django settings file<commit_after>"""
Django settings for lap project.

Generated by 'django-admin startproject' using Django 1.10.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

DEBUG = os.environ.get('DEBUG', 'on') == 'on'
SECRET_KEY = os.environ.get('SECRET_KEY', os.urandom(32))

ALLOWED_HOSTS = ['0.0.0.0']

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = BASE_DIR + '/ihc-static/'

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    BASE_DIR + '/static/',
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder'
)

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
    'analytics.apps.AnalyticsConfig',
    'authenticate',
    'django_filters',
    'rest_framework_swagger',
    'django_extensions'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'lap.urls'
LOGIN_URL = '/login/'
SESSION_EXPIRE_AT_BROWSER_CLOSE = False
SESSION_SAVE_EVERY_REQUEST = True
# Session timeout age in seconds, important to set above 15 minutes as the client-side
# Angular code is set to automatically timeout after 15 minutes, but will use the heartbeat
# API endpoint to update on any mouse or keyboard activity once a minute
SESSION_COOKIE_AGE = 1200

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            BASE_DIR + '/templates',
            BASE_DIR + '/authenticate/templates'
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'lap.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

REST_FRAMEWORK = {
    'PAGINATE_BY': None,
    'PAGINATE_BY_PARAM': 'paginate_by',
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.SessionAuthentication',
        'rest_framework.authentication.TokenAuthentication',
    ),
    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',)
}

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

MEDIA_ROOT = BASE_DIR + '/media/'
MEDIA_URL = '/media/'
<|endoftext|>


<commit_before>"""
    Schema for event objects.
"""

schema = {
    "description": "event data",
    "properties": {

        # == Basics ==

        # **_type** - All larvae events must have a _type field set to one
        # of tne entries in the enum below.
        "_type": { "enum": [ "event" ], "type": "string" },

        # **name** - A simple name of the event, such as
        # "Fiscal subcommittee hearing on pudding cups"
        "name": { "type": "string" },

        # **description** - A longer description describing the event. As
        # an example, "Topics for discussion include this that and the other
        # thing. In addition, lunch will be served".
        "description": { "type": "string", "required": False },

        # **end** - Ending date / time of the event.
        "end": { "type": ["datetime", "null"] },

        # **when** - Starting date / time of the event.
        "when": { "type": ["datetime"] },

        # **status** - Simple boolean if this event has been canceled.
        "status": { "type": ["string", "null"],
                    "enum": ["cancelled", "tentative", "confirmed"] },

        # **location** - Where the event will take place.
        "location": {
            "type": "object",
            "properties": {

                # **name** - name of the location, such as "City Hall, Boston,
                # MA, USA", or "Room E201, Dolan Science Center, 20700 North
                # Park Blvd University Heights Ohio, 44118"
                "name": { "type": "string" },

                # **note** - human readable notes regarding the location,
                # something like "The meeting will take place at the
                # Minority Whip's desk on the floor"
                "note": { "type": ["string", "null"] },

                "coordinates": {
                    "type": "object",
                    "properties": {
                        # **latitude** - latitude of the location, if any
                        "latitude": {"type": ["string", "null"]},

                        # **longitude** - longitude of the location, if any
                        "longitude": {"type": ["string", "null"]}
                    }
                },
            },
        },

        # == Linked Entities ==

        # **documents** - Links to related documents for the event. Usually,
        # this includes things like pre-written testimony, spreadsheets or
        # a slide deck that a presenter will use.
        "documents": {
            "items": {
                "properties": {
                    # * **note** - name of the document. Something like
                    # "Fiscal Report" or "John Smith's Slides".
                    "name": { "type": "string" },
                    # * **url** - URL where the content may be found.
                    "url": { "type": "string" }

                },
                "type": "object"
            },
            "type": "array"
        },

        # **links** - Links related to the event that are not documents
        # or items in the Agenda. This is filled with helpful links for the
        # event, such as a committee's homepage, reference material or
        # links to learn more about subjects related to the event.
        "links": {
            "description": "URLs for documents about the event",
            "items": {
                "properties": {

                    # * **note** - Human-readable name of the link. Something
                    # like "Historical precedent for popsicle procurement"
                    "note": {
                        "description": "A note, e.g. 'Wikipedia page'",
                        "type": ["string", "null"]
                    },

                    # * **url** - URL where the content may be found
                    "url": {
                        "description": "A URL for a link about the event",
                        "format": "uri",
                        "type": "string"
                    }
                },
                "type": "object"
            },
            "type": "array"
        },

        # **participants** - List of participants in the event. This includes
        # committees invited, legislators chairing the event or people who
        # are attending.
        #
        # Some entries:
        #
        #      { "participant": "John Q. Smith",
        #        "participant_type": "person", "type": "chair" }
        #
        # Which expresses the chair of the event will be John Q. Smith.
        #
        #     { "participant": "Ways and Means",
        #       "participant_type": "organization", "type": "host" }
        #
        # Which expresses the host of the event is the Ways and Means
        # committee.
        "participants": {
            "items": {
                "properties": {
                    # * **chamber** - Optional field storing the chamber
                    # of the related participant.
                    "chamber": {"type": ["string", "null"]},

                    # * **participant** - Human readable name of the entitity.
                    "participant": { "type": "string" },

                    # * **participant_id** - ID of the participant
                    "participant_id": { "type": "string",
                                        "required": False },

                    # * **participant_type** - What type of entity is this?
                    # `person` may be used if the person is not a Legislator,
                    # butattending the event, such as an invited speaker or one
                    # who is offering testimony.
                    "participant_type": {
                        "enum": [ "organization", "person", ],
                        "type": "string"
                    },

                    # * **type** - Role of the entity we're relating to, such
                    # as `chair` for the chair of a meeting.
                    "type": {
                        "enum": [ "host", "chair", "participant" ],
                        "type": "string"
                    },
                    "notes": {
                        "items": {
                            "properties": {
                                "description": { "type": "string" },
                            },
                            "type": "object"
                        },
                        "type": "array"
                    },
                },
                "type": "object"
            },
            "type": "array"
        },

        # **agenda** - Agenda of the event, if any. This contains information
        # about the meeting's agenda, such as bills to discuss or people to
        # present.
        "agenda": {
            "items": {
                "properties": {
                    # * **description** - Human-readable string that represents
                    # this agenda item. A good example would be something like
                    #
                    # > The Committee will consider SB 2339, HB 100
                    "description": { "type": "string" },
                    "order": {"type": "integer", "required": False },

                    # **subjects** - List of related topics of this agenda
                    # item relates to.
                    "subjects": {
                        "items": { "type": "string" },
                        "type": "array"
                    },

                    # **media** - List of media links this item relates to
                    "media": {
                        "items": {
                            "properties": {
                                "name": { "type": "string" },
                                "type": { "type": "string" },
                                "date": {
                                    "pattern": "^[0-9]{4}(-[0-9]{2}){0,2}$",
                                    "type": ["string", "null"]
                                },
                                "links": {
                                    "items": {
                                        "properties": {
                                            "mimetype": {
                                                "type": ["string", "null"]
                                            },
                                            "url": { "type": "string" },
                                            "offset": {
                                                "type": "integer",
                                                "required": False,
                                            }
                                        },
                                        "type": "object"
                                    },
                                    "type": "array"
                                },
                            },
                            "type": "object"
                        },
                        "type": "array"
                    },

                    # * **related_entities** - Entities that relate to this
                    # agenda item, such as presenters, legislative instruments,
                    # or committees.
                    "related_entities": {
                        "properties": {

                            # * * **type** - type of relation, such as
                            # `consideration` or `presenter`.
                            "type": { "type": "string" },

                            # * * **id** - ID of the related entity
                            "id": { "type": "string" },

                            # * * **name** - human readable string
                            # representing the entity, such as `John Q. Smith`.
                            "name": { "type": "string" },

                            # * * **note** - human readable string (if any)
                            # noting the relationship between the entity and
                            # the agenda item, such as "Jeff will be presenting
                            # on the effects of too much cookie dough"
                            "note": { "type": "string" },

                            "type": "object"
                        },
                        "minItems": 0,
                        "type": "array",
                    },
                },
                "type": "object"
            },
            "minItems": 0,
            "type": "array"
        },

        # == Sources ==

        # **sources** - like all larvae objects, sources is an array of one
        # or more source object.  Each source object has the following
        # properties:
        #
        # * **url** - URL to resource used to collect
        # * **note** - Note about what this URL was used to collect.
        "sources": {
            "items": {
                "properties": {
                    "url": {
                        "type": "string"
                    },
                    "note": {
                        "type": ["null", "string"],
                    }
                },
                "type": "object"
            },
            "minItems": 1,
            "type": "array"
        },

        "type": {
            "type": "string"
        },
    },
    "type": "object"
}
<commit_msg>Fixing up a few typos; adding basic votes data<commit_after>"""
    Schema for event objects.
"""

schema = {
    "description": "event data",
    "properties": {
        # == Basics ==

        # **_type** - All larvae events must have a _type field set to one
        # of tne entries in the enum below.
        "_type": { "enum": [ "event" ], "type": "string" },

        # **name** - A simple name of the event, such as
        # "Fiscal subcommittee hearing on pudding cups"
        "name": { "type": "string" },

        # **description** - A longer description describing the event. As
        # an example, "Topics for discussion include this that and the other
        # thing. In addition, lunch will be served".
        "description": { "type": "string", "required": False },

        # **end** - Ending date / time of the event.
        "end": { "type": ["datetime", "null"] },

        # **when** - Starting date / time of the event.
        "when": { "type": ["datetime"] },

        # **status** - String that denotes the status of the meeting. This is
        # useful for showing the meeting is cancelled in a machine-readable
        # way.
        "status": { "type": ["string", "null"],
                    "enum": ["cancelled", "tentative", "confirmed"] },

        # **location** - Where the event will take place.
        "location": {
            "type": "object",
            "properties": {

                # * **name** - name of the location, such as "City Hall, Boston,
                # MA, USA", or "Room E201, Dolan Science Center, 20700 North
                # Park Blvd University Heights Ohio, 44118"
                "name": { "type": "string" },

                # * **note** - human readable notes regarding the location,
                # something like "The meeting will take place at the
                # Minority Whip's desk on the floor"
                "note": { "type": ["string", "null"] },

                # * **coordinates** - coordinates where this event will take
                # place. This is purely optional.
                "coordinates": {
                    "type": "object",
                    "required": False,
                    "properties": {
                        # * * **latitude** - latitude of the location, if any
                        "latitude": {"type": ["string", "null"]},

                        # * * **longitude** - longitude of the location, if any
                        "longitude": {"type": ["string", "null"]}
                    }
                },
            },
        },

        # == Linked Entities ==

        # **documents** - Links to related documents for the event. Usually,
        # this includes things like pre-written testimony, spreadsheets or
        # a slide deck that a presenter will use.
        "documents": {
            "items": {
                "properties": {
                    # * **note** - name of the document. Something like
                    # "Fiscal Report" or "John Smith's Slides".
                    "name": { "type": "string" },
                    # * **url** - URL where the content may be found.
                    "url": { "type": "string" }

                },
                "type": "object"
            },
            "type": "array"
        },

        # **links** - Links related to the event that are not documents
        # or items in the Agenda. This is filled with helpful links for the
        # event, such as a committee's homepage, reference material or
        # links to learn more about subjects related to the event.
        "links": {
            "description": "URLs for documents about the event",
            "items": {
                "properties": {

                    # * **note** - Human-readable name of the link. Something
                    # like "Historical precedent for popsicle procurement"
                    "note": {
                        "description": "A note, e.g. 'Wikipedia page'",
                        "type": ["string", "null"]
                    },

                    # * **url** - URL where the content may be found
                    "url": {
                        "description": "A URL for a link about the event",
                        "format": "uri",
                        "type": "string"
                    }
                },
                "type": "object"
            },
            "type": "array"
        },

        # **participants** - List of participants in the event. This includes
        # committees invited, legislators chairing the event or people who
        # are attending.
        #
        # Some entries:
        #
        #      { "participant": "John Q. Smith",
        #        "participant_type": "person", "type": "chair" }
        #
        # Which expresses the chair of the event will be John Q. Smith.
        #
        #     { "participant": "Ways and Means",
        #       "participant_type": "organization", "type": "host" }
        #
        # Which expresses the host of the event is the Ways and Means
        # committee.
        "participants": {
            "items": {
                "properties": {
                    # * **chamber** - Optional field storing the chamber
                    # of the related participant.
                    "chamber": {"type": ["string", "null"]},

                    # * **participant** - Human readable name of the entitity.
                    "participant": { "type": "string" },

                    # * **participant_id** - ID of the participant
                    "participant_id": { "type": "string",
                                        "required": False },

                    # * **participant_type** - What type of entity is this?
                    # `person` may be used if the person is not a Legislator,
                    # butattending the event, such as an invited speaker or one
                    # who is offering testimony.
                    "participant_type": {
                        "enum": [ "organization", "person", ],
                        "type": "string"
                    },

                    # * **votes** - This field may be used if the person is
                    # eligible to vote, and may contain a numerical value
                    # (including decimal points, in some locales) denoting how
                    # many votes they have. This may be ommited.
                    "votes": {
                        "type": "number",
                        "required": False
                    },

                    # * **type** - Role of the entity we're relating to, such
                    # as `chair` for the chair of a meeting.
                    "type": {
                        "enum": [ "host", "chair", "participant" ],
                        "type": "string"
                    },

                },
                "type": "object"
            },
            "type": "array"
        },

        # **agenda** - Agenda of the event, if any. This contains information
        # about the meeting's agenda, such as bills to discuss or people to
        # present.
        "agenda": {
            "items": {
                "properties": {
                    # * **description** - Human-readable string that represents
                    # this agenda item. A good example would be something like
                    #
                    # > The Committee will consider SB 2339, HB 100
                    "description": { "type": "string" },
                    "order": {"type": "integer", "required": False },

                    # **subjects** - List of related topics of this agenda
                    # item relates to.
                    "subjects": {
                        "items": { "type": "string" },
                        "type": "array"
                    },

                    # **media** - List of media links this item relates to
                    "media": {
                        "items": {
                            "properties": {
                                "name": { "type": "string" },
                                "type": { "type": "string" },
                                "date": {
                                    "pattern": "^[0-9]{4}(-[0-9]{2}){0,2}$",
                                    "type": ["string", "null"]
                                },
                                "links": {
                                    "items": {
                                        "properties": {
                                            "mimetype": {
                                                "type": ["string", "null"]
                                            },
                                            "url": { "type": "string" },
                                            "offset": {
                                                "type": "integer",
                                                "required": False,
                                            }
                                        },
                                        "type": "object"
                                    },
                                    "type": "array"
                                },
                            },
                            "type": "object"
                        },
                        "type": "array"
                    },

                    # * **notes** - List of notes taken during this agenda item,
                    # may be used to construct meeting minutes.
                    "notes": {
                        "required": False,
                        "items": {
                            "properties": {
                                "description": { "type": "string" },
                            },
                            "type": "object"
                        },
                        "type": "array"
                    },

                    # * **related_entities** - Entities that relate to this
                    # agenda item, such as presenters, legislative instruments,
                    # or committees.
                    "related_entities": {
                        "properties": {

                            # * * **type** - type of relation, such as
                            # `consideration` or `presenter`.
                            "type": { "type": "string" },

                            # * * **id** - ID of the related entity
                            "id": { "type": "string" },

                            # * * **name** - human readable string
                            # representing the entity, such as `John Q. Smith`.
                            "name": { "type": "string" },

                            # * * **note** - human readable string (if any)
                            # noting the relationship between the entity and
                            # the agenda item, such as "Jeff will be presenting
                            # on the effects of too much cookie dough"
                            "note": { "type": "string" },

                            "type": "object"
                        },
                        "minItems": 0,
                        "type": "array",
                    },
                },
                "type": "object"
            },
            "minItems": 0,
            "type": "array"
        },

        # == Sources ==

        # **sources** - like all larvae objects, sources is an array of one
        # or more source object.  Each source object has the following
        # properties:
        #
        # * **url** - URL to resource used to collect
        # * **note** - Note about what this URL was used to collect.
        "sources": {
            "items": {
                "properties": {
                    "url": {
                        "type": "string"
                    },
                    "note": {
                        "type": ["null", "string"],
                    }
                },
                "type": "object"
            },
            "minItems": 1,
            "type": "array"
        },

   },
    "type": "object"
}
<|endoftext|>


<commit_before># Bootstraps the Roots instance with the default
# set of the necessary objects

import datetime

site = Site.objects.get_current()

competition = Competition(
  name="Sample competition",
  site=site
)

competition.save()

season = Season(
    competition=competition,
    name="First season",
    year=datetime.date.today().year,
    number=1,
    start=datetime.datetime.now(),
    end=datetime.datetime.now()+datetime.timedelta(days=30),
)
season.save()

problemset_series_1 = ProblemSet(
    name="Problems in the first series",
    competition=competition
)
problemset_series_1.save()

series = Series(
   season=season,
   name="First series",
   number=1,
   submission_deadline=datetime.datetime.now()+datetime.timedelta(days=30),
   problemset=problemset_series_1
)
series.save()
<commit_msg>bootstrap: Create admin superuser<commit_after># Bootstraps the Roots instance with the default
# set of the necessary objects

import datetime

site = Site.objects.get_current()

# Create the admin user
admin = User.objects.create(
    username="rootsadmin",
    email="rootsadmin@example.com"
)

# Verify his email
email = admin.emailaddress_set.get(pk=1)
email.verified = True
email.save()

season = Season(
    competition=competition,
    name="First season",
    year=datetime.date.today().year,
    number=1,
    start=datetime.datetime.now(),
    end=datetime.datetime.now()+datetime.timedelta(days=30),
)
season.save()

problemset_series_1 = ProblemSet(
    name="Problems in the first series",
    competition=competition
)
problemset_series_1.save()

series = Series(
   season=season,
   name="First series",
   number=1,
   submission_deadline=datetime.datetime.now()+datetime.timedelta(days=30),
   problemset=problemset_series_1
)
series.save()
<|endoftext|>


<commit_before>'''
Created on 28 Nov 2013

@author: jmht
'''

import os
import re

class ShelxeLogParser(object):
    """
    Class to mine information from a shelxe log
    """

    def __init__(self,logfile):

        self.logfile = logfile

        # All data refers to the best cycle
        self.CC = None
        self.avgChainLength = None
        self.maxChainLength = None
        self.numChains = None
        self.cycle = None
        self.cputime = None

        self.parse()

        return

    def parse(self):
        """Parse a shelxe log file to get the CC and average Chain length
        """

        cycleData = [] # List (CC,avgChainLength) tuples - ordered by cycle
        fh = open( self.logfile, 'r')

        line = fh.readline()
        while line:

            # find should be quicker then re match
            if line.find("residues left after pruning, divided into chains as follows:") != -1:
                (cc, avgChainLength, maxChainLength, numChains ) = self._parseCycle(fh)
                cycleData.append( (cc, avgChainLength, maxChainLength, numChains) )

            if  line.find( "Best trace (cycle" ) != -1:
                # Expecting:
                #  "Best trace (cycle   1 with CC 37.26%) was saved as shelxe-input.pdb"
                cycle = int( re.search("\s\d+\s", line).group(0) )
                cc = float( re.search("\s\d+\.\d+", line).group(0) )

                # Check it matches
                if cycleData[ cycle-1 ][0] != cc:
                    raise RuntimeError,"Error getting final CC!"

                self.CC =  cycleData[ cycle-1 ][0]
                self.avgChainLength = cycleData[ cycle-1 ][1]
                self.maxChainLength = cycleData[ cycle-1 ][2]
                self.numChains = cycleData[ cycle-1 ][3]
                self.cycle = cycle

            #+  SHELXE finished at 23:44:41      Total time:      4417.30 secs  +
            if line.find("+  SHELXE finished at") != -1:
                self.cputime = float( line.split()[7] )

            line = fh.readline()
        #End while

        fh.close()

        return

    def _parseCycle(self, fh):
        """
        Working on assumption each cycle contains something like the below:
<log>
           223 residues left after pruning, divided into chains as follows:
 A:   6   B:   7   C:   6   D:   6   E:   8   F:   7   G:  12   H:  12   I:   5
 J:  10   K:   6   L:   6   M:   6   N:   7   O:   6   P:   7   Q:   8   R:   6
 S:   5   T:   6   U:  10   V:   9   W:  12   X:  11   Y:   8   Z:   6   Z:   6
 Z:   6   Z:   7   Z:   6

 CC for partial structure against native data =  30.92 %
 </log>
 """

        lengths = []
        while True:

            line = fh.readline().strip()
            line = line.rstrip(os.linesep)
            if not line:
                # End of reading lengths
                break

            # Loop through integers & add to list
            for m in re.finditer("\s\d+", line):
                lengths.append( int(m.group(0)) )

        # Now calculate average chain length
        if not len( lengths ):
            raise RuntimeError, "Failed to read any fragment lengths"

        # Average chain lengths
        avgChainLength = sum(lengths) / int( len(lengths) )
        maxChainLength = max(lengths)
        numChains = len(lengths)

        # Here should have read the  lengths so now just get the CC
        count=0
        while True:
            line = fh.readline().strip()
            if line.startswith("CC for partial structure against native data"):
                break
            else:
                count += 1
                if count > 5:
                    raise RuntimeError,"Error parsing CC score"

        cc = float( re.search("\d+\.\d+", line).group(0) )

        return ( cc, avgChainLength, maxChainLength, numChains )

#END ShelxeLogParser

#slog = "/media/data/shared/coiled-coils/ensemble/1BYZ/ROSETTA_MR_0/MRBUMP/cluster_1/search_All_atom_trunc_0.005734_rad_1_mrbump/data/loc0_ALL_All_atom_trunc_0.005734_rad_1/unmod/mr/phaser/build/shelxe/shelxe_run.log"
#sp = ShelxeLogParser( slog )
<commit_msg>Add a shelxe pdb parser<commit_after>'''
Created on 28 Nov 2013

@author: jmht
'''

import os
import re

class ShelxePdbParser(object):
    #UNTESTED!!
    def __init__(self, pdbfile):
        self.CC             = None
        self.numResidues    = None
        self.numChains      = None

        return self.parse( pdbfile )

    def parse(self, pdbfile):
        with open(pdbfile) as f:
            line = f.readline.strip()

        tokens = line.split()
        assert line.startswith("TITLE") and len( tokens ) == 12

        self.CC             = tokens[6][:-1] # Remove % from end
        self.numResidues    = int(tokens[7])
        self.numChains      = int(tokens[10])

        return

class ShelxeLogParser(object):
    """
    Class to mine information from a shelxe log
    """

    def __init__(self,logfile):

        self.logfile = logfile

        # All data refers to the best cycle
        self.CC = None
        self.avgChainLength = None
        self.maxChainLength = None
        self.numChains = None
        self.cycle = None
        self.cputime = None

        self.parse()

        return

    def parse(self):
        """Parse a shelxe log file to get the CC and average Chain length
        """

        cycleData = [] # List (CC,avgChainLength) tuples - ordered by cycle
        fh = open( self.logfile, 'r')

        line = fh.readline()
        while line:

            # find should be quicker then re match
            if line.find("residues left after pruning, divided into chains as follows:") != -1:
                (cc, avgChainLength, maxChainLength, numChains ) = self._parseCycle(fh)
                cycleData.append( (cc, avgChainLength, maxChainLength, numChains) )

            if  line.find( "Best trace (cycle" ) != -1:
                # Expecting:
                #  "Best trace (cycle   1 with CC 37.26%) was saved as shelxe-input.pdb"
                cycle = int( re.search("\s\d+\s", line).group(0) )
                cc = float( re.search("\s\d+\.\d+", line).group(0) )

                # Check it matches
                if cycleData[ cycle-1 ][0] != cc:
                    raise RuntimeError,"Error getting final CC!"

                self.CC =  cycleData[ cycle-1 ][0]
                self.avgChainLength = cycleData[ cycle-1 ][1]
                self.maxChainLength = cycleData[ cycle-1 ][2]
                self.numChains = cycleData[ cycle-1 ][3]
                self.cycle = cycle

            #+  SHELXE finished at 23:44:41      Total time:      4417.30 secs  +
            if line.find("+  SHELXE finished at") != -1:
                self.cputime = float( line.split()[7] )

            line = fh.readline()
        #End while

        fh.close()

        return

    def _parseCycle(self, fh):
        """
        Working on assumption each cycle contains something like the below:
<log>
           223 residues left after pruning, divided into chains as follows:
 A:   6   B:   7   C:   6   D:   6   E:   8   F:   7   G:  12   H:  12   I:   5
 J:  10   K:   6   L:   6   M:   6   N:   7   O:   6   P:   7   Q:   8   R:   6
 S:   5   T:   6   U:  10   V:   9   W:  12   X:  11   Y:   8   Z:   6   Z:   6
 Z:   6   Z:   7   Z:   6

 CC for partial structure against native data =  30.92 %
 </log>
 """

        lengths = []
        while True:

            line = fh.readline().strip()
            line = line.rstrip(os.linesep)
            if not line:
                # End of reading lengths
                break

            # Loop through integers & add to list
            for m in re.finditer("\s\d+", line):
                lengths.append( int(m.group(0)) )

        # Now calculate average chain length
        if not len( lengths ):
            raise RuntimeError, "Failed to read any fragment lengths"

        # Average chain lengths
        avgChainLength = sum(lengths) / int( len(lengths) )
        maxChainLength = max(lengths)
        numChains = len(lengths)

        # Here should have read the  lengths so now just get the CC
        count=0
        while True:
            line = fh.readline().strip()
            if line.startswith("CC for partial structure against native data"):
                break
            else:
                count += 1
                if count > 5:
                    raise RuntimeError,"Error parsing CC score"

        cc = float( re.search("\d+\.\d+", line).group(0) )

        return ( cc, avgChainLength, maxChainLength, numChains )

#END ShelxeLogParser

#slog = "/media/data/shared/coiled-coils/ensemble/1BYZ/ROSETTA_MR_0/MRBUMP/cluster_1/search_All_atom_trunc_0.005734_rad_1_mrbump/data/loc0_ALL_All_atom_trunc_0.005734_rad_1/unmod/mr/phaser/build/shelxe/shelxe_run.log"
#sp = ShelxeLogParser( slog )
<|endoftext|>


<commit_before>from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import pgettext_lazy

QUANTITY_FIELD_INTEGER = 0
QUANTITY_FIELD_DECIMAL_1 = 1
QUANTITY_FIELD_DECIMAL_2 = 2

QUANTITY_FIELD_TYPES = [
    (QUANTITY_FIELD_INTEGER, _(u"Integer")),
    (QUANTITY_FIELD_DECIMAL_1, _(u"Decimal 0.1")),
    (QUANTITY_FIELD_DECIMAL_2, _(u"Decimal 0.01")),
]

CHOICES_STANDARD = 0
CHOICES_YES = 2
CHOICES_NO = 3
CHOICES = [
    (CHOICES_STANDARD, _(u"Standard")),
    (CHOICES_YES, _(u"Yes")),
    (CHOICES_NO, _(u"No")),
]

STANDARD_PRODUCT = "0"
PRODUCT_WITH_VARIANTS = "1"
VARIANT = "2"
CONFIGURABLE_PRODUCT = "3"

PRODUCT_TYPE_LOOKUP = {
    STANDARD_PRODUCT: _(u"Standard"),
    PRODUCT_WITH_VARIANTS: _(u"Product with variants"),
    VARIANT: _(u"Variant"),
    CONFIGURABLE_PRODUCT: _(u"Configurable product")
}

PRODUCT_TYPE_CHOICES = [
    (STANDARD_PRODUCT, _(u"Standard")),
    (PRODUCT_WITH_VARIANTS, _(u"Product with variants")),
    (VARIANT, _(u"Variant")),
    (CONFIGURABLE_PRODUCT, _(u"Configurable product")),
]

PRODUCT_TYPE_FORM_CHOICES = [
    (STANDARD_PRODUCT, _(u"Standard")),
    (PRODUCT_WITH_VARIANTS, _(u"Product with variants")),
    (CONFIGURABLE_PRODUCT, _(u"Configurable product")),
]

CATEGORY_VARIANT_DEFAULT = -1
CATEGORY_VARIANT_CHEAPEST_PRICE = -2
CATEGORY_VARIANT_CHEAPEST_BASE_PRICE = -3
CATEGORY_VARIANT_CHEAPEST_PRICES = -4
CATEGORY_VARIANT_CHOICES = [
    (CATEGORY_VARIANT_DEFAULT, _(u"Default")),
    (CATEGORY_VARIANT_CHEAPEST_PRICE, _(u"Cheapest price")),
    (CATEGORY_VARIANT_CHEAPEST_BASE_PRICE, _(u"Cheapest base price")),
    (CATEGORY_VARIANT_CHEAPEST_PRICES, _(u"Cheapest prices")),
]

LIST = 0
SELECT = 1
VARIANTS_DISPLAY_TYPE_CHOICES = [
    (LIST, _(u"List")),
    (SELECT, _(u"Select")),
]

CONTENT_PRODUCTS = 1
CONTENT_CATEGORIES = 2

CONTENT_CHOICES = (
    (CONTENT_PRODUCTS, _(u"Products")),
    (CONTENT_CATEGORIES, _(u"Categories")),
)

DELIVERY_TIME_UNIT_HOURS = 1
DELIVERY_TIME_UNIT_DAYS = 2
DELIVERY_TIME_UNIT_WEEKS = 3
DELIVERY_TIME_UNIT_MONTHS = 4

DELIVERY_TIME_UNIT_CHOICES = (
    (DELIVERY_TIME_UNIT_HOURS, pgettext_lazy(u"Delivery time", u"hours")),
    (DELIVERY_TIME_UNIT_DAYS, pgettext_lazy(u"Delivery time", u"days")),
    (DELIVERY_TIME_UNIT_WEEKS, pgettext_lazy(u"Delivery time", u"weeks")),
    (DELIVERY_TIME_UNIT_MONTHS, pgettext_lazy(u"Delivery time", u"months")),
)

DELIVERY_TIME_UNIT_SINGULAR = {
    DELIVERY_TIME_UNIT_HOURS: pgettext_lazy(u"Delivery time", u"hour"),
    DELIVERY_TIME_UNIT_DAYS: pgettext_lazy(u"Delivery time", u"day"),
    DELIVERY_TIME_UNIT_WEEKS: pgettext_lazy(u"Delivery time", u"week"),
    DELIVERY_TIME_UNIT_MONTHS: pgettext_lazy(u"Delivery time", u"month"),
}

PROPERTY_VALUE_TYPE_FILTER = 0
PROPERTY_VALUE_TYPE_DEFAULT = 1
PROPERTY_VALUE_TYPE_DISPLAY = 2
PROPERTY_VALUE_TYPE_VARIANT = 3

PROPERTY_NUMBER_FIELD = 1
PROPERTY_TEXT_FIELD = 2
PROPERTY_SELECT_FIELD = 3

PROPERTY_FIELD_CHOICES = (
    (PROPERTY_NUMBER_FIELD, _(u"Float field")),
    (PROPERTY_TEXT_FIELD, _(u"Text field")),
    (PROPERTY_SELECT_FIELD, _(u"Select field")),
)

PROPERTY_STEP_TYPE_AUTOMATIC = 1
PROPERTY_STEP_TYPE_FIXED_STEP = 2
PROPERTY_STEP_TYPE_MANUAL_STEPS = 3

PROPERTY_STEP_TYPE_CHOICES = (
    (PROPERTY_STEP_TYPE_AUTOMATIC, _(u"Automatic")),
    (PROPERTY_STEP_TYPE_FIXED_STEP, _(u"Fixed step")),
    (PROPERTY_STEP_TYPE_MANUAL_STEPS, _(u"Manual steps")),
)

CAT_PRODUCT_PATH = "lfs/catalog/categories/product"   # category with products
CAT_CATEGORY_PATH = "lfs/catalog/categories/category"  # category with subcategories
PRODUCT_PATH = "lfs/catalog/products"   # product templates
IMAGES_PATH = "/media/lfs/icons"  # Path to template preview images

# Template configuration for category display
CATEGORY_TEMPLATES = (
    (0, {"file": "%s/%s" % (CAT_PRODUCT_PATH, "default.html"),
         "image": IMAGES_PATH + "/product_default.png",
         "name": _(u"Category with products"),
         }),
    (1, {"file": "%s/%s" % (CAT_CATEGORY_PATH, "default.html"),
         "image": IMAGES_PATH + "/category_square.png",
         "name": _(u"Category with subcategories"),
         }),
)
CATEGORY_TEMPLATES = getattr(settings, 'CATEGORY_TEMPLATES', CATEGORY_TEMPLATES)

# Template configuration for product display
PRODUCT_TEMPLATES = (
    (0, {"file": "%s/%s" % (PRODUCT_PATH, "product_inline.html"),
         "image": IMAGES_PATH + "/product_default.png",
         "name": _(u"Default")
         },),
)
PRODUCT_TEMPLATES = getattr(settings, 'PRODUCT_TEMPLATES', PRODUCT_TEMPLATES)

THUMBNAIL_SIZES = getattr(settings, 'LFS_THUMBNAIL_SIZES', ((60, 60), (100, 100), (200, 200), (300, 300), (400, 400)))
DELETE_FILES = getattr(settings, "LFS_DELETE_FILES", True)
DELETE_IMAGES = getattr(settings, "LFS_DELETE_IMAGES", True)
if getattr(settings, 'SOLR_ENABLED', False):
    SORTING_MAP = (
        {'default': 'effective_price', 'ftx': 'price asc', 'title': _('Price ascending')},
        {'default': '-effective_price', 'ftx': 'price desc', 'title': _('Price descending')},
        {'default': 'name', 'ftx': 'name asc', 'title': _('Name ascending')},
        {'default': '-name', 'ftx': 'name desc', 'title': _('Name descending')},
    )
else:
    SORTING_MAP = (
        {'default': 'effective_price', 'ftx': 'price', 'title': _('Price ascending')},
        {'default': '-effective_price', 'ftx': '-price', 'title': _('Price descending')},
        {'default': 'name', 'ftx': 'name', 'title': _('Name ascending')},
        {'default': '-name', 'ftx': '-name', 'title': _('Name descending')},
    )
<commit_msg>settings: add more image sizes<commit_after>from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import pgettext_lazy

QUANTITY_FIELD_INTEGER = 0
QUANTITY_FIELD_DECIMAL_1 = 1
QUANTITY_FIELD_DECIMAL_2 = 2

QUANTITY_FIELD_TYPES = [
    (QUANTITY_FIELD_INTEGER, _(u"Integer")),
    (QUANTITY_FIELD_DECIMAL_1, _(u"Decimal 0.1")),
    (QUANTITY_FIELD_DECIMAL_2, _(u"Decimal 0.01")),
]

CHOICES_STANDARD = 0
CHOICES_YES = 2
CHOICES_NO = 3
CHOICES = [
    (CHOICES_STANDARD, _(u"Standard")),
    (CHOICES_YES, _(u"Yes")),
    (CHOICES_NO, _(u"No")),
]

STANDARD_PRODUCT = "0"
PRODUCT_WITH_VARIANTS = "1"
VARIANT = "2"
CONFIGURABLE_PRODUCT = "3"

PRODUCT_TYPE_LOOKUP = {
    STANDARD_PRODUCT: _(u"Standard"),
    PRODUCT_WITH_VARIANTS: _(u"Product with variants"),
    VARIANT: _(u"Variant"),
    CONFIGURABLE_PRODUCT: _(u"Configurable product")
}

PRODUCT_TYPE_CHOICES = [
    (STANDARD_PRODUCT, _(u"Standard")),
    (PRODUCT_WITH_VARIANTS, _(u"Product with variants")),
    (VARIANT, _(u"Variant")),
    (CONFIGURABLE_PRODUCT, _(u"Configurable product")),
]

PRODUCT_TYPE_FORM_CHOICES = [
    (STANDARD_PRODUCT, _(u"Standard")),
    (PRODUCT_WITH_VARIANTS, _(u"Product with variants")),
    (CONFIGURABLE_PRODUCT, _(u"Configurable product")),
]

CATEGORY_VARIANT_DEFAULT = -1
CATEGORY_VARIANT_CHEAPEST_PRICE = -2
CATEGORY_VARIANT_CHEAPEST_BASE_PRICE = -3
CATEGORY_VARIANT_CHEAPEST_PRICES = -4
CATEGORY_VARIANT_CHOICES = [
    (CATEGORY_VARIANT_DEFAULT, _(u"Default")),
    (CATEGORY_VARIANT_CHEAPEST_PRICE, _(u"Cheapest price")),
    (CATEGORY_VARIANT_CHEAPEST_BASE_PRICE, _(u"Cheapest base price")),
    (CATEGORY_VARIANT_CHEAPEST_PRICES, _(u"Cheapest prices")),
]

LIST = 0
SELECT = 1
VARIANTS_DISPLAY_TYPE_CHOICES = [
    (LIST, _(u"List")),
    (SELECT, _(u"Select")),
]

CONTENT_PRODUCTS = 1
CONTENT_CATEGORIES = 2

CONTENT_CHOICES = (
    (CONTENT_PRODUCTS, _(u"Products")),
    (CONTENT_CATEGORIES, _(u"Categories")),
)

DELIVERY_TIME_UNIT_HOURS = 1
DELIVERY_TIME_UNIT_DAYS = 2
DELIVERY_TIME_UNIT_WEEKS = 3
DELIVERY_TIME_UNIT_MONTHS = 4

DELIVERY_TIME_UNIT_CHOICES = (
    (DELIVERY_TIME_UNIT_HOURS, pgettext_lazy(u"Delivery time", u"hours")),
    (DELIVERY_TIME_UNIT_DAYS, pgettext_lazy(u"Delivery time", u"days")),
    (DELIVERY_TIME_UNIT_WEEKS, pgettext_lazy(u"Delivery time", u"weeks")),
    (DELIVERY_TIME_UNIT_MONTHS, pgettext_lazy(u"Delivery time", u"months")),
)

DELIVERY_TIME_UNIT_SINGULAR = {
    DELIVERY_TIME_UNIT_HOURS: pgettext_lazy(u"Delivery time", u"hour"),
    DELIVERY_TIME_UNIT_DAYS: pgettext_lazy(u"Delivery time", u"day"),
    DELIVERY_TIME_UNIT_WEEKS: pgettext_lazy(u"Delivery time", u"week"),
    DELIVERY_TIME_UNIT_MONTHS: pgettext_lazy(u"Delivery time", u"month"),
}

PROPERTY_VALUE_TYPE_FILTER = 0
PROPERTY_VALUE_TYPE_DEFAULT = 1
PROPERTY_VALUE_TYPE_DISPLAY = 2
PROPERTY_VALUE_TYPE_VARIANT = 3

PROPERTY_NUMBER_FIELD = 1
PROPERTY_TEXT_FIELD = 2
PROPERTY_SELECT_FIELD = 3

PROPERTY_FIELD_CHOICES = (
    (PROPERTY_NUMBER_FIELD, _(u"Float field")),
    (PROPERTY_TEXT_FIELD, _(u"Text field")),
    (PROPERTY_SELECT_FIELD, _(u"Select field")),
)

PROPERTY_STEP_TYPE_AUTOMATIC = 1
PROPERTY_STEP_TYPE_FIXED_STEP = 2
PROPERTY_STEP_TYPE_MANUAL_STEPS = 3

PROPERTY_STEP_TYPE_CHOICES = (
    (PROPERTY_STEP_TYPE_AUTOMATIC, _(u"Automatic")),
    (PROPERTY_STEP_TYPE_FIXED_STEP, _(u"Fixed step")),
    (PROPERTY_STEP_TYPE_MANUAL_STEPS, _(u"Manual steps")),
)

CAT_PRODUCT_PATH = "lfs/catalog/categories/product"   # category with products
CAT_CATEGORY_PATH = "lfs/catalog/categories/category"  # category with subcategories
PRODUCT_PATH = "lfs/catalog/products"   # product templates
IMAGES_PATH = "/media/lfs/icons"  # Path to template preview images

# Template configuration for category display
CATEGORY_TEMPLATES = (
    (0, {"file": "%s/%s" % (CAT_PRODUCT_PATH, "default.html"),
         "image": IMAGES_PATH + "/product_default.png",
         "name": _(u"Category with products"),
         }),
    (1, {"file": "%s/%s" % (CAT_CATEGORY_PATH, "default.html"),
         "image": IMAGES_PATH + "/category_square.png",
         "name": _(u"Category with subcategories"),
         }),
)
CATEGORY_TEMPLATES = getattr(settings, 'CATEGORY_TEMPLATES', CATEGORY_TEMPLATES)

# Template configuration for product display
PRODUCT_TEMPLATES = (
    (0, {"file": "%s/%s" % (PRODUCT_PATH, "product_inline.html"),
         "image": IMAGES_PATH + "/product_default.png",
         "name": _(u"Default")
         },),
)
PRODUCT_TEMPLATES = getattr(settings, 'PRODUCT_TEMPLATES', PRODUCT_TEMPLATES)

THUMBNAIL_SIZES = getattr(settings, 'LFS_THUMBNAIL_SIZES', ((60, 60), (100, 100), (200, 200), (300, 300), (400, 400), (600, 600)))
DELETE_FILES = getattr(settings, "LFS_DELETE_FILES", True)
DELETE_IMAGES = getattr(settings, "LFS_DELETE_IMAGES", True)
if getattr(settings, 'SOLR_ENABLED', False):
    SORTING_MAP = (
        {'default': 'effective_price', 'ftx': 'price asc', 'title': _('Price ascending')},
        {'default': '-effective_price', 'ftx': 'price desc', 'title': _('Price descending')},
        {'default': 'name', 'ftx': 'name asc', 'title': _('Name ascending')},
        {'default': '-name', 'ftx': 'name desc', 'title': _('Name descending')},
    )
else:
    SORTING_MAP = (
        {'default': 'effective_price', 'ftx': 'price', 'title': _('Price ascending')},
        {'default': '-effective_price', 'ftx': '-price', 'title': _('Price descending')},
        {'default': 'name', 'ftx': 'name', 'title': _('Name ascending')},
        {'default': '-name', 'ftx': '-name', 'title': _('Name descending')},
    )
<|endoftext|>


<commit_before>#!/usr/bin/python

#---------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
#---------------------------------------------------------------------------------------------

from __future__ import print_function
import sys
import os
from subprocess import check_call, CalledProcessError

root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..'))

def exec_command(command):
    try:
        print('Executing: ' + command)
        check_call(command.split(), cwd=root_dir)
    except CalledProcessError as err:
        print(err, file=sys.stderr)
        sys.exit(1)

print('Running dev setup...')
print('Root directory \'{}\''.format(root_dir))
exec_command('pip install -r requirements.txt')
# command modules have dependency on azure-cli-core so install this first
exec_command('pip install -e src/azure-cli-core')
exec_command('python scripts/command_modules/install.py')
# azure cli has dependencies on the above packages so install this one last
exec_command('pip install -e src/azure-cli')
print('Finished dev setup.')
<commit_msg>Install the requirements last (Fixes #1029) (#1038)<commit_after>#!/usr/bin/python

#---------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
#---------------------------------------------------------------------------------------------

from __future__ import print_function
import sys
import os
from subprocess import check_call, CalledProcessError

root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..'))

def exec_command(command):
    try:
        print('Executing: ' + command)
        check_call(command.split(), cwd=root_dir)
    except CalledProcessError as err:
        print(err, file=sys.stderr)
        sys.exit(1)

print('Running dev setup...')
print('Root directory \'{}\''.format(root_dir))
# command modules have dependency on azure-cli-core so install this first
exec_command('pip install -e src/azure-cli-core')
exec_command('python scripts/command_modules/install.py')
# azure cli has dependencies on the above packages so install this one last
exec_command('pip install -e src/azure-cli')
exec_command('pip install -r requirements.txt')
print('Finished dev setup.')
<|endoftext|>