<commit_before># -*- coding: utf-8 -*-
"""
    pygments.cmdline
    ~~~~~~~~~~~~~~~~

    Command line interface.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
import sys
import getopt
from textwrap import dedent

from pygments import __version__, highlight
from pygments.util import ClassNotFound, OptionError, docstring_headline
from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
     find_lexer_class, guess_lexer, TextLexer
from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
     TerminalFormatter  # pylint:disable-msg=E0611
from pygments.filters import get_all_filters, find_filter_class
from pygments.styles import get_all_styles, get_style_by_name

USAGE = """\
Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
          [-O <options>] [-P <option=value>] [-o <outfile>] [<infile>]

       %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
       %s -L [<which> ...]
       %s -N <filename>
       %s -H <type> <name>
       %s -h | -V

Highlight the input file and write the result to <outfile>.

If no input file is given, use stdin, if -o is not given, use stdout.

<lexer> is a lexer name (query all lexer names with -L). If -l is not
given, the lexer is guessed from the extension of the input file name
(this obviously doesn't work if the input is stdin).  If -g is passed,
attempt to guess the lexer from the file contents, or pass through as
plain text if this fails (this can work for stdin).

Likewise, <formatter> is a formatter name, and will be guessed from
the extension of the output file name. If no output file is given,
the terminal formatter will be used by default.

With the -O option, you can give the lexer and formatter a comma-
separated list of options, e.g. ``-O bg=light,python=cool``.

The -P option adds lexer and formatter options like the -O option, but
you can only give one option per -P. That way, the option value may
contain commas and equals signs, which it can't with -O, e.g.
``-P "heading=Pygments, the Python highlighter".

With the -F option, you can add filters to the token stream, you can
give options in the same way as for -O after a colon (note: there must
not be spaces around the colon).

The -O, -P and -F options can be given multiple times.

With the -S option, print out style definitions for style <style>
for formatter <formatter>. The argument given by -a is formatter
dependent.

The -L option lists lexers, formatters, styles or filters -- set
`which` to the thing you want to list (e.g. "styles"), or omit it to
list everything.

The -N option guesses and prints out a lexer name based solely on
the given filename. It does not take input or highlight anything.
If no specific lexer can be determined "text" is returned.

The -H option prints detailed help for the object <name> of type <type>,
where <type> is one of "lexer", "formatter" or "filter".

The -h option prints this help.
The -V option prints the package version.
"""

def _parse_options(o_strs):
    opts = {}
    if not o_strs:
        return opts
    for o_str in o_strs:
        if not o_str:
            continue
        o_args = o_str.split(',')
        for o_arg in o_args:
            o_arg = o_arg.strip()
            try:
                o_key, o_val = o_arg.split('=')
                o_key = o_key.strip()
                o_val = o_val.strip()
            except ValueError:
                opts[o_arg] = True
            else:
                opts[o_key] = o_val
    return opts

def _parse_filters(f_strs):
    filters = []
    if not f_strs:
        return filters
    for f_str in f_strs:
        if ':' in f_str:
            fname, fopts = f_str.split(':', 1)
            filters.append((fname, _parse_options([fopts])))
        else:
            filters.append((f_str, {}))
    return filters

def _print_help(what, name):
    try:
        if what == 'lexer':
            cls = find_lexer_class(name)
            print "Help on the %s lexer:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'formatter':
            cls = find_formatter_class(name)
            print "Help on the %s formatter:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'filter':
            cls = find_filter_class(name)
            print "Help on the %s filter:" % name
            print dedent(cls.__doc__)
    except AttributeError:
        print >>sys.stderr, "%s not found!" % what

def _print_list(what):
    if what == 'lexer':
        print
        print "Lexers:"
        print "~~~~~~~"

        info = []
        for fullname, names, exts, _ in get_all_lexers():
            tup = (', '.join(names)+':', fullname,
                   exts and '(filenames ' + ', '.join(exts) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'formatter':
        print
        print "Formatters:"
        print "~~~~~~~~~~~"

        info = []
        for cls in get_all_formatters():
            doc = docstring_headline(cls)
            tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
                   '(filenames ' + ', '.join(cls.filenames) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'filter':
        print
        print "Filters:"
        print "~~~~~~~~"

        for name in get_all_filters():
            cls = find_filter_class(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

    elif what == 'style':
        print
        print "Styles:"
        print "~~~~~~~"

        for name in get_all_styles():
            cls = get_style_by_name(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

def main(args=sys.argv):
    """
    Main command line entry point.
    """
    # pylint: disable-msg=R0911,R0912,R0915

    usage = USAGE % ((args[0],) * 6)

    if sys.platform in [ 'win32', 'cygwin' ] :
      try:
          # Provide coloring under Windows, if possible
          import colorama
          colorama.init()
      except:
          pass

    try:
        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg")
    except getopt.GetoptError, err:
        print >>sys.stderr, usage
        return 2
    opts = {}
    O_opts = []
    P_opts = []
    F_opts = []
    for opt, arg in popts:
        if opt == '-O':
            O_opts.append(arg)
        elif opt == '-P':
            P_opts.append(arg)
        elif opt == '-F':
            F_opts.append(arg)
        opts[opt] = arg

    if not opts and not args:
        print usage
        return 0

    if opts.pop('-h', None) is not None:
        print usage
        return 0

    if opts.pop('-V', None) is not None:
        print 'Pygments version %s, (c) 2006-2012 by Georg Brandl.' % __version__
        return 0

    # handle ``pygmentize -L``
    L_opt = opts.pop('-L', None)
    if L_opt is not None:
        if opts:
            print >>sys.stderr, usage
            return 2

        # print version
        main(['', '-V'])
        if not args:
            args = ['lexer', 'formatter', 'filter', 'style']
        for arg in args:
            _print_list(arg.rstrip('s'))
        return 0

    # handle ``pygmentize -H``
    H_opt = opts.pop('-H', None)
    if H_opt is not None:
        if opts or len(args) != 2:
            print >>sys.stderr, usage
            return 2

        what, name = args
        if what not in ('lexer', 'formatter', 'filter'):
            print >>sys.stderr, usage
            return 2

        _print_help(what, name)
        return 0

    # parse -O options
    parsed_opts = _parse_options(O_opts)
    opts.pop('-O', None)

    # parse -P options
    for p_opt in P_opts:
        try:
            name, value = p_opt.split('=', 1)
        except ValueError:
            parsed_opts[p_opt] = True
        else:
            parsed_opts[name] = value
    opts.pop('-P', None)

    # handle ``pygmentize -N``
    infn = opts.pop('-N', None)
    if infn is not None:
        try:
            lexer = get_lexer_for_filename(infn, **parsed_opts)
        except ClassNotFound, err:
            lexer = TextLexer()
        except OptionError, err:
            print >>sys.stderr, 'Error:', err
            return 1

        print lexer.aliases[0]
        return 0

    # handle ``pygmentize -S``
    S_opt = opts.pop('-S', None)
    a_opt = opts.pop('-a', None)
    if S_opt is not None:
        f_opt = opts.pop('-f', None)
        if not f_opt:
            print >>sys.stderr, usage
            return 2
        if opts or args:
            print >>sys.stderr, usage
            return 2

        try:
            parsed_opts['style'] = S_opt
            fmter = get_formatter_by_name(f_opt, **parsed_opts)
        except ClassNotFound, err:
            print >>sys.stderr, err
            return 1

        arg = a_opt or ''
        try:
            print fmter.get_style_defs(arg)
        except Exception, err:
            print >>sys.stderr, 'Error:', err
            return 1
        return 0

    # if no -S is given, -a is not allowed
    if a_opt is not None:
        print >>sys.stderr, usage
        return 2

    # parse -F options
    F_opts = _parse_filters(F_opts)
    opts.pop('-F', None)

    # select formatter
    outfn = opts.pop('-o', None)
    fmter = opts.pop('-f', None)
    if fmter:
        try:
            fmter = get_formatter_by_name(fmter, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if outfn:
        if not fmter:
            try:
                fmter = get_formatter_for_filename(outfn, **parsed_opts)
            except (OptionError, ClassNotFound), err:
                print >>sys.stderr, 'Error:', err
                return 1
        try:
            outfile = open(outfn, 'wb')
        except Exception, err:
            print >>sys.stderr, 'Error: cannot open outfile:', err
            return 1
    else:
        if not fmter:
            fmter = TerminalFormatter(**parsed_opts)
        outfile = sys.stdout

    # select lexer
    lexer = opts.pop('-l', None)
    if lexer:
        try:
            lexer = get_lexer_by_name(lexer, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if args:
        if len(args) > 1:
            print >>sys.stderr, usage
            return 2

        infn = args[0]
        try:
            code = open(infn, 'rb').read()
        except Exception, err:
            print >>sys.stderr, 'Error: cannot read infile:', err
            return 1

        if not lexer:
            try:
                lexer = get_lexer_for_filename(infn, code, **parsed_opts)
            except ClassNotFound, err:
                if '-g' in opts:
                    try:
                        lexer = guess_lexer(code)
                    except ClassNotFound:
                        lexer = TextLexer()
                else:
                    print >>sys.stderr, 'Error:', err
                    return 1
            except OptionError, err:
                print >>sys.stderr, 'Error:', err
                return 1

    else:
        if '-g' in opts:
            code = sys.stdin.read()
            try:
                lexer = guess_lexer(code)
            except ClassNotFound:
                lexer = TextLexer()
        elif not lexer:
            print >>sys.stderr, 'Error: no lexer name given and reading ' + \
                                'from stdin (try using -g or -l <lexer>)'
            return 2
        else:
            code = sys.stdin.read()

    # No encoding given? Use latin1 if output file given,
    # stdin/stdout encoding otherwise.
    # (This is a compromise, I'm not too happy with it...)
    if 'encoding' not in parsed_opts and 'outencoding' not in parsed_opts:
        if outfn:
            # encoding pass-through
            fmter.encoding = 'latin1'
        else:
            if sys.version_info < (3,):
                # use terminal encoding; Python 3's terminals already do that
                lexer.encoding = getattr(sys.stdin, 'encoding',
                                         None) or 'ascii'
                fmter.encoding = getattr(sys.stdout, 'encoding',
                                         None) or 'ascii'
    elif not outfn and sys.version_info > (3,):
        # output to terminal with encoding -> use .buffer
        outfile = sys.stdout.buffer

    # ... and do it!
    try:
        # process filters
        for fname, fopts in F_opts:
            lexer.add_filter(fname, **fopts)
        highlight(code, lexer, fmter, outfile)
    except Exception, err:
        import traceback
        info = traceback.format_exception(*sys.exc_info())
        msg = info[-1].strip()
        if len(info) >= 3:
            # extract relevant file and position info
            msg += '\n   (f%s)' % info[-2].split('\n')[0].strip()[1:]
        print >>sys.stderr
        print >>sys.stderr, '*** Error while highlighting:'
        print >>sys.stderr, msg
        return 1

    return 0
<commit_msg>Fixed code style to be PEP-8 compatible.<commit_after># -*- coding: utf-8 -*-
"""
    pygments.cmdline
    ~~~~~~~~~~~~~~~~

    Command line interface.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
import sys
import getopt
from textwrap import dedent

from pygments import __version__, highlight
from pygments.util import ClassNotFound, OptionError, docstring_headline
from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
     find_lexer_class, guess_lexer, TextLexer
from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
     TerminalFormatter  # pylint:disable-msg=E0611
from pygments.filters import get_all_filters, find_filter_class
from pygments.styles import get_all_styles, get_style_by_name

USAGE = """\
Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
          [-O <options>] [-P <option=value>] [-o <outfile>] [<infile>]

       %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
       %s -L [<which> ...]
       %s -N <filename>
       %s -H <type> <name>
       %s -h | -V

Highlight the input file and write the result to <outfile>.

If no input file is given, use stdin, if -o is not given, use stdout.

<lexer> is a lexer name (query all lexer names with -L). If -l is not
given, the lexer is guessed from the extension of the input file name
(this obviously doesn't work if the input is stdin).  If -g is passed,
attempt to guess the lexer from the file contents, or pass through as
plain text if this fails (this can work for stdin).

Likewise, <formatter> is a formatter name, and will be guessed from
the extension of the output file name. If no output file is given,
the terminal formatter will be used by default.

With the -O option, you can give the lexer and formatter a comma-
separated list of options, e.g. ``-O bg=light,python=cool``.

The -P option adds lexer and formatter options like the -O option, but
you can only give one option per -P. That way, the option value may
contain commas and equals signs, which it can't with -O, e.g.
``-P "heading=Pygments, the Python highlighter".

With the -F option, you can add filters to the token stream, you can
give options in the same way as for -O after a colon (note: there must
not be spaces around the colon).

The -O, -P and -F options can be given multiple times.

With the -S option, print out style definitions for style <style>
for formatter <formatter>. The argument given by -a is formatter
dependent.

The -L option lists lexers, formatters, styles or filters -- set
`which` to the thing you want to list (e.g. "styles"), or omit it to
list everything.

The -N option guesses and prints out a lexer name based solely on
the given filename. It does not take input or highlight anything.
If no specific lexer can be determined "text" is returned.

The -H option prints detailed help for the object <name> of type <type>,
where <type> is one of "lexer", "formatter" or "filter".

The -h option prints this help.
The -V option prints the package version.
"""

def _parse_options(o_strs):
    opts = {}
    if not o_strs:
        return opts
    for o_str in o_strs:
        if not o_str:
            continue
        o_args = o_str.split(',')
        for o_arg in o_args:
            o_arg = o_arg.strip()
            try:
                o_key, o_val = o_arg.split('=')
                o_key = o_key.strip()
                o_val = o_val.strip()
            except ValueError:
                opts[o_arg] = True
            else:
                opts[o_key] = o_val
    return opts

def _parse_filters(f_strs):
    filters = []
    if not f_strs:
        return filters
    for f_str in f_strs:
        if ':' in f_str:
            fname, fopts = f_str.split(':', 1)
            filters.append((fname, _parse_options([fopts])))
        else:
            filters.append((f_str, {}))
    return filters

def _print_help(what, name):
    try:
        if what == 'lexer':
            cls = find_lexer_class(name)
            print "Help on the %s lexer:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'formatter':
            cls = find_formatter_class(name)
            print "Help on the %s formatter:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'filter':
            cls = find_filter_class(name)
            print "Help on the %s filter:" % name
            print dedent(cls.__doc__)
    except AttributeError:
        print >>sys.stderr, "%s not found!" % what

def _print_list(what):
    if what == 'lexer':
        print
        print "Lexers:"
        print "~~~~~~~"

        info = []
        for fullname, names, exts, _ in get_all_lexers():
            tup = (', '.join(names)+':', fullname,
                   exts and '(filenames ' + ', '.join(exts) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'formatter':
        print
        print "Formatters:"
        print "~~~~~~~~~~~"

        info = []
        for cls in get_all_formatters():
            doc = docstring_headline(cls)
            tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
                   '(filenames ' + ', '.join(cls.filenames) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'filter':
        print
        print "Filters:"
        print "~~~~~~~~"

        for name in get_all_filters():
            cls = find_filter_class(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

    elif what == 'style':
        print
        print "Styles:"
        print "~~~~~~~"

        for name in get_all_styles():
            cls = get_style_by_name(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

def main(args=sys.argv):
    """
    Main command line entry point.
    """
    # pylint: disable-msg=R0911,R0912,R0915

    usage = USAGE % ((args[0],) * 6)

    if sys.platform in ['win32', 'cygwin']:
      try:
          # Provide coloring under Windows, if possible
          import colorama
          colorama.init()
      except:
          pass

    try:
        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg")
    except getopt.GetoptError, err:
        print >>sys.stderr, usage
        return 2
    opts = {}
    O_opts = []
    P_opts = []
    F_opts = []
    for opt, arg in popts:
        if opt == '-O':
            O_opts.append(arg)
        elif opt == '-P':
            P_opts.append(arg)
        elif opt == '-F':
            F_opts.append(arg)
        opts[opt] = arg

    if not opts and not args:
        print usage
        return 0

    if opts.pop('-h', None) is not None:
        print usage
        return 0

    if opts.pop('-V', None) is not None:
        print 'Pygments version %s, (c) 2006-2012 by Georg Brandl.' % __version__
        return 0

    # handle ``pygmentize -L``
    L_opt = opts.pop('-L', None)
    if L_opt is not None:
        if opts:
            print >>sys.stderr, usage
            return 2

        # print version
        main(['', '-V'])
        if not args:
            args = ['lexer', 'formatter', 'filter', 'style']
        for arg in args:
            _print_list(arg.rstrip('s'))
        return 0

    # handle ``pygmentize -H``
    H_opt = opts.pop('-H', None)
    if H_opt is not None:
        if opts or len(args) != 2:
            print >>sys.stderr, usage
            return 2

        what, name = args
        if what not in ('lexer', 'formatter', 'filter'):
            print >>sys.stderr, usage
            return 2

        _print_help(what, name)
        return 0

    # parse -O options
    parsed_opts = _parse_options(O_opts)
    opts.pop('-O', None)

    # parse -P options
    for p_opt in P_opts:
        try:
            name, value = p_opt.split('=', 1)
        except ValueError:
            parsed_opts[p_opt] = True
        else:
            parsed_opts[name] = value
    opts.pop('-P', None)

    # handle ``pygmentize -N``
    infn = opts.pop('-N', None)
    if infn is not None:
        try:
            lexer = get_lexer_for_filename(infn, **parsed_opts)
        except ClassNotFound, err:
            lexer = TextLexer()
        except OptionError, err:
            print >>sys.stderr, 'Error:', err
            return 1

        print lexer.aliases[0]
        return 0

    # handle ``pygmentize -S``
    S_opt = opts.pop('-S', None)
    a_opt = opts.pop('-a', None)
    if S_opt is not None:
        f_opt = opts.pop('-f', None)
        if not f_opt:
            print >>sys.stderr, usage
            return 2
        if opts or args:
            print >>sys.stderr, usage
            return 2

        try:
            parsed_opts['style'] = S_opt
            fmter = get_formatter_by_name(f_opt, **parsed_opts)
        except ClassNotFound, err:
            print >>sys.stderr, err
            return 1

        arg = a_opt or ''
        try:
            print fmter.get_style_defs(arg)
        except Exception, err:
            print >>sys.stderr, 'Error:', err
            return 1
        return 0

    # if no -S is given, -a is not allowed
    if a_opt is not None:
        print >>sys.stderr, usage
        return 2

    # parse -F options
    F_opts = _parse_filters(F_opts)
    opts.pop('-F', None)

    # select formatter
    outfn = opts.pop('-o', None)
    fmter = opts.pop('-f', None)
    if fmter:
        try:
            fmter = get_formatter_by_name(fmter, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if outfn:
        if not fmter:
            try:
                fmter = get_formatter_for_filename(outfn, **parsed_opts)
            except (OptionError, ClassNotFound), err:
                print >>sys.stderr, 'Error:', err
                return 1
        try:
            outfile = open(outfn, 'wb')
        except Exception, err:
            print >>sys.stderr, 'Error: cannot open outfile:', err
            return 1
    else:
        if not fmter:
            fmter = TerminalFormatter(**parsed_opts)
        outfile = sys.stdout

    # select lexer
    lexer = opts.pop('-l', None)
    if lexer:
        try:
            lexer = get_lexer_by_name(lexer, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if args:
        if len(args) > 1:
            print >>sys.stderr, usage
            return 2

        infn = args[0]
        try:
            code = open(infn, 'rb').read()
        except Exception, err:
            print >>sys.stderr, 'Error: cannot read infile:', err
            return 1

        if not lexer:
            try:
                lexer = get_lexer_for_filename(infn, code, **parsed_opts)
            except ClassNotFound, err:
                if '-g' in opts:
                    try:
                        lexer = guess_lexer(code)
                    except ClassNotFound:
                        lexer = TextLexer()
                else:
                    print >>sys.stderr, 'Error:', err
                    return 1
            except OptionError, err:
                print >>sys.stderr, 'Error:', err
                return 1

    else:
        if '-g' in opts:
            code = sys.stdin.read()
            try:
                lexer = guess_lexer(code)
            except ClassNotFound:
                lexer = TextLexer()
        elif not lexer:
            print >>sys.stderr, 'Error: no lexer name given and reading ' + \
                                'from stdin (try using -g or -l <lexer>)'
            return 2
        else:
            code = sys.stdin.read()

    # No encoding given? Use latin1 if output file given,
    # stdin/stdout encoding otherwise.
    # (This is a compromise, I'm not too happy with it...)
    if 'encoding' not in parsed_opts and 'outencoding' not in parsed_opts:
        if outfn:
            # encoding pass-through
            fmter.encoding = 'latin1'
        else:
            if sys.version_info < (3,):
                # use terminal encoding; Python 3's terminals already do that
                lexer.encoding = getattr(sys.stdin, 'encoding',
                                         None) or 'ascii'
                fmter.encoding = getattr(sys.stdout, 'encoding',
                                         None) or 'ascii'
    elif not outfn and sys.version_info > (3,):
        # output to terminal with encoding -> use .buffer
        outfile = sys.stdout.buffer

    # ... and do it!
    try:
        # process filters
        for fname, fopts in F_opts:
            lexer.add_filter(fname, **fopts)
        highlight(code, lexer, fmter, outfile)
    except Exception, err:
        import traceback
        info = traceback.format_exception(*sys.exc_info())
        msg = info[-1].strip()
        if len(info) >= 3:
            # extract relevant file and position info
            msg += '\n   (f%s)' % info[-2].split('\n')[0].strip()[1:]
        print >>sys.stderr
        print >>sys.stderr, '*** Error while highlighting:'
        print >>sys.stderr, msg
        return 1

    return 0
<|endoftext|>


<commit_before>from ._apps_base import AdminRegisteringAppConfig

class GroupsConfig(AdminRegisteringAppConfig):
    """
    Configuration for `groups`.

    Provides:
    * `default_within_days` - a default parameter for the `within_days` methods on some
      of the model managers, which return items that were posted or posted to within
      that time period.
    * `new_comment_subject` and `new_discussion_subject` - subjects for notification
      emails.  Each one will be formatted with the `{discussion}` a comment is on or the
      `{group}` a discussion belongs to, respectively.
    * `group_admin_class_path` and `discussion_admin_class_path` - these allow a project
      to override the admin behaviour of `incuna-groups`.
    """
    name = 'groups'

    default_within_days = 7

    new_comment_subject = 'New comment on {discussion}'
    new_discussion_subject = 'New discussion in {group}'

    group_admin_class_path = 'groups.admin.GroupAdmin'
    discussion_admin_class_path = 'groups.admin.DiscussionAdmin'

    def update_admin_classes(self, admin_classes):
        super(GroupsConfig, self).update_admin_classes()
        admin_classes.update(**{
            'Group': self.group_admin_class_path,
            'Discussion': self.discussion_admin_class_path,
        })
<commit_msg>Guess who forgot to click on 'run tests'...<commit_after>from ._apps_base import AdminRegisteringAppConfig

class GroupsConfig(AdminRegisteringAppConfig):
    """
    Configuration for `groups`.

    Provides:
    * `default_within_days` - a default parameter for the `within_days` methods on some
      of the model managers, which return items that were posted or posted to within
      that time period.
    * `new_comment_subject` and `new_discussion_subject` - subjects for notification
      emails.  Each one will be formatted with the `{discussion}` a comment is on or the
      `{group}` a discussion belongs to, respectively.
    * `group_admin_class_path` and `discussion_admin_class_path` - these allow a project
      to override the admin behaviour of `incuna-groups`.
    """
    name = 'groups'

    default_within_days = 7

    new_comment_subject = 'New comment on {discussion}'
    new_discussion_subject = 'New discussion in {group}'

    group_admin_class_path = 'groups.admin.GroupAdmin'
    discussion_admin_class_path = 'groups.admin.DiscussionAdmin'

    def update_admin_classes(self, admin_classes):
        super(GroupsConfig, self).update_admin_classes(admin_classes)
        admin_classes.update(**{
            'Group': self.group_admin_class_path,
            'Discussion': self.discussion_admin_class_path,
        })
<|endoftext|>


<commit_before>from pylearn2.monitor import Monitor
from pylearn2.space import VectorSpace
from pylearn2.models.model import Model
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.training_algorithms.default import DefaultTrainingAlgorithm
import numpy as np
from theano import tensor as T
from pylearn2.models.s3c import S3C, E_Step, Grad_M_Step
from nose.plugins.skip import SkipTest
from pylearn2.utils import sharedX
from pylearn2.utils.serial import to_string
from pylearn2.utils.serial import from_string
from pylearn2.utils.iteration import _iteration_schemes
from theano import shared

class DummyModel(Model):
    def  __init__(self, num_features):
        self.input_space = VectorSpace(num_features)

class DummyDataset(DenseDesignMatrix):
    def __init__(self, num_examples, num_features):
        rng = np.random.RandomState([4, 12, 17])
        super(DummyDataset, self).__init__(
            X=rng.uniform(1., 2., (num_examples, num_features))
        )

class UnserializableDataset(DummyDataset):
    """ A Dataset that raises an assertion if you try to seralize it.
        Used to test that the monitor does not serialized its dataset.
        Since the Model seralizes its Monitor, this would cause a copy
        of the dataset to get saved with ever saved Model!"""

    def __getstate__(self):
        raise AssertionError("Dataset should not be serialized")

class ArangeDataset(DenseDesignMatrix):
    """ A dataset where example i is just the number i.
    Makes it easy to track which sets of examples are visited."""
    def __init__(self, num_examples):
        X = np.zeros((num_examples,1))
        X[:,0] = np.arange(num_examples)
        super(ArangeDataset, self).__init__(X)

def test_channel_scaling_sequential():
    def channel_scaling_checker(num_examples, mode, num_batches, batch_size):
        num_features = 2
        monitor = Monitor(DummyModel(num_features))
        dataset = DummyDataset(num_examples, num_features)
        try:
            monitor.add_dataset(dataset=dataset, mode=mode,
                                num_batches=num_batches, batch_size=batch_size)
        except NotImplementedError:
            # make sure this was due to the unimplemented batch_size case
            if num_batches is None:
                assert num_examples % batch_size != 0
            else:
                assert num_examples % num_batches != 0
            raise SkipTest()
        vis_batch = T.matrix()
        mean = vis_batch.mean()
        monitor.add_channel(name='mean', ipt=vis_batch, val=mean, dataset=dataset)
        monitor()
        assert 'mean' in monitor.channels
        mean = monitor.channels['mean']
        assert len(mean.val_record) == 1
        actual = mean.val_record[0]
        X = dataset.get_design_matrix()
        if batch_size is not None and num_batches is not None:
            total = min(num_examples, num_batches * batch_size)
        else:
            total = num_examples
        expected = X[:total].mean()
        if not np.allclose(expected, actual):
            raise AssertionError("Expected monitor to contain %f but it has "
                                 "%f" % (expected, actual))

    # Specifying num_batches; even split
    yield channel_scaling_checker, 10, 'sequential', 5, None
    # Specifying num_batches; even split
    yield channel_scaling_checker, 10, 'sequential', 2, None
    # Specifying batch_size; even split
    yield channel_scaling_checker, 10, 'sequential', None, 5
    # Specifying batch_size; even split
    yield channel_scaling_checker, 10, 'sequential', None, 2
    # Specifying num_batches; uneven split
    yield channel_scaling_checker, 10, 'sequential', 4, None
    # Specifying num_batches; uneven split
    yield channel_scaling_checker, 10, 'sequential', 3, None
    # Specifying batch_size; uneven split
    yield channel_scaling_checker, 10, 'sequential', None, 3
    # Specifying batch_size; uneven split
    yield channel_scaling_checker, 10, 'sequential', None, 4
    # Specifying both, even split
    yield channel_scaling_checker, 10, 'sequential', 2, 5
    # Specifying both, even split
    yield channel_scaling_checker, 10, 'sequential', 5, 2
    # Specifying both, uneven split, dangling batch
    yield channel_scaling_checker, 10, 'sequential', 3, 4
    # Specifying both, uneven split, non-exhaustive
    yield channel_scaling_checker, 10, 'sequential', 3, 3

def test_counting():
    BATCH_SIZE = 2
    BATCHES = 3
    NUM_FEATURES = 4
    num_examples = BATCHES * BATCH_SIZE
    dataset = DummyDataset( num_examples = num_examples,
            num_features = NUM_FEATURES)
    algorithm = DefaultTrainingAlgorithm( batch_size = BATCH_SIZE,
            batches_per_iter = BATCHES)
    model = S3C( nvis = NUM_FEATURES, nhid = 1,
            irange = .01, init_bias_hid = 0., init_B = 1.,
            min_B = 1., max_B = 1., init_alpha = 1.,
            min_alpha = 1., max_alpha = 1., init_mu = 0.,
            m_step = Grad_M_Step( learning_rate = 0.),
            e_step = E_Step( h_new_coeff_schedule = [ 1. ]))
    algorithm.setup(model = model, dataset = dataset)
    algorithm.train(dataset = dataset)
    if not ( model.monitor.get_batches_seen() == BATCHES):
        raise AssertionError('Should have seen '+str(BATCHES) + \
                ' batches but saw '+str(model.monitor.get_batches_seen()))

    assert model.monitor.get_examples_seen() == num_examples
    assert isinstance(model.monitor.get_examples_seen(),int)
    assert isinstance(model.monitor.get_batches_seen(),int)

def test_prereqs():

    # Test that prereqs get run before the monitoring channels are computed

    BATCH_SIZE = 2
    num_examples = BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = DummyDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    prereq_counter = sharedX(0.)
    def prereq(X,y):
        prereq_counter.set_value(
                prereq_counter.get_value()+1.)

    name = 'num_prereq_calls'

    monitor.add_channel(name = name,
            ipt = model.input_space.make_theano_batch(),
            val = prereq_counter,
            prereqs = [ prereq ])

    channel = monitor.channels[name]

    assert len(channel.val_record) == 0
    monitor()
    assert channel.val_record == [1]
    monitor()
    assert channel.val_record == [1,2]

def test_revisit():

    # Test that each call to monitor revisits exactly the same data

    BATCH_SIZE = 3
    MAX_BATCH_SIZE = 12
    BATCH_SIZE_STRIDE = 3
    NUM_BATCHES = 10
    num_examples = NUM_BATCHES * BATCH_SIZE

    monitoring_dataset = ArangeDataset(num_examples)

    for mon_batch_size in xrange(BATCH_SIZE, MAX_BATCH_SIZE + 1,
            BATCH_SIZE_STRIDE):
        for num_mon_batches in [ 1, 3, num_examples / mon_batch_size, None ]:
            for mode in sorted(_iteration_schemes):

                if num_mon_batches is None and mode in ['random_uniform', 'random_slice']:
                    continue

                model = DummyModel(1)
                monitor = Monitor.get_monitor(model)

                try:
                    try:
                        monitor.add_dataset(monitoring_dataset, mode,
                            batch_size=mon_batch_size, num_batches=num_mon_batches)
                    except TypeError:
                        monitor.add_dataset(monitoring_dataset, mode,
                            batch_size=mon_batch_size, num_batches=num_mon_batches,
                            seed = 0)
                except NotImplementedError:
                    # Monitor does not currently support uneven iterators, so skip
                    # uneven iteration modes
                    # Check that this is what caused the error
                    if num_mon_batches is not None and mon_batch_size * num_mon_batches > num_examples:
                        continue
                    if num_mon_batches is None and num_examples % mon_batch_size != 0:
                        continue
                    print num_mon_batches, mon_batch_size, num_examples, mode
                    raise

                if num_mon_batches is None:
                    num_mon_batches = num_examples / mon_batch_size

                batches = [ None ] * num_mon_batches
                visited = [ False ] * num_mon_batches

                batch_idx = shared(0)

                class RecorderAndValidator:

                    def __init__(self):
                        self.validate = False

                    def __call__(self, X, y):
                        """ Initially, records the batches the monitor shows it.
                        When set to validate mode, makes sure the batches shown
                        on the second monitor call match those from the first."""
                        assert y is None

                        idx = batch_idx.get_value()
                        batch_idx.set_value(idx + 1)

                        # Note: if the monitor starts supporting variable batch sizes,
                        # take this out. Maybe move it to a new test that the iterator's
                        # uneven property is set accurately
                        assert X.shape[0] == mon_batch_size

                        if self.validate:
                            previous_batch = batches[idx]
                            assert not visited[idx]
                            visited[idx] = True
                            if not np.allclose(previous_batch, X):
                                print 'Visited different data in batch',idx
                                print previous_batch
                                print X
                                print 'Iteration mode', mode
                                assert False
                        else:
                            batches[idx] = X
                        # end if
                    # end __call__
                #end class

                prereq = RecorderAndValidator()

                monitor.add_channel(name = 'dummy',
                    ipt = model.input_space.make_theano_batch(),
                    val = 0.,
                    prereqs = [ prereq ])

                monitor()

                assert None not in batches

                batch_idx.set_value(0)
                prereq.validate = True

                monitor()

                assert all(visited)

def test_prereqs_batch():

    # Test that prereqs get run before each monitoring batch

    BATCH_SIZE = 2
    num_examples = 2 * BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = DummyDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    sign = sharedX(1.)
    def prereq(X,y):
        sign.set_value(
                -sign.get_value())

    name = 'batches_should_cancel_to_0'

    monitor.add_channel(name = name,
            ipt = model.input_space.make_theano_batch(),
            val = sign,
            prereqs = [ prereq ])

    channel = monitor.channels[name]

    assert len(channel.val_record) == 0
    monitor()
    assert channel.val_record == [0]
    monitor()
    assert channel.val_record == [0,0]

def test_dont_serialize_dataset():

    # Test that serializing the monitor does not serialize the dataset

    BATCH_SIZE = 2
    num_examples = 2 * BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = UnserializableDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)
    monitoring_dataset.yaml_src = ""

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    to_string(monitor)

def test_serialize_twice():

    # Test that a monitor can be serialized twice
    # with the same result

    model = DummyModel(1)
    monitor = Monitor.get_monitor(model)

    x = to_string(monitor)
    y = to_string(monitor)

    assert x == y

def test_valid_after_serialize():

    # Test that serializing the monitor does not ruin it

    BATCH_SIZE = 2
    num_examples = 2 * BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = UnserializableDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)
    monitoring_dataset.yaml_src = ""

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    to_string(monitor)

    monitor.redo_theano()

def test_deserialize():

    # Test that a monitor can be deserialized

    model = DummyModel(1)
    monitor = Monitor.get_monitor(model)

    x = to_string(monitor)
    monitor = from_string(x)
    y = to_string(monitor)

def test_prereqs_multidataset():

    # Test that prereqs are run on the right datasets

    NUM_DATASETS = 4
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    prereq_counters = []
    datasets = []
    for i in xrange(NUM_DATASETS):

        batch_size = i + 1
        num_examples = batch_size
        dataset = DummyDataset(num_examples = num_examples,
                num_features = NUM_FEATURES)
        dataset.X[:] = i
        datasets.append(dataset)

        monitor.add_dataset(dataset, 'sequential', batch_size=batch_size)

        prereq_counters.append(sharedX(0.))

    class Prereq(object):
        def __init__(self, counter_idx):
            self.counter_idx = counter_idx

        def __call__(self, X, y):
            # We set up each dataset with a different batch size
            # check here that we're getting the right one
            assert X.shape[0] == self.counter_idx + 1
            # Each dataset has different content, make sure we
            # get the right one
            assert X[0,0] == self.counter_idx
            prereq_counter = prereq_counters[self.counter_idx]
            prereq_counter.set_value(
                prereq_counter.get_value()+1)

    channels = []
    for i in xrange(NUM_DATASETS):
        monitor.add_channel(name = str(i),
                ipt = model.input_space.make_theano_batch(),
                val = prereq_counters[i],
                dataset = datasets[i],
                prereqs = [ Prereq(i) ])

        channels.append(monitor.channels[str(i)])

    for channel in channels:
        assert len(channel.val_record) == 0
    monitor()
    for channel in channels:
        assert channel.val_record == [1]
    monitor()
    for channel in channels:
        assert channel.val_record == [1,2]

<commit_msg>added test for bug where monitor tried to serialize prereqs dictionary<commit_after>from pylearn2.monitor import Monitor
from pylearn2.space import VectorSpace
from pylearn2.models.model import Model
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.training_algorithms.default import DefaultTrainingAlgorithm
import numpy as np
from theano import tensor as T
from pylearn2.models.s3c import S3C, E_Step, Grad_M_Step
from nose.plugins.skip import SkipTest
from pylearn2.utils import sharedX
from pylearn2.utils.serial import to_string
from pylearn2.utils.serial import from_string
from pylearn2.utils.iteration import _iteration_schemes
from theano import shared
from pylearn2.testing.prereqs import ReadVerifyPrereq

class DummyModel(Model):
    def  __init__(self, num_features):
        self.input_space = VectorSpace(num_features)

class DummyDataset(DenseDesignMatrix):
    def __init__(self, num_examples, num_features):
        rng = np.random.RandomState([4, 12, 17])
        super(DummyDataset, self).__init__(
            X=rng.uniform(1., 2., (num_examples, num_features))
        )

    def __getstate__(self):
        raise AssertionError("These unit tests only test Monitor "
                "functionality. If the Monitor tries to serialize a "
                "Dataset, that is an error.")

class ArangeDataset(DenseDesignMatrix):
    """ A dataset where example i is just the number i.
    Makes it easy to track which sets of examples are visited."""
    def __init__(self, num_examples):
        X = np.zeros((num_examples,1))
        X[:,0] = np.arange(num_examples)
        super(ArangeDataset, self).__init__(X)

def test_channel_scaling_sequential():
    def channel_scaling_checker(num_examples, mode, num_batches, batch_size):
        num_features = 2
        monitor = Monitor(DummyModel(num_features))
        dataset = DummyDataset(num_examples, num_features)
        try:
            monitor.add_dataset(dataset=dataset, mode=mode,
                                num_batches=num_batches, batch_size=batch_size)
        except NotImplementedError:
            # make sure this was due to the unimplemented batch_size case
            if num_batches is None:
                assert num_examples % batch_size != 0
            else:
                assert num_examples % num_batches != 0
            raise SkipTest()
        vis_batch = T.matrix()
        mean = vis_batch.mean()
        monitor.add_channel(name='mean', ipt=vis_batch, val=mean, dataset=dataset)
        monitor()
        assert 'mean' in monitor.channels
        mean = monitor.channels['mean']
        assert len(mean.val_record) == 1
        actual = mean.val_record[0]
        X = dataset.get_design_matrix()
        if batch_size is not None and num_batches is not None:
            total = min(num_examples, num_batches * batch_size)
        else:
            total = num_examples
        expected = X[:total].mean()
        if not np.allclose(expected, actual):
            raise AssertionError("Expected monitor to contain %f but it has "
                                 "%f" % (expected, actual))

    # Specifying num_batches; even split
    yield channel_scaling_checker, 10, 'sequential', 5, None
    # Specifying num_batches; even split
    yield channel_scaling_checker, 10, 'sequential', 2, None
    # Specifying batch_size; even split
    yield channel_scaling_checker, 10, 'sequential', None, 5
    # Specifying batch_size; even split
    yield channel_scaling_checker, 10, 'sequential', None, 2
    # Specifying num_batches; uneven split
    yield channel_scaling_checker, 10, 'sequential', 4, None
    # Specifying num_batches; uneven split
    yield channel_scaling_checker, 10, 'sequential', 3, None
    # Specifying batch_size; uneven split
    yield channel_scaling_checker, 10, 'sequential', None, 3
    # Specifying batch_size; uneven split
    yield channel_scaling_checker, 10, 'sequential', None, 4
    # Specifying both, even split
    yield channel_scaling_checker, 10, 'sequential', 2, 5
    # Specifying both, even split
    yield channel_scaling_checker, 10, 'sequential', 5, 2
    # Specifying both, uneven split, dangling batch
    yield channel_scaling_checker, 10, 'sequential', 3, 4
    # Specifying both, uneven split, non-exhaustive
    yield channel_scaling_checker, 10, 'sequential', 3, 3

def test_counting():
    BATCH_SIZE = 2
    BATCHES = 3
    NUM_FEATURES = 4
    num_examples = BATCHES * BATCH_SIZE
    dataset = DummyDataset( num_examples = num_examples,
            num_features = NUM_FEATURES)
    algorithm = DefaultTrainingAlgorithm( batch_size = BATCH_SIZE,
            batches_per_iter = BATCHES)
    model = S3C( nvis = NUM_FEATURES, nhid = 1,
            irange = .01, init_bias_hid = 0., init_B = 1.,
            min_B = 1., max_B = 1., init_alpha = 1.,
            min_alpha = 1., max_alpha = 1., init_mu = 0.,
            m_step = Grad_M_Step( learning_rate = 0.),
            e_step = E_Step( h_new_coeff_schedule = [ 1. ]))
    algorithm.setup(model = model, dataset = dataset)
    algorithm.train(dataset = dataset)
    if not ( model.monitor.get_batches_seen() == BATCHES):
        raise AssertionError('Should have seen '+str(BATCHES) + \
                ' batches but saw '+str(model.monitor.get_batches_seen()))

    assert model.monitor.get_examples_seen() == num_examples
    assert isinstance(model.monitor.get_examples_seen(),int)
    assert isinstance(model.monitor.get_batches_seen(),int)

def test_prereqs():

    # Test that prereqs get run before the monitoring channels are computed

    BATCH_SIZE = 2
    num_examples = BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = DummyDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    prereq_counter = sharedX(0.)
    def prereq(X,y):
        prereq_counter.set_value(
                prereq_counter.get_value()+1.)

    name = 'num_prereq_calls'

    monitor.add_channel(name = name,
            ipt = model.input_space.make_theano_batch(),
            val = prereq_counter,
            prereqs = [ prereq ])

    channel = monitor.channels[name]

    assert len(channel.val_record) == 0
    monitor()
    assert channel.val_record == [1]
    monitor()
    assert channel.val_record == [1,2]

def test_revisit():

    # Test that each call to monitor revisits exactly the same data

    BATCH_SIZE = 3
    MAX_BATCH_SIZE = 12
    BATCH_SIZE_STRIDE = 3
    NUM_BATCHES = 10
    num_examples = NUM_BATCHES * BATCH_SIZE

    monitoring_dataset = ArangeDataset(num_examples)

    for mon_batch_size in xrange(BATCH_SIZE, MAX_BATCH_SIZE + 1,
            BATCH_SIZE_STRIDE):
        for num_mon_batches in [ 1, 3, num_examples / mon_batch_size, None ]:
            for mode in sorted(_iteration_schemes):

                if num_mon_batches is None and mode in ['random_uniform', 'random_slice']:
                    continue

                model = DummyModel(1)
                monitor = Monitor.get_monitor(model)

                try:
                    try:
                        monitor.add_dataset(monitoring_dataset, mode,
                            batch_size=mon_batch_size, num_batches=num_mon_batches)
                    except TypeError:
                        monitor.add_dataset(monitoring_dataset, mode,
                            batch_size=mon_batch_size, num_batches=num_mon_batches,
                            seed = 0)
                except NotImplementedError:
                    # Monitor does not currently support uneven iterators, so skip
                    # uneven iteration modes
                    # Check that this is what caused the error
                    if num_mon_batches is not None and mon_batch_size * num_mon_batches > num_examples:
                        continue
                    if num_mon_batches is None and num_examples % mon_batch_size != 0:
                        continue
                    print num_mon_batches, mon_batch_size, num_examples, mode
                    raise

                if num_mon_batches is None:
                    num_mon_batches = num_examples / mon_batch_size

                batches = [ None ] * num_mon_batches
                visited = [ False ] * num_mon_batches

                batch_idx = shared(0)

                class RecorderAndValidator:

                    def __init__(self):
                        self.validate = False

                    def __call__(self, X, y):
                        """ Initially, records the batches the monitor shows it.
                        When set to validate mode, makes sure the batches shown
                        on the second monitor call match those from the first."""
                        assert y is None

                        idx = batch_idx.get_value()
                        batch_idx.set_value(idx + 1)

                        # Note: if the monitor starts supporting variable batch sizes,
                        # take this out. Maybe move it to a new test that the iterator's
                        # uneven property is set accurately
                        assert X.shape[0] == mon_batch_size

                        if self.validate:
                            previous_batch = batches[idx]
                            assert not visited[idx]
                            visited[idx] = True
                            if not np.allclose(previous_batch, X):
                                print 'Visited different data in batch',idx
                                print previous_batch
                                print X
                                print 'Iteration mode', mode
                                assert False
                        else:
                            batches[idx] = X
                        # end if
                    # end __call__
                #end class

                prereq = RecorderAndValidator()

                monitor.add_channel(name = 'dummy',
                    ipt = model.input_space.make_theano_batch(),
                    val = 0.,
                    prereqs = [ prereq ])

                monitor()

                assert None not in batches

                batch_idx.set_value(0)
                prereq.validate = True

                monitor()

                assert all(visited)

def test_prereqs_batch():

    # Test that prereqs get run before each monitoring batch

    BATCH_SIZE = 2
    num_examples = 2 * BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = DummyDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    sign = sharedX(1.)
    def prereq(X,y):
        sign.set_value(
                -sign.get_value())

    name = 'batches_should_cancel_to_0'

    monitor.add_channel(name = name,
            ipt = model.input_space.make_theano_batch(),
            val = sign,
            prereqs = [ prereq ])

    channel = monitor.channels[name]

    assert len(channel.val_record) == 0
    monitor()
    assert channel.val_record == [0]
    monitor()
    assert channel.val_record == [0,0]

def test_dont_serialize_dataset():

    # Test that serializing the monitor does not serialize the dataset

    BATCH_SIZE = 2
    num_examples = 2 * BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = DummyDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)
    monitoring_dataset.yaml_src = ""

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    monitor()

    to_string(monitor)

def test_serialize_twice():

    # Test that a monitor can be serialized twice
    # with the same result

    model = DummyModel(1)
    monitor = Monitor.get_monitor(model)

    x = to_string(monitor)
    y = to_string(monitor)

    assert x == y

def test_valid_after_serialize():

    # Test that serializing the monitor does not ruin it

    BATCH_SIZE = 2
    num_examples = 2 * BATCH_SIZE
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    monitoring_dataset = DummyDataset(num_examples = num_examples,
            num_features = NUM_FEATURES)
    monitoring_dataset.yaml_src = ""

    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)

    to_string(monitor)

    monitor.redo_theano()

def test_deserialize():

    # Test that a monitor can be deserialized

    model = DummyModel(1)
    monitor = Monitor.get_monitor(model)

    x = to_string(monitor)
    monitor = from_string(x)
    y = to_string(monitor)

def test_prereqs_multidataset():

    # Test that prereqs are run on the right datasets

    NUM_DATASETS = 4
    NUM_FEATURES = 3

    model = DummyModel(NUM_FEATURES)
    monitor = Monitor.get_monitor(model)

    prereq_counters = []
    datasets = []
    for i in xrange(NUM_DATASETS):

        batch_size = i + 1
        num_examples = batch_size
        dataset = DummyDataset(num_examples = num_examples,
                num_features = NUM_FEATURES)
        dataset.X[:] = i
        datasets.append(dataset)

        monitor.add_dataset(dataset, 'sequential', batch_size=batch_size)

        prereq_counters.append(sharedX(0.))

    channels = []
    for i in xrange(NUM_DATASETS):
        monitor.add_channel(name = str(i),
                ipt = model.input_space.make_theano_batch(),
                val = prereq_counters[i],
                dataset = datasets[i],
                prereqs = [ ReadVerifyPrereq(i, prereq_counters[i]) ])

        channels.append(monitor.channels[str(i)])

    for channel in channels:
        assert len(channel.val_record) == 0
    monitor()
    for channel in channels:
        assert channel.val_record == [1]
    monitor()
    for channel in channels:
        assert channel.val_record == [1,2]

    # check that handling all these datasets did not
    # result in them getting serialized
    to_string(monitor)

<|endoftext|>


<commit_before>from parcels import Grid, Particle, JITParticle, AdvectionRK4
import numpy as np
import pytest
from datetime import timedelta as delta

ptype = {'scipy': Particle, 'jit': JITParticle}

@pytest.fixture
def lon(xdim=200):
    return np.linspace(-170, 170, xdim, dtype=np.float32)

@pytest.fixture
def lat(ydim=100):
    return np.linspace(-80, 80, ydim, dtype=np.float32)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_advection_zonal(lon, lat, mode, npart=10):
    """ Particles at high latitude move geographically faster due to
        the pole correction in `GeographicPolar`.
    """
    U = np.ones((lon.size, lat.size), dtype=np.float32)
    V = np.zeros((lon.size, lat.size), dtype=np.float32)
    grid = Grid.from_data(U, lon, lat, V, lon, lat, mesh='spherical')

    pset = grid.ParticleSet(npart, pclass=ptype[mode],
                            lon=np.zeros(npart, dtype=np.float32) + 20.,
                            lat=np.linspace(0, 80, npart, dtype=np.float32))
    pset.execute(AdvectionRK4, endtime=delta(hours=2), dt=delta(seconds=30))
    assert (np.diff(np.array([p.lon for p in pset])) > 1.e-4).all()

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_advection_meridional(lon, lat, mode, npart=10):
    """ Particles at high latitude move geographically faster due to
        the pole correction in `GeographicPolar`.
    """
    U = np.zeros((lon.size, lat.size), dtype=np.float32)
    V = np.ones((lon.size, lat.size), dtype=np.float32)
    grid = Grid.from_data(U, lon, lat, V, lon, lat, mesh='spherical')

    pset = grid.ParticleSet(npart, pclass=ptype[mode],
                            lon=np.linspace(-60, 60, npart, dtype=np.float32),
                            lat=np.linspace(0, 30, npart, dtype=np.float32))
    delta_lat = np.diff(np.array([p.lat for p in pset]))
    pset.execute(AdvectionRK4, endtime=delta(hours=2), dt=delta(seconds=30))
    assert np.allclose(np.diff(np.array([p.lat for p in pset])), delta_lat, rtol=1.e-4)
<commit_msg>Advection: Adding eddy test cases from Fabbroni, 2009, to test_advection<commit_after>from parcels import Grid, Particle, JITParticle, AdvectionRK4
import numpy as np
import pytest
import math
from datetime import timedelta as delta

ptype = {'scipy': Particle, 'jit': JITParticle}

# Some constants
f = 1.e-4
u_0 = 0.3
u_g = 0.04
gamma = 1/(86400. * 2.89)
gamma_g = 1/(86400. * 28.9)

@pytest.fixture
def lon(xdim=200):
    return np.linspace(-170, 170, xdim, dtype=np.float32)

@pytest.fixture
def lat(ydim=100):
    return np.linspace(-80, 80, ydim, dtype=np.float32)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_advection_zonal(lon, lat, mode, npart=10):
    """ Particles at high latitude move geographically faster due to
        the pole correction in `GeographicPolar`.
    """
    U = np.ones((lon.size, lat.size), dtype=np.float32)
    V = np.zeros((lon.size, lat.size), dtype=np.float32)
    grid = Grid.from_data(U, lon, lat, V, lon, lat, mesh='spherical')

    pset = grid.ParticleSet(npart, pclass=ptype[mode],
                            lon=np.zeros(npart, dtype=np.float32) + 20.,
                            lat=np.linspace(0, 80, npart, dtype=np.float32))
    pset.execute(AdvectionRK4, endtime=delta(hours=2), dt=delta(seconds=30))
    assert (np.diff(np.array([p.lon for p in pset])) > 1.e-4).all()

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_advection_meridional(lon, lat, mode, npart=10):
    """ Particles at high latitude move geographically faster due to
        the pole correction in `GeographicPolar`.
    """
    U = np.zeros((lon.size, lat.size), dtype=np.float32)
    V = np.ones((lon.size, lat.size), dtype=np.float32)
    grid = Grid.from_data(U, lon, lat, V, lon, lat, mesh='spherical')

    pset = grid.ParticleSet(npart, pclass=ptype[mode],
                            lon=np.linspace(-60, 60, npart, dtype=np.float32),
                            lat=np.linspace(0, 30, npart, dtype=np.float32))
    delta_lat = np.diff(np.array([p.lat for p in pset]))
    pset.execute(AdvectionRK4, endtime=delta(hours=2), dt=delta(seconds=30))
    assert np.allclose(np.diff(np.array([p.lat for p in pset])), delta_lat, rtol=1.e-4)

def truth_stationary(x_0, y_0, t):
    lat = y_0 - u_0 / f * (1 - math.cos(f * t))
    lon = x_0 + u_0 / f * math.sin(f * t)
    return lon, lat

@pytest.fixture
def grid_stationary(xdim=100, ydim=100, maxtime=delta(hours=6)):
    """Generate a grid encapsulating the flow field of a stationary eddy.

    Reference: N. Fabbroni, 2009, "Numerical simulations of passive
    tracers dispersion in the sea"
    """
    lon = np.linspace(0, 25000, xdim, dtype=np.float32)
    lat = np.linspace(0, 25000, ydim, dtype=np.float32)
    time = np.arange(0., maxtime.total_seconds(), 60., dtype=np.float64)
    U = np.ones((xdim, ydim, 1), dtype=np.float32) * u_0 * np.cos(f * time)
    V = np.ones((xdim, ydim, 1), dtype=np.float32) * -u_0 * np.sin(f * time)
    return Grid.from_data(U, lon, lat, V, lon, lat, time=time, mesh='flat')

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_stationary_eddy(grid_stationary, mode, npart=1):
    grid = grid_stationary
    lon = np.linspace(12000, 21000, npart, dtype=np.float32)
    lat = np.linspace(12500, 12500, npart, dtype=np.float32)
    pset = grid.ParticleSet(size=npart, pclass=ptype[mode], lon=lon, lat=lat)
    endtime = delta(hours=6).total_seconds()
    pset.execute(AdvectionRK4, dt=delta(minutes=3), endtime=endtime)
    exp_lon = [truth_stationary(x, y, endtime)[0] for x, y, in zip(lon, lat)]
    exp_lat = [truth_stationary(x, y, endtime)[1] for x, y, in zip(lon, lat)]
    assert np.allclose(np.array([p.lon for p in pset]), exp_lon, rtol=1e-5)
    assert np.allclose(np.array([p.lat for p in pset]), exp_lat, rtol=1e-5)

def truth_moving(x_0, y_0, t):
    lat = y_0 - (u_0 - u_g) / f * (1 - math.cos(f * t))
    lon = x_0 + u_g * t + (u_0 - u_g) / f * math.sin(f * t)
    return lon, lat

@pytest.fixture
def grid_moving(xdim=100, ydim=100, maxtime=delta(hours=6)):
    """Generate a grid encapsulating the flow field of a moving eddy.

    Reference: N. Fabbroni, 2009, "Numerical simulations of passive
    tracers dispersion in the sea"
    """
    lon = np.linspace(0, 25000, xdim, dtype=np.float32)
    lat = np.linspace(0, 25000, ydim, dtype=np.float32)
    time = np.arange(0., maxtime.total_seconds(), 60., dtype=np.float64)
    U = np.ones((xdim, ydim, 1), dtype=np.float32) * u_g + (u_0 - u_g) * np.cos(f * time)
    V = np.ones((xdim, ydim, 1), dtype=np.float32) * -(u_0 - u_g) * np.sin(f * time)
    return Grid.from_data(U, lon, lat, V, lon, lat, time=time, mesh='flat')

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_moving_eddy(grid_moving, mode, npart=1):
    grid = grid_moving
    lon = np.linspace(12000, 21000, npart, dtype=np.float32)
    lat = np.linspace(12500, 12500, npart, dtype=np.float32)
    pset = grid.ParticleSet(size=npart, pclass=ptype[mode], lon=lon, lat=lat)
    endtime = delta(hours=6).total_seconds()
    pset.execute(AdvectionRK4, dt=delta(minutes=3), endtime=endtime)
    exp_lon = [truth_moving(x, y, endtime)[0] for x, y, in zip(lon, lat)]
    exp_lat = [truth_moving(x, y, endtime)[1] for x, y, in zip(lon, lat)]
    assert np.allclose(np.array([p.lon for p in pset]), exp_lon, rtol=1e-5)
    assert np.allclose(np.array([p.lat for p in pset]), exp_lat, rtol=1e-5)

def truth_decaying(x_0, y_0, t):
    lat = y_0 - ((u_0 - u_g) * f / (f ** 2 + gamma ** 2) *
                 (1 - np.exp(-gamma * t) * (np.cos(f * t) + gamma / f * np.sin(f * t))))
    lon = x_0 + (u_g / gamma_g * (1 - np.exp(-gamma_g * t)) +
                 (u_0 - u_g) * f / (f ** 2 + gamma ** 2) *
                 (gamma / f + np.exp(-gamma * t) *
                  (math.sin(f * t) - gamma / f * math.cos(f * t))))
    return lon, lat

@pytest.fixture
def grid_decaying(xdim=100, ydim=100, maxtime=delta(hours=6)):
    """Generate a grid encapsulating the flow field of a decaying eddy.

    Reference: N. Fabbroni, 2009, "Numerical simulations of passive
    tracers dispersion in the sea"
    """
    lon = np.linspace(0, 25000, xdim, dtype=np.float32)
    lat = np.linspace(0, 25000, ydim, dtype=np.float32)
    time = np.arange(0., maxtime.total_seconds(), 60., dtype=np.float64)
    U = np.ones((xdim, ydim, 1), dtype=np.float32) * u_g *\
        np.exp(-gamma_g * time) + (u_0 - u_g) * np.exp(-gamma * time) * np.cos(f * time)
    V = np.ones((xdim, ydim, 1), dtype=np.float32) * -(u_0 - u_g) *\
        np.exp(-gamma * time) * np.sin(f * time)
    return Grid.from_data(U, lon, lat, V, lon, lat, time=time, mesh='flat')

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_decaying_eddy(grid_decaying, mode, npart=1):
    grid = grid_decaying
    lon = np.linspace(12000, 21000, npart, dtype=np.float32)
    lat = np.linspace(12500, 12500, npart, dtype=np.float32)
    pset = grid.ParticleSet(size=npart, pclass=ptype[mode], lon=lon, lat=lat)
    endtime = delta(hours=6).total_seconds()
    pset.execute(AdvectionRK4, dt=delta(minutes=3), endtime=endtime)
    exp_lon = [truth_decaying(x, y, endtime)[0] for x, y, in zip(lon, lat)]
    exp_lat = [truth_decaying(x, y, endtime)[1] for x, y, in zip(lon, lat)]
    assert np.allclose(np.array([p.lon for p in pset]), exp_lon, rtol=1e-5)
    assert np.allclose(np.array([p.lat for p in pset]), exp_lat, rtol=1e-5)
<|endoftext|>


<commit_before># Copyright (c) 2017, DjaoDjin inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import logging, os, shutil
from collections import OrderedDict
from functools import reduce #pylint:disable=redefined-builtin

from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
from django.core.files.storage import get_storage_class, FileSystemStorage
from django.http import Http404
from django.db.models import Q
from django.utils._os import safe_join
from django.utils import six
from rest_framework.generics import get_object_or_404

from . import settings
from .models import MediaTag, PageElement, ThemePackage
from .extras import AccountMixinBase

#pylint:disable=no-name-in-module,import-error
from django.utils.six.moves.urllib.parse import urljoin, urlsplit

LOGGER = logging.getLogger(__name__)

class AccountMixin(AccountMixinBase, settings.EXTRA_MIXIN):
    pass

class TrailMixin(object):
    """
    Generate a trail of PageElement based on a path.
    """

    @staticmethod
    def get_full_element_path(path):
        if not path:
            return []
        parts = path.split('/')
        if not parts[0]:
            parts.pop(0)
        results = []
        if parts:
            element = get_object_or_404(
                PageElement.objects.all(), slug=parts[-1])
            candidates = element.get_parent_paths(hints=parts[:-1])
            if not candidates:
                raise Http404("%s could not be found." % path)
            # XXX Implementation Note: if we have multiple candidates,
            # it means the hints were not enough to select a single path.
            # This is still OK to pick the first candidate as the breadcrumbs
            # should take a user back to the top-level page.
            if len(candidates) > 1:
                LOGGER.info("get_full_element_path has multiple candidates"\
                    " for '%s': %s", path, candidates)
            results = candidates[0]
        return results

class PageElementMixin(AccountMixin):

    lookup_field = 'slug'
    lookup_url_kwarg = 'slug'

    def get_queryset(self):
        return PageElement.objects.filter(account=self.account)

class UploadedImageMixin(object):

    def build_filter_list(self, validated_data):
        items = validated_data.get('items')
        filter_list = []
        if items:
            for item in items:
                location = item['location']
                parts = urlsplit(location)
                if parts.netloc == self.request.get_host():
                    location = parts.path
                filter_list += [location]
        return filter_list

    def list_media(self, storage, filter_list, prefix='.'):
        """
        Return a list of media from default storage
        """
        results = []
        total_count = 0
        if prefix.startswith('/'):
            prefix = prefix[1:]
        try:
            dirs, files = storage.listdir(prefix)
            for media in files:
                if prefix and prefix != '.':
                    media = prefix + '/' + media
                if not media.endswith('/') and media != "":
                    total_count += 1
                    location = storage.url(media)
                    try:
                        updated_at = storage.get_modified_time(media)
                    except AttributeError: # Django<2.0
                        updated_at = storage.modified_time(media)
                    normalized_location = location.split('?')[0]
                    if (filter_list is None
                        or normalized_location in filter_list):
                        results += [
                            {'location': location,
                            'tags': MediaTag.objects.filter(
                                location=normalized_location).values_list(
                                'tag', flat=True),
                            'updated_at': updated_at
                            }]
            for asset_dir in dirs:
                dir_results, dir_total_count = self.list_media(
                    storage, filter_list, prefix=prefix + '/' + asset_dir)
                results += dir_results
                total_count += dir_total_count
        except OSError:
            if storage.exists('.'):
                LOGGER.exception(
                    "Unable to list objects in %s.", storage.__class__.__name__)
        except S3ResponseError:
            LOGGER.exception(
                "Unable to list objects in %s bucket.", storage.bucket_name)

        # sort results by updated_at to sort by created_at.
        # Media are not updated, so updated_at = created_at
        return results, total_count

    def get_default_storage(self, account=None):
        storage_class = get_storage_class()
        try:
            _ = storage_class.bucket_name
            kwargs = {}
            for key in ['access_key', 'secret_key', 'security_token']:
                if key in self.request.session:
                    kwargs[key] = self.request.session[key]
            return storage_class(
                bucket=get_bucket_name(account),
                location=get_media_prefix(account),
                **kwargs)
        except AttributeError:
            LOGGER.debug("``%s`` does not contain a ``bucket_name``"\
                " field, default to FileSystemStorage.", storage_class)
        return self.get_file_system_storage(account)

    @staticmethod
    def get_file_system_storage(account=None):
        location = settings.MEDIA_ROOT
        base_url = settings.MEDIA_URL
        prefix = get_media_prefix(account)
        parts = location.split(os.sep)
        if prefix and prefix != parts[-1]:
            location = os.sep.join(parts[:-1] + [prefix, parts[-1]])
            if base_url.startswith('/'):
                base_url = base_url[1:]
            base_url = urljoin("/%s/" % prefix, base_url)
        return FileSystemStorage(location=location, base_url=base_url)

def get_bucket_name(account=None):
    if account:
        for bucket_field in settings.BUCKET_NAME_FROM_FIELDS:
            try:
                bucket_name = getattr(account, bucket_field)
                if bucket_name:
                    return bucket_name
            except AttributeError:
                pass
    return settings.AWS_STORAGE_BUCKET_NAME

def get_media_prefix(account=None):
    media_prefix = settings.MEDIA_PREFIX
    if account:
        try:
            media_prefix = account.media_prefix
        except AttributeError:
            LOGGER.debug("``%s`` does not contain a ``media_prefix``"\
                " field.", account.__class__)
        if not media_prefix:
            media_prefix = str(account)
    return media_prefix

class ThemePackageMixin(AccountMixin):

    @staticmethod
    def get_templates_dir(theme):
        if isinstance(settings.THEME_DIR_CALLABLE, six.string_types):
            from ..compat import import_string
            settings.THEME_DIR_CALLABLE = import_string(
                settings.THEME_DIR_CALLABLE)
        theme_dir = settings.THEME_DIR_CALLABLE(theme.slug)
        return safe_join(theme_dir, 'templates')

    @staticmethod
    def get_statics_dir(theme):
        return safe_join(settings.PUBLIC_ROOT, theme.slug, 'static')

    @staticmethod
    def get_file_tree(root_path):
        tree = OrderedDict()
        root_path = root_path.rstrip(os.sep)
        start = root_path.rfind(os.sep) + 1
        for (dirpath, _, filenames) in os.walk(root_path):
            folders = dirpath[start:].split(os.sep)
            subdir = dict.fromkeys(filenames)
            parent = reduce(dict.get, folders[:-1], tree)
            parent[folders[-1]] = OrderedDict(subdir)
        return tree

    @staticmethod
    def find_file(root_path, file_path):
        abs_file_path = None
        for (dirpath, _, filenames) in os.walk(root_path):
            for filename in filenames:
                if file_path in os.path.join(dirpath, filename):
                    abs_file_path = os.path.join(dirpath, filename)
        return abs_file_path

    def get_queryset(self):
        queryset = ThemePackage.objects.filter(
            Q(account=self.account)|Q(account=None)).order_by('-created_at')
        return queryset

    @staticmethod
    def get_file_from_s3(bucket, orig, dest):
        conn = S3Connection()
        bucket = conn.get_bucket(bucket)
        key = bucket.get_key(orig)

        if not key:
            return None
        else:# Save file from S3 into tmp_dir
            key.get_contents_to_filename(dest)
            return dest

    @staticmethod
    def create_file(to_path):
        if not os.path.exists(os.path.dirname(to_path)):
            os.makedirs(os.path.dirname(to_path))
        if not os.path.exists(to_path):
            open(to_path, 'w').close()

    @staticmethod
    def copy_file(from_path, to_path):
        if not os.path.exists(os.path.dirname(to_path)):
            os.makedirs(os.path.dirname(to_path))
        if not os.path.exists(to_path):
            shutil.copyfile(
                from_path,
                to_path)

    def copy_files(self, from_dir, to_dir):
        for (dirpath, _, filenames) in os.walk(from_dir):
            for filename in filenames:
                from_path = os.path.join(dirpath, filename)
                to_path = os.path.join(
                    to_dir,
                    from_path.replace(os.path.join(from_dir, ''), ''))
                self.copy_file(
                    from_path,
                    to_path)

    @staticmethod
    def write_zipfile(zipf, dir_path, dir_option=""):
        for dirname, _, files in os.walk(dir_path):
            for filename in files:
                abs_file_path = os.path.join(
                    dirname, filename)
                rel_file_path = os.path.join(
                    dir_option,
                    abs_file_path.replace("%s/" % dir_path, ''))

                zipf.write(abs_file_path,
                    rel_file_path)
        return zipf
<commit_msg>enables to pass acl parameter to storage<commit_after># Copyright (c) 2017, DjaoDjin inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import logging, os, shutil
from collections import OrderedDict
from functools import reduce #pylint:disable=redefined-builtin

from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
from django.core.files.storage import get_storage_class, FileSystemStorage
from django.http import Http404
from django.db.models import Q
from django.utils._os import safe_join
from django.utils import six
from rest_framework.generics import get_object_or_404

from . import settings
from .models import MediaTag, PageElement, ThemePackage
from .extras import AccountMixinBase

#pylint:disable=no-name-in-module,import-error
from django.utils.six.moves.urllib.parse import urljoin, urlsplit

LOGGER = logging.getLogger(__name__)

class AccountMixin(AccountMixinBase, settings.EXTRA_MIXIN):
    pass

class TrailMixin(object):
    """
    Generate a trail of PageElement based on a path.
    """

    @staticmethod
    def get_full_element_path(path):
        if not path:
            return []
        parts = path.split('/')
        if not parts[0]:
            parts.pop(0)
        results = []
        if parts:
            element = get_object_or_404(
                PageElement.objects.all(), slug=parts[-1])
            candidates = element.get_parent_paths(hints=parts[:-1])
            if not candidates:
                raise Http404("%s could not be found." % path)
            # XXX Implementation Note: if we have multiple candidates,
            # it means the hints were not enough to select a single path.
            # This is still OK to pick the first candidate as the breadcrumbs
            # should take a user back to the top-level page.
            if len(candidates) > 1:
                LOGGER.info("get_full_element_path has multiple candidates"\
                    " for '%s': %s", path, candidates)
            results = candidates[0]
        return results

class PageElementMixin(AccountMixin):

    lookup_field = 'slug'
    lookup_url_kwarg = 'slug'

    def get_queryset(self):
        return PageElement.objects.filter(account=self.account)

class UploadedImageMixin(object):

    def build_filter_list(self, validated_data):
        items = validated_data.get('items')
        filter_list = []
        if items:
            for item in items:
                location = item['location']
                parts = urlsplit(location)
                if parts.netloc == self.request.get_host():
                    location = parts.path
                filter_list += [location]
        return filter_list

    def list_media(self, storage, filter_list, prefix='.'):
        """
        Return a list of media from default storage
        """
        results = []
        total_count = 0
        if prefix.startswith('/'):
            prefix = prefix[1:]
        try:
            dirs, files = storage.listdir(prefix)
            for media in files:
                if prefix and prefix != '.':
                    media = prefix + '/' + media
                if not media.endswith('/') and media != "":
                    total_count += 1
                    location = storage.url(media)
                    try:
                        updated_at = storage.get_modified_time(media)
                    except AttributeError: # Django<2.0
                        updated_at = storage.modified_time(media)
                    normalized_location = location.split('?')[0]
                    if (filter_list is None
                        or normalized_location in filter_list):
                        results += [
                            {'location': location,
                            'tags': MediaTag.objects.filter(
                                location=normalized_location).values_list(
                                'tag', flat=True),
                            'updated_at': updated_at
                            }]
            for asset_dir in dirs:
                dir_results, dir_total_count = self.list_media(
                    storage, filter_list, prefix=prefix + '/' + asset_dir)
                results += dir_results
                total_count += dir_total_count
        except OSError:
            if storage.exists('.'):
                LOGGER.exception(
                    "Unable to list objects in %s.", storage.__class__.__name__)
        except S3ResponseError:
            LOGGER.exception(
                "Unable to list objects in %s bucket.", storage.bucket_name)

        # sort results by updated_at to sort by created_at.
        # Media are not updated, so updated_at = created_at
        return results, total_count

    def get_default_storage(self, account=None, **kwargs):
        storage_class = get_storage_class()
        try:
            _ = storage_class.bucket_name
            storage_kwargs = {}
            storage_kwargs.update(**kwargs)
            for key in ['access_key', 'secret_key', 'security_token']:
                if key in self.request.session:
                    storage_kwargs[key] = self.request.session[key]
            return storage_class(
                bucket=get_bucket_name(account),
                location=get_media_prefix(account),
                **storage_kwargs)
        except AttributeError:
            LOGGER.debug("``%s`` does not contain a ``bucket_name``"\
                " field, default to FileSystemStorage.", storage_class)
        return self.get_file_system_storage(account)

    @staticmethod
    def get_file_system_storage(account=None):
        location = settings.MEDIA_ROOT
        base_url = settings.MEDIA_URL
        prefix = get_media_prefix(account)
        parts = location.split(os.sep)
        if prefix and prefix != parts[-1]:
            location = os.sep.join(parts[:-1] + [prefix, parts[-1]])
            if base_url.startswith('/'):
                base_url = base_url[1:]
            base_url = urljoin("/%s/" % prefix, base_url)
        return FileSystemStorage(location=location, base_url=base_url)

def get_bucket_name(account=None):
    if account:
        for bucket_field in settings.BUCKET_NAME_FROM_FIELDS:
            try:
                bucket_name = getattr(account, bucket_field)
                if bucket_name:
                    return bucket_name
            except AttributeError:
                pass
    return settings.AWS_STORAGE_BUCKET_NAME

def get_media_prefix(account=None):
    media_prefix = settings.MEDIA_PREFIX
    if account:
        try:
            media_prefix = account.media_prefix
        except AttributeError:
            LOGGER.debug("``%s`` does not contain a ``media_prefix``"\
                " field.", account.__class__)
        if not media_prefix:
            media_prefix = str(account)
    return media_prefix

class ThemePackageMixin(AccountMixin):

    @staticmethod
    def get_templates_dir(theme):
        if isinstance(settings.THEME_DIR_CALLABLE, six.string_types):
            from ..compat import import_string
            settings.THEME_DIR_CALLABLE = import_string(
                settings.THEME_DIR_CALLABLE)
        theme_dir = settings.THEME_DIR_CALLABLE(theme.slug)
        return safe_join(theme_dir, 'templates')

    @staticmethod
    def get_statics_dir(theme):
        return safe_join(settings.PUBLIC_ROOT, theme.slug, 'static')

    @staticmethod
    def get_file_tree(root_path):
        tree = OrderedDict()
        root_path = root_path.rstrip(os.sep)
        start = root_path.rfind(os.sep) + 1
        for (dirpath, _, filenames) in os.walk(root_path):
            folders = dirpath[start:].split(os.sep)
            subdir = dict.fromkeys(filenames)
            parent = reduce(dict.get, folders[:-1], tree)
            parent[folders[-1]] = OrderedDict(subdir)
        return tree

    @staticmethod
    def find_file(root_path, file_path):
        abs_file_path = None
        for (dirpath, _, filenames) in os.walk(root_path):
            for filename in filenames:
                if file_path in os.path.join(dirpath, filename):
                    abs_file_path = os.path.join(dirpath, filename)
        return abs_file_path

    def get_queryset(self):
        queryset = ThemePackage.objects.filter(
            Q(account=self.account)|Q(account=None)).order_by('-created_at')
        return queryset

    @staticmethod
    def get_file_from_s3(bucket, orig, dest):
        conn = S3Connection()
        bucket = conn.get_bucket(bucket)
        key = bucket.get_key(orig)

        if not key:
            return None
        else:# Save file from S3 into tmp_dir
            key.get_contents_to_filename(dest)
            return dest

    @staticmethod
    def create_file(to_path):
        if not os.path.exists(os.path.dirname(to_path)):
            os.makedirs(os.path.dirname(to_path))
        if not os.path.exists(to_path):
            open(to_path, 'w').close()

    @staticmethod
    def copy_file(from_path, to_path):
        if not os.path.exists(os.path.dirname(to_path)):
            os.makedirs(os.path.dirname(to_path))
        if not os.path.exists(to_path):
            shutil.copyfile(
                from_path,
                to_path)

    def copy_files(self, from_dir, to_dir):
        for (dirpath, _, filenames) in os.walk(from_dir):
            for filename in filenames:
                from_path = os.path.join(dirpath, filename)
                to_path = os.path.join(
                    to_dir,
                    from_path.replace(os.path.join(from_dir, ''), ''))
                self.copy_file(
                    from_path,
                    to_path)

    @staticmethod
    def write_zipfile(zipf, dir_path, dir_option=""):
        for dirname, _, files in os.walk(dir_path):
            for filename in files:
                abs_file_path = os.path.join(
                    dirname, filename)
                rel_file_path = os.path.join(
                    dir_option,
                    abs_file_path.replace("%s/" % dir_path, ''))

                zipf.write(abs_file_path,
                    rel_file_path)
        return zipf
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#

__version__ = '4.0.5'
__author__ = u'Nico Schlömer'
__author_email__ = 'nico.schloemer@gmail.com'
__website__ = 'https://github.com/nschloe/pygmsh'
__license__ = 'License :: OSI Approved :: MIT License'
__status__ = 'Development Status :: 5 - Production'
<commit_msg>fix status<commit_after># -*- coding: utf-8 -*-
#

__version__ = '4.0.5'
__author__ = u'Nico Schlömer'
__author_email__ = 'nico.schloemer@gmail.com'
__website__ = 'https://github.com/nschloe/pygmsh'
__license__ = 'License :: OSI Approved :: MIT License'
__status__ = 'Development Status :: 5 - Production/Stable'
<|endoftext|>