<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function, unicode_literals
from fsed.utils import open_file
import click
import fsed.ahocorasick
import logging
import re
import sys

logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                    stream=sys.stderr, level=logging.INFO)
LOGGER = logging.getLogger(__name__)

def set_log_level(verbose, quiet):
    '''Sets the logging level.'''
    if quiet:
        verbose = -1
    if verbose < 0:
        verbose = logging.CRITICAL
    elif verbose == 1:
        verbose = logging.INFO
    elif 1 < verbose:
        verbose = logging.DEBUG
    LOGGER.setLevel(verbose)
    fsed.ahocorasick.LOGGER.setLevel(verbose)

def detect_pattern_format(pattern_filename, encoding, on_word_boundaries):
    tsv = True
    boundaries = on_word_boundaries
    with open_file(pattern_filename) as input_file:
        for line in input_file:
            line = line.decode(encoding)
            if line.count('\t') != 1:
                tsv = False
            if '\\b' in line:
                boundaries = True
            if boundaries and not tsv:
                break
    return tsv, boundaries

def sub_escapes(sval):
    sval = sval.replace('\\a', '\a')
    sval = sval.replace('\\b', '\x00')
    sval = sval.replace('\\f', '\f')
    sval = sval.replace('\\n', '\n')
    sval = sval.replace('\\r', '\r')
    sval = sval.replace('\\t', '\t')
    sval = sval.replace('\\v', '\v')
    sval = sval.replace('\\\\', '\\')
    return sval

def build_trie(pattern_filename, pattern_format, encoding, on_word_boundaries):
    boundaries = on_word_boundaries
    if pattern_format == 'auto' or not on_word_boundaries:
        tsv, boundaries = detect_pattern_format(pattern_filename, encoding,
                                                on_word_boundaries)
    if pattern_format == 'auto':
        if tsv:
            pattern_format = 'tsv'
        else:
            pattern_format = 'sed'
    trie = fsed.ahocorasick.AhoCorasickTrie()
    num_candidates = 0
    with open_file(pattern_filename) as pattern_file:
        for lineno, line in enumerate(pattern_file):
            line = line.decode(encoding).rstrip('\n')
            if not line.strip():
                continue
            # decode the line
            if pattern_format == 'tsv':
                fields = line.split('\t')
                if len(fields) != 2:
                    LOGGER.warning(('skipping line {} of pattern file (not '
                                    'in tab-separated format): {}').format(lineno, line))
                    continue
                before, after = fields
            elif pattern_format == 'sed':
                before = after = None
                line = line.lstrip()
                if line[0] == 's':
                    delim = line[1]
                    fields = re.split(r'(?<!\\){}'.format(delim), line)
                    if len(fields) == 4:
                        before, after = fields[1], fields[2]
                        before = re.sub(r'(?<!\\)\\{}'.format(delim), delim, before)
                        after = re.sub(r'(?<!\\)\\{}'.format(delim), delim, after)
                if before is None or after is None:
                    LOGGER.warning(('skipping line {} of pattern file (not '
                                    'in sed format): {}').format(lineno, line))
                    continue
            num_candidates += 1
            if on_word_boundaries and before != before.strip():
                LOGGER.warning(('before pattern on line {} padded whitespace; '
                                'this may interact strangely with the --words '
                                'option: {}').format(lineno, line))
            before = sub_escapes(before)
            after = sub_escapes(after)
            if boundaries:
                before = fsed.ahocorasick.boundary_transform(before, on_word_boundaries)
            trie[before] = after
    LOGGER.info('{} patterns loaded from {}'.format(num_candidates,
                                                    pattern_filename))
    return trie, boundaries

@click.command()
@click.argument('pattern_filename', type=click.Path(exists=True),
                metavar='PATTERN_FILE')
@click.argument('input_filenames', default='-', nargs=-1,
                type=click.Path(exists=True), metavar='[INPUT_FILES]')
@click.option('--pattern-format', type=click.Choice(['auto', 'tsv', 'sed']),
              default='auto', show_default=True,
              help='Specify the format of PATTERN_FILE')
@click.option('-o', '--output', 'output_filename', type=click.Path(),
              help='Program output is written '
              'to this file. Default is to write '
              'to standard output.')
@click.option('-e', '--encoding', default='utf-8', show_default=True,
              help='The character encoding to use')
@click.option('-w', '--words', is_flag=True,
              help='Match only on word boundaries: '
              'appends "\\b" to the beginning and '
              'end of every pattern in PATTERN_FILE.')
@click.option('--by-line/--across-lines', default=False,
              help='Process the input line by '
              'line or character by character; the default is --across-lines.')
@click.option('--slow', is_flag=True,
              help='Try very hard to '
              'find the longest matches on the input; this is very slow, '
              'and forces --by-line.')
@click.option('-v', '--verbose', default=0, count=True,
              help='Turns on debugging output.')
@click.option('-q', '--quiet', is_flag=True,
              help='Quiet operation, do not emit warnings.')
def main(pattern_filename, input_filenames, pattern_format,
         output_filename,
         encoding, words, by_line, slow, verbose, quiet):
    '''
    Search and replace on INPUT_FILE(s) (or standard input), with
    matching on fixed strings.
    '''
    set_log_level(verbose, quiet)
    if slow:
        by_line = True
    by_line = True # TODO: implement non-line-based rewriting
    # load the patterns
    LOGGER.info('fsed {} input {} output {}'.format(pattern_filename,
                                                    input_filenames,
                                                    output_filename))
    if not input_filenames:
        input_filenames = ('-',)
    if not output_filename:
        output_filename = '-'
    # build trie machine for matching
    trie, boundaries = build_trie(pattern_filename, pattern_format, encoding, words)
    LOGGER.info('writing to {}'.format(output_filename))
    with open_file(output_filename, 'w') as output_file:
        for input_filename in input_filenames:
            # search and replace
            with open_file(input_filename) as input_file:
                LOGGER.info('reading {}'.format(input_filename))
                if by_line:
                    num_lines = 0
                    for line in input_file:
                        line = line.decode(encoding).rstrip('\n')
                        if boundaries:
                            line = fsed.ahocorasick.boundary_transform(line)
                        if slow:
                            line = trie.replace(line)
                        else:
                            line = trie.greedy_replace(line)
                        if boundaries:
                            line = ''.join(fsed.ahocorasick.boundary_untransform(line))
                        output_file.write((line + '\n').encode(encoding))
                        num_lines += 1
                    LOGGER.info('{} lines written'.format(num_lines))
                else:
                    raise NotImplementedError

if __name__ == '__main__':
    main()
<commit_msg>fsed/fsed: docstrings<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function, unicode_literals
from fsed.utils import open_file
import click
import fsed.ahocorasick
import logging
import re
import sys

logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                    stream=sys.stderr, level=logging.INFO)
LOGGER = logging.getLogger(__name__)

def set_log_level(verbose, quiet):
    '''
    Ses the logging level of the script based on command line options.

    Arguments:
    - `verbose`:
    - `quiet`:
    '''
    '''Sets the logging level.'''
    if quiet:
        verbose = -1
    if verbose < 0:
        verbose = logging.CRITICAL
    elif verbose == 1:
        verbose = logging.INFO
    elif 1 < verbose:
        verbose = logging.DEBUG
    LOGGER.setLevel(verbose)
    fsed.ahocorasick.LOGGER.setLevel(verbose)

def detect_pattern_format(pattern_filename, encoding, on_word_boundaries):
    '''
    Automatically detects the pattern file format, and determines
    whether the Aho-Corasick string matching should pay attention to
    word boundaries or not.

    Arguments:
    - `pattern_filename`:
    - `encoding`:
    - `on_word_boundaries`:
    '''
    tsv = True
    boundaries = on_word_boundaries
    with open_file(pattern_filename) as input_file:
        for line in input_file:
            line = line.decode(encoding)
            if line.count('\t') != 1:
                tsv = False
            if '\\b' in line:
                boundaries = True
            if boundaries and not tsv:
                break
    return tsv, boundaries

def sub_escapes(sval):
    '''
    Process escaped characters in ``sval``.

    Arguments:
    - `sval`:
    '''
    sval = sval.replace('\\a', '\a')
    sval = sval.replace('\\b', '\x00')
    sval = sval.replace('\\f', '\f')
    sval = sval.replace('\\n', '\n')
    sval = sval.replace('\\r', '\r')
    sval = sval.replace('\\t', '\t')
    sval = sval.replace('\\v', '\v')
    sval = sval.replace('\\\\', '\\')
    return sval

def build_trie(pattern_filename, pattern_format, encoding, on_word_boundaries):
    '''
    Constructs a finite state machine for performing string rewriting.

    Arguments:
    - `pattern_filename`:
    - `pattern_format`:
    - `encoding`:
    - `on_word_boundaries`:
    '''
    boundaries = on_word_boundaries
    if pattern_format == 'auto' or not on_word_boundaries:
        tsv, boundaries = detect_pattern_format(pattern_filename, encoding,
                                                on_word_boundaries)
    if pattern_format == 'auto':
        if tsv:
            pattern_format = 'tsv'
        else:
            pattern_format = 'sed'
    trie = fsed.ahocorasick.AhoCorasickTrie()
    num_candidates = 0
    with open_file(pattern_filename) as pattern_file:
        for lineno, line in enumerate(pattern_file):
            line = line.decode(encoding).rstrip('\n')
            if not line.strip():
                continue
            # decode the line
            if pattern_format == 'tsv':
                fields = line.split('\t')
                if len(fields) != 2:
                    LOGGER.warning(('skipping line {} of pattern file (not '
                                    'in tab-separated format): {}').format(lineno, line))
                    continue
                before, after = fields
            elif pattern_format == 'sed':
                before = after = None
                line = line.lstrip()
                if line[0] == 's':
                    delim = line[1]
                    fields = re.split(r'(?<!\\){}'.format(delim), line)
                    if len(fields) == 4:
                        before, after = fields[1], fields[2]
                        before = re.sub(r'(?<!\\)\\{}'.format(delim), delim, before)
                        after = re.sub(r'(?<!\\)\\{}'.format(delim), delim, after)
                if before is None or after is None:
                    LOGGER.warning(('skipping line {} of pattern file (not '
                                    'in sed format): {}').format(lineno, line))
                    continue
            num_candidates += 1
            if on_word_boundaries and before != before.strip():
                LOGGER.warning(('before pattern on line {} padded whitespace; '
                                'this may interact strangely with the --words '
                                'option: {}').format(lineno, line))
            before = sub_escapes(before)
            after = sub_escapes(after)
            if boundaries:
                before = fsed.ahocorasick.boundary_transform(before, on_word_boundaries)
            trie[before] = after
    LOGGER.info('{} patterns loaded from {}'.format(num_candidates,
                                                    pattern_filename))
    return trie, boundaries

@click.command()
@click.argument('pattern_filename', type=click.Path(exists=True),
                metavar='PATTERN_FILE')
@click.argument('input_filenames', default='-', nargs=-1,
                type=click.Path(exists=True), metavar='[INPUT_FILES]')
@click.option('--pattern-format', type=click.Choice(['auto', 'tsv', 'sed']),
              default='auto', show_default=True,
              help='Specify the format of PATTERN_FILE')
@click.option('-o', '--output', 'output_filename', type=click.Path(),
              help='Program output is written '
              'to this file. Default is to write '
              'to standard output.')
@click.option('-e', '--encoding', default='utf-8', show_default=True,
              help='The character encoding to use')
@click.option('-w', '--words', is_flag=True,
              help='Match only on word boundaries: '
              'appends "\\b" to the beginning and '
              'end of every pattern in PATTERN_FILE.')
@click.option('--by-line/--across-lines', default=False,
              help='Process the input line by '
              'line or character by character; the default is --across-lines.')
@click.option('--slow', is_flag=True,
              help='Try very hard to '
              'find the longest matches on the input; this is very slow, '
              'and forces --by-line.')
@click.option('-v', '--verbose', default=0, count=True,
              help='Turns on debugging output.')
@click.option('-q', '--quiet', is_flag=True,
              help='Quiet operation, do not emit warnings.')
def main(pattern_filename, input_filenames, pattern_format,
         output_filename,
         encoding, words, by_line, slow, verbose, quiet):
    '''
    Search and replace on INPUT_FILE(s) (or standard input), with
    matching on fixed strings.
    '''
    set_log_level(verbose, quiet)
    if slow:
        by_line = True
    by_line = True # TODO: implement non-line-based rewriting
    # load the patterns
    LOGGER.info('fsed {} input {} output {}'.format(pattern_filename,
                                                    input_filenames,
                                                    output_filename))
    if not input_filenames:
        input_filenames = ('-',)
    if not output_filename:
        output_filename = '-'
    # build trie machine for matching
    trie, boundaries = build_trie(pattern_filename, pattern_format, encoding, words)
    LOGGER.info('writing to {}'.format(output_filename))
    with open_file(output_filename, 'w') as output_file:
        for input_filename in input_filenames:
            # search and replace
            with open_file(input_filename) as input_file:
                LOGGER.info('reading {}'.format(input_filename))
                if by_line:
                    num_lines = 0
                    for line in input_file:
                        line = line.decode(encoding).rstrip('\n')
                        if boundaries:
                            line = fsed.ahocorasick.boundary_transform(line)
                        if slow:
                            line = trie.replace(line)
                        else:
                            line = trie.greedy_replace(line)
                        if boundaries:
                            line = ''.join(fsed.ahocorasick.boundary_untransform(line))
                        output_file.write((line + '\n').encode(encoding))
                        num_lines += 1
                    LOGGER.info('{} lines written'.format(num_lines))
                else:
                    raise NotImplementedError

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import math
import random
from mxnet import nd, autograd
from mxnet.test_utils import assert_almost_equal, random_arrays, rand_shape_nd
from common import with_seed

@with_seed()
def test_sin():
    def sin(x):
        return nd.sin(x)

    def grad_grad_op(x):
        return -nd.sin(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, sin, grad_grad_op)

@with_seed()
def test_cos():
    def cos(x):
        return nd.cos(x)

    def grad_grad_op(x):
        return -nd.cos(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, cos, grad_grad_op)

@with_seed()
def test_tan():
    def tan(x):
        return nd.tan(x)

    def grad_op(x):
        return 1 / nd.cos(x)**2

    def grad_grad_op(x):
        return 2 * tan(x) * grad_op(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, tan, grad_grad_op)

@with_seed()
def test_sinh():
    def sinh(x):
        return nd.sinh(x)

    def grad_grad_op(x):
        return sinh(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, sinh, grad_grad_op)

@with_seed()
def test_cosh():
    def cosh(x):
        return nd.cosh(x)

    def grad_grad_op(x):
        return cosh(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, cosh, grad_grad_op)

@with_seed()
def test_tanh():
    def tanh(x):
        return nd.tanh(x)

    def grad_op(x):
        return 1 / nd.cosh(x)**2

    def grad_grad_op(x):
        return -2 * tanh(x) * grad_op(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(
            array, tanh, grad_grad_op, rtol=1e-6, atol=1e-6)

@with_seed()
def test_arctan():
    def arctan(x):
        return nd.arctan(x)

    def grad_grad_op(x):
        return (-2 * x)/((1 + x**2)**2)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        # Domain of arctan is all real numbers.
        # Scale std_dev
        array *= random.randint(500, 10000)
        check_second_order_unary(array, arctan, grad_grad_op)

@with_seed()
def test_arctanh():
    def arctanh(x):
        return nd.arctanh(x)

    def grad_grad_op(x):
        return (2 * x)/((1 - x**2)**2)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, arctanh, grad_grad_op)

@with_seed()
def test_radians():
    def radians(x):
        return nd.radians(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, radians, grad_grad_op)

@with_seed()
def test_relu():
    def relu(x):
        return nd.relu(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, relu, grad_grad_op)

@with_seed()
def test_log():
    def log(x):
        return nd.log(x)

    def grad_grad_op(x):
        return -1/(x**2)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, log, grad_grad_op)

@with_seed()
def test_log2():
    def log2(x):
        return nd.log2(x)

    def grad_grad_op(x):
        return -1/((x**2) * math.log(2))

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, log2, grad_grad_op)

@with_seed()
def test_log10():
    def log10(x):
        return nd.log10(x)

    def grad_grad_op(x):
        return -1/((x**2) * math.log(10))

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, log10, grad_grad_op)

@with_seed()
def test_reciprocal():
    def reciprocal(x):
        return nd.reciprocal(x)

    def grad_grad_op(x):
        return 2 / x**3

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, reciprocal, grad_grad_op)

@with_seed()
def test_abs():
    def abs(x):
        return nd.abs(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, abs, grad_grad_op)

@with_seed()
def test_clip():
    def clip(x):
        a_min, a_max = sorted([random.random(), random.random()])

        return nd.clip(x, a_min, a_max)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, clip, grad_grad_op)

@with_seed()
def test_dropout():
    def dropout(x):
        return nd.Dropout(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, dropout, grad_grad_op)

@with_seed()
def test_sigmoid():
    def sigmoid(x):
        return nd.sigmoid(x)

    def grad_op(x):
        return sigmoid(x) * (1 - sigmoid(x))

    def grad_grad_op(x):
        return grad_op(x) * (1 - 2 * sigmoid(x))

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, sigmoid, grad_grad_op)

@with_seed()
def test_sqrt():
    def sqrt(x):
        return nd.sqrt(x)

    def grad_grad_op(x):
        return -1/(4 * sqrt(x**3))

    sigma = random.randint(25, 100)
    mu = random.randint(500, 1000)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        array = sigma * array + mu
        # Only positive numbers
        assert((array > 0).all())
        check_second_order_unary(array, sqrt, grad_grad_op)

@with_seed()
def test_cbrt():
    def cbrt(x):
        return nd.cbrt(x)

    def grad_grad_op(x):
        return -2/(9 * cbrt(x**5))

    sigma = random.randint(25, 100)
    mu = random.randint(500, 1000)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        array = sigma * array + mu
        # Only positive numbers
        assert((array > 0).all())
        check_second_order_unary(array, cbrt, grad_grad_op)

def check_second_order_unary(x, op, grad_grad_op, rtol=None, atol=None):
    x = nd.array(x)
    grad_grad_x = grad_grad_op(x)
    x.attach_grad()

    # Manual head_grads.
    y_grad = nd.random.normal(shape=x.shape)
    head_grad_grads = nd.random.normal(shape=x.shape)

    # Perform compute.
    with autograd.record():
        y = op(x)
        x_grad = autograd.grad(heads=y, variables=x, head_grads=y_grad,
                               create_graph=True, retain_graph=True)[0]
    x_grad.backward(head_grad_grads)

    # Compute expected values.
    expected_grad_grad = grad_grad_x.asnumpy() * head_grad_grads.asnumpy() * \
        y_grad.asnumpy()

    # Validate the gradients.
    assert_almost_equal(expected_grad_grad,
                        x.grad.asnumpy(), rtol=rtol, atol=atol)

if __name__ == '__main__':
    import nose
    nose.runmodule()
<commit_msg>[MXNET-978] n-th order gradient test support. (#15611)<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import math
import random
from mxnet import nd, autograd
from mxnet.test_utils import assert_almost_equal, random_arrays, rand_shape_nd
from common import with_seed

@with_seed()
def test_sin():
    def sin(x):
        return nd.sin(x)

    def grad_grad_op(x):
        return -nd.sin(x)

    def grad_grad_grad_op(x):
        return -nd.cos(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, sin, grad_grad_op)
        # TODO(kshitij12345): Remove
        check_nth_order_unary(array, sin,
                              [grad_grad_op, grad_grad_grad_op], [2, 3])

@with_seed()
def test_cos():
    def cos(x):
        return nd.cos(x)

    def grad_grad_op(x):
        return -nd.cos(x)

    def grad_grad_grad_op(x):
        return nd.sin(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, cos, grad_grad_op)
        # TODO(kshitij12345): Remove
        check_nth_order_unary(array, cos,
                              [grad_grad_op, grad_grad_grad_op], [2, 3])

@with_seed()
def test_tan():
    def tan(x):
        return nd.tan(x)

    def grad_op(x):
        return 1 / nd.cos(x)**2

    def grad_grad_op(x):
        return 2 * tan(x) * grad_op(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, tan, grad_grad_op)

@with_seed()
def test_sinh():
    def sinh(x):
        return nd.sinh(x)

    def grad_grad_op(x):
        return sinh(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, sinh, grad_grad_op)

@with_seed()
def test_cosh():
    def cosh(x):
        return nd.cosh(x)

    def grad_grad_op(x):
        return cosh(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, cosh, grad_grad_op)

@with_seed()
def test_tanh():
    def tanh(x):
        return nd.tanh(x)

    def grad_op(x):
        return 1 / nd.cosh(x)**2

    def grad_grad_op(x):
        return -2 * tanh(x) * grad_op(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(
            array, tanh, grad_grad_op, rtol=1e-6, atol=1e-6)

@with_seed()
def test_arctan():
    def arctan(x):
        return nd.arctan(x)

    def grad_grad_op(x):
        return (-2 * x)/((1 + x**2)**2)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        # Domain of arctan is all real numbers.
        # Scale std_dev
        array *= random.randint(500, 10000)
        check_second_order_unary(array, arctan, grad_grad_op)

@with_seed()
def test_arctanh():
    def arctanh(x):
        return nd.arctanh(x)

    def grad_grad_op(x):
        return (2 * x)/((1 - x**2)**2)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, arctanh, grad_grad_op)

@with_seed()
def test_radians():
    def radians(x):
        return nd.radians(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, radians, grad_grad_op)

@with_seed()
def test_relu():
    def relu(x):
        return nd.relu(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, relu, grad_grad_op)

@with_seed()
def test_log():
    def log(x):
        return nd.log(x)

    def grad_op(x):
        return 1/x

    def grad_grad_op(x):
        return -1/(x**2)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, log, grad_grad_op)
        # TODO(kshitij12345): Remove
        check_nth_order_unary(array, log, [grad_op, grad_grad_op], [1, 2])

@with_seed()
def test_log2():
    def log2(x):
        return nd.log2(x)

    def grad_grad_op(x):
        return -1/((x**2) * math.log(2))

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, log2, grad_grad_op)

@with_seed()
def test_log10():
    def log10(x):
        return nd.log10(x)

    def grad_grad_op(x):
        return -1/((x**2) * math.log(10))

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, log10, grad_grad_op)

@with_seed()
def test_reciprocal():
    def reciprocal(x):
        return nd.reciprocal(x)

    def grad_grad_op(x):
        return 2 / x**3

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, reciprocal, grad_grad_op)

@with_seed()
def test_abs():
    def abs(x):
        return nd.abs(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, abs, grad_grad_op)

@with_seed()
def test_clip():
    def clip(x):
        a_min, a_max = sorted([random.random(), random.random()])

        return nd.clip(x, a_min, a_max)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, clip, grad_grad_op)

@with_seed()
def test_dropout():
    def dropout(x):
        return nd.Dropout(x)

    def grad_grad_op(x):
        return nd.zeros_like(x)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, dropout, grad_grad_op)

@with_seed()
def test_sigmoid():
    def sigmoid(x):
        return nd.sigmoid(x)

    def grad_op(x):
        return sigmoid(x) * (1 - sigmoid(x))

    def grad_grad_op(x):
        return grad_op(x) * (1 - 2 * sigmoid(x))

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        check_second_order_unary(array, sigmoid, grad_grad_op)
        # TODO(kshitij12345): Remove
        check_nth_order_unary(array, sigmoid, [grad_op, grad_grad_op], [1, 2])
        check_nth_order_unary(array, sigmoid, grad_grad_op, 2)

@with_seed()
def test_sqrt():
    def sqrt(x):
        return nd.sqrt(x)

    def grad_grad_op(x):
        return -1/(4 * sqrt(x**3))

    sigma = random.randint(25, 100)
    mu = random.randint(500, 1000)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        array = sigma * array + mu
        # Only positive numbers
        assert((array > 0).all())
        check_second_order_unary(array, sqrt, grad_grad_op)

@with_seed()
def test_cbrt():
    def cbrt(x):
        return nd.cbrt(x)

    def grad_grad_op(x):
        return -2/(9 * cbrt(x**5))

    sigma = random.randint(25, 100)
    mu = random.randint(500, 1000)

    for dim in range(1, 5):
        shape = rand_shape_nd(dim)
        array = random_arrays(shape)
        array = sigma * array + mu
        # Only positive numbers
        assert((array > 0).all())
        check_second_order_unary(array, cbrt, grad_grad_op)

def check_second_order_unary(x, op, grad_grad_op, rtol=None, atol=None):
    check_nth_order_unary(x, op, grad_grad_op, 2, rtol, atol)

def check_nth_order_unary(x, op, grad_ops, orders, rtol=None, atol=None):
    """Assert n-th order autograd gradient against expected gradient.

    Multiple order of gradients can be checked by passing list of
    function computing the particular order gradient and passing the
    corresponding list of order.

    Note
    ----
    1. Orders should always be monotonically increasing.
    2. Elements of grads_ops should correspond to elements of orders
    i.e. grads_op = [grad_op, grad_grad_grad_op] should be passed with
         orders = [1, 3]

    Parameters
    ----------
    x : mxnet.NDArray
        Input Array.
    op : Callable
        Operation to perform on Input Array.
    grad_ops : Callable or List of Callable
        Function to compute and assert gradient of given order.
    orders : int or List of int
        Order/s to assert expected and computed gradients.

    Returns
    -------
    None

    """
    if isinstance(orders, int):
        orders = [orders]
        grad_ops = [grad_ops]

    assert all(i < j for i, j in zip(orders[0:-1], orders[1:])), \
        "orders should be monotonically increasing"
    assert len(set(orders)) == len(orders), \
        "orders should have unique elements"
    highest_order = max(orders)

    x = nd.array(x)
    x.attach_grad()

    expected_grads = [grad_op(x) for grad_op in grad_ops]
    computed_grads = []
    head_grads = []

    # Perform compute.
    with autograd.record():
        y = op(x)
        for current_order in range(1, highest_order+1):
            head_grad = nd.random.normal(shape=x.shape)
            y = autograd.grad(heads=y, variables=x, head_grads=head_grad,
                              create_graph=True, retain_graph=True)[0]
            if current_order in orders:
                computed_grads.append(y)
            head_grads.append(head_grad)

    # Validate all the gradients.
    for order, grad, computed_grad in \
            zip(orders, expected_grads, computed_grads):
        # Compute expected values.
        expected_grad = grad.asnumpy()
        for head_grad in head_grads[:order]:
            expected_grad *= head_grad.asnumpy()

        assert_almost_equal(
            expected_grad, computed_grad.asnumpy(), rtol=rtol, atol=atol)

if __name__ == '__main__':
    import nose
    nose.runmodule()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (C) 2015-2016 GEM Foundation

# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import time
import uuid
from datetime import datetime
import multiprocessing
from multiprocessing.connection import Listener, Client

import numpy
import h5py

from openquake.baselib.general import humansize
from openquake.baselib.hdf5 import Hdf5Dataset

import psutil
if psutil.__version__ > '2.0.0':  # Ubuntu 14.10
    def virtual_memory():
        return psutil.virtual_memory()

    def memory_info(proc):
        return proc.memory_info()

elif psutil.__version__ >= '1.2.1':  # Ubuntu 14.04
    def virtual_memory():
        return psutil.virtual_memory()

    def memory_info(proc):
        return proc.get_memory_info()

else:  # Ubuntu 12.04
    def virtual_memory():
        return psutil.phymem_usage()

    def memory_info(proc):
        return proc.get_memory_info()

perf_dt = numpy.dtype([('operation', (bytes, 50)), ('time_sec', float),
                       ('memory_mb', float), ('counts', int)])

class Process(multiprocessing.Process):
    def __init__(self, target, *args):
        multiprocessing.Process.__init__(self, target=target, args=args)

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args):
        self.join()

def read_all(listener):
    """
    A listener with an associated monitor that can send back messages.
    The sent messages can be read by iterating on the listener.

    >>> def greetings(monitor):
    ...     with monitor:
    ...         monitor.send('hello')
    ...         monitor.send('world')
    ...         monitor.send(SystemExit())
    >>> monitor = PerformanceMonitor('test')
    >>> listener = monitor.make_listener('')
    >>> with Process(greetings, monitor):
    ...     list(read_all(listener))
    ['hello', 'world']
    """
    while True:
        conn = listener.accept()
        try:
            got = conn.recv()
            if isinstance(got, SystemExit):
                break
            yield got
        finally:
            conn.close()
    listener.close()

# this is not thread-safe
class PerformanceMonitor(object):
    """
    Measure the resident memory occupied by a list of processes during
    the execution of a block of code. Should be used as a context manager,
    as follows::

     with PerformanceMonitor('do_something') as mon:
         do_something()
     print mon.mem

    At the end of the block the PerformanceMonitor object will have the
    following 5 public attributes:

    .start_time: when the monitor started (a datetime object)
    .duration: time elapsed between start and stop (in seconds)
    .exc: usually None; otherwise the exception happened in the `with` block
    .mem: the memory delta in bytes

    The behaviour of the PerformanceMonitor can be customized by subclassing it
    and by overriding the method on_exit(), called at end and used to display
    or store the results of the analysis.

    NB: if the .address and .authkey attributes are set, it is possible for
    the monitor to send objects to that address, assuming there is
    :class:`multiprocessing.connection.Listener` listening.
    """
    def __init__(self, operation, hdf5path=None,
                 autoflush=False, measuremem=False):
        self.operation = operation
        self.hdf5path = hdf5path
        self.autoflush = autoflush
        self.measuremem = measuremem
        self.mem = 0
        self.duration = 0
        self._start_time = self._stop_time = time.time()
        self.children = []
        self.counts = 0
        self.address = None

    @property
    def dt(self):
        """Last time interval measured"""
        return self._stop_time - self._start_time

    def measure_mem(self):
        """A memory measurement (in bytes)"""
        proc = psutil.Process(os.getpid())
        try:
            return memory_info(proc).rss
        except psutil.AccessDenied:
            # no access to information about this process
            pass

    @property
    def start_time(self):
        """
        Datetime instance recording when the monitoring started
        """
        return datetime.fromtimestamp(self._start_time)

    def get_data(self):
        """
        :returns:
            an array of dtype perf_dt, with the information
            of the monitor (operation, time_sec, memory_mb, counts);
            the lenght of the array can be 0 (for counts=0) or 1 (otherwise).
        """
        data = []
        if self.counts:
            time_sec = self.duration
            memory_mb = self.mem / 1024. / 1024. if self.measuremem else 0
            data.append((self.operation, time_sec, memory_mb, self.counts))
        return numpy.array(data, perf_dt)

    def __enter__(self):
        self.exc = None  # exception
        self._start_time = time.time()
        if self.measuremem:
            self.start_mem = self.measure_mem()
        return self

    def __exit__(self, etype, exc, tb):
        self.exc = exc
        if self.measuremem:
            self.stop_mem = self.measure_mem()
            self.mem += self.stop_mem - self.start_mem
        self._stop_time = time.time()
        self.duration += self._stop_time - self._start_time
        self.counts += 1
        self.on_exit()

    def on_exit(self):
        "To be overridden in subclasses"
        if self.autoflush:
            self.flush()

    def make_listener(self, hostname):
        """
        :param hostname: name of the server host, as seen by the clients
        :returns: a Listener object listening on `hostname`
        """
        authkey = uuid.uuid1().bytes
        listener = Listener((hostname, 0), authkey=authkey)
        self.address = listener.address
        self.authkey = authkey
        return listener

    def send(self, obj):
        """
        Send back an object to the listener: this is only defined
        if an .address has been defined.
        """
        client = Client(self.address, authkey=self.authkey)
        try:
            res = client.send(obj)
        finally:
            client.close()
        return res

    def flush(self):
        """
        Save the measurements on the performance file (or on stdout)
        """
        for child in self.children:
            child.flush()
        data = self.get_data()
        if len(data) == 0:  # no information
            return []

        # reset monitor
        self.duration = 0
        self.mem = 0
        self.counts = 0

        if self.hdf5path:
            h5 = h5py.File(self.hdf5path)
            try:
                pdata = Hdf5Dataset(h5['performance_data'])
            except KeyError:
                pdata = Hdf5Dataset.create(h5, 'performance_data', perf_dt)
            pdata.extend(data)
            h5.close()
        else:  # print on stddout
            print(data[0])

        return data

    # TODO: rename this as spawn; see what will break
    def __call__(self, operation='no operation', **kw):
        """
        Return a child of the monitor usable for a different operation.
        """
        child = self.new(operation, **kw)
        self.children.append(child)
        return child

    def new(self, operation='no operation', **kw):
        """
        Return a copy of the monitor usable for a different operation.
        """
        self_vars = vars(self).copy()
        del self_vars['operation']
        del self_vars['children']
        del self_vars['counts']
        new = self.__class__(operation)
        vars(new).update(self_vars)
        vars(new).update(kw)
        return new

    def __repr__(self):
        if self.measuremem:
            return '<%s %s, duration=%ss, memory=%s>' % (
                self.__class__.__name__, self.operation, self.duration,
                humansize(self.mem))
        return '<%s %s, duration=%ss>' % (self.__class__.__name__,
                                          self.operation, self.duration)

class DummyMonitor(PerformanceMonitor):
    """
    This class makes it easy to disable the monitoring in client code.
    Disabling the monitor can improve the performance.
    """
    def __init__(self, operation='dummy', *args, **kw):
        self.operation = operation
        self.hdf5path = None
        self.children = []
        self.counts = 0
        self._start_time = self._stop_time = time.time()

    def __enter__(self):
        self._start_time = time.time()
        return self

    def __exit__(self, etype, exc, tb):
        self._stop_time = time.time()

    def flush(self):
        pass

    def __repr__(self):
        return '<%s>' % self.__class__.__name__
<commit_msg>Cleanup<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (C) 2015-2016 GEM Foundation

# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
import os
import time
import uuid
import socket
from datetime import datetime
from multiprocessing.connection import Listener, Client

import numpy
import h5py

from openquake.baselib.general import humansize
from openquake.baselib.hdf5 import Hdf5Dataset

import psutil
if psutil.__version__ > '2.0.0':  # Ubuntu 14.10
    def virtual_memory():
        return psutil.virtual_memory()

    def memory_info(proc):
        return proc.memory_info()

elif psutil.__version__ >= '1.2.1':  # Ubuntu 14.04
    def virtual_memory():
        return psutil.virtual_memory()

    def memory_info(proc):
        return proc.get_memory_info()

else:  # Ubuntu 12.04
    def virtual_memory():
        return psutil.phymem_usage()

    def memory_info(proc):
        return proc.get_memory_info()

perf_dt = numpy.dtype([('operation', (bytes, 50)), ('time_sec', float),
                       ('memory_mb', float), ('counts', int)])

# this is not thread-safe
class PerformanceMonitor(object):
    """
    Measure the resident memory occupied by a list of processes during
    the execution of a block of code. Should be used as a context manager,
    as follows::

     with PerformanceMonitor('do_something') as mon:
         do_something()
     print mon.mem

    At the end of the block the PerformanceMonitor object will have the
    following 5 public attributes:

    .start_time: when the monitor started (a datetime object)
    .duration: time elapsed between start and stop (in seconds)
    .exc: usually None; otherwise the exception happened in the `with` block
    .mem: the memory delta in bytes

    The behaviour of the PerformanceMonitor can be customized by subclassing it
    and by overriding the method on_exit(), called at end and used to display
    or store the results of the analysis.

    NB: if the .address and .authkey attributes are set, it is possible for
    the monitor to send commands to that address, assuming there is a
    :class:`multiprocessing.connection.Listener` listening.
    """
    def __init__(self, operation, hdf5path=None,
                 autoflush=False, measuremem=False):
        self.operation = operation
        self.hdf5path = hdf5path
        self.autoflush = autoflush
        self.measuremem = measuremem
        self.mem = 0
        self.duration = 0
        self._start_time = self._stop_time = time.time()
        self.children = []
        self.counts = 0
        self.address = None

    @property
    def dt(self):
        """Last time interval measured"""
        return self._stop_time - self._start_time

    def measure_mem(self):
        """A memory measurement (in bytes)"""
        proc = psutil.Process(os.getpid())
        try:
            return memory_info(proc).rss
        except psutil.AccessDenied:
            # no access to information about this process
            pass

    @property
    def start_time(self):
        """
        Datetime instance recording when the monitoring started
        """
        return datetime.fromtimestamp(self._start_time)

    def get_data(self):
        """
        :returns:
            an array of dtype perf_dt, with the information
            of the monitor (operation, time_sec, memory_mb, counts);
            the lenght of the array can be 0 (for counts=0) or 1 (otherwise).
        """
        data = []
        if self.counts:
            time_sec = self.duration
            memory_mb = self.mem / 1024. / 1024. if self.measuremem else 0
            data.append((self.operation, time_sec, memory_mb, self.counts))
        return numpy.array(data, perf_dt)

    def __enter__(self):
        self.exc = None  # exception
        self._start_time = time.time()
        if self.measuremem:
            self.start_mem = self.measure_mem()
        return self

    def __exit__(self, etype, exc, tb):
        self.exc = exc
        if self.measuremem:
            self.stop_mem = self.measure_mem()
            self.mem += self.stop_mem - self.start_mem
        self._stop_time = time.time()
        self.duration += self._stop_time - self._start_time
        self.counts += 1
        self.on_exit()

    def on_exit(self):
        "To be overridden in subclasses"
        if self.autoflush:
            self.flush()

    def make_listener(self, hostname=None):
        """
        :param hostname:
            name of the server host, as seen by the clients
        :returns:
            a Listener object listening on `hostname` at a random port;
            as a side effect, set the attributes .address and .authkey
        """
        if hostname is None:
            hostname = socket.gethostname()
        authkey = uuid.uuid1().bytes
        authkey = None
        listener = Listener((hostname, 0), authkey=authkey)
        self.address = listener.address
        self.authkey = authkey
        return listener

    def send(self, *cmd):
        """
        Send a command to the listener, for instance `.send(print, 'hello')`
        """
        client = Client(self.address, authkey=self.authkey)
        try:
            res = client.send(cmd)
        finally:
            client.close()
        return res

    def flush(self):
        """
        Save the measurements on the performance file (or on stdout)
        """
        for child in self.children:
            child.flush()
        data = self.get_data()
        if len(data) == 0:  # no information
            return []

        # reset monitor
        self.duration = 0
        self.mem = 0
        self.counts = 0

        if self.hdf5path:
            h5 = h5py.File(self.hdf5path)
            try:
                pdata = Hdf5Dataset(h5['performance_data'])
            except KeyError:
                pdata = Hdf5Dataset.create(h5, 'performance_data', perf_dt)
            pdata.extend(data)
            h5.close()
        else:  # print on stddout
            print(data[0])

        return data

    # TODO: rename this as spawn; see what will break
    def __call__(self, operation='no operation', **kw):
        """
        Return a child of the monitor usable for a different operation.
        """
        child = self.new(operation, **kw)
        self.children.append(child)
        return child

    def new(self, operation='no operation', **kw):
        """
        Return a copy of the monitor usable for a different operation.
        """
        self_vars = vars(self).copy()
        del self_vars['operation']
        del self_vars['children']
        del self_vars['counts']
        new = self.__class__(operation)
        vars(new).update(self_vars)
        vars(new).update(kw)
        return new

    def __repr__(self):
        if self.measuremem:
            return '<%s %s, duration=%ss, memory=%s>' % (
                self.__class__.__name__, self.operation, self.duration,
                humansize(self.mem))
        return '<%s %s, duration=%ss>' % (self.__class__.__name__,
                                          self.operation, self.duration)

class DummyMonitor(PerformanceMonitor):
    """
    This class makes it easy to disable the monitoring in client code.
    Disabling the monitor can improve the performance.
    """
    def __init__(self, operation='dummy', *args, **kw):
        self.operation = operation
        self.hdf5path = None
        self.children = []
        self.counts = 0
        self._start_time = self._stop_time = time.time()

    def __enter__(self):
        self._start_time = time.time()
        return self

    def __exit__(self, etype, exc, tb):
        self._stop_time = time.time()

    def flush(self):
        pass

    def __repr__(self):
        return '<%s>' % self.__class__.__name__
<|endoftext|>


<commit_before>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

from __future__ import print_function, unicode_literals

import json
import os
from pipes import quote
import re
import shlex
import shutil
import subprocess
import tempfile
import logging
import uuid
from dock.constants import DOCKERFILE_FILENAME, PY2

logger = logging.getLogger(__name__)

class ImageName(object):
    def __init__(self, registry=None, namespace=None, repo=None, tag=None):
        self.registry = registry
        self.namespace = namespace
        self.repo = repo
        self.tag = tag

    @classmethod
    def parse(cls, image_name):
        result = cls()

        # registry.org/namespace/repo:tag
        s = image_name.split('/', 2)

        if len(s) == 2:
            if '.' in s[0] or ':' in s[0]:
                result.registry = s[0]
            else:
                result.namespace = s[0]
        elif len(s) == 3:
            result.registry = s[0]
            result.namespace = s[1]
        if result.namespace == 'library':
            # https://github.com/DBuildService/dock/issues/45
            logger.debug("namespace 'library' -> ''")
            result.namespace = None
        result.repo = s[-1]

        try:
            result.repo, result.tag = result.repo.rsplit(':', 1)
        except ValueError:
            pass

        return result

    def to_str(self, registry=True, tag=True, explicit_tag=False,
               explicit_namespace=False):
        if self.repo is None:
            raise RuntimeError('No image repository specified')

        result = self.repo

        if tag and self.tag:
            result = '{0}:{1}'.format(result, self.tag)
        elif tag and explicit_tag:
            result = '{0}:{1}'.format(result, 'latest')

        if self.namespace:
            result = '{0}/{1}'.format(self.namespace, result)
        elif explicit_namespace:
            result = '{0}/{1}'.format('library', result)

        if registry and self.registry:
            result = '{0}/{1}'.format(self.registry, result)

        return result

    def __str__(self):
        return self.to_str(registry=True, tag=True)

    def __eq__(self, other):
        return type(self) == type(other) and self.__dict__ == other.__dict__

    def copy(self):
        return ImageName(
            registry=self.registry,
            namespace=self.namespace,
            repo=self.repo,
            tag=self.tag)

class DockerfileParser(object):
    def __init__(self, git_path, path=''):
        if git_path.endswith(DOCKERFILE_FILENAME):
            self.dockerfile_path = git_path
        else:
            if path.endswith(DOCKERFILE_FILENAME):
                self.dockerfile_path = os.path.join(git_path, path)
            else:
                self.dockerfile_path = os.path.join(git_path, path, DOCKERFILE_FILENAME)

    @staticmethod
    def b2u(string):
        """ bytes to unicode """
        if isinstance(string, bytes):
            return string.decode('utf-8')
        return string

    @staticmethod
    def u2b(string):
        """ unicode to bytes (Python 2 only) """
        if PY2 and isinstance(string, unicode):
            return string.encode('utf-8')
        return string

    @property
    def lines(self):
        try:
            with open(self.dockerfile_path, 'r') as dockerfile:
                return [self.b2u(l) for l in dockerfile.readlines()]
        except (IOError, OSError) as ex:
            logger.error("Couldn't retrieve lines from dockerfile: %s" % repr(ex))
            raise

    @lines.setter
    def lines(self, lines):
        try:
            with open(self.dockerfile_path, 'w') as dockerfile:
                dockerfile.writelines([self.u2b(l) for l in lines])
        except (IOError, OSError) as ex:
            logger.error("Couldn't write lines to dockerfile: %s" % repr(ex))
            raise

    @property
    def content(self):
        try:
            with open(self.dockerfile_path, 'r') as dockerfile:
                return self.b2u(dockerfile.read())
        except (IOError, OSError) as ex:
            logger.error("Couldn't retrieve content of dockerfile: %s" % repr(ex))
            raise

    @content.setter
    def content(self, content):
        try:
            with open(self.dockerfile_path, 'w') as dockerfile:
                dockerfile.write(self.u2b(content))
        except (IOError, OSError) as ex:
            logger.error("Couldn't write content to dockerfile: %s" % repr(ex))
            raise

    def get_baseimage(self):
        for line in self.lines:
            if line.startswith("FROM"):
                return line.split()[1]

    def _split(self, string):
        if PY2 and isinstance(string, unicode):
            # Python2's shlex doesn't like unicode
            string = self.u2b(string)
            splits = shlex.split(string)
            return map(self.b2u, splits)
        else:
            return shlex.split(string)

    def get_labels(self):
        """ opposite of AddLabelsPlugin, i.e. return dict of labels from dockerfile
        :return: dictionary of label:value or label:'' if there's no value
        """
        labels = {}
        multiline = False
        processed_instr = ""
        for line in self.lines:
            line = line.rstrip()  # docker does this
            logger.debug("processing line %s", repr(line))
            if multiline:
                processed_instr += line
                if line.endswith("\\"):  # does multiline continue?
                    # docker strips single \
                    processed_instr = processed_instr[:-1]
                    continue
                else:
                    multiline = False
            else:
                processed_instr = line
            if processed_instr.startswith("LABEL"):
                if processed_instr.endswith("\\"):
                    logger.debug("multiline LABEL")
                    # docker strips single \
                    processed_instr = processed_instr[:-1]
                    multiline = True
                    continue
                for token in self._split(processed_instr[len("LABEL "):]):
                    key_val = token.split("=", 1)
                    if len(key_val) == 2:
                        labels[key_val[0]] = key_val[1]
                    else:
                        labels[key_val[0]] = ''
                    logger.debug("new label %s=%s", repr(key_val[0]), repr(labels[key_val[0]]))
        return labels

def figure_out_dockerfile(absolute_path, local_path=None):
    """
    try to figure out dockerfile from provided path and optionally from relative local path
    this is meant to be used with git repo: absolute_path is path to git repo,
    local_path is path to dockerfile within git repo

    :param absolute_path:
    :param local_path:
    :return: tuple, (dockerfile_path, dir_with_dockerfile_path)
    """
    logger.info("find dockerfile")
    logger.debug("abs path = '%s', local path = '%s'", absolute_path, local_path)
    if local_path:
        if local_path.endswith(DOCKERFILE_FILENAME):
            git_df_dir = os.path.dirname(local_path)
            df_dir = os.path.abspath(os.path.join(absolute_path, git_df_dir))
        else:
            df_dir = os.path.abspath(os.path.join(absolute_path, local_path))
    else:
        df_dir = os.path.abspath(absolute_path)
    if not os.path.isdir(df_dir):
        raise IOError("Directory '%s' doesn't exist." % df_dir)
    df_path = os.path.join(df_dir, DOCKERFILE_FILENAME)
    if not os.path.isfile(df_path):
        raise IOError("Dockerfile '%s' doesn't exist." % df_path)
    logger.debug("dockerfile found: '%s'", df_path)
    return df_path, df_dir

class CommandResult(object):
    def __init__(self, logs, error=None, error_detail=None):
        self._logs = logs
        self._error = error
        self._error_detail = error_detail

    @property
    def logs(self):
        return self._logs

    @property
    def error(self):
        return self._error

    @property
    def error_detail(self):
        return self._error_detail

    def is_failed(self):
        return bool(self.error) or bool(self.error_detail)

def wait_for_command(logs_generator):
    """
    using given generator, wait for it to raise StopIteration, which
    indicates that docker has finished with processing

    :return: list of str, logs
    """
    # FIXME: this function is getting pretty big, let's break it down a bit
    #        and merge it into CommandResult
    logger.info("wait_for_command")
    logs = []
    error = None
    error_message = None
    while True:
        try:
            parsed_item = None
            item = next(logs_generator)  # py2 & 3 compat
            item = item.decode("utf-8")
            try:
                parsed_item = json.loads(item)
            except ValueError:
                pass

            # make sure the json is an object
            if isinstance(parsed_item, dict):
                line = parsed_item.get("stream", "")
            else:
                parsed_item = None
                line = item

            for l in re.split(r"\r?\n", line, re.MULTILINE):
                # line = line.replace("\r\n", " ").replace("\n", " ").strip()
                l = l.strip()
                if l:
                    logger.debug(l)
            logs.append(item)
            if parsed_item is not None:
                error = parsed_item.get("error", None)
                error_message = parsed_item.get("errorDetail", None)
                if error:
                    logger.error(item.strip())
        except StopIteration:
            logger.info("no more logs")
            break
    cr = CommandResult(logs=logs, error=error, error_detail=error_message)
    return cr

def backported_check_output(*popenargs, **kwargs):
    """
    Run command with arguments and return its output as a byte string.

    Backported from Python 2.7 as it's implemented as pure python on stdlib.

    https://gist.github.com/edufelipe/1027906
    """
    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
    output, _ = process.communicate()
    retcode = process.poll()
    if retcode:
        cmd = kwargs.get("args")
        if cmd is None:
            cmd = popenargs[0]
        error = subprocess.CalledProcessError(retcode, cmd)
        error.output = output
        raise error
    return output

def clone_git_repo(git_url, target_dir, commit=None):
    """
    clone provided git repo to target_dir, optionally checkout provided commit

    :param git_url: str, git repo to clone
    :param target_dir: str, filesystem path where the repo should be cloned
    :param commit: str, commit to checkout, SHA-1 or ref
    :return: str, commit ID of HEAD
    """
    commit = commit or "master"
    logger.info("clone git repo")
    logger.debug("url = '%s', dir = '%s', commit = '%s'",
                 git_url, target_dir, commit)

    # http://stackoverflow.com/questions/1911109/clone-a-specific-git-branch/4568323#4568323
    # -b takes only refs, not SHA-1
    cmd = ["git", "clone", "-b", commit, "--single-branch", git_url, quote(target_dir)]
    logger.debug("Cloning single branch: %s", cmd)
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as ex:
        logger.warning(repr(ex))
        # let's try again with plain `git clone $url && git checkout`
        cmd = ["git", "clone", git_url, quote(target_dir)]
        logger.debug("Cloning: %s", cmd)
        subprocess.check_call(cmd)
        cmd = ["git", "checkout", commit]
        logger.debug("Checking out branch: %s", cmd)
        subprocess.check_call(cmd, cwd=target_dir)
    cmd = ["git", "rev-parse", "HEAD"]
    logger.debug("getting SHA-1 of provided ref: %s", cmd)
    try:
        commit_id = subprocess.check_output(cmd, cwd=target_dir)  # py 2.7
    except AttributeError:
        commit_id = backported_check_output(cmd, cwd=target_dir)  # py 2.6
    commit_id = commit_id.strip()
    logger.info("commit ID = %s", commit_id)
    return commit_id

class LazyGit(object):
    """
    usage:

        lazy_git = LazyGit(git_url="...")
        with lazy_git:
            laze_git.git_path

    or

        lazy_git = LazyGit(git_url="...", tmpdir=tmp_dir)
        lazy_git.git_path
    """
    def __init__(self, git_url, commit=None, tmpdir=None):
        self.git_url = git_url
        # provided commit ID/reference to check out
        self.commit = commit
        # commit ID of HEAD; we'll figure this out ourselves
        self._commit_id = None
        self.provided_tmpdir = tmpdir
        self._git_path = None

    @property
    def _tmpdir(self):
        return self.provided_tmpdir or self.our_tmpdir

    @property
    def commit_id(self):
        return self._commit_id

    @property
    def git_path(self):
        if self._git_path is None:
            self._commit_id = clone_git_repo(self.git_url, self._tmpdir, self.commit)
            self._git_path = self._tmpdir
        return self._git_path

    def __enter__(self):
        if not self.provided_tmpdir:
            self.our_tmpdir = tempfile.mkdtemp()

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not self.provided_tmpdir:
            if self.our_tmpdir:
                shutil.rmtree(self.our_tmpdir)

def escape_dollar(v):
    try:
        str_type = unicode
    except NameError:
        str_type = str
    if isinstance(v, str_type):
        return v.replace('$', r'\$')
    else:
        return v

def render_yum_repo(repo, escape_dollars=True):
    repo.setdefault("name", str(uuid.uuid4().hex[:6]))
    repo_name = repo["name"]
    logger.info("rendering repo '%s'", repo_name)
    rendered_repo = '[%s]\n' % repo_name
    for key, value in repo.items():
        if escape_dollars:
            value = escape_dollar(value)
        rendered_repo += "%s=%s\n" % (key, value)
    logger.info("rendered repo: %s", repr(rendered_repo))
    return rendered_repo
<commit_msg>ImageName: add __repr__<commit_after>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

from __future__ import print_function, unicode_literals

import json
import os
from pipes import quote
import re
import shlex
import shutil
import subprocess
import tempfile
import logging
import uuid
from dock.constants import DOCKERFILE_FILENAME, PY2

logger = logging.getLogger(__name__)

class ImageName(object):
    def __init__(self, registry=None, namespace=None, repo=None, tag=None):
        self.registry = registry
        self.namespace = namespace
        self.repo = repo
        self.tag = tag

    @classmethod
    def parse(cls, image_name):
        result = cls()

        # registry.org/namespace/repo:tag
        s = image_name.split('/', 2)

        if len(s) == 2:
            if '.' in s[0] or ':' in s[0]:
                result.registry = s[0]
            else:
                result.namespace = s[0]
        elif len(s) == 3:
            result.registry = s[0]
            result.namespace = s[1]
        if result.namespace == 'library':
            # https://github.com/DBuildService/dock/issues/45
            logger.debug("namespace 'library' -> ''")
            result.namespace = None
        result.repo = s[-1]

        try:
            result.repo, result.tag = result.repo.rsplit(':', 1)
        except ValueError:
            pass

        return result

    def to_str(self, registry=True, tag=True, explicit_tag=False,
               explicit_namespace=False):
        if self.repo is None:
            raise RuntimeError('No image repository specified')

        result = self.repo

        if tag and self.tag:
            result = '{0}:{1}'.format(result, self.tag)
        elif tag and explicit_tag:
            result = '{0}:{1}'.format(result, 'latest')

        if self.namespace:
            result = '{0}/{1}'.format(self.namespace, result)
        elif explicit_namespace:
            result = '{0}/{1}'.format('library', result)

        if registry and self.registry:
            result = '{0}/{1}'.format(self.registry, result)

        return result

    def __str__(self):
        return self.to_str(registry=True, tag=True)

    def __repr__(self):
        return "ImageName(image=%s)" % repr(self.to_str())

    def __eq__(self, other):
        return type(self) == type(other) and self.__dict__ == other.__dict__

    def copy(self):
        return ImageName(
            registry=self.registry,
            namespace=self.namespace,
            repo=self.repo,
            tag=self.tag)

class DockerfileParser(object):
    def __init__(self, git_path, path=''):
        if git_path.endswith(DOCKERFILE_FILENAME):
            self.dockerfile_path = git_path
        else:
            if path.endswith(DOCKERFILE_FILENAME):
                self.dockerfile_path = os.path.join(git_path, path)
            else:
                self.dockerfile_path = os.path.join(git_path, path, DOCKERFILE_FILENAME)

    @staticmethod
    def b2u(string):
        """ bytes to unicode """
        if isinstance(string, bytes):
            return string.decode('utf-8')
        return string

    @staticmethod
    def u2b(string):
        """ unicode to bytes (Python 2 only) """
        if PY2 and isinstance(string, unicode):
            return string.encode('utf-8')
        return string

    @property
    def lines(self):
        try:
            with open(self.dockerfile_path, 'r') as dockerfile:
                return [self.b2u(l) for l in dockerfile.readlines()]
        except (IOError, OSError) as ex:
            logger.error("Couldn't retrieve lines from dockerfile: %s" % repr(ex))
            raise

    @lines.setter
    def lines(self, lines):
        try:
            with open(self.dockerfile_path, 'w') as dockerfile:
                dockerfile.writelines([self.u2b(l) for l in lines])
        except (IOError, OSError) as ex:
            logger.error("Couldn't write lines to dockerfile: %s" % repr(ex))
            raise

    @property
    def content(self):
        try:
            with open(self.dockerfile_path, 'r') as dockerfile:
                return self.b2u(dockerfile.read())
        except (IOError, OSError) as ex:
            logger.error("Couldn't retrieve content of dockerfile: %s" % repr(ex))
            raise

    @content.setter
    def content(self, content):
        try:
            with open(self.dockerfile_path, 'w') as dockerfile:
                dockerfile.write(self.u2b(content))
        except (IOError, OSError) as ex:
            logger.error("Couldn't write content to dockerfile: %s" % repr(ex))
            raise

    def get_baseimage(self):
        for line in self.lines:
            if line.startswith("FROM"):
                return line.split()[1]

    def _split(self, string):
        if PY2 and isinstance(string, unicode):
            # Python2's shlex doesn't like unicode
            string = self.u2b(string)
            splits = shlex.split(string)
            return map(self.b2u, splits)
        else:
            return shlex.split(string)

    def get_labels(self):
        """ opposite of AddLabelsPlugin, i.e. return dict of labels from dockerfile
        :return: dictionary of label:value or label:'' if there's no value
        """
        labels = {}
        multiline = False
        processed_instr = ""
        for line in self.lines:
            line = line.rstrip()  # docker does this
            logger.debug("processing line %s", repr(line))
            if multiline:
                processed_instr += line
                if line.endswith("\\"):  # does multiline continue?
                    # docker strips single \
                    processed_instr = processed_instr[:-1]
                    continue
                else:
                    multiline = False
            else:
                processed_instr = line
            if processed_instr.startswith("LABEL"):
                if processed_instr.endswith("\\"):
                    logger.debug("multiline LABEL")
                    # docker strips single \
                    processed_instr = processed_instr[:-1]
                    multiline = True
                    continue
                for token in self._split(processed_instr[len("LABEL "):]):
                    key_val = token.split("=", 1)
                    if len(key_val) == 2:
                        labels[key_val[0]] = key_val[1]
                    else:
                        labels[key_val[0]] = ''
                    logger.debug("new label %s=%s", repr(key_val[0]), repr(labels[key_val[0]]))
        return labels

def figure_out_dockerfile(absolute_path, local_path=None):
    """
    try to figure out dockerfile from provided path and optionally from relative local path
    this is meant to be used with git repo: absolute_path is path to git repo,
    local_path is path to dockerfile within git repo

    :param absolute_path:
    :param local_path:
    :return: tuple, (dockerfile_path, dir_with_dockerfile_path)
    """
    logger.info("find dockerfile")
    logger.debug("abs path = '%s', local path = '%s'", absolute_path, local_path)
    if local_path:
        if local_path.endswith(DOCKERFILE_FILENAME):
            git_df_dir = os.path.dirname(local_path)
            df_dir = os.path.abspath(os.path.join(absolute_path, git_df_dir))
        else:
            df_dir = os.path.abspath(os.path.join(absolute_path, local_path))
    else:
        df_dir = os.path.abspath(absolute_path)
    if not os.path.isdir(df_dir):
        raise IOError("Directory '%s' doesn't exist." % df_dir)
    df_path = os.path.join(df_dir, DOCKERFILE_FILENAME)
    if not os.path.isfile(df_path):
        raise IOError("Dockerfile '%s' doesn't exist." % df_path)
    logger.debug("dockerfile found: '%s'", df_path)
    return df_path, df_dir

class CommandResult(object):
    def __init__(self, logs, error=None, error_detail=None):
        self._logs = logs
        self._error = error
        self._error_detail = error_detail

    @property
    def logs(self):
        return self._logs

    @property
    def error(self):
        return self._error

    @property
    def error_detail(self):
        return self._error_detail

    def is_failed(self):
        return bool(self.error) or bool(self.error_detail)

def wait_for_command(logs_generator):
    """
    using given generator, wait for it to raise StopIteration, which
    indicates that docker has finished with processing

    :return: list of str, logs
    """
    # FIXME: this function is getting pretty big, let's break it down a bit
    #        and merge it into CommandResult
    logger.info("wait_for_command")
    logs = []
    error = None
    error_message = None
    while True:
        try:
            parsed_item = None
            item = next(logs_generator)  # py2 & 3 compat
            item = item.decode("utf-8")
            try:
                parsed_item = json.loads(item)
            except ValueError:
                pass

            # make sure the json is an object
            if isinstance(parsed_item, dict):
                line = parsed_item.get("stream", "")
            else:
                parsed_item = None
                line = item

            for l in re.split(r"\r?\n", line, re.MULTILINE):
                # line = line.replace("\r\n", " ").replace("\n", " ").strip()
                l = l.strip()
                if l:
                    logger.debug(l)
            logs.append(item)
            if parsed_item is not None:
                error = parsed_item.get("error", None)
                error_message = parsed_item.get("errorDetail", None)
                if error:
                    logger.error(item.strip())
        except StopIteration:
            logger.info("no more logs")
            break
    cr = CommandResult(logs=logs, error=error, error_detail=error_message)
    return cr

def backported_check_output(*popenargs, **kwargs):
    """
    Run command with arguments and return its output as a byte string.

    Backported from Python 2.7 as it's implemented as pure python on stdlib.

    https://gist.github.com/edufelipe/1027906
    """
    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
    output, _ = process.communicate()
    retcode = process.poll()
    if retcode:
        cmd = kwargs.get("args")
        if cmd is None:
            cmd = popenargs[0]
        error = subprocess.CalledProcessError(retcode, cmd)
        error.output = output
        raise error
    return output

def clone_git_repo(git_url, target_dir, commit=None):
    """
    clone provided git repo to target_dir, optionally checkout provided commit

    :param git_url: str, git repo to clone
    :param target_dir: str, filesystem path where the repo should be cloned
    :param commit: str, commit to checkout, SHA-1 or ref
    :return: str, commit ID of HEAD
    """
    commit = commit or "master"
    logger.info("clone git repo")
    logger.debug("url = '%s', dir = '%s', commit = '%s'",
                 git_url, target_dir, commit)

    # http://stackoverflow.com/questions/1911109/clone-a-specific-git-branch/4568323#4568323
    # -b takes only refs, not SHA-1
    cmd = ["git", "clone", "-b", commit, "--single-branch", git_url, quote(target_dir)]
    logger.debug("Cloning single branch: %s", cmd)
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as ex:
        logger.warning(repr(ex))
        # let's try again with plain `git clone $url && git checkout`
        cmd = ["git", "clone", git_url, quote(target_dir)]
        logger.debug("Cloning: %s", cmd)
        subprocess.check_call(cmd)
        cmd = ["git", "checkout", commit]
        logger.debug("Checking out branch: %s", cmd)
        subprocess.check_call(cmd, cwd=target_dir)
    cmd = ["git", "rev-parse", "HEAD"]
    logger.debug("getting SHA-1 of provided ref: %s", cmd)
    try:
        commit_id = subprocess.check_output(cmd, cwd=target_dir)  # py 2.7
    except AttributeError:
        commit_id = backported_check_output(cmd, cwd=target_dir)  # py 2.6
    commit_id = commit_id.strip()
    logger.info("commit ID = %s", commit_id)
    return commit_id

class LazyGit(object):
    """
    usage:

        lazy_git = LazyGit(git_url="...")
        with lazy_git:
            laze_git.git_path

    or

        lazy_git = LazyGit(git_url="...", tmpdir=tmp_dir)
        lazy_git.git_path
    """
    def __init__(self, git_url, commit=None, tmpdir=None):
        self.git_url = git_url
        # provided commit ID/reference to check out
        self.commit = commit
        # commit ID of HEAD; we'll figure this out ourselves
        self._commit_id = None
        self.provided_tmpdir = tmpdir
        self._git_path = None

    @property
    def _tmpdir(self):
        return self.provided_tmpdir or self.our_tmpdir

    @property
    def commit_id(self):
        return self._commit_id

    @property
    def git_path(self):
        if self._git_path is None:
            self._commit_id = clone_git_repo(self.git_url, self._tmpdir, self.commit)
            self._git_path = self._tmpdir
        return self._git_path

    def __enter__(self):
        if not self.provided_tmpdir:
            self.our_tmpdir = tempfile.mkdtemp()

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not self.provided_tmpdir:
            if self.our_tmpdir:
                shutil.rmtree(self.our_tmpdir)

def escape_dollar(v):
    try:
        str_type = unicode
    except NameError:
        str_type = str
    if isinstance(v, str_type):
        return v.replace('$', r'\$')
    else:
        return v

def render_yum_repo(repo, escape_dollars=True):
    repo.setdefault("name", str(uuid.uuid4().hex[:6]))
    repo_name = repo["name"]
    logger.info("rendering repo '%s'", repo_name)
    rendered_repo = '[%s]\n' % repo_name
    for key, value in repo.items():
        if escape_dollars:
            value = escape_dollar(value)
        rendered_repo += "%s=%s\n" % (key, value)
    logger.info("rendered repo: %s", repr(rendered_repo))
    return rendered_repo
<|endoftext|>


<commit_before>from django.conf import settings
from django.urls import reverse
from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import TemplateView, ListView, \
    DetailView, CreateView, UpdateView

from .models import Ambulance, AmbulanceCapability, AmbulanceStatus, \
    Call, Base, AmbulanceRoute

from .forms import AmbulanceCreateForm, AmbulanceUpdateForm

from .serializers import AmbulanceSerializer

from emstrack.mixins import BasePermissionMixin

# Django views

class AmbulancePermissionMixin(BasePermissionMixin):

    filter_field = 'id'
    profile_field = 'ambulances'
    profile_values = 'ambulance_id'
    queryset = Ambulance.objects.all()

class AmbulanceActionMixin:

    @property
    def success_message(self):
        return NotImplemented

    def form_valid(self, form):

        # add message
        messages.info(self.request, self.success_message)

        # add updated_by to form and save
        form.instance.updated_by = self.request.user

        # call super
        return super().form_valid(form)

class AmbulanceDetailView(LoginRequiredMixin,
                          AmbulancePermissionMixin,
                          DetailView):

    model = Ambulance

class AmbulanceListView(LoginRequiredMixin,
                        AmbulancePermissionMixin,
                        ListView):

    model = Ambulance;

class AmbulanceCreateView(LoginRequiredMixin,
                          AmbulancePermissionMixin,
                          AmbulanceActionMixin,
                          CreateView):

    model = Ambulance
    form_class = AmbulanceCreateForm

    def get_success_url(self):
        return self.object.get_absolute_url()

class AmbulanceUpdateView(LoginRequiredMixin,
                          AmbulancePermissionMixin,
                          AmbulanceActionMixin,
                          UpdateView):

    model = Ambulance
    form_class = AmbulanceUpdateForm

    def get_success_url(self):
        return self.object.get_absolute_url()

class AmbulanceMap(TemplateView):
     template_name = 'ambulance/map.html'

     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context['ambulance_status'] = {m.name: m.value
                                        for m in AmbulanceStatus}
         context['ambulance_capability'] = {m.name: m.value
                                            for m in AmbulanceCapability}
         context['broker_websockets_host'] = settings.MQTT['BROKER_WEBSOCKETS_HOST']
         context['broker_websockets_port'] = settings.MQTT['BROKER_WEBSOCKETS_PORT']
         context['api_url'] = reverse('api')
         return context

# NEED REVISING

# Call list page
class CallView(ListView):
    model = Call
    template_name = 'ambulance/dispatch_list.html'
    context_object_name = "ambulance_call"

# Admin page
class AdminView(ListView):
    model = Call
    template_name = 'ambulance/dispatch_list.html'
    context_object_name = "ambulance_call"

# # AmbulanceStatus list page
# class AmbulanceStatusCreateView(CreateView):
#     model = AmbulanceStatus
#     context_object_name = "ambulance_status_form"
#     form_class = AmbulanceStatusCreateForm
#     success_url = reverse_lazy('status')

#     def get_context_data(self, **kwargs):
#         context = super(AmbulanceStatusCreateView, self).get_context_data(**kwargs)
#         context['statuses'] = AmbulanceStatus.objects.all().order_by('id')
#         return context

# # AmbulanceStatus update page
# class AmbulanceStatusUpdateView(UpdateView):
#     model = AmbulanceStatus
#     form_class = AmbulanceStatusUpdateForm
#     template_name = 'ambulances/ambulance_status_edit.html'
#     success_url = reverse_lazy('status')

#     def get_object(self, queryset=None):
#         obj = AmbulanceStatus.objects.get(id=self.kwargs['pk'])
#         return obj

#     def get_context_data(self, **kwargs):
#         context = super(AmbulanceStatusUpdateView, self).get_context_data(**kwargs)
#         context['id'] = self.kwargs['pk']
#         return context

# # Ambulance map page
# class AmbulanceMap(views.JSONResponseMixin, views.AjaxResponseMixin, ListView):
#     template_name = 'ambulances/ambulance_map.html'

#     def get_queryset(self):
#         return Ambulance.objects.all()
<commit_msg>Dynamic mqtt and api server information on templates<commit_after>from django.conf import settings
from django.urls import reverse
from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import TemplateView, ListView, \
    DetailView, CreateView, UpdateView

from .models import Ambulance, AmbulanceCapability, AmbulanceStatus, \
    Call, Base, AmbulanceRoute

from .forms import AmbulanceCreateForm, AmbulanceUpdateForm

from .serializers import AmbulanceSerializer

from emstrack.mixins import BasePermissionMixin

# Django views

class AmbulancePermissionMixin(BasePermissionMixin):

    filter_field = 'id'
    profile_field = 'ambulances'
    profile_values = 'ambulance_id'
    queryset = Ambulance.objects.all()

class AmbulanceActionMixin:

    @property
    def success_message(self):
        return NotImplemented

    def form_valid(self, form):

        # add message
        messages.info(self.request, self.success_message)

        # add updated_by to form and save
        form.instance.updated_by = self.request.user

        # call super
        return super().form_valid(form)

class AmbulanceDetailView(LoginRequiredMixin,
                          AmbulancePermissionMixin,
                          DetailView):

    model = Ambulance

class AmbulanceListView(LoginRequiredMixin,
                        AmbulancePermissionMixin,
                        ListView):

    model = Ambulance;

class AmbulanceCreateView(LoginRequiredMixin,
                          AmbulancePermissionMixin,
                          AmbulanceActionMixin,
                          CreateView):

    model = Ambulance
    form_class = AmbulanceCreateForm

    def get_success_url(self):
        return self.object.get_absolute_url()

class AmbulanceUpdateView(LoginRequiredMixin,
                          AmbulancePermissionMixin,
                          AmbulanceActionMixin,
                          UpdateView):

    model = Ambulance
    form_class = AmbulanceUpdateForm

    def get_success_url(self):
        return self.object.get_absolute_url()

class AmbulanceMap(TemplateView):
     template_name = 'ambulance/map.html'

     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context['ambulance_status'] = {m.name: m.value
                                        for m in AmbulanceStatus}
         context['ambulance_capability'] = {m.name: m.value
                                            for m in AmbulanceCapability}
         context['broker_websockets_host'] = settings.MQTT['BROKER_WEBSOCKETS_HOST']
         context['broker_websockets_port'] = settings.MQTT['BROKER_WEBSOCKETS_PORT']
         return context

# NEED REVISING

# Call list page
class CallView(ListView):
    model = Call
    template_name = 'ambulance/dispatch_list.html'
    context_object_name = "ambulance_call"

# Admin page
class AdminView(ListView):
    model = Call
    template_name = 'ambulance/dispatch_list.html'
    context_object_name = "ambulance_call"

# # AmbulanceStatus list page
# class AmbulanceStatusCreateView(CreateView):
#     model = AmbulanceStatus
#     context_object_name = "ambulance_status_form"
#     form_class = AmbulanceStatusCreateForm
#     success_url = reverse_lazy('status')

#     def get_context_data(self, **kwargs):
#         context = super(AmbulanceStatusCreateView, self).get_context_data(**kwargs)
#         context['statuses'] = AmbulanceStatus.objects.all().order_by('id')
#         return context

# # AmbulanceStatus update page
# class AmbulanceStatusUpdateView(UpdateView):
#     model = AmbulanceStatus
#     form_class = AmbulanceStatusUpdateForm
#     template_name = 'ambulances/ambulance_status_edit.html'
#     success_url = reverse_lazy('status')

#     def get_object(self, queryset=None):
#         obj = AmbulanceStatus.objects.get(id=self.kwargs['pk'])
#         return obj

#     def get_context_data(self, **kwargs):
#         context = super(AmbulanceStatusUpdateView, self).get_context_data(**kwargs)
#         context['id'] = self.kwargs['pk']
#         return context

# # Ambulance map page
# class AmbulanceMap(views.JSONResponseMixin, views.AjaxResponseMixin, ListView):
#     template_name = 'ambulances/ambulance_map.html'

#     def get_queryset(self):
#         return Ambulance.objects.all()
<|endoftext|>