<commit_before># -*- coding: utf-8 -*-
import os
import tempfile
import unittest
from operator import itemgetter
from unittest import TestCase

from os3.fs.directory import Dir
from os3.fs.file import File
from os3.tests.base import MockTreeNode

class TestDirectory(MockTreeNode):

    def test_deep(self):
        self.assertEqual(set(Dir(self.directory).ls(deep=True).value_list('path')), self.deep_list_dir())

    def test_not_repeated(self):
        items = Dir(self.directory).ls(deep=True).value_list('path')
        self.assertEqual(len(items), len(set(items)))

    def test_max_deep(self):
        self.assertEqual(set(Dir(self.directory).ls(deep=0).value_list('path')), set(self.list_dir(full_path=True)))
        self.assertNotIn(os.path.join(self.directory, 'dir01/subdir01/subsubdir01'),
                         Dir(self.directory).ls(deep=1).value_list('path'))
        self.assertIn(os.path.join(self.directory, 'dir01/subdir01/subsubdir01'),
                      Dir(self.directory).ls(deep=2).value_list('path'))

    def test_filters_deep(self):
        # print(len([x for x in Dir(self.directory).ls(deep=True).filter(type='f')]))
        files = list(Dir(self.directory).ls(deep=True).filter(type='f').value_list('path'))
        self.assertCountEqual(files, set(files))
        self.assertEqual(len(files), (len(self.tree) + 1) * self.files_by_dir)

    def test_pre_filters_deep(self):
        from os3.fs.file import File
        files = list(Dir(self.directory).ls(type='f', deep=True))
        self.assertEqual(len(files), self.files_by_dir)
        for file in files:
            self.assertIsInstance(file, File)

    def test_repr(self):
        self.assertEqual(repr(Dir(self.directory)), Dir(self.directory).name)

    def test_print_format(self):
        self.assertIn(Dir(self.directory).name, Dir(self.directory).print_format())

class TestFile(TestCase):
    def setUp(self):
        self.filename = tempfile.NamedTemporaryFile().name

    def test_read(self):
        data = 'FOO' * 10
        with open(self.filename, 'w') as f:
            f.write(data)
        self.assertEqual(File(self.filename).read(), data)

    def test_repeat_read(self):
        data = 'SPAM' * 10
        with open(self.filename, 'w') as f:
            f.write(data)
        f = File(self.filename)
        self.assertEqual(f.read(), data)
        self.assertEqual(f.read(), data)

    def test_readlines(self):
        lines = list(map(str, range(10)))
        with open(self.filename, 'w') as f:
            f.write('\n'.join(lines))
        self.assertEqual(File(self.filename).readlines(breaklines=False), lines)
        with open(self.filename, 'wb') as f:
            f.write(b'\r\n'.join(map(lambda x: x.encode('utf-8'), lines)))
        self.assertEqual(File(self.filename).readlines(breaklines=False), lines)

    def test_readline(self):
        lines = list(map(str, range(10)))
        with open(self.filename, 'w') as f:
            f.write('\n'.join(lines))
        f = File(self.filename)
        read_lines = []
        while True:
            line = f.readline(breakline=False)
            if line is None:
                break
            read_lines.append(line)
        self.assertEqual(read_lines, lines)

    def test_read_n_data(self):
        data = u'ññññ'
        with open(self.filename, 'w') as f:
            f.write(data)
        self.assertEqual(File(self.filename).read(4), data)

    def test_read_n_bytes_data(self):
        data = u'ññññ'
        with open(self.filename, 'wb') as f:
            f.write(data.encode('utf-8'))
        self.assertEqual(File(self.filename).read(4, 'rb'), data.encode('utf-8')[:4])

    def tearDown(self):
        os.remove(self.filename)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Python2 unittest support.<commit_after># -*- coding: utf-8 -*-
import os
import tempfile
import unittest
from operator import itemgetter
from unittest import TestCase

import six

from os3.fs.directory import Dir
from os3.fs.file import File
from os3.tests.base import MockTreeNode

class TestDirectory(MockTreeNode):

    def test_deep(self):
        self.assertEqual(set(Dir(self.directory).ls(deep=True).value_list('path')), self.deep_list_dir())

    def test_not_repeated(self):
        items = Dir(self.directory).ls(deep=True).value_list('path')
        self.assertEqual(len(items), len(set(items)))

    def test_max_deep(self):
        self.assertEqual(set(Dir(self.directory).ls(deep=0).value_list('path')), set(self.list_dir(full_path=True)))
        self.assertNotIn(os.path.join(self.directory, 'dir01/subdir01/subsubdir01'),
                         Dir(self.directory).ls(deep=1).value_list('path'))
        self.assertIn(os.path.join(self.directory, 'dir01/subdir01/subsubdir01'),
                      Dir(self.directory).ls(deep=2).value_list('path'))

    def test_filters_deep(self):
        # print(len([x for x in Dir(self.directory).ls(deep=True).filter(type='f')]))
        files = list(Dir(self.directory).ls(deep=True).filter(type='f').value_list('path'))
        six.assertCountEqual(files, set(files))
        self.assertEqual(len(files), (len(self.tree) + 1) * self.files_by_dir)

    def test_pre_filters_deep(self):
        from os3.fs.file import File
        files = list(Dir(self.directory).ls(type='f', deep=True))
        self.assertEqual(len(files), self.files_by_dir)
        for file in files:
            self.assertIsInstance(file, File)

    def test_repr(self):
        self.assertEqual(repr(Dir(self.directory)), Dir(self.directory).name)

    def test_print_format(self):
        self.assertIn(Dir(self.directory).name, Dir(self.directory).print_format())

class TestFile(TestCase):
    def setUp(self):
        self.filename = tempfile.NamedTemporaryFile().name

    def test_read(self):
        data = 'FOO' * 10
        with open(self.filename, 'w') as f:
            f.write(data)
        self.assertEqual(File(self.filename).read(), data)

    def test_repeat_read(self):
        data = 'SPAM' * 10
        with open(self.filename, 'w') as f:
            f.write(data)
        f = File(self.filename)
        self.assertEqual(f.read(), data)
        self.assertEqual(f.read(), data)

    def test_readlines(self):
        lines = list(map(str, range(10)))
        with open(self.filename, 'w') as f:
            f.write('\n'.join(lines))
        self.assertEqual(File(self.filename).readlines(breaklines=False), lines)
        with open(self.filename, 'wb') as f:
            f.write(b'\r\n'.join(map(lambda x: x.encode('utf-8'), lines)))
        self.assertEqual(File(self.filename).readlines(breaklines=False), lines)

    def test_readline(self):
        lines = list(map(str, range(10)))
        with open(self.filename, 'w') as f:
            f.write('\n'.join(lines))
        f = File(self.filename)
        read_lines = []
        while True:
            line = f.readline(breakline=False)
            if line is None:
                break
            read_lines.append(line)
        self.assertEqual(read_lines, lines)

    def test_read_n_data(self):
        data = u'ññññ'
        with open(self.filename, 'w') as f:
            f.write(data)
        self.assertEqual(File(self.filename).read(4), data)

    def test_read_n_bytes_data(self):
        data = u'ññññ'
        with open(self.filename, 'wb') as f:
            f.write(data.encode('utf-8'))
        self.assertEqual(File(self.filename).read(4, 'rb'), data.encode('utf-8')[:4])

    def tearDown(self):
        os.remove(self.filename)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>
import eHive.Params

import os
import sys
import json
import numbers
import warnings
import traceback

__doc__ = """
This module mainly implements python's counterpart of ForeignProcess. Read
the later for more information about the JSON protocol used to communicate.
"""

class Job(object):
    """Dummy class to hold job-related information"""
    pass

class CompleteEarlyException(Exception):
    """Can be raised by a derived class of BaseRunnable to indicate an early successful termination"""
    pass
class JobFailedException(Exception):
    """Can be raised by a derived class of BaseRunnable to indicate an early unsuccessful termination"""
    pass
class HiveJSONMessageException(Exception):
    """Raised when we could not parse the JSON message coming from ForeignProcess"""
    pass

class BaseRunnable(object):

    # Private BaseRunnable interface
    #################################

    def __init__(self, read_fileno, write_fileno):
        # We need the binary mode to disable the buffering
        self.read_pipe = os.fdopen(read_fileno, mode='rb', buffering=0)
        self.write_pipe = os.fdopen(write_fileno, mode='wb', buffering=0)
        self.pid = os.getpid()
        self.__process_life_cycle()

    def __print_debug(self, *args):
        print("PYTHON {0}".format(self.pid), *args, file=sys.stderr)

    def __send_message(self, event, content):
        def default_json_encoder(o):
            self.__print_debug("Cannot serialize {0} (type {1}) in JSON".format(o, type(o)))
            return 'UNSERIALIZABLE OBJECT'
        j = json.dumps({'event': event, 'content': content}, indent=None, default=default_json_encoder)
        self.__print_debug('__send_message:', j)
        self.write_pipe.write(bytes(j+"\n", 'utf-8'))

    def __read_message(self):
        try:
            self.__print_debug("__read_message ...")
            l = self.read_pipe.readline()
            self.__print_debug(" ... -> ", l[:-1].decode())
            return json.loads(l.decode())
        except ValueError as e:
            raise HiveJSONMessageException from e

    def __send_message_and_wait_for_OK(self, event, content):
        self.__send_message(event, content)
        response = self.__read_message()
        if response['response'] != 'OK':
            raise HiveJSONMessageException("Received '{0}' instead of OK".format(response))

    def __process_life_cycle(self):
        self.__send_message_and_wait_for_OK('PARAM_DEFAULTS', self.param_defaults())
        while True:
            self.__print_debug("waiting for instructions")
            config = self.__read_message()
            if 'input_job' not in config:
                self.__print_debug("no params")
                return
            self.__job_life_cycle(config)

    def __job_life_cycle(self, config):

        self.__print_debug("__life_cycle")

        # Params
        self.p = eHive.Params.ParamContainer(config['input_job']['parameters'])

        # Job attributes
        self.input_job = Job()
        for x in ['dbID', 'input_id', 'retry_count']:
            setattr(self.input_job, x, config['input_job'][x])
        self.input_job.autoflow = True
        self.input_job.lethal_for_worker = False
        self.input_job.transient_error = True

        # Worker attributes
        self.debug = config['debug']

        # Which methods should be run
        steps = [ 'fetch_input', 'run' ]
        if self.input_job.retry_count > 0:
            steps.insert(0, 'pre_cleanup')
        if config['execute_writes']:
            steps.append('write_output')
        self.__print_debug("steps to run:", steps)

        # The actual life-cycle
        died_somewhere = False
        try:
            for s in steps:
                self.__run_method_if_exists(s)
        except CompleteEarlyException as e:
            self.warning(e.args[0] if len(e.args) else repr(e), False)
        except:
            died_somewhere = True
            self.warning( self.__traceback(2), True)

        try:
            self.__run_method_if_exists('post_cleanup')
        except:
            died_somewhere = True
            self.warning( self.__traceback(2), True)

        job_end_structure = {'complete' : not died_somewhere, 'job': {}, 'params': {'substituted': self.p._param_hash, 'unsubstituted': self.p._unsubstituted_param_hash}}
        for x in [ 'autoflow', 'lethal_for_worker', 'transient_error' ]:
            job_end_structure['job'][x] = getattr(self.input_job, x)
        self.__send_message_and_wait_for_OK('JOB_END', job_end_structure)

    def __run_method_if_exists(self, method):
        if hasattr(self, method):
            self.__send_message_and_wait_for_OK('JOB_STATUS_UPDATE', method)
            getattr(self, method)()

    def __traceback(self, skipped_traces):
        """Remove "skipped_traces" lines from the stack trace (the eHive part)"""
        (etype, value, tb) = sys.exc_info()
        s1 = traceback.format_exception_only(etype, value)
        l = traceback.extract_tb(tb)[skipped_traces:]
        s2 = traceback.format_list(l)
        return "".join(s1+s2)

    # Public BaseRunnable interface
    ################################

    def warning(self, message, is_error = False):
        self.__send_message_and_wait_for_OK('WARNING', {'message': message, 'is_error': is_error})

    def dataflow(self, output_ids, branch_name_or_code = 1):
        self.__send_message('DATAFLOW', {'output_ids': output_ids, 'branch_name_or_code': branch_name_or_code, 'params': {'substituted': self.p._param_hash, 'unsubstituted': self.p._unsubstituted_param_hash}})
        return self.__read_message()

    def worker_temp_directory(self):
        if not hasattr(self, '_created_worker_temp_directory'):
            template_name = self.worker_temp_directory_name() if hasattr(self, 'worker_temp_directory_name') else None
            self.__send_message('WORKER_TEMP_DIRECTORY', template_name)
            self._created_worker_temp_directory = self.__read_message()
        return self._created_worker_temp_directory

    # Param interface
    ##################

    def param_defaults(self):
        return {}

    def param_required(self, param_name):
        """Returns the value of the parameter "param_name" or raises an exception if anything wrong happens. The exception is marked as non-transient."""
        t = self.input_job.transient_error
        self.input_job.transient_error = False
        v = self.p.get_param(param_name)
        self.input_job.transient_error = t
        return v

    def param(self, param_name, *args):
        """When called as a setter: sets the value of the parameter "param_name".
        When called as a getter: returns the value of the parameter "param_name".
        It does not raise an exception if the parameter (or another one in the substitution stack) is undefined"""
        # As a setter
        if len(args):
            return self.p.set_param(param_name, args[0])

        # As a getter
        try:
            return self.p.get_param(param_name)
        except KeyError as e:
            warnings.warn("parameter '{0}' cannot be initialized because {1} is not defined !\n".format(param_name, e), Params.ParamWarning, 2)
            return None

    def param_exists(self, param_name):
        return self.p.has_param(param_name)

    def param_is_defined(self, param_name):
        if not self.param_exists(param_name):
            return False
        try:
            return self.p.get_param(param_name) is not None
        except KeyError:
            return False

<commit_msg>Only in debug mode<commit_after>
import eHive.Params

import os
import sys
import json
import numbers
import warnings
import traceback

__doc__ = """
This module mainly implements python's counterpart of ForeignProcess. Read
the later for more information about the JSON protocol used to communicate.
"""

class Job(object):
    """Dummy class to hold job-related information"""
    pass

class CompleteEarlyException(Exception):
    """Can be raised by a derived class of BaseRunnable to indicate an early successful termination"""
    pass
class JobFailedException(Exception):
    """Can be raised by a derived class of BaseRunnable to indicate an early unsuccessful termination"""
    pass
class HiveJSONMessageException(Exception):
    """Raised when we could not parse the JSON message coming from ForeignProcess"""
    pass

class BaseRunnable(object):

    # Private BaseRunnable interface
    #################################

    def __init__(self, read_fileno, write_fileno):
        # We need the binary mode to disable the buffering
        self.read_pipe = os.fdopen(read_fileno, mode='rb', buffering=0)
        self.write_pipe = os.fdopen(write_fileno, mode='wb', buffering=0)
        self.pid = os.getpid()
        self.debug = 0
        self.__process_life_cycle()

    def __print_debug(self, *args):
        if self.debug > 1:
            print("PYTHON {0}".format(self.pid), *args, file=sys.stderr)

    def __send_message(self, event, content):
        def default_json_encoder(o):
            self.__print_debug("Cannot serialize {0} (type {1}) in JSON".format(o, type(o)))
            return 'UNSERIALIZABLE OBJECT'
        j = json.dumps({'event': event, 'content': content}, indent=None, default=default_json_encoder)
        self.__print_debug('__send_message:', j)
        self.write_pipe.write(bytes(j+"\n", 'utf-8'))

    def __read_message(self):
        try:
            self.__print_debug("__read_message ...")
            l = self.read_pipe.readline()
            self.__print_debug(" ... -> ", l[:-1].decode())
            return json.loads(l.decode())
        except ValueError as e:
            raise HiveJSONMessageException from e

    def __send_message_and_wait_for_OK(self, event, content):
        self.__send_message(event, content)
        response = self.__read_message()
        if response['response'] != 'OK':
            raise HiveJSONMessageException("Received '{0}' instead of OK".format(response))

    def __process_life_cycle(self):
        self.__send_message_and_wait_for_OK('PARAM_DEFAULTS', self.param_defaults())
        while True:
            self.__print_debug("waiting for instructions")
            config = self.__read_message()
            if 'input_job' not in config:
                self.__print_debug("no params")
                return
            self.__job_life_cycle(config)

    def __job_life_cycle(self, config):

        self.__print_debug("__life_cycle")

        # Params
        self.p = eHive.Params.ParamContainer(config['input_job']['parameters'])

        # Job attributes
        self.input_job = Job()
        for x in ['dbID', 'input_id', 'retry_count']:
            setattr(self.input_job, x, config['input_job'][x])
        self.input_job.autoflow = True
        self.input_job.lethal_for_worker = False
        self.input_job.transient_error = True

        # Worker attributes
        self.debug = config['debug']

        # Which methods should be run
        steps = [ 'fetch_input', 'run' ]
        if self.input_job.retry_count > 0:
            steps.insert(0, 'pre_cleanup')
        if config['execute_writes']:
            steps.append('write_output')
        self.__print_debug("steps to run:", steps)

        # The actual life-cycle
        died_somewhere = False
        try:
            for s in steps:
                self.__run_method_if_exists(s)
        except CompleteEarlyException as e:
            self.warning(e.args[0] if len(e.args) else repr(e), False)
        except:
            died_somewhere = True
            self.warning( self.__traceback(2), True)

        try:
            self.__run_method_if_exists('post_cleanup')
        except:
            died_somewhere = True
            self.warning( self.__traceback(2), True)

        job_end_structure = {'complete' : not died_somewhere, 'job': {}, 'params': {'substituted': self.p._param_hash, 'unsubstituted': self.p._unsubstituted_param_hash}}
        for x in [ 'autoflow', 'lethal_for_worker', 'transient_error' ]:
            job_end_structure['job'][x] = getattr(self.input_job, x)
        self.__send_message_and_wait_for_OK('JOB_END', job_end_structure)

    def __run_method_if_exists(self, method):
        if hasattr(self, method):
            self.__send_message_and_wait_for_OK('JOB_STATUS_UPDATE', method)
            getattr(self, method)()

    def __traceback(self, skipped_traces):
        """Remove "skipped_traces" lines from the stack trace (the eHive part)"""
        (etype, value, tb) = sys.exc_info()
        s1 = traceback.format_exception_only(etype, value)
        l = traceback.extract_tb(tb)[skipped_traces:]
        s2 = traceback.format_list(l)
        return "".join(s1+s2)

    # Public BaseRunnable interface
    ################################

    def warning(self, message, is_error = False):
        self.__send_message_and_wait_for_OK('WARNING', {'message': message, 'is_error': is_error})

    def dataflow(self, output_ids, branch_name_or_code = 1):
        self.__send_message('DATAFLOW', {'output_ids': output_ids, 'branch_name_or_code': branch_name_or_code, 'params': {'substituted': self.p._param_hash, 'unsubstituted': self.p._unsubstituted_param_hash}})
        return self.__read_message()

    def worker_temp_directory(self):
        if not hasattr(self, '_created_worker_temp_directory'):
            template_name = self.worker_temp_directory_name() if hasattr(self, 'worker_temp_directory_name') else None
            self.__send_message('WORKER_TEMP_DIRECTORY', template_name)
            self._created_worker_temp_directory = self.__read_message()
        return self._created_worker_temp_directory

    # Param interface
    ##################

    def param_defaults(self):
        return {}

    def param_required(self, param_name):
        """Returns the value of the parameter "param_name" or raises an exception if anything wrong happens. The exception is marked as non-transient."""
        t = self.input_job.transient_error
        self.input_job.transient_error = False
        v = self.p.get_param(param_name)
        self.input_job.transient_error = t
        return v

    def param(self, param_name, *args):
        """When called as a setter: sets the value of the parameter "param_name".
        When called as a getter: returns the value of the parameter "param_name".
        It does not raise an exception if the parameter (or another one in the substitution stack) is undefined"""
        # As a setter
        if len(args):
            return self.p.set_param(param_name, args[0])

        # As a getter
        try:
            return self.p.get_param(param_name)
        except KeyError as e:
            warnings.warn("parameter '{0}' cannot be initialized because {1} is not defined !\n".format(param_name, e), Params.ParamWarning, 2)
            return None

    def param_exists(self, param_name):
        return self.p.has_param(param_name)

    def param_is_defined(self, param_name):
        if not self.param_exists(param_name):
            return False
        try:
            return self.p.get_param(param_name) is not None
        except KeyError:
            return False

<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import cgi
import fileinput
import os.path
import re
import sys
import wsgiref.util

# which logs support severity
SUPPORTS_SEV = '(screen-(n-|c-|q-|g-|h-|ceil|key))'

DATEFMT = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}((\.|\,)\d{3})?'
STATUSFMT = '(DEBUG|INFO|WARNING|ERROR|TRACE|AUDIT)'
KEY_COMPONENT = '\([^\)]+\):'

OSLO_LOGMATCH = '^(?P<date>%s)(?P<pid> \d+)? (?P<status>%s)' % \
    (DATEFMT, STATUSFMT)
KEY_LOGMATCH = '^(?P<comp>%s) (?P<date>%s) (?P<status>%s)' % \
    (KEY_COMPONENT, DATEFMT, STATUSFMT)

SEVS = {
    'NONE': 0,
    'DEBUG': 1,
    'INFO': 2,
    'AUDIT': 3,
    'TRACE': 4,
    'WARNING': 5,
    'ERROR': 6
    }

def _html_close():
    return ("
\n")

def _css_preamble(supports_sev):
    """Write a valid html start with css that we need."""
    header = """

"""
    if supports_sev:
        header = header + """

Display level: [
ALL |
DEBUG |
INFO |
AUDIT |
TRACE |
WARNING |
ERROR ]
"""

    header = header + "
"
    return header

def file_supports_sev(fname):
    m = re.search(SUPPORTS_SEV, fname)
    return m is not None

def not_html(fname):
    return re.search('(\.html(\.gz)?)$', fname) is None

def sev_of_line(line, oldsev="NONE"):
    m = re.match(OSLO_LOGMATCH, line)
    if m:
        return m.group('status')

    m = re.match(KEY_LOGMATCH, line)
    if m:
        return m.group('status')

    return oldsev

def color_by_sev(line, sev):
    """Wrap a line in a span whose class matches it's severity."""
    return "%s" % (sev, line)

def escape_html(line):
    """Escape the html in a line.

    We need to do this because we dump xml into the logs, and if we don't
    escape the xml we end up with invisible parts of the logs in turning it
    into html.
    """
    return cgi.escape(line)

def link_timestamp(line):
    m = re.match(
        '(<span class=\'(?P<class>[^\']+)\'>)?(?P<comp>%s )?'
        '(?P<date>%s)(?P<rest>.*)' % (KEY_COMPONENT, DATEFMT),
        line)
    if m:
        date = "_" + re.sub('[\s\:\.\,]', '_', m.group('date'))

        # everyone that got this far had a date
        line = "%s%s\n" % (
            date, date, m.group('date'), m.group('rest'))
        # if we found a keystone component, add it back
        if m.group('comp'):
            line = ("%s" % m.group('comp')) + line

        # if we found a severity class, put the spans back
        if m.group('class'):
            line = "%s" % (
                m.group('class'), date, line)

    return line

def skip_line_by_sev(sev, minsev):
    """should we skip this line?

    If the line severity is less than our minimum severity,
    yes we should.
    """
    return SEVS.get(sev, 0) < SEVS.get(minsev, 0)

def passthrough_filter(fname, minsev):
    sev = "NONE"
    supports_sev = file_supports_sev(fname)

    for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
        if supports_sev:
            sev = sev_of_line(line, sev)

            if skip_line_by_sev(sev, minsev):
                continue

        yield line

def does_file_exist(fname):
    """Figure out if we'll be able to read this file.

    Because we are handling the file streams as generators, we actually raise
    an exception too late for us to be able to handle it before apache has
    completely control. This attempts to do the same open outside of the
    generator to trigger the IOError early enough for us to catch it, without
    completely changing the logic flow, as we really want the generator
    pipeline for performance reasons.

    This does open us up to a small chance for a race where the file comes
    or goes between this call and the next, however that is a vanishingly
    small possibility.
    """
    f = open(fname)
    f.close()

def html_filter(fname, minsev):
    """Generator to read logs and output html in a stream.

    This produces a stream of the htmlified logs which lets us return
    data quickly to the user, and use minimal memory in the process.
    """

    supports_sev = file_supports_sev(fname)
    sev = "NONE"
    should_escape = not_html(fname)

    yield _css_preamble(supports_sev)

    for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
        if should_escape:
            newline = escape_html(line)
        else:
            newline = line

        if supports_sev:
            sev = sev_of_line(newline, sev)
            if skip_line_by_sev(sev, minsev):
                continue
            newline = color_by_sev(newline, sev)

        newline = link_timestamp(newline)
        yield newline
    yield _html_close()

def htmlify_stdin():
    minsev = "NONE"
    out = sys.stdout
    out.write(_css_preamble())
    for line in fileinput.FileInput():
        newline = escape_html(line)
        newline = color_by_sev(newline, minsev)
        newline = link_timestamp(newline)
        out.write(newline)
    out.write(_html_close())

def safe_path(root, environ):
    """Pull out a safe path from a url.

    Basically we need to ensure that the final computed path
    remains under the root path. If not, we return None to indicate
    that we are very sad.
    """
    path = wsgiref.util.request_uri(environ, include_query=0)
    match = re.search('htmlify/(.*)', path)
    if match:
        raw = match.groups(1)[0]
        newpath = os.path.abspath(os.path.join(root, raw))
        if newpath.find(root) == 0:
            return newpath

    return None

def should_be_html(environ):
    """Simple content negotiation.

    If the client supports content negotiation, and asks for text/html,
    we give it to them, unless they also specifically want to override
    by passing ?content-type=text/plain in the query.

    This should be able to handle the case of dumb clients defaulting to
    html, but also let devs override the text format when 35 MB html
    log files kill their browser (as per a nova-api log).
    """
    text_override = False
    accepts_html = ('HTTP_ACCEPT' in environ and
                    'text/html' in environ['HTTP_ACCEPT'])
    parameters = cgi.parse_qs(environ.get('QUERY_STRING', ''))
    if 'content-type' in parameters:
        ct = cgi.escape(parameters['content-type'][0])
        if ct == 'text/plain':
            text_override = True

    return accepts_html and not text_override

def get_min_sev(environ):
    print environ.get('QUERY_STRING')
    parameters = cgi.parse_qs(environ.get('QUERY_STRING', ''))
    if 'level' in parameters:
        return cgi.escape(parameters['level'][0])
    else:
        return "NONE"

def application(environ, start_response, root_path='/srv/static/logs/'):
    status = '200 OK'

    logpath = safe_path(root_path, environ)
    if not logpath:
        status = '400 Bad Request'
        response_headers = [('Content-type', 'text/plain')]
        start_response(status, response_headers)
        return ['Invalid file url']

    try:
        minsev = get_min_sev(environ)
        if should_be_html(environ):
            response_headers = [('Content-type', 'text/html')]
            does_file_exist(logpath)
            generator = html_filter(logpath, minsev)
            start_response(status, response_headers)
            return generator
        else:
            response_headers = [('Content-type', 'text/plain')]
            does_file_exist(logpath)
            generator = passthrough_filter(logpath, minsev)
            start_response(status, response_headers)
            return generator
    except IOError:
        status = "404 Not Found"
        response_headers = [('Content-type', 'text/plain')]
        start_response(status, response_headers)
        return ['File Not Found']

# for development purposes, makes it easy to test the filter output
if __name__ == "__main__":
    htmlify_stdin()
<commit_msg>add tempest log to the list of level supporting logs<commit_after>#!/usr/bin/python
#
# Copyright (c) 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import cgi
import fileinput
import os.path
import re
import sys
import wsgiref.util

# which logs support severity
SUPPORTS_SEV = '(screen-(n-|c-|q-|g-|h-|ceil|key)|tempest\.txt)'

DATEFMT = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}((\.|\,)\d{3})?'
STATUSFMT = '(DEBUG|INFO|WARNING|ERROR|TRACE|AUDIT)'
KEY_COMPONENT = '\([^\)]+\):'

OSLO_LOGMATCH = '^(?P<date>%s)(?P<pid> \d+)? (?P<status>%s)' % \
    (DATEFMT, STATUSFMT)
KEY_LOGMATCH = '^(?P<comp>%s) (?P<date>%s) (?P<status>%s)' % \
    (KEY_COMPONENT, DATEFMT, STATUSFMT)

SEVS = {
    'NONE': 0,
    'DEBUG': 1,
    'INFO': 2,
    'AUDIT': 3,
    'TRACE': 4,
    'WARNING': 5,
    'ERROR': 6
    }

def _html_close():
    return ("
\n")

def _css_preamble(supports_sev):
    """Write a valid html start with css that we need."""
    header = """

"""
    if supports_sev:
        header = header + """

Display level: [
ALL |
DEBUG |
INFO |
AUDIT |
TRACE |
WARNING |
ERROR ]
"""

    header = header + "
"
    return header

def file_supports_sev(fname):
    m = re.search(SUPPORTS_SEV, fname)
    return m is not None

def not_html(fname):
    return re.search('(\.html(\.gz)?)$', fname) is None

def sev_of_line(line, oldsev="NONE"):
    m = re.match(OSLO_LOGMATCH, line)
    if m:
        return m.group('status')

    m = re.match(KEY_LOGMATCH, line)
    if m:
        return m.group('status')

    return oldsev

def color_by_sev(line, sev):
    """Wrap a line in a span whose class matches it's severity."""
    return "%s" % (sev, line)

def escape_html(line):
    """Escape the html in a line.

    We need to do this because we dump xml into the logs, and if we don't
    escape the xml we end up with invisible parts of the logs in turning it
    into html.
    """
    return cgi.escape(line)

def link_timestamp(line):
    m = re.match(
        '(<span class=\'(?P<class>[^\']+)\'>)?(?P<comp>%s )?'
        '(?P<date>%s)(?P<rest>.*)' % (KEY_COMPONENT, DATEFMT),
        line)
    if m:
        date = "_" + re.sub('[\s\:\.\,]', '_', m.group('date'))

        # everyone that got this far had a date
        line = "%s%s\n" % (
            date, date, m.group('date'), m.group('rest'))
        # if we found a keystone component, add it back
        if m.group('comp'):
            line = ("%s" % m.group('comp')) + line

        # if we found a severity class, put the spans back
        if m.group('class'):
            line = "%s" % (
                m.group('class'), date, line)

    return line

def skip_line_by_sev(sev, minsev):
    """should we skip this line?

    If the line severity is less than our minimum severity,
    yes we should.
    """
    return SEVS.get(sev, 0) < SEVS.get(minsev, 0)

def passthrough_filter(fname, minsev):
    sev = "NONE"
    supports_sev = file_supports_sev(fname)

    for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
        if supports_sev:
            sev = sev_of_line(line, sev)

            if skip_line_by_sev(sev, minsev):
                continue

        yield line

def does_file_exist(fname):
    """Figure out if we'll be able to read this file.

    Because we are handling the file streams as generators, we actually raise
    an exception too late for us to be able to handle it before apache has
    completely control. This attempts to do the same open outside of the
    generator to trigger the IOError early enough for us to catch it, without
    completely changing the logic flow, as we really want the generator
    pipeline for performance reasons.

    This does open us up to a small chance for a race where the file comes
    or goes between this call and the next, however that is a vanishingly
    small possibility.
    """
    f = open(fname)
    f.close()

def html_filter(fname, minsev):
    """Generator to read logs and output html in a stream.

    This produces a stream of the htmlified logs which lets us return
    data quickly to the user, and use minimal memory in the process.
    """

    supports_sev = file_supports_sev(fname)
    sev = "NONE"
    should_escape = not_html(fname)

    yield _css_preamble(supports_sev)

    for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
        if should_escape:
            newline = escape_html(line)
        else:
            newline = line

        if supports_sev:
            sev = sev_of_line(newline, sev)
            if skip_line_by_sev(sev, minsev):
                continue
            newline = color_by_sev(newline, sev)

        newline = link_timestamp(newline)
        yield newline
    yield _html_close()

def htmlify_stdin():
    minsev = "NONE"
    out = sys.stdout
    out.write(_css_preamble())
    for line in fileinput.FileInput():
        newline = escape_html(line)
        newline = color_by_sev(newline, minsev)
        newline = link_timestamp(newline)
        out.write(newline)
    out.write(_html_close())

def safe_path(root, environ):
    """Pull out a safe path from a url.

    Basically we need to ensure that the final computed path
    remains under the root path. If not, we return None to indicate
    that we are very sad.
    """
    path = wsgiref.util.request_uri(environ, include_query=0)
    match = re.search('htmlify/(.*)', path)
    if match:
        raw = match.groups(1)[0]
        newpath = os.path.abspath(os.path.join(root, raw))
        if newpath.find(root) == 0:
            return newpath

    return None

def should_be_html(environ):
    """Simple content negotiation.

    If the client supports content negotiation, and asks for text/html,
    we give it to them, unless they also specifically want to override
    by passing ?content-type=text/plain in the query.

    This should be able to handle the case of dumb clients defaulting to
    html, but also let devs override the text format when 35 MB html
    log files kill their browser (as per a nova-api log).
    """
    text_override = False
    accepts_html = ('HTTP_ACCEPT' in environ and
                    'text/html' in environ['HTTP_ACCEPT'])
    parameters = cgi.parse_qs(environ.get('QUERY_STRING', ''))
    if 'content-type' in parameters:
        ct = cgi.escape(parameters['content-type'][0])
        if ct == 'text/plain':
            text_override = True

    return accepts_html and not text_override

def get_min_sev(environ):
    print environ.get('QUERY_STRING')
    parameters = cgi.parse_qs(environ.get('QUERY_STRING', ''))
    if 'level' in parameters:
        return cgi.escape(parameters['level'][0])
    else:
        return "NONE"

def application(environ, start_response, root_path='/srv/static/logs/'):
    status = '200 OK'

    logpath = safe_path(root_path, environ)
    if not logpath:
        status = '400 Bad Request'
        response_headers = [('Content-type', 'text/plain')]
        start_response(status, response_headers)
        return ['Invalid file url']

    try:
        minsev = get_min_sev(environ)
        if should_be_html(environ):
            response_headers = [('Content-type', 'text/html')]
            does_file_exist(logpath)
            generator = html_filter(logpath, minsev)
            start_response(status, response_headers)
            return generator
        else:
            response_headers = [('Content-type', 'text/plain')]
            does_file_exist(logpath)
            generator = passthrough_filter(logpath, minsev)
            start_response(status, response_headers)
            return generator
    except IOError:
        status = "404 Not Found"
        response_headers = [('Content-type', 'text/plain')]
        start_response(status, response_headers)
        return ['File Not Found']

# for development purposes, makes it easy to test the filter output
if __name__ == "__main__":
    htmlify_stdin()
<|endoftext|>


<commit_before>"""A collection of pandas data interfaces to a project instance."""
from __future__ import absolute_import
import os.path as osp

try:
    import pandas as pd
except ImportError:
    raise ImportError('The pandas package is required for this plugin. '
                      'Try pip install pandas')

class ProjectOrRunData(pd.DataFrame):
    """
    A representation of data read from either the SWIM project or Run instance.
    """
    swim_path = None
    plugin = []

    def __init__(self, projectorrun):
        from swimpy.project import Project  # avoid circular import
        # init DataFrame
        pd.DataFrame.__init__(self)
        self.name = self.__class__.__name__
        # instantiated with project
        if isinstance(projectorrun, Project):
            self.project = projectorrun
            self.path = osp.join(self.project.projectdir, self.swim_path)
            self.read = self.from_project
        # instantiated with run
        elif hasattr(projectorrun, 'resultfiles'):
            self.run = projectorrun
            self.path = self.find_resultfile()
            self.read = self.from_run
        else:
            raise IOError('Run includes no saved files.')
        # read file
        pd.DataFrame.__init__(self, self.read(self.path))
        return

    def find_resultfile(self):
        # find file
        fileqs = (self.run.resultfiles.filter(tags__contains=self.name) or
                  self.run.resultfiles.filter(file__contains=self.name))
        if fileqs.count() > 1:
            print('Found two resultfiles for %s, using last!' % self.name)
        elif fileqs.count() == 0:
            raise IOError('No resultfile found for %s!' % self.name)
        fileobj = fileqs.last()
        return fileobj.file.path

    def from_run(self, path, **readkwargs):
        """
        Read data from a run instance with resultfiles.
        """
        reader = self.reader_by_ext(path)
        return reader(path, **readkwargs)

    def from_project(self, path, **kw):
        """!Overwrite me!"""
        raise NotImplementedError('Cant read this ProjectOrRunData from '
                                  'project, define a from_project method!')

    def from_gzip(self, path, **readkwargs):
        readkwargs['compression'] = 'gzip'
        return self.reader_by_ext(osp.splitext(path)[0])(path, **readkwargs)

    def reader_by_ext(self, path):
        """
        Return the read method from_* using the self.path extension.
        Raises a NotImplementedError if none found.
        """
        ext = osp.splitext(path)[1][1:]  # no dot
        readmethodname = 'from_' + ext
        if not hasattr(self, readmethodname):
            raise NotImplementedError('No method %s to read file %s defined!' %
                                      (readmethodname, path))
        return getattr(self, readmethodname)

class ReadWriteDataFrame(pd.DataFrame):
    """
    A representation of data read and written to file.

    The ``read`` method has to reinitialise the dataframe. An example use as a
    plugin (instantiated on project instantiation)::

        class ProjectData(ReadWriteData):
            path = 'some/relative/path.csv'

            def read(self, **kw):
                data = pd.read_table(self.path)
                pd.DataFrame.__init__(self, data)
                return
            def write(self, **kw):
                # possible error/consistency checking
                assert len(self.columns) == 2, 'must have only 2 columns'
                self.to_csv(self.path)
        # add to project
        p = swimpy.Project('project/')
        p.settings(ProjectData)
        # access the DataFrame
        p.projectdata
        # or modify and write out again
        p.projectdata.write()

    To defer reading of the dataframe until it is actually accessed, decorate
    the class with a ``@modelmanager.utils.propertyplugin``.
    """
    path = None
    plugin = []

    def __init__(self, project, oninit='read', **kwargs):
        # init DataFrame
        pd.DataFrame.__init__(self)
        self.name = self.__class__.__name__
        self.project = project
        self.path = osp.join(self.project.projectdir, self.path)
        errmsg = self.name + 'file does not exist: ' + self.path
        assert osp.exists(self.path), errmsg
        if oninit:
            getattr(self, oninit)(**kwargs)
        return

    def __call__(self, data=None, **set):
        """
        Assign read data from file and optionally set and write new values.

        data: <2D-array-like>
            Set entire dataframe.
        **set: <array-like> | <dict>
            Set columns or rows by key. Subset of values can be set by parsing
            a dict. Creates new row if key is neither in columns or index.
        """
        if data is not None:
            pd.DataFrame.__init__(self, data)
            self.write()
        elif set:
            self.read()
            for k, v in set.items():
                ix = slice(None)
                if type(v) == dict:
                    ix, v = zip(*v.items())
                if k in self.columns:
                    self.loc[ix, k] = v
                else:
                    self.loc[k, ix] = v
            self.write()
        else:
            self.read()
        return self

    def __repr__(self):
        rpr = '<%s: %s >\n' % (self.name, osp.relpath(self.path))
        return rpr + pd.DataFrame.__repr__(self)

    def read(self, **kwargs):
        """
        Override me and reinitialise the data by calling:
            pd.DataFrame.__init__(self, data)
        """
        raise NotImplementedError('Reading of %s not implemented.' % self.name)

    def write(self, **kwargs):
        """
        Override me. Error checking and writing to file should be done here.
        """
        raise NotImplementedError('Writing of %s not implemented.' % self.name)
<commit_msg>Ensure ProjectOrRunData has a project attribute.<commit_after>"""A collection of pandas data interfaces to a project instance."""
from __future__ import absolute_import
import os.path as osp

try:
    import pandas as pd
except ImportError:
    raise ImportError('The pandas package is required for this plugin. '
                      'Try pip install pandas')

class ProjectOrRunData(pd.DataFrame):
    """
    A representation of data read from either the SWIM project or Run instance.
    """
    swim_path = None
    plugin = []

    def __init__(self, projectorrun):
        from swimpy.project import Project  # avoid circular import
        # init DataFrame
        pd.DataFrame.__init__(self)
        self.name = self.__class__.__name__
        # instantiated with project
        if isinstance(projectorrun, Project):
            self.project = projectorrun
            self.path = osp.join(self.project.projectdir, self.swim_path)
            self.read = self.from_project
        # instantiated with run
        elif hasattr(projectorrun, 'resultfiles'):
            from django.conf import settings
            self.project = settings.PROJECT
            self.run = projectorrun
            self.path = self.find_resultfile()
            self.read = self.from_run
        else:
            raise IOError('Run includes no saved files.')
        # read file
        pd.DataFrame.__init__(self, self.read(self.path))
        return

    def find_resultfile(self):
        # find file
        fileqs = (self.run.resultfiles.filter(tags__contains=self.name) or
                  self.run.resultfiles.filter(file__contains=self.name))
        if fileqs.count() > 1:
            print('Found two resultfiles for %s, using last!' % self.name)
        elif fileqs.count() == 0:
            raise IOError('No resultfile found for %s!' % self.name)
        fileobj = fileqs.last()
        return fileobj.file.path

    def from_run(self, path, **readkwargs):
        """
        Read data from a run instance with resultfiles.
        """
        reader = self.reader_by_ext(path)
        return reader(path, **readkwargs)

    def from_project(self, path, **kw):
        """!Overwrite me!"""
        raise NotImplementedError('Cant read this ProjectOrRunData from '
                                  'project, define a from_project method!')

    def from_gzip(self, path, **readkwargs):
        readkwargs['compression'] = 'gzip'
        return self.reader_by_ext(osp.splitext(path)[0])(path, **readkwargs)

    def reader_by_ext(self, path):
        """
        Return the read method from_* using the self.path extension.
        Raises a NotImplementedError if none found.
        """
        ext = osp.splitext(path)[1][1:]  # no dot
        readmethodname = 'from_' + ext
        if not hasattr(self, readmethodname):
            raise NotImplementedError('No method %s to read file %s defined!' %
                                      (readmethodname, path))
        return getattr(self, readmethodname)

class ReadWriteDataFrame(pd.DataFrame):
    """
    A representation of data read and written to file.

    The ``read`` method has to reinitialise the dataframe. An example use as a
    plugin (instantiated on project instantiation)::

        class ProjectData(ReadWriteData):
            path = 'some/relative/path.csv'

            def read(self, **kw):
                data = pd.read_table(self.path)
                pd.DataFrame.__init__(self, data)
                return
            def write(self, **kw):
                # possible error/consistency checking
                assert len(self.columns) == 2, 'must have only 2 columns'
                self.to_csv(self.path)
        # add to project
        p = swimpy.Project('project/')
        p.settings(ProjectData)
        # access the DataFrame
        p.projectdata
        # or modify and write out again
        p.projectdata.write()

    To defer reading of the dataframe until it is actually accessed, decorate
    the class with a ``@modelmanager.utils.propertyplugin``.
    """
    path = None
    plugin = []

    def __init__(self, project, oninit='read', **kwargs):
        # init DataFrame
        pd.DataFrame.__init__(self)
        self.name = self.__class__.__name__
        self.project = project
        self.path = osp.join(self.project.projectdir, self.path)
        errmsg = self.name + 'file does not exist: ' + self.path
        assert osp.exists(self.path), errmsg
        if oninit:
            getattr(self, oninit)(**kwargs)
        return

    def __call__(self, data=None, **set):
        """
        Assign read data from file and optionally set and write new values.

        data: <2D-array-like>
            Set entire dataframe.
        **set: <array-like> | <dict>
            Set columns or rows by key. Subset of values can be set by parsing
            a dict. Creates new row if key is neither in columns or index.
        """
        if data is not None:
            pd.DataFrame.__init__(self, data)
            self.write()
        elif set:
            self.read()
            for k, v in set.items():
                ix = slice(None)
                if type(v) == dict:
                    ix, v = zip(*v.items())
                if k in self.columns:
                    self.loc[ix, k] = v
                else:
                    self.loc[k, ix] = v
            self.write()
        else:
            self.read()
        return self

    def __repr__(self):
        rpr = '<%s: %s >\n' % (self.name, osp.relpath(self.path))
        return rpr + pd.DataFrame.__repr__(self)

    def read(self, **kwargs):
        """
        Override me and reinitialise the data by calling:
            pd.DataFrame.__init__(self, data)
        """
        raise NotImplementedError('Reading of %s not implemented.' % self.name)

    def write(self, **kwargs):
        """
        Override me. Error checking and writing to file should be done here.
        """
        raise NotImplementedError('Writing of %s not implemented.' % self.name)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import unittest
import six

from dockermap.map.base import ContainerImageResolver
from dockermap.map.container import ContainerMap
from dockermap.map.policy.dep import ContainerDependencyResolver

TEST_MAP_DATA = {
    'a': dict(uses=['b', 'c']),
    'b': dict(uses=['dv', 'f'], links='e.1'),
    'c': dict(),
    'd': dict(uses='e', instances=['1', '2'], attaches='dv'),
    'e': dict(instances=['1', '2']),
    'f': dict(),
    'x': dict(uses=['b', 'f']),
}

TEST_IMG_DATA = [
    ('a', 'b'),
    ('b', 'c'),
    ('c', 'e'),
    ('d', 'e'),
    ('f', 'x'),
]
TEST_CONTAINER_IMAGES = {'a', 'c', 'f'}

class ContainerDependencyTest(unittest.TestCase):
    def setUp(self):
        test_map = ContainerMap('test_map', initial=TEST_MAP_DATA, check_integrity=False)
        self.f_res = ContainerDependencyResolver(test_map)
        self.r_res = ContainerDependencyResolver()
        self.r_res.update_backward(test_map)

    def assertOrder(self, dependency_list, *items):
        iterator = iter(items)
        last_item = six.next(iterator)
        last_idx = dependency_list.index(last_item)
        for item in iterator:
            index = dependency_list.index(item)
            if index < last_idx:
                self.fail("{0} found before {1}, should be later.".format(item, last_item))

    def test_forward_resolution_order(self):
        a_dep = self.f_res.get_container_dependencies('test_map', 'a')
        self.assertOrder(a_dep,
                         ('test_map', 'd', None),
                         ('test_map', 'b', None))
        self.assertOrder(a_dep,
                         ('test_map', 'e', None),
                         ('test_map', 'd', None))
        self.assertOrder(a_dep,
                         ('test_map', 'e', '1'),
                         ('test_map', 'b', None))
        self.assertOrder(a_dep,
                         ('test_map', 'f', None),
                         ('test_map', 'b', None))
        x_dep = self.f_res.get_container_dependencies('test_map', 'x')
        self.assertOrder(x_dep,
                         ('test_map', 'f', None),
                         ('test_map', 'b', None))

    def test_backward_resolution_order(self):
        f_dep = self.r_res.get_container_dependencies('test_map', 'f')
        self.assertOrder(f_dep,
                         ('test_map', 'x', None),
                         ('test_map', 'b', None))
        self.assertOrder(f_dep,
                         ('test_map', 'a', None),
                         ('test_map', 'b', None))
        e_dep = self.r_res.get_container_dependencies('test_map', 'e')
        self.assertOrder(e_dep,
                         ('test_map', 'a', None),
                         ('test_map', 'b', None),
                         ('test_map', 'd', None))

class ImageDependencyTest(unittest.TestCase):
    def setUp(self):
        self.res = ContainerImageResolver(TEST_CONTAINER_IMAGES, TEST_IMG_DATA)

    def test_iamge_dependencies(self):
        self.assertTrue(self.res.get_dependencies('a'))
        self.assertTrue(self.res.get_dependencies('b'))
        self.assertTrue(self.res.get_dependencies('c'))
        self.assertTrue(self.res.get_dependencies('f'))
        self.assertFalse(self.res.get_dependencies('d'))
        self.assertFalse(self.res.get_dependencies('e'))
        self.assertFalse(self.res.get_dependencies('x'))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix typo in test name.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import unittest
import six

from dockermap.map.base import ContainerImageResolver
from dockermap.map.container import ContainerMap
from dockermap.map.policy.dep import ContainerDependencyResolver

TEST_MAP_DATA = {
    'a': dict(uses=['b', 'c']),
    'b': dict(uses=['dv', 'f'], links='e.1'),
    'c': dict(),
    'd': dict(uses='e', instances=['1', '2'], attaches='dv'),
    'e': dict(instances=['1', '2']),
    'f': dict(),
    'x': dict(uses=['b', 'f']),
}

TEST_IMG_DATA = [
    ('a', 'b'),
    ('b', 'c'),
    ('c', 'e'),
    ('d', 'e'),
    ('f', 'x'),
]
TEST_CONTAINER_IMAGES = {'a', 'c', 'f'}

class ContainerDependencyTest(unittest.TestCase):
    def setUp(self):
        test_map = ContainerMap('test_map', initial=TEST_MAP_DATA, check_integrity=False)
        self.f_res = ContainerDependencyResolver(test_map)
        self.r_res = ContainerDependencyResolver()
        self.r_res.update_backward(test_map)

    def assertOrder(self, dependency_list, *items):
        iterator = iter(items)
        last_item = six.next(iterator)
        last_idx = dependency_list.index(last_item)
        for item in iterator:
            index = dependency_list.index(item)
            if index < last_idx:
                self.fail("{0} found before {1}, should be later.".format(item, last_item))

    def test_forward_resolution_order(self):
        a_dep = self.f_res.get_container_dependencies('test_map', 'a')
        self.assertOrder(a_dep,
                         ('test_map', 'd', None),
                         ('test_map', 'b', None))
        self.assertOrder(a_dep,
                         ('test_map', 'e', None),
                         ('test_map', 'd', None))
        self.assertOrder(a_dep,
                         ('test_map', 'e', '1'),
                         ('test_map', 'b', None))
        self.assertOrder(a_dep,
                         ('test_map', 'f', None),
                         ('test_map', 'b', None))
        x_dep = self.f_res.get_container_dependencies('test_map', 'x')
        self.assertOrder(x_dep,
                         ('test_map', 'f', None),
                         ('test_map', 'b', None))

    def test_backward_resolution_order(self):
        f_dep = self.r_res.get_container_dependencies('test_map', 'f')
        self.assertOrder(f_dep,
                         ('test_map', 'x', None),
                         ('test_map', 'b', None))
        self.assertOrder(f_dep,
                         ('test_map', 'a', None),
                         ('test_map', 'b', None))
        e_dep = self.r_res.get_container_dependencies('test_map', 'e')
        self.assertOrder(e_dep,
                         ('test_map', 'a', None),
                         ('test_map', 'b', None),
                         ('test_map', 'd', None))

class ImageDependencyTest(unittest.TestCase):
    def setUp(self):
        self.res = ContainerImageResolver(TEST_CONTAINER_IMAGES, TEST_IMG_DATA)

    def test_image_dependencies(self):
        self.assertTrue(self.res.get_dependencies('a'))
        self.assertTrue(self.res.get_dependencies('b'))
        self.assertTrue(self.res.get_dependencies('c'))
        self.assertTrue(self.res.get_dependencies('f'))
        self.assertFalse(self.res.get_dependencies('d'))
        self.assertFalse(self.res.get_dependencies('e'))
        self.assertFalse(self.res.get_dependencies('x'))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import os
import re

def some_little_modify(s):
	"""在字符串中的(及)前面加上\, 方便转换成正则表达式"""
	s_1 = s.replace('(', '\(')
	s_2 = s_1.replace(')', '\)')
	s_3 = s_2.replace('+', '\+')
	return s_3

def find_all(substring, string):
	start = 0
	while True:
		start  = string.find(substring, start)
		if start == -1:
			return -1
		yield start
		start += len(substring)

def find_mod(path, dic):
	"""用字典中的词发现文本模式"""
	dic =  open(dic, 'r', encoding= 'utf8')
	word_list = dic.readlines()
	file_list = os.listdir(path)

	word_count = {}
	#用一个字典保存, key为发现的文本模式, 键值为匹配该模式的词典中的词的数目
	mod_list = []
	#文本模式以列表形式保存

	num = 0
	for file in file_list:
		txt_fr = open(os.path.join(path, file), 'r', encoding='utf8')
		p = 5
		q = 5
		txt_file = txt_fr.read()
		txt_fr.close()
		if len(txt_file) > 0:
			for word in word_list:
				inde = word.split('\t')[0]
				loc_list = [w.start() for w in re.finditer(inde, txt_file)]
				for loc in loc_list:
					num += 1
					for i in range(1, (p+1)):
						for j in range(1,(q+1)):
							ext_word = txt_file[loc - i:loc + len(inde) + j]
							ext_wd = some_little_modify(ext_word)
							local_ind = ext_wd.index(inde)
							mod = re.compile(ext_wd[:local_ind]+'(\S{%d})'%len(inde)+ext_wd[local_ind+len(inde):])
							if mod not in mod_list:
								mod_list.append(mod)
								word_count[mod] = 1
							else:
								word_count[mod] += 1
	dic.close()
	return mod_list, word_count

def find_word(path, mod_list):
	file_list = os.listdir(path)
	mod_dict = {}
	for mod in mod_list:
		wor_set = set()
		for file in file_list:
			with open(os.path.join(path, file), 'r', encoding='utf8') as txt_fr:
				txt_file = txt_fr.read()
				wor_set = wor_set.union(set(re.findall(mod, txt_file)))
		num_extract = len(wor_set)
		mod_dict[mod] = num_extract
	return mod_dict

def score_mod(mod, mod_dict, word_count):
	import math
	return float(mod_dict[mod])/float(word_count[mod])*math.log(float(mod_dict[mod]) + 1, 2)

path = 'E:/病例特点'
dic = 'C:/Users/yingying.zhu/Documents/dicts/disease.txt'
print (dic)
mod_list, count = find_mod(path, dic)
print (mod_list[:15])<commit_msg>change some function<commit_after>import os
import re

def some_little_modify(s):
    """在字符串中的(及)前面加上\, 方便转换成正则表达式"""
    to_be_removed = ['(', ')', '+', '*']
    for signal in to_be_removed:
        s = s.replace(signal, '\\'+signal)
    return s

def find_all(substring, string):
    start = 0
    while True:
        start  = string.find(substring, start)
        if start == -1:
            return -1
        yield start
        start += len(substring)

def find_mod(path, dic):
    """用字典中的词发现文本模式"""
    dic =  open(dic, 'r', encoding= 'utf8')
    word_list = dic.readlines()
    file_list = os.listdir(path)

    word_count = {}
    #用一个字典保存, key为发现的文本模式, 键值为匹配该模式的词典中的词的数目
    mod_list = []
    #文本模式以列表形式保存

    num = 0
    for file in file_list:
        txt_fr = open(os.path.join(path, file), 'r', encoding='utf8')
        p = 5
        q = 5
        txt_file = txt_fr.read()
        txt_fr.close()
        if len(txt_file) > 0:
            for word in word_list:
                inde = word.split('\t')[0]
                loc_list = [w.start() for w in re.finditer(inde, txt_file)]
                for loc in loc_list:
                    num += 1
                    for i in range(1, (p+1)):
                        for j in range(1,(q+1)):
                            ext_word = txt_file[max(0, loc - i): min(loc + len(word) + j, len(txt_file)-1)]
                            ext_wd = some_little_modify(ext_word)
                            local_ind = ext_wd.index(some_little_modify(inde))
                            try:
                                mod = re.compile(ext_wd[:local_ind]+'(\S{%d})'%len(inde)+ext_wd[local_ind+len(inde):])
                            except re.error:
                                print (inde + '\t\t' + ext_word + '\n')
                            if mod not in mod_list:
                                mod_list.append(mod)
                                word_count[mod] = 1
                            else:
                                word_count[mod] += 1
    dic.close()
    return mod_list, word_count

def find_word(path, mod_list):
    file_list = os.listdir(path)
    mod_dict = {}
    for mod in mod_list:
        wor_set = set()
        for file in file_list:
            with open(os.path.join(path, file), 'r', encoding='utf8') as txt_fr:
                txt_file = txt_fr.read()
                wor_set = wor_set.union(set(re.findall(mod, txt_file)))
        num_extract = len(wor_set)
        mod_dict[mod] = num_extract
    return mod_dict

def score_mod(mod, mod_dict, word_count):
    import math
    return float(mod_dict[mod])/float(word_count[mod])*math.log(float(mod_dict[mod]) + 1, 2)

path = 'E:/病例特点'
dic = 'C:/Users/yingying.zhu/Documents/dicts/disease.txt'
print (dic)
mod_list, count = find_mod(path, dic)
print (mod_list[:15])<|endoftext|>


<commit_before>import pure_interface

import abc
import unittest

import six

class IAnimal(pure_interface.PureInterface):
    @pure_interface.abstractproperty
    def height(self):
        return None

class IGrowingAnimal(pure_interface.PureInterface):
    def get_height(self):
        return None

    def set_height(self, height):
        pass

    height = pure_interface.abstractproperty(get_height, set_height)

class IPlant(pure_interface.PureInterface):
    @property
    def height(self):
        return None

class IGrowingPlant(pure_interface.PureInterface):
    @property
    def height(self):
        return None

    @height.setter
    def height(self, height):
        pass

class TestImplementationChecks(unittest.TestCase):
    def test_instantiation_fails(self):
        with self.assertRaises(TypeError):
            pure_interface.PureInterface()
        with self.assertRaises(TypeError):
            IPlant()
        with self.assertRaises(TypeError):
            IAnimal()

    def test_concrete_base_detection(self):
        class Concrete(object, pure_interface.PureInterface):
            pass

        self.assertFalse(Concrete._type_is_pure_interface)
        try:
            c = Concrete()
        except Exception as exc:
            self.fail('Instantiation failed {}'.format(exc))

    def test_concrete_base_detection2(self):
        class B(object):
            def __init__(self):
                self.foo = 'bar'

        class Concrete(B, pure_interface.PureInterface):
            pass

        self.assertFalse(Concrete._type_is_pure_interface)
        try:
            c = Concrete()
        except Exception as exc:
            self.fail('Instantiation failed {}'.format(exc))

    def test_concrete_abc_detection(self):
        @six.add_metaclass(abc.ABCMeta)
        class B(object):
            def __init__(self):
                self.foo = 'bar'

        class Concrete(B, pure_interface.PureInterface):
            pass

        self.assertFalse(Concrete._type_is_pure_interface)
        try:
            c = Concrete()
        except Exception as exc:
            self.fail('Instantiation failed {}'.format(exc))

    def test_interface_abc_detection(self):
        @six.add_metaclass(abc.ABCMeta)
        class IABC(object):

            @abc.abstractmethod
            def foo(self):
                pass

        class EmptyABCPI(IABC, pure_interface.PureInterface):
            pass

        class PIEmptyABC(pure_interface.PureInterface, IABC):
            pass

        self.assertTrue(EmptyABCPI._type_is_pure_interface)
        self.assertTrue(PIEmptyABC._type_is_pure_interface)
        with self.assertRaises(TypeError):
            EmptyABCPI()
        with self.assertRaises(TypeError):
            PIEmptyABC()

class TestPropertyImplementations(unittest.TestCase):
    def test_abstract_property_override_passes(self):
        class Animal(object, IGrowingAnimal):
            def get_height(self):
                return 10

            def set_height(self, height):
                pass

            height = property(get_height, set_height)

        a = Animal()
        self.assertEqual(a.height, 10)

    def test_abstract_attribute_override_passes(self):
        class Animal(object, IGrowingAnimal):
            def __init__(self):
                self.height = 5

        a = Animal()
        self.assertEqual(a.height, 5)

    def test_property_override_passes(self):
        class Plant(object, IGrowingPlant):
            @property
            def height(self):
                return 10

            @height.setter
            def height(self, height):
                pass

        a = Plant()
        self.assertEqual(a.height, 10)

    def test_attribute_override_passes(self):
        class Plant(object, IGrowingPlant):
            def __init__(self):
                self.height = 5

        a = Plant()
        self.assertEqual(a.height, 5)

    def test_ro_abstract_property_override_passes(self):
        class Animal(object, IAnimal):
            @property
            def height(self):
                return 10

        a = Animal()
        self.assertEqual(a.height, 10)

    def test_ro_abstract_attribute_override_passes(self):
        class Animal(object, IAnimal):
            def __init__(self):
                self.height = 5

        a = Animal()
        self.assertEqual(a.height, 5)

    def test_ro_property_override_passes(self):
        class Plant(object, IPlant):
            @property
            def height(self):
                return 10

        a = Plant()
        self.assertEqual(a.height, 10)

    def test_ro_attribute_override_passes(self):
        class Plant(object, IPlant):
            def __init__(self):
                self.height = 5

        a = Plant()
        self.assertEqual(a.height, 5)

    def test_missing_abstract_property_fails(self):
        class Animal(object, IAnimal):
            pass

        with self.assertRaises(TypeError):
            Animal()

    def test_missing_property_fails(self):
        class Plant(object, IPlant):
            pass

        with self.assertRaises(TypeError):
            Plant()
<commit_msg>Added __getattr__ property implementation test<commit_after>import pure_interface

import abc
import unittest

import six

class IAnimal(pure_interface.PureInterface):
    @pure_interface.abstractproperty
    def height(self):
        return None

class IGrowingAnimal(pure_interface.PureInterface):
    def get_height(self):
        return None

    def set_height(self, height):
        pass

    height = pure_interface.abstractproperty(get_height, set_height)

class IPlant(pure_interface.PureInterface):
    @property
    def height(self):
        return None

class IGrowingPlant(pure_interface.PureInterface):
    @property
    def height(self):
        return None

    @height.setter
    def height(self, height):
        pass

class TestImplementationChecks(unittest.TestCase):
    def test_instantiation_fails(self):
        with self.assertRaises(TypeError):
            pure_interface.PureInterface()
        with self.assertRaises(TypeError):
            IPlant()
        with self.assertRaises(TypeError):
            IAnimal()

    def test_concrete_base_detection(self):
        class Concrete(object, pure_interface.PureInterface):
            pass

        self.assertFalse(Concrete._type_is_pure_interface)
        try:
            c = Concrete()
        except Exception as exc:
            self.fail('Instantiation failed {}'.format(exc))

    def test_concrete_base_detection2(self):
        class B(object):
            def __init__(self):
                self.foo = 'bar'

        class Concrete(B, pure_interface.PureInterface):
            pass

        self.assertFalse(Concrete._type_is_pure_interface)
        try:
            c = Concrete()
        except Exception as exc:
            self.fail('Instantiation failed {}'.format(exc))

    def test_concrete_abc_detection(self):
        @six.add_metaclass(abc.ABCMeta)
        class B(object):
            def __init__(self):
                self.foo = 'bar'

        class Concrete(B, pure_interface.PureInterface):
            pass

        self.assertFalse(Concrete._type_is_pure_interface)
        try:
            c = Concrete()
        except Exception as exc:
            self.fail('Instantiation failed {}'.format(exc))

    def test_interface_abc_detection(self):
        @six.add_metaclass(abc.ABCMeta)
        class IABC(object):

            @abc.abstractmethod
            def foo(self):
                pass

        class EmptyABCPI(IABC, pure_interface.PureInterface):
            pass

        class PIEmptyABC(pure_interface.PureInterface, IABC):
            pass

        self.assertTrue(EmptyABCPI._type_is_pure_interface)
        self.assertTrue(PIEmptyABC._type_is_pure_interface)
        with self.assertRaises(TypeError):
            EmptyABCPI()
        with self.assertRaises(TypeError):
            PIEmptyABC()

class TestPropertyImplementations(unittest.TestCase):
    def test_abstract_property_override_passes(self):
        class Animal(object, IGrowingAnimal):
            def get_height(self):
                return 10

            def set_height(self, height):
                pass

            height = property(get_height, set_height)

        a = Animal()
        self.assertEqual(a.height, 10)

    def test_abstract_attribute_override_passes(self):
        class Animal(object, IGrowingAnimal):
            def __init__(self):
                self.height = 5

        a = Animal()
        self.assertEqual(a.height, 5)

    def test_property_override_passes(self):
        class Plant(object, IGrowingPlant):
            @property
            def height(self):
                return 10

            @height.setter
            def height(self, height):
                pass

        a = Plant()
        self.assertEqual(a.height, 10)

    def test_attribute_override_passes(self):
        class Plant(object, IGrowingPlant):
            def __init__(self):
                self.height = 5

        a = Plant()
        self.assertEqual(a.height, 5)

    def test_ro_abstract_property_override_passes(self):
        class Animal(object, IAnimal):
            @property
            def height(self):
                return 10

        a = Animal()
        self.assertEqual(a.height, 10)

    def test_ro_abstract_attribute_override_passes(self):
        class Animal(object, IAnimal):
            def __init__(self):
                self.height = 5

        a = Animal()
        self.assertEqual(a.height, 5)

    def test_ro_property_override_passes(self):
        class Plant(object, IPlant):
            @property
            def height(self):
                return 10

        a = Plant()
        self.assertEqual(a.height, 10)

    def test_ro_attribute_override_passes(self):
        class Plant(object, IPlant):
            def __init__(self):
                self.height = 5

        a = Plant()
        self.assertEqual(a.height, 5)

    def test_missing_abstract_property_fails(self):
        class Animal(object, IAnimal):
            pass

        with self.assertRaises(TypeError):
            Animal()

    def test_missing_property_fails(self):
        class Plant(object, IPlant):
            pass

        with self.assertRaises(TypeError):
            Plant()

    def test_getattr_property_passes(self):
        class Plant(object, IPlant):
            def __getattr__(self, item):
                if item == 'height':
                    return 10
                else:
                    raise AttributeError(item)

        a = Plant()
        self.assertEqual(a.height, 10)
<|endoftext|>


<commit_before>import inspect
from mock import Mock

__all__ = ('mocksignature', 'MagicMock')

"""
extendmock adds additional features to the mock module.

The mocksignature function creates wrapper functions that call a mock whilst
preserving the signature of the original function.

Note that when a function signature is mocked with mocksignature it will *always* be
called with positional arguments and not keyword arguments (except where it
collects arguments with \*\*). Defaults will explicitly be supplied where the function
is called with missing arguments (as normal).

MagicMock is a class that adds magic method support to Mock. You add magic methods
to mock instances in the same way you mock other attributes::

    mock = Mock()
    mock.__repr__ = lambda self: 'some string'

Note that functions (or callable objects like a Mock instance) that are used for
magic methods must take self as the first argument.

The only unsupported magic methods (that I'm aware of) are:

* Used by Mock: __init__, __new__, __getattr__, __setattr__, __delattr__
* Rare: __dir__,
* Can cause bad interactions with other functionality:

    - __reversed__, __missing__, __del__, __unicode__, __getattribute__
    - __get__, __set__, __delete__

* Context manager methods are not directly supported (__enter__ and __exit__) as
  Python doesn't lookup these methods on the class as it should (in Python 2.5 / 2.6
  anyway).

For more examples of how to use mocksignature and MagicMock see the tests.
"""

# getsignature and mocksignature heavily "inspired" by
# the decorator module: http://pypi.python.org/pypi/decorator/
# by Michele Simionato

DELEGATE = MISSING = object()

def getsignature(func):
    assert inspect.ismethod(func) or inspect.isfunction(func)
    regargs, varargs, varkwargs, defaults = inspect.getargspec(func)

    # instance methods need to lose the self argument
    im_self = getattr(func, 'im_self', None)
    if im_self is not None:
        regargs = regargs[1:]

    argnames = list(regargs)
    if varargs:
        argnames.append(varargs)
    if varkwargs:
        argnames.append(varkwargs)
    assert '_mock_' not in argnames, ("_mock_ is a reserved argument name, can't mock signatures using _mock_")
    signature = inspect.formatargspec(regargs, varargs, varkwargs, defaults, formatvalue=lambda value: "")
    return signature[1:-1], varargs, defaults

def mocksignature(func, mock):
    signature = getsignature(func)
    src = "lambda %(signature)s: _mock_(%(signature)s)" % {'signature': signature}

    funcopy = eval(src, dict(_mock_=mock))
    funcopy.__name__ = func.__name__
    funcopy.__doc__ = func.__doc__
    funcopy.__dict__.update(func.__dict__)
    funcopy.__module__ = func.__module__
    funcopy.func_defaults = func.func_defaults
    return funcopy

class MagicMock(Mock):
    pass

magic_methods = [
    ("lt le gt ge eq ne", NotImplemented),
    ("getitem setitem delitem", TypeError),
    ("len contains iter", TypeError),
    ("hash repr str", DELEGATE),
    ("nonzero", True),
    ("divmod neg pos abs invert", TypeError),
    ("complex int long float oct hex index", TypeError)
]

numerics = "add sub mul div truediv floordiv mod lshift rshift and xor or"
inplace = ' '.join('i%s' % n for n in numerics.split())
right = ' '.join('r%s' % n for n in numerics.split())

magic_methods.extend([
    (numerics, NotImplemented), (inplace, NotImplemented), (right, NotImplemented)
])

def get_method(name, action):
    def func(self, *args, **kw):
        real = self.__dict__.get(name, MISSING)
        if real is not MISSING:
            return real(self, *args, **kw)
        if action is NotImplemented:
            return NotImplemented
        elif isinstance(action, type) and issubclass(action, Exception):
            raise action
        elif action is DELEGATE:
            return getattr(Mock, name)(self, *args, **kw)
        return action
    func.__name__ = name
    return func

for methods, action in magic_methods:
    for method in methods.split():
        name = '__%s__' % method
        setattr(MagicMock, name, get_method(name, action))

<commit_msg>Correct horrible error accidentally left in previous commit.<commit_after>import inspect
from mock import Mock

__all__ = ('mocksignature', 'MagicMock')

"""
extendmock adds additional features to the mock module.

The mocksignature function creates wrapper functions that call a mock whilst
preserving the signature of the original function.

Note that when a function signature is mocked with mocksignature it will *always* be
called with positional arguments and not keyword arguments (except where it
collects arguments with \*\*). Defaults will explicitly be supplied where the function
is called with missing arguments (as normal).

MagicMock is a class that adds magic method support to Mock. You add magic methods
to mock instances in the same way you mock other attributes::

    mock = Mock()
    mock.__repr__ = lambda self: 'some string'

Note that functions (or callable objects like a Mock instance) that are used for
magic methods must take self as the first argument.

The only unsupported magic methods (that I'm aware of) are:

* Used by Mock: __init__, __new__, __getattr__, __setattr__, __delattr__
* Rare: __dir__,
* Can cause bad interactions with other functionality:

    - __reversed__, __missing__, __del__, __unicode__, __getattribute__
    - __get__, __set__, __delete__

* Context manager methods are not directly supported (__enter__ and __exit__) as
  Python doesn't lookup these methods on the class as it should (in Python 2.5 / 2.6
  anyway).

For more examples of how to use mocksignature and MagicMock see the tests.
"""

# getsignature and mocksignature heavily "inspired" by
# the decorator module: http://pypi.python.org/pypi/decorator/
# by Michele Simionato

DELEGATE = MISSING = object()

def getsignature(func):
    assert inspect.ismethod(func) or inspect.isfunction(func)
    regargs, varargs, varkwargs, defaults = inspect.getargspec(func)

    # instance methods need to lose the self argument
    im_self = getattr(func, 'im_self', None)
    if im_self is not None:
        regargs = regargs[1:]

    argnames = list(regargs)
    if varargs:
        argnames.append(varargs)
    if varkwargs:
        argnames.append(varkwargs)
    assert '_mock_' not in argnames, ("_mock_ is a reserved argument name, can't mock signatures using _mock_")
    signature = inspect.formatargspec(regargs, varargs, varkwargs, defaults, formatvalue=lambda value: "")
    return signature[1:-1]

def mocksignature(func, mock):
    signature = getsignature(func)
    src = "lambda %(signature)s: _mock_(%(signature)s)" % {'signature': signature}

    funcopy = eval(src, dict(_mock_=mock))
    funcopy.__name__ = func.__name__
    funcopy.__doc__ = func.__doc__
    funcopy.__dict__.update(func.__dict__)
    funcopy.__module__ = func.__module__
    funcopy.func_defaults = func.func_defaults
    return funcopy

class MagicMock(Mock):
    pass

magic_methods = [
    ("lt le gt ge eq ne", NotImplemented),
    ("getitem setitem delitem", TypeError),
    ("len contains iter", TypeError),
    ("hash repr str", DELEGATE),
    ("nonzero", True),
    ("divmod neg pos abs invert", TypeError),
    ("complex int long float oct hex index", TypeError)
]

numerics = "add sub mul div truediv floordiv mod lshift rshift and xor or"
inplace = ' '.join('i%s' % n for n in numerics.split())
right = ' '.join('r%s' % n for n in numerics.split())

magic_methods.extend([
    (numerics, NotImplemented), (inplace, NotImplemented), (right, NotImplemented)
])

def get_method(name, action):
    def func(self, *args, **kw):
        real = self.__dict__.get(name, MISSING)
        if real is not MISSING:
            return real(self, *args, **kw)
        if action is NotImplemented:
            return NotImplemented
        elif isinstance(action, type) and issubclass(action, Exception):
            raise action
        elif action is DELEGATE:
            return getattr(Mock, name)(self, *args, **kw)
        return action
    func.__name__ = name
    return func

for methods, action in magic_methods:
    for method in methods.split():
        name = '__%s__' % method
        setattr(MagicMock, name, get_method(name, action))

<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    url(r'^admin/', include(admin.site.urls), name='admin'),
)
<commit_msg>Fix Django 1.10 url patterns warning<commit_after>from django.conf.urls import include, url
from django.contrib import admin
admin.autodiscover()

urlpatterns = [
    url(r'^admin/', include(admin.site.urls), name='admin'),
]
<|endoftext|>