<commit_before># -*- coding: utf-8 -*-
import os

from jinja2 import Environment, PackageLoader

from django_deployer.helpers import _write_file

from fabric.operations import local
from fabric.context_managers import shell_env

template_env = Environment(loader=PackageLoader('django_deployer', 'paas_templates'))

class PaaSProvider(object):
    """
    Base PaasProvider class. PaaS providers should inherit from this
    class and override all methods.
    """

    # Subclasses should override these
    name = ""
    setup_instructions = ""
    PYVERSIONS = {}
    provider_yml_name = "%s.yml" % name

    @classmethod
    def init(cls, site):
        cls._create_configs(site)
        print cls.setup_instructions

    @classmethod
    def deploy(cls):
        raise NotImplementedError()

    @classmethod
    def delete(cls):
        raise NotImplementedError()

    @classmethod
    def _create_configs(cls, site):
        """
        This is going to generate the following configuration:
        * wsgi.py
        * <provider>.yml
        * settings_<provider>.py
        """
        provider = cls.name

        cls._render_config('wsgi.py', 'wsgi.py', site)

        # create yaml file
        yaml_template_name = os.path.join(provider, cls.provider_yml_name)
        cls._render_config(cls.provider_yml_name, yaml_template_name, site)

        # create requirements file
        # don't do anything if the requirements file is called requirements.txt and in the root of the project
        requirements_filename = "requirements.txt"
        if site['requirements'] != requirements_filename:   # providers expect the file to be called requirements.txt
            requirements_template_name = os.path.join(provider, requirements_filename)
            cls._render_config(requirements_filename, requirements_template_name, site)

        # create settings file
        settings_template_name = os.path.join(provider, 'settings_%s.py' % provider)
        settings_path = site['django_settings'].replace('.', '/') + '_%s.py' % provider
        cls._render_config(settings_path, settings_template_name, site)

    @classmethod
    def _render_config(cls, dest, template_name, template_args):
        """
        Renders and writes a template_name to a dest given some template_args.

        This is for platform-specific configurations
        """
        template_args = template_args.copy()

        # Substitute values here
        pyversion = template_args['pyversion']
        template_args['pyversion'] = cls.PYVERSIONS[pyversion]

        template = template_env.get_template(template_name)
        contents = template.render(**template_args)
        _write_file(dest, contents)

class Stackato(PaaSProvider):
    """
    ActiveState Stackato PaaSProvider.
    """
    name = "stackato"

    PYVERSIONS = {
        "Python2.7": "python27",
        "Python3.2": "python32",
    }

    setup_instructions = """
Just a few more steps before you're ready to deploy your app!

1. Go to http://www.activestate.com/stackato/download_client to download
   the Stackato client, and then add the executable somewhere in your PATH.
   If you're not sure where to place it, you can simply drop it in your
   project's root directory (the same directory as the fabfile.py created
   by django-deployer).

2. Once you've done that, target the stackato api with:

       $ stackato target api.stacka.to

   and then login. You can find your sandbox password at
   https://account.activestate.com, which you'll need when
   using the command:

       $ stackato login --email <email>

3. You can push your app the first time with:

       $ stackato push -n

   and make subsequent updates with:

       $ stackato update

"""

    provider_yml_name = "stackato.yml"

    def init():
        pass

    def deploy():
        pass

    def delete():
        pass

class DotCloud(PaaSProvider):
    """
    Dotcloud PaaSProvider.
    """

    name = "dotcloud"

    PYVERSIONS = {
        "Python2.6": "v2.6",
        "Python2.7": "v2.7",
        "Python3.2": "v3.2",
    }

    setup_instructions = """
        Just a few more steps before you're ready to deploy your app!

        1. Install the dotcloud command line tool with:

                $ pip install dotcloud

        2. Once you've done that, setup your Dotcloud environment for the first time:

                $ dotcloud setup
                dotCloud username or email: appsembler
                Password:
                ==> dotCloud authentication is complete! You are recommended to run `dotcloud check` now.

                $ dotcloud check
                ==> Checking the authentication status
                ==> Client is authenticated as appsembler

        3. You can create the app with:

               $ dotcloud create myapp

           and deploy it with:

               $ dotcloud push

        """

    provider_yml_name = "dotcloud.yml"

    @classmethod
    def init(cls, site):
        super(DotCloud, cls).init(site)

        # config_list: files to put in project folder, django_config_list: files to put in django project folder
        config_list = [
            'createdb.py',
            'mkadmin.py',
            'nginx.conf',
            'postinstall',
            'wsgi.py',
        ]

        # for rendering configs under root
        get_config = lambda filename: cls._render_config(filename, os.path.join(cls.name, filename), site)
        map(get_config, config_list)

    def deploy():
        pass

    def delete():
        pass

class AppEngine(PaaSProvider):
    """
    AppEngine PaaSProvider
    """

    name = 'appengine'

    PYVERSIONS = {
        "Python2.7": "v2.7"
    }

    setup_instructions = """
Just a few more steps before you're ready to deploy your app!

1. Run this command to create the virtualenv with all the packages and deploy:

        $ fab deploy

2. Create and sync the db on the Cloud SQL:

        $ sh manage.sh cloudcreatedb
        $ sh manage.sh cloudsyncdb

3. Everything is set up now, you can run other commands that will execute on your remotely deployed app, such as:

        $ sh manage.sh dbshell

"""

    provider_yml_name = "app.yaml"

    @classmethod
    def init(cls, site):
        super(AppEngine, cls).init(site)

        # config_list: files to put in project folder, django_config_list: files to put in django project folder
        config_list = ['requirements_deploy.txt', 'manage.sh']
        django_config_list = ['urls_appengine.py']

        # for rendering configs under root
        get_config = lambda filename: cls._render_config(filename, os.path.join(cls.name, filename), site)
        map(get_config, config_list)

        # for rendering configs under django project folder
        get_django_config = lambda filename: cls._render_config("%s/%s"%(site['project_name'], filename), os.path.join(cls.name, filename), site)
        map(get_django_config, django_config_list)

    @classmethod
    def deploy(cls, site):
        """
        tasks:
            * collect statics
            * fetch all the required libraries from pip (require_deploy.txt)
            * deploy
        """
        # Create virtual environment
        # TODO: detect whether it is a virtualenv
        if not os.path.isdir("env"):
            print "Didn't find a virtual environment, so we're going to create one..."
            local("virtualenv --no-site-packages env")

        # install requirements for deployment
        local("env/bin/pip install -r %(requirements)s" % site)
        local("env/bin/pip install -r requirements_deploy.txt")

        # Collects static files into static folder
        local("sh manage.sh collectstatic --noinput")

        # deploy
        local("sh manage.sh deploy")

    def delete():
        pass

PROVIDERS = {
    'stackato': Stackato,
    'dotcloud': DotCloud,
    'appengine': AppEngine
}
<commit_msg>create the requirements file the same way as with dotcloud<commit_after># -*- coding: utf-8 -*-
import os

from jinja2 import Environment, PackageLoader

from django_deployer.helpers import _write_file

from fabric.operations import local
from fabric.context_managers import shell_env

template_env = Environment(loader=PackageLoader('django_deployer', 'paas_templates'))

class PaaSProvider(object):
    """
    Base PaasProvider class. PaaS providers should inherit from this
    class and override all methods.
    """

    # Subclasses should override these
    name = ""
    setup_instructions = ""
    PYVERSIONS = {}
    provider_yml_name = "%s.yml" % name

    @classmethod
    def init(cls, site):
        cls._create_configs(site)
        print cls.setup_instructions

    @classmethod
    def deploy(cls):
        raise NotImplementedError()

    @classmethod
    def delete(cls):
        raise NotImplementedError()

    @classmethod
    def _create_configs(cls, site):
        """
        This is going to generate the following configuration:
        * wsgi.py
        * <provider>.yml
        * settings_<provider>.py
        """
        provider = cls.name

        cls._render_config('wsgi.py', 'wsgi.py', site)

        # create yaml file
        yaml_template_name = os.path.join(provider, cls.provider_yml_name)
        cls._render_config(cls.provider_yml_name, yaml_template_name, site)

        # create requirements file
        # don't do anything if the requirements file is called requirements.txt and in the root of the project
        requirements_filename = "requirements.txt"
        if site['requirements'] != requirements_filename:   # providers expect the file to be called requirements.txt
            requirements_template_name = os.path.join(provider, requirements_filename)
            cls._render_config(requirements_filename, requirements_template_name, site)

        # create settings file
        settings_template_name = os.path.join(provider, 'settings_%s.py' % provider)
        settings_path = site['django_settings'].replace('.', '/') + '_%s.py' % provider
        cls._render_config(settings_path, settings_template_name, site)

    @classmethod
    def _render_config(cls, dest, template_name, template_args):
        """
        Renders and writes a template_name to a dest given some template_args.

        This is for platform-specific configurations
        """
        template_args = template_args.copy()

        # Substitute values here
        pyversion = template_args['pyversion']
        template_args['pyversion'] = cls.PYVERSIONS[pyversion]

        template = template_env.get_template(template_name)
        contents = template.render(**template_args)
        _write_file(dest, contents)

class Stackato(PaaSProvider):
    """
    ActiveState Stackato PaaSProvider.
    """
    name = "stackato"

    PYVERSIONS = {
        "Python2.7": "python27",
        "Python3.2": "python32",
    }

    setup_instructions = """
Just a few more steps before you're ready to deploy your app!

1. Go to http://www.activestate.com/stackato/download_client to download
   the Stackato client, and then add the executable somewhere in your PATH.
   If you're not sure where to place it, you can simply drop it in your
   project's root directory (the same directory as the fabfile.py created
   by django-deployer).

2. Once you've done that, target the stackato api with:

       $ stackato target api.stacka.to

   and then login. You can find your sandbox password at
   https://account.activestate.com, which you'll need when
   using the command:

       $ stackato login --email <email>

3. You can push your app the first time with:

       $ stackato push -n

   and make subsequent updates with:

       $ stackato update

"""

    provider_yml_name = "stackato.yml"

    def init():
        pass

    def deploy():
        pass

    def delete():
        pass

class DotCloud(PaaSProvider):
    """
    Dotcloud PaaSProvider.
    """

    name = "dotcloud"

    PYVERSIONS = {
        "Python2.6": "v2.6",
        "Python2.7": "v2.7",
        "Python3.2": "v3.2",
    }

    setup_instructions = """
        Just a few more steps before you're ready to deploy your app!

        1. Install the dotcloud command line tool with:

                $ pip install dotcloud

        2. Once you've done that, setup your Dotcloud environment for the first time:

                $ dotcloud setup
                dotCloud username or email: appsembler
                Password:
                ==> dotCloud authentication is complete! You are recommended to run `dotcloud check` now.

                $ dotcloud check
                ==> Checking the authentication status
                ==> Client is authenticated as appsembler

        3. You can create the app with:

               $ dotcloud create myapp

           and deploy it with:

               $ dotcloud push

        """

    provider_yml_name = "dotcloud.yml"

    @classmethod
    def init(cls, site):
        super(DotCloud, cls).init(site)

        # config_list: files to put in project folder, django_config_list: files to put in django project folder
        config_list = [
            'createdb.py',
            'mkadmin.py',
            'nginx.conf',
            'postinstall',
            'wsgi.py',
        ]

        # for rendering configs under root
        get_config = lambda filename: cls._render_config(filename, os.path.join(cls.name, filename), site)
        map(get_config, config_list)

    def deploy():
        pass

    def delete():
        pass

class AppEngine(PaaSProvider):
    """
    AppEngine PaaSProvider
    """

    name = 'appengine'

    PYVERSIONS = {
        "Python2.7": "v2.7"
    }

    setup_instructions = """
Just a few more steps before you're ready to deploy your app!

1. Run this command to create the virtualenv with all the packages and deploy:

        $ fab deploy

2. Create and sync the db on the Cloud SQL:

        $ sh manage.sh cloudcreatedb
        $ sh manage.sh cloudsyncdb

3. Everything is set up now, you can run other commands that will execute on your remotely deployed app, such as:

        $ sh manage.sh dbshell

"""

    provider_yml_name = "app.yaml"

    @classmethod
    def init(cls, site):
        super(AppEngine, cls).init(site)

        # config_list: files to put in project folder, django_config_list: files to put in django project folder
        config_list = ['requirements_deploy.txt', 'manage.sh']
        django_config_list = ['urls_appengine.py']

        # for rendering configs under root
        get_config = lambda filename: cls._render_config(filename, os.path.join(cls.name, filename), site)
        map(get_config, config_list)

        # for rendering configs under django project folder
        get_django_config = lambda filename: cls._render_config("%s/%s" % (site['project_name'], filename), os.path.join(cls.name, filename), site)
        map(get_django_config, django_config_list)

    @classmethod
    def deploy(cls, site):
        """
        tasks:
            * collect statics
            * fetch all the required libraries from pip (require_deploy.txt)
            * deploy
        """
        # Create virtual environment
        # TODO: detect whether it is a virtualenv
        if not os.path.isdir("env"):
            print "Didn't find a virtual environment, so we're going to create one..."
            local("virtualenv --no-site-packages env")

        # install requirements for deployment
        local("env/bin/pip install -r requirements.txt" % site)
        local("env/bin/pip install -r requirements_deploy.txt")

        # Collects static files into static folder
        local("sh manage.sh collectstatic --noinput")

        # deploy
        local("sh manage.sh deploy")

    def delete():
        pass

PROVIDERS = {
    'stackato': Stackato,
    'dotcloud': DotCloud,
    'appengine': AppEngine
}
<|endoftext|>


<commit_before># Don't import unicode_literals here, otherwise set_cookie / delete_cookie will fail (as of django 1.4)
from __future__ import print_function

import hashlib
import re

from django.utils.http import http_date
from django.conf import settings
from django.core.exceptions import MiddlewareNotUsed
from django.contrib.staticfiles.handlers import StaticFilesHandler

HTML_CONTENT_TYPES = ("text/html", "application/xhtml+xml")
COOKIE_NAME = "DJANGO_LIVEJS2"

def inject_script_if_necessary(response):
    if (not hasattr(response, "content") or
        getattr(response, "status_code", 0) != 200):

        return

    ctype = response.get("Content-Type", "").split(";")[0].strip().lower()
    if ctype not in HTML_CONTENT_TYPES:
        return

    src = getattr(settings,
                  "LIVEJS2_SCRIPT_SRC",
                  settings.STATIC_URL + "django_livejs2/live.js")

    response.content = re.sub(
        r"()",
        r"""\1""".format(src),
        response.content,
        count=1,
        flags=re.I,
    )

    if response.get("Content-Length", None):
        response["Content-Length"] = len(response.content)

def process(request, response):
    if request.GET.get("live", None) == "0":
        response.delete_cookie(COOKIE_NAME)
    elif (request.COOKIES.get("live", False) or
        request.GET.get(COOKIE_NAME, False)):

        cookie_age = getattr(settings,
                             "LIVEJS2_DISABLE_AFTER_INACTIVITY",
                             24*60*60)
        response.set_cookie(COOKIE_NAME, "1", max_age=cookie_age)

        inject_script_if_necessary(response)

        # We need to explicitly state that files are uncacheable, otherwise
        # Firefox (and possibly other browsers) may cache them for 1 minute
        response["Pragma"] = "no-cache"
        response["Cache-Control"] = "no-cache"
        response["Expires"] = http_date()
        if "ETag" not in response:
            response["ETag"] = '"{}"'.format(
                hashlib.md5(response.content).hexdigest()
            )

    return response

def patch_static_handler():
    # Static files won't go through middleware pipeline, so we have to do this
    old_serve = StaticFilesHandler.serve

    def _patched_serve(self, request):
        return process(request, old_serve(self, request))

    StaticFilesHandler.serve = _patched_serve

if settings.DEBUG:
    patch_static_handler()

class LiveJsMiddleware(object):
    def __init__(self):
        if not settings.DEBUG:
            raise MiddlewareNotUsed()

    def process_response(self, request, response):
        return process(request, response)
<commit_msg>Increase default cookie age<commit_after># Don't import unicode_literals here, otherwise set_cookie / delete_cookie will fail (as of django 1.4)
from __future__ import print_function

import hashlib
import re

from django.utils.http import http_date
from django.conf import settings
from django.core.exceptions import MiddlewareNotUsed
from django.contrib.staticfiles.handlers import StaticFilesHandler

HTML_CONTENT_TYPES = ("text/html", "application/xhtml+xml")
COOKIE_NAME = "DJANGO_LIVEJS2"

def inject_script_if_necessary(response):
    if (not hasattr(response, "content") or
        getattr(response, "status_code", 0) != 200):

        return

    ctype = response.get("Content-Type", "").split(";")[0].strip().lower()
    if ctype not in HTML_CONTENT_TYPES:
        return

    src = getattr(settings,
                  "LIVEJS2_SCRIPT_SRC",
                  settings.STATIC_URL + "django_livejs2/live.js")

    response.content = re.sub(
        r"()",
        r"""\1""".format(src),
        response.content,
        count=1,
        flags=re.I,
    )

    if response.get("Content-Length", None):
        response["Content-Length"] = len(response.content)

def process(request, response):
    if request.GET.get("live", None) == "0":
        response.delete_cookie(COOKIE_NAME)
    elif (request.COOKIES.get("live", False) or
        request.GET.get(COOKIE_NAME, False)):

        # After a bit of thinking, I can't find any real use of this setting.
        # But since it has been written, just leave it here.
        cookie_age = getattr(settings,
                             "LIVEJS2_DISABLE_AFTER_INACTIVITY",
                             60*60*24*365*10)
        response.set_cookie(COOKIE_NAME, "1", max_age=cookie_age)

        inject_script_if_necessary(response)

        # We need to explicitly state that files are uncacheable, otherwise
        # Firefox (and possibly other browsers) may cache them for 1 minute
        response["Pragma"] = "no-cache"
        response["Cache-Control"] = "no-cache"
        response["Expires"] = http_date()
        if "ETag" not in response:
            response["ETag"] = '"{}"'.format(
                hashlib.md5(response.content).hexdigest()
            )

    return response

def patch_static_handler():
    # Static files won't go through middleware pipeline, so we have to do this
    old_serve = StaticFilesHandler.serve

    def _patched_serve(self, request):
        return process(request, old_serve(self, request))

    StaticFilesHandler.serve = _patched_serve

if settings.DEBUG:
    patch_static_handler()

class LiveJsMiddleware(object):
    def __init__(self):
        if not settings.DEBUG:
            raise MiddlewareNotUsed()

    def process_response(self, request, response):
        return process(request, response)
<|endoftext|>


<commit_before>__version__ = '2.2.7'
<commit_msg>template search path now includes the project directory by default, making cross-app inheritance easier<commit_after>__version__ = '2.2.8'
<|endoftext|>


<commit_before>from django.http import HttpResponse
from django.conf import settings
import os
import prometheus_client

def SetupPrometheusEndpointOnPort(port, addr=''):
    """Exports Prometheus metrics on an HTTPServer running in its own thread.

    The server runs on the given port and is by default listenning on
    all interfaces. This HTTPServer is fully independent of Django and
    its stack. This offers the advantage that even if Django becomes
    unable to respond, the HTTPServer will continue to function and
    export metrics. However, this also means that none of the features
    offered by Django (like middlewares or WSGI) can't be used.

    Now here's the really weird part. When Django runs with the
    auto-reloader enabled (which is the default, you can disable it
    with `manage.py runserver --noreload`), it forks and executes
    manage.py twice. That's wasteful but usually OK. It starts being a
    problem when you try to open a port, like we do. We can detect
    that we're running under an autoreloader through the presence of
    the RUN_MAIN environment variable, so we abort if we're trying to
    export under an autoreloader and trying to open a port.
    """
    assert os.environ.get('RUN_MAIN') != 'true', (
        'The thread-based exporter can\'t be safely used when django\'s '
        'autoreloader is active. Use the URL exporter, or start django '
        'with --noreload. See documentation/exports.md.')
    prometheus_client.start_http_server(port, addr=addr)

def SetupPrometheusExportsFromConfig():
    """Exports metrics so Prometheus can collect them."""
    port = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_PORT', None)
    addr = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_ADDRESS', None)
    if port:
        SetupPrometheusEndpointOnPort(port, addr)

def ExportToDjangoView(request):
    """Exports /metrics as a Django view.

    You can use django_prometheus.urls to map /metrics to this view.
    """
    metrics_page = prometheus_client.generate_latest()
    return HttpResponse(
        metrics_page,
        content_type=prometheus_client.CONTENT_TYPE_LATEST)
<commit_msg>Fix docstring typo.<commit_after>from django.http import HttpResponse
from django.conf import settings
import os
import prometheus_client

def SetupPrometheusEndpointOnPort(port, addr=''):
    """Exports Prometheus metrics on an HTTPServer running in its own thread.

    The server runs on the given port and is by default listenning on
    all interfaces. This HTTPServer is fully independent of Django and
    its stack. This offers the advantage that even if Django becomes
    unable to respond, the HTTPServer will continue to function and
    export metrics. However, this also means that the features
    offered by Django (like middlewares or WSGI) can't be used.

    Now here's the really weird part. When Django runs with the
    auto-reloader enabled (which is the default, you can disable it
    with `manage.py runserver --noreload`), it forks and executes
    manage.py twice. That's wasteful but usually OK. It starts being a
    problem when you try to open a port, like we do. We can detect
    that we're running under an autoreloader through the presence of
    the RUN_MAIN environment variable, so we abort if we're trying to
    export under an autoreloader and trying to open a port.
    """
    assert os.environ.get('RUN_MAIN') != 'true', (
        'The thread-based exporter can\'t be safely used when django\'s '
        'autoreloader is active. Use the URL exporter, or start django '
        'with --noreload. See documentation/exports.md.')
    prometheus_client.start_http_server(port, addr=addr)

def SetupPrometheusExportsFromConfig():
    """Exports metrics so Prometheus can collect them."""
    port = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_PORT', None)
    addr = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_ADDRESS', None)
    if port:
        SetupPrometheusEndpointOnPort(port, addr)

def ExportToDjangoView(request):
    """Exports /metrics as a Django view.

    You can use django_prometheus.urls to map /metrics to this view.
    """
    metrics_page = prometheus_client.generate_latest()
    return HttpResponse(
        metrics_page,
        content_type=prometheus_client.CONTENT_TYPE_LATEST)
<|endoftext|>


<commit_before>import haystack.forms
from django import forms
from .models import DocumentRelease

# Right now this just does version because we don't really have
# multiple languages. If we get them, we'll need to deal with that.

class DocSearchForm(haystack.forms.SearchForm):

    def __init__(self, *args, **kwargs):
        super(DocSearchForm, self).__init__(*args, **kwargs)
        self.fields['q'].widget = SearchInput()
        self.fields['release'] = DocumentReleaseChoiceField(
            queryset = DocumentRelease.objects.all().order_by('version'),
            initial = DocumentRelease.objects.default(),
            empty_label = None,
        )

    def search(self):
        sqs = super(DocSearchForm, self).search()
        if self.is_valid():
            rel = self.cleaned_data['release']
            sqs = sqs.filter(lang=rel.lang, version=rel.version)
        return sqs

class DocumentReleaseChoiceField(forms.ModelChoiceField):
    def label_from_instance(self, obj):
        return obj.human_version

class SearchInput(forms.TextInput):
    input_type = 'search'<commit_msg>So here's to you, Mr. URL-By-Hand-Constructor.<commit_after>import haystack.forms
from django import forms
from .models import DocumentRelease

# Right now this just does version because we don't really have
# multiple languages. If we get them, we'll need to deal with that.

class DocSearchForm(haystack.forms.SearchForm):

    def __init__(self, *args, **kwargs):
        super(DocSearchForm, self).__init__(*args, **kwargs)
        self.fields['q'].widget = SearchInput()
        self.fields['release'] = DocumentReleaseChoiceField(
            queryset = DocumentRelease.objects.all().order_by('version'),
            initial = DocumentRelease.objects.default(),
            empty_label = None,
            required = False,
        )

    def search(self):
        sqs = super(DocSearchForm, self).search()
        if self.is_valid():
            rel = self.cleaned_data['release'] or DocumentRelease.objects.default()
            sqs = sqs.filter(lang=rel.lang, version=rel.version)
        return sqs

class DocumentReleaseChoiceField(forms.ModelChoiceField):
    def label_from_instance(self, obj):
        return obj.human_version

class SearchInput(forms.TextInput):
    input_type = 'search'<|endoftext|>


<commit_before>"""
The :mod:`views` module provides the Views you will most probably
be subclassing in your implementation.

By setting or modifying class attributes on your view, you change it's predefined behaviour.
"""

from django.core.urlresolvers import set_script_prefix
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt

from djangorestframework.compat import View as DjangoView
from djangorestframework.response import Response, ErrorResponse
from djangorestframework.mixins import *
from djangorestframework import resources, renderers, parsers, authentication, permissions, status

__all__ = (
    'View',
    'ModelView',
    'InstanceModelView',
    'ListModelView',
    'ListOrCreateModelView'
)

class View(ResourceMixin, RequestMixin, ResponseMixin, AuthMixin, DjangoView):
    """
    Handles incoming requests and maps them to REST operations.
    Performs request deserialization, response serialization, authentication and input validation.
    """

    """
    The resource to use when validating requests and filtering responses,
    or `None` to use default behaviour.
    """
    resource = None

    """
    List of renderers the resource can serialize the response with, ordered by preference.
    """
    renderers = renderers.DEFAULT_RENDERERS

    """
    List of parsers the resource can parse the request with.
    """
    parsers = parsers.DEFAULT_PARSERS

    """
    List of all authenticating methods to attempt.
    """
    authentication = ( authentication.UserLoggedInAuthentication,
                       authentication.BasicAuthentication )

    """
    List of all permissions that must be checked.
    """
    permissions = ( permissions.FullAnonAccess, )

    @classmethod
    def as_view(cls, **initkwargs):
        """
        Override the default :meth:`as_view` to store an instance of the view
        as an attribute on the callable function.  This allows us to discover
        information about the view when we do URL reverse lookups.
        """
        view = super(View, cls).as_view(**initkwargs)
        view.cls_instance = cls(**initkwargs)
        return view

    @property
    def allowed_methods(self):
        """
        Return the list of allowed HTTP methods, uppercased.
        """
        return [method.upper() for method in self.http_method_names if hasattr(self, method)]

    def http_method_not_allowed(self, request, *args, **kwargs):
        """
        Return an HTTP 405 error if an operation is called which does not have a handler method.
        """
        raise ErrorResponse(status.HTTP_405_METHOD_NOT_ALLOWED,
                            {'detail': 'Method \'%s\' not allowed on this resource.' % self.method})

    def initial(self, request, *args, **kargs):
        """
        Hook for any code that needs to run prior to anything else.
        Required if you want to do things like set `request.upload_handlers` before
        the authentication and dispatch handling is run.
        """
        pass

    def add_header(self, field, value):
        """
        Add *field* and *value* to the :attr:`headers` attribute of the :class:`View` class.
        """
        self.headers[field] = value

    # Note: session based authentication is explicitly CSRF validated,
    # all other authentication is CSRF exempt.
    @csrf_exempt
    def dispatch(self, request, *args, **kwargs):
        self.request = request
        self.args = args
        self.kwargs = kwargs
        self.headers = {}

        # Calls to 'reverse' will not be fully qualified unless we set the scheme/host/port here.
        prefix = '%s://%s' % (request.is_secure() and 'https' or 'http', request.get_host())
        set_script_prefix(prefix)

        try:
            self.initial(request, *args, **kwargs)

            # Authenticate and check request has the relevant permissions
            self._check_permissions()

            # Get the appropriate handler method
            if self.method.lower() in self.http_method_names:
                handler = getattr(self, self.method.lower(), self.http_method_not_allowed)
            else:
                handler = self.http_method_not_allowed

            response_obj = handler(request, *args, **kwargs)

            # Allow return value to be either HttpResponse, Response, or an object, or None
            if isinstance(response_obj, HttpResponse):
                return response_obj
            elif isinstance(response_obj, Response):
                response = response_obj
            elif response_obj is not None:
                response = Response(status.HTTP_200_OK, response_obj)
            else:
                response = Response(status.HTTP_204_NO_CONTENT)

            # Pre-serialize filtering (eg filter complex objects into natively serializable types)
            response.cleaned_content = self.filter_response(response.raw_content)

        except ErrorResponse, exc:
            response = exc.response

        # Always add these headers.
        #
        # TODO - this isn't actually the correct way to set the vary header,
        # also it's currently sub-obtimal for HTTP caching - need to sort that out.
        response.headers['Allow'] = ', '.join(self.allowed_methods)
        response.headers['Vary'] = 'Authenticate, Accept'

        # merge with headers possibly set at some point in the view
        response.headers.update(self.headers)

        return self.render(response)

class ModelView(View):
    """A RESTful view that maps to a model in the database."""
    resource = resources.ModelResource

class InstanceModelView(InstanceMixin, ReadModelMixin, UpdateModelMixin, DeleteModelMixin, ModelView):
    """A view which provides default operations for read/update/delete against a model instance."""
    _suffix = 'Instance'

class ListModelView(ListModelMixin, ModelView):
    """A view which provides default operations for list, against a model in the database."""
    _suffix = 'List'

class ListOrCreateModelView(ListModelMixin, CreateModelMixin, ModelView):
    """A view which provides default operations for list and create, against a model in the database."""
    _suffix = 'List'
<commit_msg>Whitespace tweaks<commit_after>"""
The :mod:`views` module provides the Views you will most probably
be subclassing in your implementation.

By setting or modifying class attributes on your view, you change it's predefined behaviour.
"""

from django.core.urlresolvers import set_script_prefix
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt

from djangorestframework.compat import View as DjangoView
from djangorestframework.response import Response, ErrorResponse
from djangorestframework.mixins import *
from djangorestframework import resources, renderers, parsers, authentication, permissions, status

__all__ = (
    'View',
    'ModelView',
    'InstanceModelView',
    'ListModelView',
    'ListOrCreateModelView'
)

class View(ResourceMixin, RequestMixin, ResponseMixin, AuthMixin, DjangoView):
    """
    Handles incoming requests and maps them to REST operations.
    Performs request deserialization, response serialization, authentication and input validation.
    """

    """
    The resource to use when validating requests and filtering responses,
    or `None` to use default behaviour.
    """
    resource = None

    """
    List of renderers the resource can serialize the response with, ordered by preference.
    """
    renderers = renderers.DEFAULT_RENDERERS

    """
    List of parsers the resource can parse the request with.
    """
    parsers = parsers.DEFAULT_PARSERS

    """
    List of all authenticating methods to attempt.
    """
    authentication = ( authentication.UserLoggedInAuthentication,
                       authentication.BasicAuthentication )

    """
    List of all permissions that must be checked.
    """
    permissions = ( permissions.FullAnonAccess, )

    @classmethod
    def as_view(cls, **initkwargs):
        """
        Override the default :meth:`as_view` to store an instance of the view
        as an attribute on the callable function.  This allows us to discover
        information about the view when we do URL reverse lookups.
        """
        view = super(View, cls).as_view(**initkwargs)
        view.cls_instance = cls(**initkwargs)
        return view

    @property
    def allowed_methods(self):
        """
        Return the list of allowed HTTP methods, uppercased.
        """
        return [method.upper() for method in self.http_method_names if hasattr(self, method)]

    def http_method_not_allowed(self, request, *args, **kwargs):
        """
        Return an HTTP 405 error if an operation is called which does not have a handler method.
        """
        raise ErrorResponse(status.HTTP_405_METHOD_NOT_ALLOWED,
                            {'detail': 'Method \'%s\' not allowed on this resource.' % self.method})

    def initial(self, request, *args, **kargs):
        """
        Hook for any code that needs to run prior to anything else.
        Required if you want to do things like set `request.upload_handlers` before
        the authentication and dispatch handling is run.
        """
        pass

    def add_header(self, field, value):
        """
        Add *field* and *value* to the :attr:`headers` attribute of the :class:`View` class.
        """
        self.headers[field] = value

    # Note: session based authentication is explicitly CSRF validated,
    # all other authentication is CSRF exempt.
    @csrf_exempt
    def dispatch(self, request, *args, **kwargs):
        self.request = request
        self.args = args
        self.kwargs = kwargs
        self.headers = {}

        # Calls to 'reverse' will not be fully qualified unless we set the scheme/host/port here.
        prefix = '%s://%s' % (request.is_secure() and 'https' or 'http', request.get_host())
        set_script_prefix(prefix)

        try:
            self.initial(request, *args, **kwargs)

            # Authenticate and check request has the relevant permissions
            self._check_permissions()

            # Get the appropriate handler method
            if self.method.lower() in self.http_method_names:
                handler = getattr(self, self.method.lower(), self.http_method_not_allowed)
            else:
                handler = self.http_method_not_allowed

            response_obj = handler(request, *args, **kwargs)

            # Allow return value to be either HttpResponse, Response, or an object, or None
            if isinstance(response_obj, HttpResponse):
                return response_obj
            elif isinstance(response_obj, Response):
                response = response_obj
            elif response_obj is not None:
                response = Response(status.HTTP_200_OK, response_obj)
            else:
                response = Response(status.HTTP_204_NO_CONTENT)

            # Pre-serialize filtering (eg filter complex objects into natively serializable types)
            response.cleaned_content = self.filter_response(response.raw_content)

        except ErrorResponse, exc:
            response = exc.response

        # Always add these headers.
        #
        # TODO - this isn't actually the correct way to set the vary header,
        # also it's currently sub-obtimal for HTTP caching - need to sort that out.
        response.headers['Allow'] = ', '.join(self.allowed_methods)
        response.headers['Vary'] = 'Authenticate, Accept'

        # merge with headers possibly set at some point in the view
        response.headers.update(self.headers)

        return self.render(response)

class ModelView(View):
    """
    A RESTful view that maps to a model in the database.
    """
    resource = resources.ModelResource

class InstanceModelView(InstanceMixin, ReadModelMixin, UpdateModelMixin, DeleteModelMixin, ModelView):
    """
    A view which provides default operations for read/update/delete against a model instance.
    """
    _suffix = 'Instance'

class ListModelView(ListModelMixin, ModelView):
    """
    A view which provides default operations for list, against a model in the database.
    """
    _suffix = 'List'

class ListOrCreateModelView(ListModelMixin, CreateModelMixin, ModelView):
    """
    A view which provides default operations for list and create, against a model in the database.
    """
    _suffix = 'List'
<|endoftext|>


<commit_before>import os
import locale
import codecs
import subprocess

import six

import pytest

import click

import django
from django.core.management import get_commands, call_command
from django.core.management import execute_from_command_line

import djclick

todo = pytest.mark.xfail(reason='TODO')

@pytest.mark.skipif(not six.PY3, reason='Only necessary on Python3')
def test_not_ascii():  # NOCOV
    """
    Make sure that the systems preferred encoding is not `ascii`.

    Otherwise `click` is raising a RuntimeError for Python3. For a detailed
    description of this very problem please consult the following gist:
    https://gist.github.com/hackebrot/937245251887197ef542

    This test also checks that `tox.ini` explicitly copies the according
    system environment variables to the test environments.
    """
    try:
        preferred_encoding = locale.getpreferredencoding()
        fs_enc = codecs.lookup(preferred_encoding).name
    except Exception:
        fs_enc = 'ascii'
    assert fs_enc != 'ascii'

def test_attributes():
    for attr in click.__all__:
        assert hasattr(djclick, attr)

def test_command_recognized():
    assert 'testcmd' in get_commands()

def test_call_cli():
    execute_from_command_line(['./manage.py', 'testcmd'])
    with pytest.raises(RuntimeError):
        execute_from_command_line(['./manage.py', 'testcmd', '--raise'])

def test_call_command_args():
    call_command('testcmd')
    with pytest.raises(RuntimeError):
        call_command('testcmd', '--raise')

def test_call_command_kwargs():
    call_command('testcmd', raise_when_called=False)
    with pytest.raises(RuntimeError):
        call_command('testcmd', raise_when_called=True)

def test_call_command_kwargs_rename():
    call_command('testcmd', **{'raise': False})
    with pytest.raises(RuntimeError):
        call_command('testcmd', **{'raise': True})

def test_call_directly():
    from testapp.management.commands.testcmd import command

    command(raise_when_called=False)

    with pytest.raises(RuntimeError):
        command(raise_when_called=True)

    with pytest.raises(RuntimeError):
        command(**{'raise': True})

def test_django_verbosity(capsys):
    # Make sure any command can be called, even if it does not explictly
    # accept the --verbosity option
    with pytest.raises(RuntimeError):
        execute_from_command_line([
            './manage.py', 'testcmd', '--raise', '--verbosity', '1'])

    # Default
    execute_from_command_line([
        './manage.py', 'ctxverbositycmd'])
    out, err = capsys.readouterr()
    assert out == '1'

    # Explicit
    execute_from_command_line([
        './manage.py', 'ctxverbositycmd', '--verbosity', '2'])
    out, err = capsys.readouterr()
    assert out == '2'

    # Invalid
    with pytest.raises(click.BadParameter):
        execute_from_command_line([
            './manage.py', 'ctxverbositycmd', '--verbosity', '4'])

    # Default (option)
    execute_from_command_line([
        './manage.py', 'argverbositycommand'])
    out, err = capsys.readouterr()
    assert out == '1'

    # Explicit (option)
    execute_from_command_line([
        './manage.py', 'argverbositycommand', '--verbosity', '2'])
    out, err = capsys.readouterr()
    assert out == '2'

def test_django_pythonpath(manage):
    with pytest.raises(subprocess.CalledProcessError):
        manage('pathcmd')

    manage('pathcmd', '--pythonpath',
           os.path.join(os.path.dirname(__file__), 'testdir')) == b'1'

def test_django_traceback(manage):
    try:
        manage('errcmd')
    except subprocess.CalledProcessError as e:
        assert e.output == b'CommandError: Raised error description\n'
        assert e.returncode == 1
    else:
        assert False  # NOCOV

    try:
        manage('errcmd', '--traceback')
    except subprocess.CalledProcessError as e:
        lines = e.output.splitlines()
        assert lines[0] == b'Traceback (most recent call last):'
        for line in lines[1:-1]:
            assert line.startswith(b'  ')
        # Use `.endswith()` because of differences between CPython and pypy
        assert lines[-1].endswith(b'CommandError: Raised error description')
        assert e.returncode == 1
    else:
        assert False  # NOCOV

def test_django_settings(manage):
    # The --settings switch only works from the command line (or if the django
    # settings where not setup before)... this means that we have to call it
    # in a subprocess.
    cmd = 'settingscmd'
    assert manage(cmd) == b'default'
    assert manage(cmd, '--settings', 'testprj.settings') == b'default'
    assert manage(cmd, '--settings', 'testprj.settings_alt') == b'alternative'

def test_django_color(capsys):
    call_command('colorcmd')
s    out, err = capsys.readouterr()
    # Not passing a --color/--no-color flag defaults to autodetection. As the
    # command is run through the test suite, the autodetection defaults to not
    # colorizing the output.
    assert out == 'stdout'
    assert err == 'stderr'

    call_command('colorcmd', '--color')
    out, err = capsys.readouterr()
    assert out == click.style('stdout', fg='blue')
    assert err == click.style('stderr', bg='red')

    call_command('colorcmd', '--no-color')
    out, err = capsys.readouterr()
    assert out == 'stdout'
    assert err == 'stderr'

def test_django_help(manage):
    # The -h/--help switches cause the program to exit. Invoking the command
    # through execute_from_command_line would cause the test suit to exit as
    # well... this means that we have to call it in a subprocess.
    helps = [
        manage('helpcmd', '-h'),
        manage('helpcmd', '--help'),
        manage('help', 'helpcmd'),
    ]
    assert len(set(helps)) == 1

    help_text = helps[0]
    assert b'HELP_CALLED' not in help_text
    assert help_text.startswith(b'Usage: manage.py helpcmd ')

def test_django_version(manage):
    django_version = django.get_version().encode('ascii') + b'\n'
    if django.VERSION < (1, 8):
        prefix = django_version
    else:
        prefix = b''
    assert manage('testcmd', '--version') == prefix + django_version
    assert manage('versioncmd', '--version') == prefix + b'20.0\n'

def test_group_command(capsys):
    execute_from_command_line(['./manage.py', 'groupcmd'])
    out, err = capsys.readouterr()
    assert out == 'group_command\n'

    execute_from_command_line(['./manage.py', 'groupcmd', 'subcmd1'])
    out, err = capsys.readouterr()
    assert out == 'group_command\nSUB1\n'

    execute_from_command_line(['./manage.py', 'groupcmd', 'subcmd3'])
    out, err = capsys.readouterr()
    assert out == 'group_command\nSUB2\n'
<commit_msg>Remove typo<commit_after>import os
import locale
import codecs
import subprocess

import six

import pytest

import click

import django
from django.core.management import get_commands, call_command
from django.core.management import execute_from_command_line

import djclick

todo = pytest.mark.xfail(reason='TODO')

@pytest.mark.skipif(not six.PY3, reason='Only necessary on Python3')
def test_not_ascii():  # NOCOV
    """
    Make sure that the systems preferred encoding is not `ascii`.

    Otherwise `click` is raising a RuntimeError for Python3. For a detailed
    description of this very problem please consult the following gist:
    https://gist.github.com/hackebrot/937245251887197ef542

    This test also checks that `tox.ini` explicitly copies the according
    system environment variables to the test environments.
    """
    try:
        preferred_encoding = locale.getpreferredencoding()
        fs_enc = codecs.lookup(preferred_encoding).name
    except Exception:
        fs_enc = 'ascii'
    assert fs_enc != 'ascii'

def test_attributes():
    for attr in click.__all__:
        assert hasattr(djclick, attr)

def test_command_recognized():
    assert 'testcmd' in get_commands()

def test_call_cli():
    execute_from_command_line(['./manage.py', 'testcmd'])
    with pytest.raises(RuntimeError):
        execute_from_command_line(['./manage.py', 'testcmd', '--raise'])

def test_call_command_args():
    call_command('testcmd')
    with pytest.raises(RuntimeError):
        call_command('testcmd', '--raise')

def test_call_command_kwargs():
    call_command('testcmd', raise_when_called=False)
    with pytest.raises(RuntimeError):
        call_command('testcmd', raise_when_called=True)

def test_call_command_kwargs_rename():
    call_command('testcmd', **{'raise': False})
    with pytest.raises(RuntimeError):
        call_command('testcmd', **{'raise': True})

def test_call_directly():
    from testapp.management.commands.testcmd import command

    command(raise_when_called=False)

    with pytest.raises(RuntimeError):
        command(raise_when_called=True)

    with pytest.raises(RuntimeError):
        command(**{'raise': True})

def test_django_verbosity(capsys):
    # Make sure any command can be called, even if it does not explictly
    # accept the --verbosity option
    with pytest.raises(RuntimeError):
        execute_from_command_line([
            './manage.py', 'testcmd', '--raise', '--verbosity', '1'])

    # Default
    execute_from_command_line([
        './manage.py', 'ctxverbositycmd'])
    out, err = capsys.readouterr()
    assert out == '1'

    # Explicit
    execute_from_command_line([
        './manage.py', 'ctxverbositycmd', '--verbosity', '2'])
    out, err = capsys.readouterr()
    assert out == '2'

    # Invalid
    with pytest.raises(click.BadParameter):
        execute_from_command_line([
            './manage.py', 'ctxverbositycmd', '--verbosity', '4'])

    # Default (option)
    execute_from_command_line([
        './manage.py', 'argverbositycommand'])
    out, err = capsys.readouterr()
    assert out == '1'

    # Explicit (option)
    execute_from_command_line([
        './manage.py', 'argverbositycommand', '--verbosity', '2'])
    out, err = capsys.readouterr()
    assert out == '2'

def test_django_pythonpath(manage):
    with pytest.raises(subprocess.CalledProcessError):
        manage('pathcmd')

    manage('pathcmd', '--pythonpath',
           os.path.join(os.path.dirname(__file__), 'testdir')) == b'1'

def test_django_traceback(manage):
    try:
        manage('errcmd')
    except subprocess.CalledProcessError as e:
        assert e.output == b'CommandError: Raised error description\n'
        assert e.returncode == 1
    else:
        assert False  # NOCOV

    try:
        manage('errcmd', '--traceback')
    except subprocess.CalledProcessError as e:
        lines = e.output.splitlines()
        assert lines[0] == b'Traceback (most recent call last):'
        for line in lines[1:-1]:
            assert line.startswith(b'  ')
        # Use `.endswith()` because of differences between CPython and pypy
        assert lines[-1].endswith(b'CommandError: Raised error description')
        assert e.returncode == 1
    else:
        assert False  # NOCOV

def test_django_settings(manage):
    # The --settings switch only works from the command line (or if the django
    # settings where not setup before)... this means that we have to call it
    # in a subprocess.
    cmd = 'settingscmd'
    assert manage(cmd) == b'default'
    assert manage(cmd, '--settings', 'testprj.settings') == b'default'
    assert manage(cmd, '--settings', 'testprj.settings_alt') == b'alternative'

def test_django_color(capsys):
    call_command('colorcmd')
    out, err = capsys.readouterr()
    # Not passing a --color/--no-color flag defaults to autodetection. As the
    # command is run through the test suite, the autodetection defaults to not
    # colorizing the output.
    assert out == 'stdout'
    assert err == 'stderr'

    call_command('colorcmd', '--color')
    out, err = capsys.readouterr()
    assert out == click.style('stdout', fg='blue')
    assert err == click.style('stderr', bg='red')

    call_command('colorcmd', '--no-color')
    out, err = capsys.readouterr()
    assert out == 'stdout'
    assert err == 'stderr'

def test_django_help(manage):
    # The -h/--help switches cause the program to exit. Invoking the command
    # through execute_from_command_line would cause the test suit to exit as
    # well... this means that we have to call it in a subprocess.
    helps = [
        manage('helpcmd', '-h'),
        manage('helpcmd', '--help'),
        manage('help', 'helpcmd'),
    ]
    assert len(set(helps)) == 1

    help_text = helps[0]
    assert b'HELP_CALLED' not in help_text
    assert help_text.startswith(b'Usage: manage.py helpcmd ')

def test_django_version(manage):
    django_version = django.get_version().encode('ascii') + b'\n'
    if django.VERSION < (1, 8):
        prefix = django_version
    else:
        prefix = b''
    assert manage('testcmd', '--version') == prefix + django_version
    assert manage('versioncmd', '--version') == prefix + b'20.0\n'

def test_group_command(capsys):
    execute_from_command_line(['./manage.py', 'groupcmd'])
    out, err = capsys.readouterr()
    assert out == 'group_command\n'

    execute_from_command_line(['./manage.py', 'groupcmd', 'subcmd1'])
    out, err = capsys.readouterr()
    assert out == 'group_command\nSUB1\n'

    execute_from_command_line(['./manage.py', 'groupcmd', 'subcmd3'])
    out, err = capsys.readouterr()
    assert out == 'group_command\nSUB2\n'
<|endoftext|>


<commit_before>import unittest

import numpy as np
import numpy.testing as npt
import tensorflow as tf

from astroNN.config import MAGIC_NUMBER, keras_import_manager
from astroNN.nn import magic_correction_term, reduce_var
from astroNN.nn.losses import mean_absolute_error, mean_squared_error, categorical_crossentropy, binary_crossentropy, \
    nll
from astroNN.nn.metrics import categorical_accuracy, binary_accuracy, mean_absolute_percentage_error, \
    mean_squared_logarithmic_error

keras = keras_import_manager()
get_session = keras.backend.get_session

# force the test to use CPU, using GPU will be much slower for such small test
sess = tf.Session(config=tf.ConfigProto(device_count={'GPU': 0}))
keras.backend.set_session(sess)

class LossFuncTestCase(unittest.TestCase):
    def test_loss_func(self):
        # make sure custom reduce_var works
        var_array = [1, 2, 3, 4, 5]
        self.assertEqual(reduce_var(tf.Variable(var_array)).eval(session=get_session()), np.var(var_array))

        # =============Magic correction term============= #
        y_true = tf.Variable([[2., MAGIC_NUMBER, MAGIC_NUMBER], [2., MAGIC_NUMBER, 4.]])
        npt.assert_array_equal(magic_correction_term(y_true).eval(session=get_session()), [3., 1.5])

        # =============MSE/MAE============= #
        y_pred = tf.Variable([[2., 3., 4.], [2., 3., 7.]])
        y_pred_2 = tf.Variable([[2., 9., 4.], [2., 0., 7.]])
        y_true = tf.Variable([[2., MAGIC_NUMBER, 4.], [2., MAGIC_NUMBER, 4.]])
        npt.assert_almost_equal(mean_absolute_error(y_true, y_pred).eval(session=get_session()), [0., 3. / 2.])
        npt.assert_almost_equal(mean_squared_error(y_true, y_pred).eval(session=get_session()), [0., 9. / 2])

        # make sure neural network prediction won't matter for magic number term
        npt.assert_almost_equal(mean_absolute_error(y_true, y_pred).eval(session=get_session()),
                                mean_absolute_error(y_true, y_pred_2).eval(session=get_session()))
        npt.assert_almost_equal(mean_squared_error(y_true, y_pred).eval(session=get_session()),
                                mean_squared_error(y_true, y_pred_2).eval(session=get_session()))

        # =============Accuracy============= #
        y_pred = tf.Variable([[1., 0., 0.], [1., 0., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [0., MAGIC_NUMBER, 1.]])
        npt.assert_array_equal(categorical_accuracy(y_true, y_pred).eval(session=get_session()), [1., 0.])
        npt.assert_almost_equal(binary_accuracy(from_logits=False)(y_true, y_pred).eval(session=get_session()),
                                [1. / 2., 0.])

        # =============Percentage Accuracy============= #
        y_pred = tf.Variable([[1., 0., 0.], [1., 0., 0.]])
        y_pred_2 = tf.Variable([[1., 9., 0.], [1., -1., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 1.]])

        npt.assert_array_almost_equal(mean_absolute_percentage_error(y_true, y_pred).eval(session=get_session()),
                                      [50., 50.], decimal=3)
        # make sure neural network prediction won't matter for magic number term
        npt.assert_array_almost_equal(mean_absolute_percentage_error(y_true, y_pred).eval(session=get_session()),
                                      mean_absolute_percentage_error(y_true, y_pred_2).eval(session=get_session()),
                                      decimal=3)

        # =============Mean Squared Log Error============= #
        y_pred = tf.Variable([[1., 0., 0.], [1., 0., 0.]])
        y_pred_2 = tf.Variable([[1., 9., 0.], [1., -1., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 1.]])
        npt.assert_array_almost_equal(mean_squared_logarithmic_error(y_true, y_pred).eval(session=get_session()),
                                      [0.24, 0.24], decimal=3)
        # make sure neural network prediction won't matter for magic number term
        npt.assert_array_almost_equal(mean_squared_logarithmic_error(y_true, y_pred).eval(session=get_session()),
                                      mean_squared_logarithmic_error(y_true, y_pred_2).eval(session=get_session()),
                                      decimal=3)

    def test_categorical_crossentropy(self):
        y_pred = tf.Variable([[1., 0., 1.], [2., 1., 0.]])
        y_pred_2 = tf.Variable([[1., 2., 1.], [2., 2., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 0.]])
        y_pred_softmax = tf.nn.softmax(y_pred)
        y_pred_2_softmax = tf.nn.softmax(y_pred_2)
        # Truth with Magic number is wrong
        npt.assert_array_almost_equal(categorical_crossentropy(y_true, y_pred_softmax).eval(session=get_session()),
                                      categorical_crossentropy(y_true, y_pred, from_logits=True).eval(
                                          session=get_session()), decimal=3)

    def test_binary_crossentropy(self):
        y_pred = tf.Variable([[0.5, 0., 1.], [2., 0., -1.]])
        y_pred_2 = tf.Variable([[0.5, 2., 1.], [2., 2., -1.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 0.]])
        y_pred_sigmoid = tf.nn.sigmoid(y_pred)
        y_pred_2_sigmoid = tf.nn.sigmoid(y_pred_2)
        # Truth with Magic number is wrong
        npt.assert_array_almost_equal(binary_crossentropy(y_true, y_pred_sigmoid).eval(session=get_session()),
                                      binary_crosssentropy(y_true, y_pred, from_logits=True).eval(
                                          session=get_session()), decimal=3)
        # make sure neural network prediction won't matter for magic number term
        npt.assert_array_almost_equal(
            binary_crossentropy(y_true, y_pred_2, from_logits=True).eval(session=get_session()),
            binary_crossentropy(y_true, y_pred, from_logits=True).eval(session=get_session())
            , decimal=3)
        npt.assert_array_almost_equal(binary_crossentropy(y_true, y_pred_sigmoid).eval(session=get_session()),
                                      binary_crossentropy(y_true, y_pred_2_sigmoid).eval(
                                          session=get_session()), decimal=3)

    def test_negative_log_likelihood(self):
        y_pred = tf.Variable([[0.5, 0., 1.], [2., 0., -1.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 0.]])
        # Truth with Magic number is wrong
        npt.assert_array_almost_equal(nll(y_true, y_pred).eval(session=get_session()), 0.34657377, decimal=3)

if __name__ == '__main__':
    unittest.main()
<commit_msg>ops typo<commit_after>import unittest

import numpy as np
import numpy.testing as npt
import tensorflow as tf

from astroNN.config import MAGIC_NUMBER, keras_import_manager
from astroNN.nn import magic_correction_term, reduce_var
from astroNN.nn.losses import mean_absolute_error, mean_squared_error, categorical_crossentropy, binary_crossentropy, \
    nll
from astroNN.nn.metrics import categorical_accuracy, binary_accuracy, mean_absolute_percentage_error, \
    mean_squared_logarithmic_error

keras = keras_import_manager()
get_session = keras.backend.get_session

# force the test to use CPU, using GPU will be much slower for such small test
sess = tf.Session(config=tf.ConfigProto(device_count={'GPU': 0}))
keras.backend.set_session(sess)

class LossFuncTestCase(unittest.TestCase):
    def test_loss_func(self):
        # make sure custom reduce_var works
        var_array = [1, 2, 3, 4, 5]
        self.assertEqual(reduce_var(tf.Variable(var_array)).eval(session=get_session()), np.var(var_array))

        # =============Magic correction term============= #
        y_true = tf.Variable([[2., MAGIC_NUMBER, MAGIC_NUMBER], [2., MAGIC_NUMBER, 4.]])
        npt.assert_array_equal(magic_correction_term(y_true).eval(session=get_session()), [3., 1.5])

        # =============MSE/MAE============= #
        y_pred = tf.Variable([[2., 3., 4.], [2., 3., 7.]])
        y_pred_2 = tf.Variable([[2., 9., 4.], [2., 0., 7.]])
        y_true = tf.Variable([[2., MAGIC_NUMBER, 4.], [2., MAGIC_NUMBER, 4.]])
        npt.assert_almost_equal(mean_absolute_error(y_true, y_pred).eval(session=get_session()), [0., 3. / 2.])
        npt.assert_almost_equal(mean_squared_error(y_true, y_pred).eval(session=get_session()), [0., 9. / 2])

        # make sure neural network prediction won't matter for magic number term
        npt.assert_almost_equal(mean_absolute_error(y_true, y_pred).eval(session=get_session()),
                                mean_absolute_error(y_true, y_pred_2).eval(session=get_session()))
        npt.assert_almost_equal(mean_squared_error(y_true, y_pred).eval(session=get_session()),
                                mean_squared_error(y_true, y_pred_2).eval(session=get_session()))

        # =============Accuracy============= #
        y_pred = tf.Variable([[1., 0., 0.], [1., 0., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [0., MAGIC_NUMBER, 1.]])
        npt.assert_array_equal(categorical_accuracy(y_true, y_pred).eval(session=get_session()), [1., 0.])
        npt.assert_almost_equal(binary_accuracy(from_logits=False)(y_true, y_pred).eval(session=get_session()),
                                [1. / 2., 0.])

        # =============Percentage Accuracy============= #
        y_pred = tf.Variable([[1., 0., 0.], [1., 0., 0.]])
        y_pred_2 = tf.Variable([[1., 9., 0.], [1., -1., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 1.]])

        npt.assert_array_almost_equal(mean_absolute_percentage_error(y_true, y_pred).eval(session=get_session()),
                                      [50., 50.], decimal=3)
        # make sure neural network prediction won't matter for magic number term
        npt.assert_array_almost_equal(mean_absolute_percentage_error(y_true, y_pred).eval(session=get_session()),
                                      mean_absolute_percentage_error(y_true, y_pred_2).eval(session=get_session()),
                                      decimal=3)

        # =============Mean Squared Log Error============= #
        y_pred = tf.Variable([[1., 0., 0.], [1., 0., 0.]])
        y_pred_2 = tf.Variable([[1., 9., 0.], [1., -1., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 1.]])
        npt.assert_array_almost_equal(mean_squared_logarithmic_error(y_true, y_pred).eval(session=get_session()),
                                      [0.24, 0.24], decimal=3)
        # make sure neural network prediction won't matter for magic number term
        npt.assert_array_almost_equal(mean_squared_logarithmic_error(y_true, y_pred).eval(session=get_session()),
                                      mean_squared_logarithmic_error(y_true, y_pred_2).eval(session=get_session()),
                                      decimal=3)

    def test_categorical_crossentropy(self):
        y_pred = tf.Variable([[1., 0., 1.], [2., 1., 0.]])
        y_pred_2 = tf.Variable([[1., 2., 1.], [2., 2., 0.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 0.]])
        y_pred_softmax = tf.nn.softmax(y_pred)
        y_pred_2_softmax = tf.nn.softmax(y_pred_2)
        # Truth with Magic number is wrong
        npt.assert_array_almost_equal(categorical_crossentropy(y_true, y_pred_softmax).eval(session=get_session()),
                                      categorical_crossentropy(y_true, y_pred, from_logits=True).eval(
                                          session=get_session()), decimal=3)

    def test_binary_crossentropy(self):
        y_pred = tf.Variable([[0.5, 0., 1.], [2., 0., -1.]])
        y_pred_2 = tf.Variable([[0.5, 2., 1.], [2., 2., -1.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 0.]])
        y_pred_sigmoid = tf.nn.sigmoid(y_pred)
        y_pred_2_sigmoid = tf.nn.sigmoid(y_pred_2)
        # Truth with Magic number is wrong
        npt.assert_array_almost_equal(binary_crossentropy(y_true, y_pred_sigmoid).eval(session=get_session()),
                                      binary_crossentropy(y_true, y_pred, from_logits=True).eval(
                                          session=get_session()), decimal=3)
        # make sure neural network prediction won't matter for magic number term
        npt.assert_array_almost_equal(
            binary_crossentropy(y_true, y_pred_2, from_logits=True).eval(session=get_session()),
            binary_crossentropy(y_true, y_pred, from_logits=True).eval(session=get_session())
            , decimal=3)
        npt.assert_array_almost_equal(binary_crossentropy(y_true, y_pred_sigmoid).eval(session=get_session()),
                                      binary_crossentropy(y_true, y_pred_2_sigmoid).eval(
                                          session=get_session()), decimal=3)

    def test_negative_log_likelihood(self):
        y_pred = tf.Variable([[0.5, 0., 1.], [2., 0., -1.]])
        y_true = tf.Variable([[1., MAGIC_NUMBER, 1.], [1., MAGIC_NUMBER, 0.]])
        # Truth with Magic number is wrong
        npt.assert_array_almost_equal(nll(y_true, y_pred).eval(session=get_session()), 0.34657377, decimal=3)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.db import models
from django.db.models import PROTECT
from django.utils.encoding import python_2_unicode_compatible

@python_2_unicode_compatible
class Province(models.Model):
    name = models.CharField(max_length=255)
    province_id = models.CharField(max_length=80, unique=True)

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Regency(models.Model):
    name = models.CharField(max_length=255)
    regency_id = models.CharField(max_length=80, unique=True)
    province = models.ForeignKey(Province, on_delete=PROTECT)

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class District(models.Model):
    name = models.CharField(max_length=255)
    district_id = models.CharField(max_length=80, unique=True)
    regency = models.ForeignKey(Regency, on_delete=PROTECT)

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Village(models.Model):
    name = models.CharField(max_length=255)
    village_id = models.CharField(max_length=80, unique=True)
    district = models.ForeignKey(District, on_delete=PROTECT)

    def __str__(self):
        return self.name
<commit_msg>Py2encodingerror (#5)<commit_after>from __future__ import unicode_literals

from django.db import models
from django.db.models import PROTECT
from django.utils import version

# fix import error for django 1.5 and up
vtuple = version.get_complete_version()
if vtuple[0]< 2 and vtuple[1]<6:
    from django.utils.encoding import python_2_unicode_compatible
else:
    from six import python_2_unicode_compatible

@python_2_unicode_compatible
class Province(models.Model):
    name = models.CharField(max_length=255)
    province_id = models.CharField(max_length=80, unique=True)

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Regency(models.Model):
    name = models.CharField(max_length=255)
    regency_id = models.CharField(max_length=80, unique=True)
    province = models.ForeignKey(Province, on_delete=PROTECT)

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class District(models.Model):
    name = models.CharField(max_length=255)
    district_id = models.CharField(max_length=80, unique=True)
    regency = models.ForeignKey(Regency, on_delete=PROTECT)

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Village(models.Model):
    name = models.CharField(max_length=255)
    village_id = models.CharField(max_length=80, unique=True)
    district = models.ForeignKey(District, on_delete=PROTECT)

    def __str__(self):
        return self.name
<|endoftext|>


<commit_before>"""
Unit tests for mpath dmp
"""
import errno
import os
import unittest
import mock
import testlib
import mpath_dmp
import SR

# pylint: disable=W0613; mocks don't need to be accessed
# pylint: disable=R0201; methods must be instance for nose to work
# pylint: disable=W0212; unit tests are permitted to snoop
class TestMpathDmp(unittest.TestCase):
    """
    Unit tests for mpath dmp
    """

    @testlib.with_context
    @mock.patch('mpath_dmp.util', autospec=True)
    @mock.patch('mpath_dmp.os', autospec=True)
    def test_is_valid_multipath_device(self, context, mock_os, util_mod):
        """
        Tests for checking validity of multipath device
        """

        # Setup errors codes
        context.setup_error_codes()

        # Test 'multipath -ll' success
        util_mod.doexec.side_effect = [(0, "out", "err")]
        self.assertTrue(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test 'multipath -a' failure
        util_mod.doexec.side_effect = [(0, "", ""), (1, "out", "err")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test failure when device is not available
        mock_os.path.exists.return_value = False
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test 'multipath -c' with error and empty output
        mock_os.path.exists.return_value = True
        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "", ""), OSError()]
        with self.assertRaises(SR.SROSError) as exc:
            mpath_dmp._is_valid_multipath_device("xx")
        self.assertEqual(exc.exception.errno, 431)

        # Test 'multipath -c' with error but some output
        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "xx", "")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "xx", "yy")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "", "yy")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test when everything is fine
        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (0, "out", "err")]
        self.assertTrue(mpath_dmp._is_valid_multipath_device("fake_dev"))

    @mock.patch('util.pread2', autospec=True)
    @mock.patch('mpath_dmp.os.mkdir', autospec=True)
    def test_activate_no_exception(self, mock_mkdir, pread2):
        """
        Test that activeate MPDev works if directory does not exist
        """
        mpath_dmp.activate_MPdev("sid", "dst")
        pread2.assert_called_with(['ln', '-sf', "dst", os.path.join(mpath_dmp.MP_INUSEDIR, "sid")])

    @mock.patch('util.pread2', autospec=True)
    @mock.patch('mpath_dmp.os.mkdir', autospec=True)
    def test_activate_exists_success(self, mock_mkdir, pread2):
        """
        Test that activeate MPDev works if directory exists
        """
        mock_mkdir.side_effect = [OSError(errno.EEXIST, "Directory exists")]
        mpath_dmp.activate_MPdev("sid", "dst")
        pread2.assert_called_with(['ln', '-sf', "dst", os.path.join(mpath_dmp.MP_INUSEDIR, "sid")])

    @mock.patch('mpath_dmp.os.mkdir', autospec=True)
    def test_activate_permission_denied(self, mock_mkdir):
        """
        Test that activeate MPDev works if mkdir returns permission denied
        """
        mock_mkdir.side_effect = [OSError(errno.EPERM, "Permission denied")]
        with self.assertRaises(OSError) as context:
            mpath_dmp.activate_MPdev("sid", "dst")

        self.assertEqual(errno.EPERM, context.exception.errno)
<commit_msg>CP-34042: Add unit tests for mpath utils<commit_after>"""
Unit tests for mpath dmp
"""
import errno
import os
import unittest
import mock
import testlib
import util

import mpath_dmp
import SR
from SR import SROSError

from Queue import Queue

import xs_errors

# pylint: disable=W0613; mocks don't need to be accessed
# pylint: disable=R0201; methods must be instance for nose to work
# pylint: disable=W0212; unit tests are permitted to snoop
class TestMpathDmp(unittest.TestCase):
    """
    Unit tests for mpath dmp
    """

    def setUp(self):
        time_patcher = mock.patch('mpath_dmp.time', autospec=True)
        self.mock_time = time_patcher.start()

        mpath_cli_patcher = mock.patch('mpath_dmp.mpath_cli', autospec=True)
        self.mock_mpath_cli = mpath_cli_patcher.start()

        self.addCleanup(mock.patch.stopall)

    @testlib.with_context
    @mock.patch('mpath_dmp.util', autospec=True)
    @mock.patch('mpath_dmp.os', autospec=True)
    def test_is_valid_multipath_device(self, context, mock_os, util_mod):
        """
        Tests for checking validity of multipath device
        """

        # Setup errors codes
        context.setup_error_codes()

        # Test 'multipath -ll' success
        util_mod.doexec.side_effect = [(0, "out", "err")]
        self.assertTrue(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test 'multipath -a' failure
        util_mod.doexec.side_effect = [(0, "", ""), (1, "out", "err")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test failure when device is not available
        mock_os.path.exists.return_value = False
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test 'multipath -c' with error and empty output
        mock_os.path.exists.return_value = True
        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "", ""), OSError()]
        with self.assertRaises(SR.SROSError) as exc:
            mpath_dmp._is_valid_multipath_device("xx")
        self.assertEqual(exc.exception.errno, 431)

        # Test 'multipath -c' with error but some output
        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "xx", "")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "xx", "yy")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (1, "", "yy")]
        self.assertFalse(mpath_dmp._is_valid_multipath_device("fake_dev"))

        # Test when everything is fine
        mock_os.listdir.side_effect = [['sdc']]
        util_mod.doexec.side_effect = [(0, "", ""), (0, "out", "err"),
                                       (0, "out", "err")]
        self.assertTrue(mpath_dmp._is_valid_multipath_device("fake_dev"))

    @mock.patch('util.pread2', autospec=True)
    @mock.patch('mpath_dmp.os.mkdir', autospec=True)
    def test_activate_no_exception(self, mock_mkdir, pread2):
        """
        Test that activate MPDev works if directory does not exist
        """
        mpath_dmp.activate_MPdev("sid", "dst")
        pread2.assert_called_with(['ln', '-sf', "dst", os.path.join(mpath_dmp.MP_INUSEDIR, "sid")])

    @mock.patch('util.pread2', autospec=True)
    @mock.patch('mpath_dmp.os.mkdir', autospec=True)
    def test_activate_exists_success(self, mock_mkdir, pread2):
        """
        Test that activate MPDev works if directory exists
        """
        mock_mkdir.side_effect = [OSError(errno.EEXIST, "Directory exists")]
        mpath_dmp.activate_MPdev("sid", "dst")
        pread2.assert_called_with(['ln', '-sf', "dst", os.path.join(mpath_dmp.MP_INUSEDIR, "sid")])

    @mock.patch('mpath_dmp.os.mkdir', autospec=True)
    def test_activate_permission_denied(self, mock_mkdir):
        """
        Test that activate MPDev works if mkdir returns permission denied
        """
        mock_mkdir.side_effect = [OSError(errno.EPERM, "Permission denied")]
        with self.assertRaises(OSError) as context:
            mpath_dmp.activate_MPdev("sid", "dst")

        self.assertEqual(errno.EPERM, context.exception.errno)

    @testlib.with_context
    @mock.patch('mpath_dmp._is_valid_multipath_device', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    @mock.patch('mpath_dmp.os.path.exists', autospec=True)
    def test_refresh_dmp_success(self, context, mock_exists, mock_util, mock_valid):
        """
        Test refresh DMP in success case
        """
        mock_valid.return_value = True

        test_id='360871234'

        mock_exists.return_value = True

        with mock.patch('mpath_dmp.activate_MPdev') as mock_activate:
            mpath_dmp._refresh_DMP(test_id, 4)

        # util retry around multipath should not be called
        self.assertEqual(0, mock_util.retry.call_count)

        self.assertTrue(
            mock_util.wait_for_path.call_args_list[0][0][0].endswith(
                '%s/mapper' % test_id),
            msg='wait_for_path not called with expected mapper path')
        mock_activate.assert_called_with(test_id, mock.ANY)

    @testlib.with_context
    @mock.patch('mpath_dmp._is_valid_multipath_device', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_refresh_dmp_device_not_found(self, context, mock_util, mock_valid):
        """
        Test refresh DMP device not found
        """
        # Setup error codes
        context.setup_error_codes()

        mock_valid.return_value = True

        test_id='360871234'

        with self.assertRaises(SROSError):
            mpath_dmp._refresh_DMP(test_id, 4)

        self.assertEqual(1, mock_util.wait_for_path.call_count)
        self.assertTrue(
            mock_util.wait_for_path.call_args_list[0][0][0].endswith(test_id))

    @mock.patch('mpath_dmp._is_valid_multipath_device', autospec=True)
    @mock.patch('mpath_dmp.util.pread2', autospec=True)
    @mock.patch('mpath_dmp.util.wait_for_path', autospec=True)
    @mock.patch('mpath_dmp.os.path.exists', autospec=True)
    def test_refresh_dmp_reload_required(
            self, mock_exists, mock_wait_for_path, mock_pread, mock_valid):
        """
        Test refresh DMP device reload
        """
        mock_valid.return_value = True

        test_id='360871234'

        mapper_exists = Queue()
        mapper_exists.put(False)
        mapper_exists.put(True)

        exists_data = {'/dev/mapper/%s' % test_id: mapper_exists}

        def exists(path):
            assert(path in exists_data)
            return exists_data[path].get()

        mock_exists.side_effect = exists

        mock_pread.return_value = 0

        with mock.patch('mpath_dmp.activate_MPdev') as mock_activate:
            mpath_dmp._refresh_DMP(test_id, 4)

        self.assertTrue(
            mock_wait_for_path.call_args_list[0][0][0].endswith(test_id))

    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_activate_noiscsi_success(
            self, mock_util, mock_iscsilib):
        """
        MPATH activate, no iscsi, success
        """
        mock_iscsilib.is_iscsi_daemon_running.return_value = False
        mock_util.doexec.return_value = (0, "", "")
        self.mock_mpath_cli.is_working.side_effect = [False, False, True]

        mpath_dmp.activate()

        self.assertEqual(0, mock_util.pread2.call_count)

    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_activate_noiscsi_start_mpath(
            self, mock_util, mock_iscsilib):
        """
        MPATH activate, no iscsi, start mpath
        """
        mock_iscsilib.is_iscsi_daemon_running.return_value = False
        mock_util.doexec.return_value = (1, "", "")
        self.mock_mpath_cli.is_working.side_effect = [False, False, True]

        mpath_dmp.activate()

        self.assertEqual(1, mock_util.pread2.call_count)
        mock_util.pread2.assert_called_once_with(
            ['service', 'multipathd', 'start'])

    @testlib.with_context
    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_activate_noiscsi_mpath_not_working(
            self, context, mock_util, mock_iscsilib):
        """
        MPATH activate, mpath not running
        """
        # Setup error codes
        context.setup_error_codes()

        mock_iscsilib.is_iscsi_daemon_running.return_value = False
        mock_util.doexec.return_value = (0, "", "")
        self.mock_mpath_cli.is_working.side_effect = [False] * 120

        with self.assertRaises(SROSError) as soe:
            mpath_dmp.activate()

        self.assertEqual(430, soe.exception.errno)

    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_activate_active_iscsi_success(
            self, mock_util, mock_iscsilib):
        """
        MPATH activate, active iscsi, success
        """
        mock_iscsilib.is_iscsi_daemon_running.return_value = True
        mock_iscsilib._checkAnyTGT.return_value = True
        mock_util.doexec.return_value = (0, "", "")
        self.mock_mpath_cli.is_working.side_effect = [False, False, True]

        mpath_dmp.activate()

        self.assertEqual(0, mock_iscsilib.restart_daemon.call_count)

    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_activate_iscsi_no_targets_success(
            self, mock_util, mock_iscsilib):
        """
        MPATH activate, iscsi, no_targets, success
        """
        mock_iscsilib.is_iscsi_daemon_running.return_value = True
        mock_iscsilib._checkAnyTGT.return_value = False
        mock_util.doexec.return_value = (0, "", "")
        self.mock_mpath_cli.is_working.side_effect = [False, False, True]

        mpath_dmp.activate()

        self.assertEqual(0, mock_util.pread2.call_count)
        self.assertEqual(1, mock_iscsilib.restart_daemon.call_count)

    @mock.patch('mpath_dmp.glob.glob', autospec=True)
    @mock.patch('mpath_dmp.os.path.realpath', autospec=True)
    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_deactivate_mpath_running(
            self, mock_util, mock_iscsilib, mock_realpath, mock_glob):
        """
        MPATH deactivate, running, success
        """
        mock_util.doexec.return_value = (0, "", "")
        self.mock_mpath_cli.list_maps.return_value = [
            '360a98000534b4f4e46704f5270674d70',
            '3600140582622313e8dc4270a4a897b4e']
        mock_realpath.return_value = '/dev/disk/by-id/scsi-34564'
        mock_util.retry.side_effect = util.retry

        mpath_dmp.deactivate()

        # Check that the mpath maps were removed
        mock_util.pread2.assert_has_calls([
            mock.call(['/usr/sbin/multipath', '-f', '360a98000534b4f4e46704f5270674d70']),
            mock.call(['/usr/sbin/multipath', '-W']),
            mock.call(['/usr/sbin/multipath', '-f', '3600140582622313e8dc4270a4a897b4e']),
            mock.call(['/usr/sbin/multipath', '-W'])])

    @mock.patch('mpath_dmp.glob.glob', autospec=True)
    @mock.patch('mpath_dmp.os.path.realpath', autospec=True)
    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_deactivate_mpath_root(
            self, mock_util, mock_iscsilib, mock_realpath, mock_glob):
        """
        MPATH deactivate, mpathed root
        """
        mock_util.doexec.return_value = (0, "", "")
        self.mock_mpath_cli.list_maps.return_value = [
            '360a98000534b4f4e46704f5270674d70',
            '3600140582622313e8dc4270a4a897b4e']
        mock_realpath.side_effect = iter(
            ['/dev/mapper/34564', '/dev/mapper/34564',
             '/dev/mapper/360a98000534b4f4e46704f5270674d70'])
        mock_glob.return_value = [
            '/dev/mapper/34564',
            '/dev/mapper/360a98000534b4f4e46704f5270674d70']
        mock_util.retry.side_effect = util.retry

        mpath_dmp.deactivate()

        # Check that the mpath maps were removed
        mock_util.pread2.assert_has_calls([
            mock.call(['/usr/sbin/multipath', '-f', '360a98000534b4f4e46704f5270674d70']),
            mock.call(['/usr/sbin/multipath', '-W']),
            mock.call(['/usr/sbin/multipath', '-f', '3600140582622313e8dc4270a4a897b4e']),
            mock.call(['/usr/sbin/multipath', '-W'])])

    @mock.patch('mpath_dmp.glob.glob', autospec=True)
    @mock.patch('mpath_dmp.os.path.realpath', autospec=True)
    @mock.patch('mpath_dmp.iscsilib', autospec=True)
    @mock.patch('mpath_dmp.util', autospec=True)
    def test_deactivate_mpath_no_iscsi_targets(
            self, mock_util, mock_iscsilib, mock_realpath, mock_glob):
        """
        MPATH deactivate, running, success
        """
        mock_util.doexec.return_value = (0, "", "")
        self.mock_mpath_cli.list_maps.return_value = [
            '360a98000534b4f4e46704f5270674d70',
            '3600140582622313e8dc4270a4a897b4e']
        mock_realpath.return_value = '/dev/disk/by-id/scsi-34564'
        mock_util.retry.side_effect = util.retry
        mock_iscsilib.is_iscsi_daemon_running.return_value = True
        mock_iscsilib._checkAnyTGT.return_value = False

        mpath_dmp.deactivate()

        # Check that the mpath maps were removed
        mock_util.pread2.assert_has_calls([
            mock.call(['/usr/sbin/multipath', '-f', '360a98000534b4f4e46704f5270674d70']),
            mock.call(['/usr/sbin/multipath', '-W']),
            mock.call(['/usr/sbin/multipath', '-f', '3600140582622313e8dc4270a4a897b4e']),
            mock.call(['/usr/sbin/multipath', '-W'])])

        self.assertEqual(1, mock_iscsilib.restart_daemon.call_count)

    @testlib.with_context
    def test_refresh_no_sid(self, context):
        # Setup error codes
        context.setup_error_codes()

        with self.assertRaises(SR.SROSError):
            mpath_dmp.refresh("", 0)

    @mock.patch('mpath_dmp._refresh_DMP', autospec=True)
    @mock.patch('mpath_dmp.os.path.exists', autospec=True)
    def test_refresh_path_exists(self, mock_exists, mock_refresh):

        mock_exists.return_value = True

        mpath_dmp.refresh('360a98000534b4f4e46704f5270674d70', 0)

        mock_refresh.assert_called_once_with(
            '360a98000534b4f4e46704f5270674d70', 0)

        mock_exists.assert_called_once_with(
            '/dev/disk/by-id/scsi-360a98000534b4f4e46704f5270674d70')

    @mock.patch('mpath_dmp.util.wait_for_path', autospec=True)
    @mock.patch('mpath_dmp.scsiutil', autospec=True)
    @mock.patch('mpath_dmp._refresh_DMP', autospec=True)
    @mock.patch('mpath_dmp.os.path.exists', autospec=True)
    def test_refresh_refresh_scsi(
            self, mock_exists, mock_refresh, mock_scsiutil, mock_wait):

        mock_exists.return_value = False
        mock_wait.return_value = True

        mpath_dmp.refresh('360a98000534b4f4e46704f5270674d70', 0)

        mock_refresh.assert_called_once_with(
            '360a98000534b4f4e46704f5270674d70', 0)

        mock_exists.assert_called_once_with(
            '/dev/disk/by-id/scsi-360a98000534b4f4e46704f5270674d70')

    @testlib.with_context
    @mock.patch('mpath_dmp.util.wait_for_path', autospec=True)
    @mock.patch('mpath_dmp.scsiutil', autospec=True)
    @mock.patch('mpath_dmp.os.path.exists', autospec=True)
    def test_refresh_refresh_error(
            self, context, mock_exists, mock_scsiutil, mock_wait):

        # Setup error codes
        context.setup_error_codes()

        def exists(path):
            print 'Exists %s' % path
            if path.startswith('/dev/'):
                return False

            return True

        mock_exists.side_effect = exists
        mock_wait.return_value = False

        with self.assertRaises(SR.SROSError):
            mpath_dmp.refresh('360a98000534b4f4e46704f5270674d70', 0)
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2015 John Dewey
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from ddt import data
from ddt import ddt
from ddt import unpack
import netaddr
import unittest2 as unittest

from demeter.namespace import Namespace
from demeter.namespace import NetworkNotAllowedException
from tests import helper

@ddt
class TestNamespace(unittest.TestCase):
    def setUp(self):
        self._namespace = Namespace()
        self._name, self._cidr, self._family = helper.namespace_data()

    def setup_teardown_namespace(func):
        def wrapper(self, *args, **kwargs):
            ns = self._namespace.create(self._name, self._cidr, self._family)

            func(self, *args, **kwargs)

            self._namespace.delete(ns)
        return wrapper

    @setup_teardown_namespace
    def test_all(self):
        result = self._namespace.all()
        assert 1 <= len(result)

    @unpack
    @data(helper.namespace_data())
    def test_all_is_empty(self, name, cidr, family):
        result = self._namespace.all()
        self.assertEquals([], result)

    @setup_teardown_namespace
    def test_create(self):
        result = self._namespace.find_by_name(self._name)
        self.assertEquals(self._name, result.name)

    @unpack
    @data(helper.namespace_data(cidr='198.51.100.0/36'))
    def test_create_not_allowed(self, name, cidr, family):
        with self.assertRaises(netaddr.AddrFormatError):
            self._namespace.create(name, cidr, family)

    @unpack
    @data(helper.namespace_data())
    def test_delete(self, name, cidr, family):
        ns = self._namespace.create(name, cidr, family)

        result = self._namespace.delete(ns)
        assert result
        result = self._namespace.find_by_name(name)
        assert not result

        self._namespace.delete(ns)

    @unpack
    @data(helper.namespace_data())
    def test_delete_by_name(self, name, cidr, family):
        ns = self._namespace.create(name, cidr, family)

        result = self._namespace.delete_by_name(name)
        assert result

        self._namespace.delete(ns)

    @data('name-not-found')
    def test_delete_by_name_is_false_when_not_found(self, name):
        result = self._namespace.delete_by_name(name)
        assert not result

    @setup_teardown_namespace
    def test_find_by_name(self):
        result = self._namespace.find_by_name(self._name)
        self.assertEquals(self._name, result.name)

    @data('name-not-found')
    def test_find_by_name_is_false_when_not_found(self, name):
        result = self._namespace.find_by_name(name)
        assert not result

    @data('198.51.100.0/24')
    def test_allowed_cidr(self, cidr):
        result = self._namespace._allowed_cidr(cidr)
        assert result

    @data('198.51.100.0/36')
    def test_allowed_cidr_raises_on_invalid_cidr(self, cidr):
        with self.assertRaises(netaddr.AddrFormatError):
            self._namespace._allowed_cidr(cidr)

    @data('198.51.100.0/15')
    def test_allowed_cidr_raises_on_disallowed(self, cidr):
        with self.assertRaises(NetworkNotAllowedException):
            self._namespace._allowed_cidr(cidr)
<commit_msg>Cleaned up namespace tests<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2015 John Dewey
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from ddt import data
from ddt import ddt
from ddt import unpack
import netaddr
import unittest2 as unittest

from demeter.namespace import Namespace
from demeter.namespace import NetworkNotAllowedException
from tests import helper

@ddt
class TestNamespace(unittest.TestCase):
    def setUp(self):
        self._namespace = Namespace()
        self._name, self._cidr, self._family = helper.namespace_data()

    def setup_teardown_namespace(func):
        def wrapper(self, *args, **kwargs):
            ns = self._namespace.create(self._name, self._cidr, self._family)

            func(self, *args, **kwargs)

            self._namespace.delete(ns)
        return wrapper

    @setup_teardown_namespace
    def test_all(self):
        result = self._namespace.all()
        assert 1 <= len(result)

    def test_all_is_empty(self):
        result = self._namespace.all()
        self.assertEquals([], result)

    @setup_teardown_namespace
    def test_create(self):
        result = self._namespace.find_by_name(self._name)
        self.assertEquals(self._name, result.name)

    @unpack
    @data(helper.namespace_data(cidr='198.51.100.0/36'))
    def test_create_not_allowed(self, name, cidr, family):
        with self.assertRaises(netaddr.AddrFormatError):
            self._namespace.create(name, cidr, family)

    @unpack
    @data(helper.namespace_data())
    def test_delete(self, name, cidr, family):
        ns = self._namespace.create(name, cidr, family)

        result = self._namespace.delete(ns)
        assert result
        result = self._namespace.find_by_name(name)
        assert not result

        self._namespace.delete(ns)

    @setup_teardown_namespace
    def test_delete_by_name(self):
        result = self._namespace.delete_by_name(self._name)
        assert result

    @data('name-not-found')
    def test_delete_by_name_is_false_when_not_found(self, name):
        result = self._namespace.delete_by_name(name)
        assert not result

    @setup_teardown_namespace
    def test_find_by_name(self):
        result = self._namespace.find_by_name(self._name)
        self.assertEquals(self._name, result.name)

    @data('name-not-found')
    def test_find_by_name_is_false_when_not_found(self, name):
        result = self._namespace.find_by_name(name)
        assert not result

    @data('198.51.100.0/24')
    def test_allowed_cidr(self, cidr):
        result = self._namespace._allowed_cidr(cidr)
        assert result

    @data('198.51.100.0/36')
    def test_allowed_cidr_raises_on_invalid_cidr(self, cidr):
        with self.assertRaises(netaddr.AddrFormatError):
            self._namespace._allowed_cidr(cidr)

    @data('198.51.100.0/15')
    def test_allowed_cidr_raises_on_disallowed(self, cidr):
        with self.assertRaises(NetworkNotAllowedException):
            self._namespace._allowed_cidr(cidr)
<|endoftext|>