<commit_before># Copyright 2014, 2015 Red Hat
#
# Authors: Chris Dent <chdent@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import json
import os
import re

import jsonpath_rw
import testtools
from six.moves.urllib import parse as urlparse

class HTTPTestCase(testtools.TestCase):
    """Encapsulate a single HTTP request as a TestCase.

    If the test is a member of a sequence of requests, ensure that prior
    tests are run.

    To keep the test harness happy we need to make sure the setUp and
    tearDown are only run once.
    """

    def setUp(self):
        if not self.has_run:
            super(HTTPTestCase, self).setUp()

    def tearDown(self):
        if not self.has_run:
            super(HTTPTestCase, self).tearDown()
        self.has_run = True

    def test_request(self):
        """Run this request if it has not yet run.

        If there is a prior test in the sequence, run it first.
        """
        if self.has_run:
            return
        if self.prior and not self.prior.has_run:
            self.prior.run()
        self._run_test()

    def _assert_response(self, response, content, status, headers=None,
                         expected=None, json_paths=None):
        """Compare the response with expected data."""

        # Never accept a 500
        if response['status'] == '500':
            raise ServerError(content)

        # Always test status
        self._test_status(status, response['status'])

        if headers:
            self._test_headers(headers, response)

        decoded_output = self._decode_content(response, content)

        # Compare strings in response body
        if expected:
            for expect in expected:
                self.assertIn(expect, decoded_output)

        # Decode body as JSON and compare.
        # NOTE(chdent): This just here for now to see if it is workable.
        if json_paths:
            response_data = json.loads(decoded_output)
            self.json_data = response_data
            for path in json_paths:
                match = self._extract_json_path_value(response_data, path)
                self.assertEqual(json_paths[path], match)

    def _decode_content(self, response, content):
        """Decode content to a proper string."""
        content_type = response.get('content-type',
                                    'application/binary').lower()
        if ';' in content_type:
            content_type, charset = (attr.strip() for attr in
                                     content_type.split(';'))
            charset = charset.split('=')[1].strip()
        else:
            charset = 'utf-8'

        if self._not_binary(content_type):
            return content.decode(charset)
        else:
            return content

    @staticmethod
    def _extract_json_path_value(data, path):
        """Extract the value at JSON Path path from the data.

        The input data is a Python datastructre, not a JSON string.
        """
        path_expr = jsonpath_rw.parse(path)
        matches = [match.value for match in path_expr.find(data)]
        return matches[0]

    def _load_data_file(self, filename):
        """Read a file from the current test directory."""
        path = os.path.join(self.test_directory, os.path.basename(filename))
        with open(path, mode='rb') as data_file:
            return data_file.read()

    @staticmethod
    def _not_binary(content_type):
        """Decide if something is content we'd like to treat as a string."""
        return (content_type.startswith('text/')
                or content_type.endswith('+xml')
                or content_type.endswith('+json')
                or content_type == 'application/javascript'
                or content_type == 'application/json')

    def _parse_url(self, url, ssl=False):
        """Create a url from test data.

        If provided with a full URL, just return that. If SSL is requested
        set the scheme appropriately.

        Scheme and netloc are saved for later use in comparisons.
        """
        parsed_url = urlparse.urlsplit(url)
        url_scheme = parsed_url[0]
        scheme = 'http'
        netloc = self.host

        if not url_scheme:
            if self.port:
                netloc = '%s:%s' % (self.host, self.port)
            if ssl:
                scheme = 'https'
            full_url = urlparse.urlunsplit((scheme, netloc, parsed_url[2],
                                            parsed_url[3], ''))
            self.scheme = scheme
            self.netloc = netloc
        else:
            full_url = url
            self.scheme = url_scheme
            self.netloc = parsed_url[1]

        return full_url

    def _replacer(self, match):
        """Replace a regex match with the value of a JSON Path."""
        path = match.group(1)
        return str(self._extract_json_path_value(self.prior.json_data, path))

    def _replace_response_values(self, template):
        """Replace a JSON Path in a template string with its value."""
        return re.sub(r"\$RESPONSE\['([^']+)'\]", self._replacer, template)

    def _run_test(self):
        """Make an HTTP request and compare the response with expectations."""
        test = self.test_data
        http = self.http
        base_url = test['url']

        # Reset follow_redirects with every go.
        http.follow_redirects = False
        if test['redirects']:
            http.follow_redirects = True

        if '$LOCATION' in base_url:
            # Let AttributeError raise
            base_url = base_url.replace('$LOCATION', self.prior.location)

        if '$RESPONSE' in base_url:
            base_url = self._replace_response_values(base_url)

        full_url = self._parse_url(base_url, test['ssl'])
        method = test['method'].upper()
        headers = test['request_headers']

        body = test['data']
        if body:
            body, is_str = self._test_data_to_string(body)
            if self._not_binary(headers['content-type']):
                if is_str:
                    try:
                        body = str(body, 'UTF-8')
                    except TypeError:
                        body = body.encode('UTF-8')
                body = self._replace_response_values(body)

        response, content = http.request(
            full_url,
            method=method,
            headers=headers,
            body=body
        )

        # Set location attribute for follow on requests
        if 'location' in response:
            self.location = response['location']

        self._assert_response(response, content, test['status'],
                              headers=test['response_headers'],
                              expected=test['response_strings'],
                              json_paths=test['response_json_paths'])

    def _test_data_to_string(self, data):
        """Turn the request data into a string."""
        if isinstance(data, str):
            if data.startswith('<@'):
                info = self._load_data_file(data.replace('<@', '', 1))
                return info, True
            else:
                return data, False
        else:
            return json.dumps(data), False

    def _test_headers(self, headers, response):
        """Compare expected headers with actual headers.

        If a header value is wrapped in ``/`` it is treated as a raw
        regular expression.
        """
        for header in headers:
            header_value = headers[header].replace('$SCHEME', self.scheme)
            header_value = header_value.replace('$NETLOC', self.netloc)
            response_value = response[header]
            if header_value.startswith('/') and header_value.endswith('/'):
                header_value = header_value.strip('/').rstrip('/')
                self.assertRegexpMatches(
                    response_value, header_value,
                    'Expect header %s to match /%s/, got %s' %
                    (header, header_value, response_value))
            else:
                self.assertEqual(header_value, response[header],
                                 'Expect header %s with value %s, got %s' %
                                 (header, header_value, response[header]))

    def _test_status(self, expected_status, observed_status):
        """Confirm we got the expected status.

        If the status contains one or more || then it is treated as a
        list of acceptable statuses.
        """
        expected_status = str(expected_status)
        if '||' in expected_status:
            statii = [stat.strip() for stat in expected_status.split('||')]
        else:
            statii = [expected_status.strip()]
        self.assertIn(observed_status, statii)

class ServerError(Exception):
    """A catchall ServerError."""
    pass
<commit_msg>Be more informative when JSONpaths do not match<commit_after># Copyright 2014, 2015 Red Hat
#
# Authors: Chris Dent <chdent@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import json
import os
import re

import jsonpath_rw
import testtools
from six.moves.urllib import parse as urlparse

class HTTPTestCase(testtools.TestCase):
    """Encapsulate a single HTTP request as a TestCase.

    If the test is a member of a sequence of requests, ensure that prior
    tests are run.

    To keep the test harness happy we need to make sure the setUp and
    tearDown are only run once.
    """

    def setUp(self):
        if not self.has_run:
            super(HTTPTestCase, self).setUp()

    def tearDown(self):
        if not self.has_run:
            super(HTTPTestCase, self).tearDown()
        self.has_run = True

    def test_request(self):
        """Run this request if it has not yet run.

        If there is a prior test in the sequence, run it first.
        """
        if self.has_run:
            return
        if self.prior and not self.prior.has_run:
            self.prior.run()
        self._run_test()

    def _assert_response(self, response, content, status, headers=None,
                         expected=None, json_paths=None):
        """Compare the response with expected data."""

        # Never accept a 500
        if response['status'] == '500':
            raise ServerError(content)

        # Always test status
        self._test_status(status, response['status'])

        if headers:
            self._test_headers(headers, response)

        decoded_output = self._decode_content(response, content)

        # Compare strings in response body
        if expected:
            for expect in expected:
                self.assertIn(expect, decoded_output)

        # Decode body as JSON and compare.
        # NOTE(chdent): This just here for now to see if it is workable.
        if json_paths:
            response_data = json.loads(decoded_output)
            self.json_data = response_data
            for path in json_paths:
                match = self._extract_json_path_value(response_data, path)
                self.assertEqual(json_paths[path], match,
                                 'Unable to match %s as %s'
                                 % (path, json_paths[path]))

    def _decode_content(self, response, content):
        """Decode content to a proper string."""
        content_type = response.get('content-type',
                                    'application/binary').lower()
        if ';' in content_type:
            content_type, charset = (attr.strip() for attr in
                                     content_type.split(';'))
            charset = charset.split('=')[1].strip()
        else:
            charset = 'utf-8'

        if self._not_binary(content_type):
            return content.decode(charset)
        else:
            return content

    @staticmethod
    def _extract_json_path_value(data, path):
        """Extract the value at JSON Path path from the data.

        The input data is a Python datastructre, not a JSON string.
        """
        path_expr = jsonpath_rw.parse(path)
        matches = [match.value for match in path_expr.find(data)]
        try:
            return matches[0]
        except IndexError:
            raise ValueError(
                    "JSONPath '%s' failed to match on data: '%s'"
                    % (path, data))

    def _load_data_file(self, filename):
        """Read a file from the current test directory."""
        path = os.path.join(self.test_directory, os.path.basename(filename))
        with open(path, mode='rb') as data_file:
            return data_file.read()

    @staticmethod
    def _not_binary(content_type):
        """Decide if something is content we'd like to treat as a string."""
        return (content_type.startswith('text/')
                or content_type.endswith('+xml')
                or content_type.endswith('+json')
                or content_type == 'application/javascript'
                or content_type == 'application/json')

    def _parse_url(self, url, ssl=False):
        """Create a url from test data.

        If provided with a full URL, just return that. If SSL is requested
        set the scheme appropriately.

        Scheme and netloc are saved for later use in comparisons.
        """
        parsed_url = urlparse.urlsplit(url)
        url_scheme = parsed_url[0]
        scheme = 'http'
        netloc = self.host

        if not url_scheme:
            if self.port:
                netloc = '%s:%s' % (self.host, self.port)
            if ssl:
                scheme = 'https'
            full_url = urlparse.urlunsplit((scheme, netloc, parsed_url[2],
                                            parsed_url[3], ''))
            self.scheme = scheme
            self.netloc = netloc
        else:
            full_url = url
            self.scheme = url_scheme
            self.netloc = parsed_url[1]

        return full_url

    def _replacer(self, match):
        """Replace a regex match with the value of a JSON Path."""
        path = match.group(1)
        return str(self._extract_json_path_value(self.prior.json_data, path))

    def _replace_response_values(self, template):
        """Replace a JSON Path in a template string with its value."""
        return re.sub(r"\$RESPONSE\['([^']+)'\]", self._replacer, template)

    def _run_test(self):
        """Make an HTTP request and compare the response with expectations."""
        test = self.test_data
        http = self.http
        base_url = test['url']

        # Reset follow_redirects with every go.
        http.follow_redirects = False
        if test['redirects']:
            http.follow_redirects = True

        if '$LOCATION' in base_url:
            # Let AttributeError raise
            base_url = base_url.replace('$LOCATION', self.prior.location)

        if '$RESPONSE' in base_url:
            base_url = self._replace_response_values(base_url)

        full_url = self._parse_url(base_url, test['ssl'])
        method = test['method'].upper()
        headers = test['request_headers']

        body = test['data']
        if body:
            body, is_str = self._test_data_to_string(body)
            if self._not_binary(headers['content-type']):
                if is_str:
                    try:
                        body = str(body, 'UTF-8')
                    except TypeError:
                        body = body.encode('UTF-8')
                body = self._replace_response_values(body)

        response, content = http.request(
            full_url,
            method=method,
            headers=headers,
            body=body
        )

        # Set location attribute for follow on requests
        if 'location' in response:
            self.location = response['location']

        self._assert_response(response, content, test['status'],
                              headers=test['response_headers'],
                              expected=test['response_strings'],
                              json_paths=test['response_json_paths'])

    def _test_data_to_string(self, data):
        """Turn the request data into a string."""
        if isinstance(data, str):
            if data.startswith('<@'):
                info = self._load_data_file(data.replace('<@', '', 1))
                return info, True
            else:
                return data, False
        else:
            return json.dumps(data), False

    def _test_headers(self, headers, response):
        """Compare expected headers with actual headers.

        If a header value is wrapped in ``/`` it is treated as a raw
        regular expression.
        """
        for header in headers:
            header_value = headers[header].replace('$SCHEME', self.scheme)
            header_value = header_value.replace('$NETLOC', self.netloc)
            response_value = response[header]
            if header_value.startswith('/') and header_value.endswith('/'):
                header_value = header_value.strip('/').rstrip('/')
                self.assertRegexpMatches(
                    response_value, header_value,
                    'Expect header %s to match /%s/, got %s' %
                    (header, header_value, response_value))
            else:
                self.assertEqual(header_value, response[header],
                                 'Expect header %s with value %s, got %s' %
                                 (header, header_value, response[header]))

    def _test_status(self, expected_status, observed_status):
        """Confirm we got the expected status.

        If the status contains one or more || then it is treated as a
        list of acceptable statuses.
        """
        expected_status = str(expected_status)
        if '||' in expected_status:
            statii = [stat.strip() for stat in expected_status.split('||')]
        else:
            statii = [expected_status.strip()]
        self.assertIn(observed_status, statii)

class ServerError(Exception):
    """A catchall ServerError."""
    pass
<|endoftext|>


<commit_before>from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import RequestContext
from django.db.models import Count
from django.http import HttpResponse
from django.core.urlresolvers import reverse

from poll.models import Poll, PollOption, Vote

from django_twilio.decorators import twilio_view
from twilio.twiml import Response

import json

def poll_default(request):
    try:
        poll = Poll.objects.get(is_active=True)
        return redirect(reverse('poll_results', args=[poll.pk, poll.slug]))
    except Poll.DoesNotExist:
        return redirect('no_active_polls')

def poll_results(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

    # total_votes = Vote.objects.filter(poll_option__poll=poll).count()

    return render_to_response(
        'poll/result.html',
        context_instance=RequestContext(
            request,
            {
                'poll': poll, 'poll_options': poll_options,
                # 'total_votes': total_votes
            }
            )
        )

def poll_refresh(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

    refresh_dict = {'max_votes': poll_options[0].vote_count}
    for po in poll_options:
        refresh_dict[po.title] = po.vote_count

    return HttpResponse(json.dumps(refresh_dict), content_type='application/json')

@twilio_view
def sms_inbound(request):
    r = Response()

    body = request.GET.get('Body')
    sender = request.GET.get('From')

    if body.lower() == 'help':
        r.message(poll.long_title)
        return r

    if body.lower() == 'help':
        r.message(poll.long_title)
        return r

    try:
        poll = Poll.objects.get(is_active=True)

        try:
            poll_option = PollOption.objects.get(poll=poll, title__iexact=body)

            try:
                existing_vote = Vote.objects.get(
                    poll_option__poll=poll, sender_number=sender
                    )

                if existing_vote.poll_option.title.lower() == body.lower():
                    r.message("You've already voted for %s. You're drunk!" % body)
                    return r
                else:
                    existing_vote.poll_option = poll_option
                    existing_vote.save()
                    r.message(u'Your vote has been updated! Cheers. \U0001F37B')
                    return r

            except Vote.DoesNotExist:
                vote = Vote()
                vote.poll_option = poll_option
                vote.sender_number = sender
                vote.save()
                r.message(u'Thanks for your vote! Cheers. \U0001F37B')
                return r

        except PollOption.DoesNotExist:
            r.message('How drunk are you? That option does not exist!')
            return r

    except Poll.DoesNotExist:
        r.message('You must be drunk, there are no open polls!')
        return r

    <commit_msg>Added catch-all response to SMS.<commit_after>from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import RequestContext
from django.db.models import Count
from django.http import HttpResponse
from django.core.urlresolvers import reverse

from poll.models import Poll, PollOption, Vote

from django_twilio.decorators import twilio_view
from twilio.twiml import Response

import json

def poll_default(request):
    try:
        poll = Poll.objects.get(is_active=True)
        return redirect(reverse('poll_results', args=[poll.pk, poll.slug]))
    except Poll.DoesNotExist:
        return redirect('no_active_polls')

def poll_results(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

    # total_votes = Vote.objects.filter(poll_option__poll=poll).count()

    return render_to_response(
        'poll/result.html',
        context_instance=RequestContext(
            request,
            {
                'poll': poll, 'poll_options': poll_options,
                # 'total_votes': total_votes
            }
            )
        )

def poll_refresh(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

    refresh_dict = {'max_votes': poll_options[0].vote_count}
    for po in poll_options:
        refresh_dict[po.title] = po.vote_count

    return HttpResponse(json.dumps(refresh_dict), content_type='application/json')

@twilio_view
def sms_inbound(request):
    r = Response()

    body = request.GET.get('Body')
    sender = request.GET.get('From')

    if body.lower() == 'help':
        r.message(poll.long_title)
        return r

    if body.lower() == 'help':
        r.message(poll.long_title)
        return r

    try:
        poll = Poll.objects.get(is_active=True)

        try:
            poll_option = PollOption.objects.get(poll=poll, title__iexact=body)

            try:
                existing_vote = Vote.objects.get(
                    poll_option__poll=poll, sender_number=sender
                    )

                if existing_vote.poll_option.title.lower() == body.lower():
                    r.message("You've already voted for %s. You're drunk!" % body)
                    return r
                else:
                    existing_vote.poll_option = poll_option
                    existing_vote.save()
                    r.message(u'Your vote has been updated! Cheers. \U0001F37B')
                    return r

            except Vote.DoesNotExist:
                vote = Vote()
                vote.poll_option = poll_option
                vote.sender_number = sender
                vote.save()
                r.message(u'Thanks for your vote! Cheers. \U0001F37B')
                return r

        except PollOption.DoesNotExist:
            r.message('How drunk are you? That option does not exist!')
            return r

    except Poll.DoesNotExist:
        r.message('You must be drunk, there are no open polls!')
        return r

    r.message('What happened?!')
    return r<|endoftext|>


<commit_before># converts vim documentation to html

import sys
import re
import cgi
import urllib

CONTENT_TYPE = "Content-Type: text/html; charset={charset}\n\n"

HEADER1 = """\
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

<title>Vim: {filename}</title>

<link rel="stylesheet" href="vimhelp.css" type="text/css">

"""

START_HEADER = """

Vim help files

This is an HTML version of the Vim help pages. They are kept up-to-date automatically from
the Vim source repository. Also included is the Vim FAQ, kept up to date from its github
repository.

"""

SITENAVI = """

Quick links:
help overview ·
quick reference ·
user manual toc ·
reference manual toc ·
faq

"""

SITESEARCH = """

Loading Google custom search

"""

HEADER2 = """

"""

FOOTER = '
'

FOOTER2 = """

This site is maintained by Carlo Teubner ((my first name) dot (my last name) at gmail dot com).

"""

VIM_FAQ_LINE = '' \
	       'vim_faq.txt   Frequently Asked Questions\n'

RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)')

PAT_HEADER   = r'(?P^.*~$)'
PAT_GRAPHIC  = r'(?P<graphic>^.* `$)'
PAT_PIPEWORD = r'(?P<pipe>(?<!\\)\|[#-)!+-~]+\|)'
PAT_STARWORD = r'(?P<star>\*[#-)!+-~]+\*(?:(?=\s)|$))'
PAT_OPTWORD  = r"(?P<opt>'(?:[a-z]{2,}|t_..)')"
PAT_CTRL     = r'(?P<ctrl>CTRL-(?:W_)?(?:[\w\[\]^+-<>=@]|<[A-Za-z]+?>)?)'
PAT_SPECIAL  = r'(?P<special><.*?>|\{.*?}|' + \
	       r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' + \
	       r'arg|arguments|ident|addr|group)]|' + \
	       r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])'
PAT_TITLE    = r'(?P<title>Vim version [0-9.a-z]+|VIM REFERENCE.*)'
PAT_NOTE     = r'(?P<note>Notes?:?)'
PAT_URL      = r'(?P<url>(?:https?|ftp)://[^\'"<> \t]+[a-zA-Z0-9/])'
PAT_WORD     = r'(?P<word>[!#-)+-{}~]+)'
RE_LINKWORD = re.compile(
	PAT_OPTWORD  + '|' + \
	PAT_CTRL     + '|' + \
	PAT_SPECIAL)
RE_TAGWORD = re.compile(
	PAT_HEADER   + '|' + \
        PAT_GRAPHIC  + '|' + \
	PAT_PIPEWORD + '|' + \
	PAT_STARWORD + '|' + \
	PAT_OPTWORD  + '|' + \
	PAT_CTRL     + '|' + \
	PAT_SPECIAL  + '|' + \
	PAT_TITLE    + '|' + \
	PAT_NOTE     + '|' + \
	PAT_URL      + '|' + \
	PAT_WORD)
RE_NEWLINE   = re.compile(r'[\r\n]')
RE_HRULE     = re.compile(r'[-=]{3,}.*[-=]{3,3}$')
RE_EG_START  = re.compile(r'(?:.* )?>$')
RE_EG_END    = re.compile(r'\S')
RE_SECTION   = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)')
RE_STARTAG   = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)')
RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$')

class Link:
    def __init__(self, link_pipe, link_plain):
	self.link_pipe = link_pipe
	self.link_plain = link_plain

class VimH2H:
    urls = { }

    def __init__(self, tags):
	for line in RE_NEWLINE.split(tags):
	    m = RE_TAGLINE.match(line)
	    if m:
		tag, filename = m.group(1, 2)
		self.do_add_tag(filename, tag)

    def add_tags(self, filename, contents):
	for match in RE_STARTAG.finditer(contents):
	    tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/')
	    self.do_add_tag(filename, tag)

    def do_add_tag(self, filename, tag):
	part1 = '' + cgi.escape(tag) + ''
	link_pipe = part1 + ' class="l"' + part2
	classattr = ' class="d"'
	m = RE_LINKWORD.match(tag)
	if m:
	    opt, ctrl, special = m.group('opt', 'ctrl', 'special')
	    if opt is not None: classattr = ' class="o"'
	    elif ctrl is not None: classattr = ' class="k"'
	    elif special is not None: classattr = ' class="s"'
	link_plain = part1 + classattr + part2
	self.urls[tag] = Link(link_pipe, link_plain)

    def maplink(self, tag, css_class = None):
	links = self.urls.get(tag)
	if links is not None:
	    if css_class == 'l': return links.link_pipe
	    else: return links.link_plain
	elif css_class is not None:
	    return '' + cgi.escape(tag) + \
		    ''
	else: return cgi.escape(tag)

    def to_html(self, filename, contents, encoding = None,
            include_sitesearch = True, include_faq = True):

	out = [ ]

	inexample = 0
	faq_line = False
	for line in RE_NEWLINE.split(contents):
	    line = line.rstrip('\r\n')
	    line_tabs = line
	    line = line.expandtabs()
	    if RE_HRULE.match(line):
		out.append('' + line + '\n')
		continue
	    if inexample == 2:
		if RE_EG_END.match(line):
		    inexample = 0
		    if line[0] == '<': line = line[1:]
		else:
		    out.append('' + cgi.escape(line) +
			    '\n')
		    continue
	    if RE_EG_START.match(line_tabs):
		inexample = 1
		line = line[0:-1]
	    if RE_SECTION.match(line_tabs):
		m = RE_SECTION.match(line)
		out.append(m.expand(r'\g<0>'))
		line = line[m.end():]
	    if filename == 'help.txt' and RE_LOCAL_ADD.match(line_tabs):
		faq_line = True
	    lastpos = 0
	    for match in RE_TAGWORD.finditer(line):
		pos = match.start()
		if pos > lastpos:
		    out.append(cgi.escape(line[lastpos:pos]))
		lastpos = match.end()
		pipeword, starword, opt, ctrl, special, title, note, \
			header, graphic, url, word = \
			match.group('pipe', 'star', 'opt', 'ctrl',
			'special', 'title', 'note', 'header', 'graphic', 'url', 'word')
		if pipeword is not None:
		    out.append(self.maplink(pipeword[1:-1], 'l'))
		elif starword is not None:
		    tag = starword[1:-1]
		    out.append('' + cgi.escape(tag) + '')
		elif opt is not None:
		    out.append(self.maplink(opt, 'o'))
		elif ctrl is not None:
		    out.append(self.maplink(ctrl, 'k'))
		elif special is not None:
		    out.append(self.maplink(special, 's'))
		elif title is not None:
		    out.append('' +
			    cgi.escape(title) + '')
		elif note is not None:
		    out.append('' +
			    cgi.escape(note) + '')
		elif header is not None:
		    out.append('' +
			    cgi.escape(header[:-1]) + '')
                elif graphic is not None:
                    out.append(cgi.escape(graphic[:-2]))
		elif url is not None:
		    out.append('' +
			    cgi.escape(url) + '')
		elif word is not None:
		    out.append(self.maplink(word))
	    if lastpos < len(line):
		out.append(cgi.escape(line[lastpos:]))
	    out.append('\n')
	    if inexample == 1: inexample = 2
	    if faq_line:
		out.append(VIM_FAQ_LINE)
		faq_line = False

        return (CONTENT_TYPE.replace('{charset}', encoding) if encoding else '') + \
	        HEADER1.replace('{filename}', filename). \
                       replace('{charset}', encoding) + \
		(START_HEADER if filename == 'help.txt' else '') + \
		SITENAVI + \
		(SITESEARCH if include_sitesearch else '') + \
		HEADER2 + \
		''.join(out) + \
		FOOTER + \
		SITENAVI + \
		FOOTER2

<commit_msg>Fix a traceback when encoding is False.<commit_after># converts vim documentation to html

import sys
import re
import cgi
import urllib

CONTENT_TYPE = "Content-Type: text/html; charset={charset}\n\n"

HEADER1 = """\
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

<title>Vim: {filename}</title>

<link rel="stylesheet" href="vimhelp.css" type="text/css">

"""

START_HEADER = """

Vim help files

This is an HTML version of the Vim help pages. They are kept up-to-date automatically from
the Vim source repository. Also included is the Vim FAQ, kept up to date from its github
repository.

"""

SITENAVI = """

Quick links:
help overview ·
quick reference ·
user manual toc ·
reference manual toc ·
faq

"""

SITESEARCH = """

Loading Google custom search

"""

HEADER2 = """

"""

FOOTER = '
'

FOOTER2 = """

This site is maintained by Carlo Teubner ((my first name) dot (my last name) at gmail dot com).

"""

VIM_FAQ_LINE = '' \
	       'vim_faq.txt   Frequently Asked Questions\n'

RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)')

PAT_HEADER   = r'(?P^.*~$)'
PAT_GRAPHIC  = r'(?P<graphic>^.* `$)'
PAT_PIPEWORD = r'(?P<pipe>(?<!\\)\|[#-)!+-~]+\|)'
PAT_STARWORD = r'(?P<star>\*[#-)!+-~]+\*(?:(?=\s)|$))'
PAT_OPTWORD  = r"(?P<opt>'(?:[a-z]{2,}|t_..)')"
PAT_CTRL     = r'(?P<ctrl>CTRL-(?:W_)?(?:[\w\[\]^+-<>=@]|<[A-Za-z]+?>)?)'
PAT_SPECIAL  = r'(?P<special><.*?>|\{.*?}|' + \
	       r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' + \
	       r'arg|arguments|ident|addr|group)]|' + \
	       r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])'
PAT_TITLE    = r'(?P<title>Vim version [0-9.a-z]+|VIM REFERENCE.*)'
PAT_NOTE     = r'(?P<note>Notes?:?)'
PAT_URL      = r'(?P<url>(?:https?|ftp)://[^\'"<> \t]+[a-zA-Z0-9/])'
PAT_WORD     = r'(?P<word>[!#-)+-{}~]+)'
RE_LINKWORD = re.compile(
	PAT_OPTWORD  + '|' + \
	PAT_CTRL     + '|' + \
	PAT_SPECIAL)
RE_TAGWORD = re.compile(
	PAT_HEADER   + '|' + \
        PAT_GRAPHIC  + '|' + \
	PAT_PIPEWORD + '|' + \
	PAT_STARWORD + '|' + \
	PAT_OPTWORD  + '|' + \
	PAT_CTRL     + '|' + \
	PAT_SPECIAL  + '|' + \
	PAT_TITLE    + '|' + \
	PAT_NOTE     + '|' + \
	PAT_URL      + '|' + \
	PAT_WORD)
RE_NEWLINE   = re.compile(r'[\r\n]')
RE_HRULE     = re.compile(r'[-=]{3,}.*[-=]{3,3}$')
RE_EG_START  = re.compile(r'(?:.* )?>$')
RE_EG_END    = re.compile(r'\S')
RE_SECTION   = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)')
RE_STARTAG   = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)')
RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$')

class Link:
    def __init__(self, link_pipe, link_plain):
	self.link_pipe = link_pipe
	self.link_plain = link_plain

class VimH2H:
    urls = { }

    def __init__(self, tags):
	for line in RE_NEWLINE.split(tags):
	    m = RE_TAGLINE.match(line)
	    if m:
		tag, filename = m.group(1, 2)
		self.do_add_tag(filename, tag)

    def add_tags(self, filename, contents):
	for match in RE_STARTAG.finditer(contents):
	    tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/')
	    self.do_add_tag(filename, tag)

    def do_add_tag(self, filename, tag):
	part1 = '' + cgi.escape(tag) + ''
	link_pipe = part1 + ' class="l"' + part2
	classattr = ' class="d"'
	m = RE_LINKWORD.match(tag)
	if m:
	    opt, ctrl, special = m.group('opt', 'ctrl', 'special')
	    if opt is not None: classattr = ' class="o"'
	    elif ctrl is not None: classattr = ' class="k"'
	    elif special is not None: classattr = ' class="s"'
	link_plain = part1 + classattr + part2
	self.urls[tag] = Link(link_pipe, link_plain)

    def maplink(self, tag, css_class = None):
	links = self.urls.get(tag)
	if links is not None:
	    if css_class == 'l': return links.link_pipe
	    else: return links.link_plain
	elif css_class is not None:
	    return '' + cgi.escape(tag) + \
		    ''
	else: return cgi.escape(tag)

    def to_html(self, filename, contents, encoding = None,
            include_sitesearch = True, include_faq = True):

	out = [ ]

	inexample = 0
	faq_line = False
	for line in RE_NEWLINE.split(contents):
	    line = line.rstrip('\r\n')
	    line_tabs = line
	    line = line.expandtabs()
	    if RE_HRULE.match(line):
		out.append('' + line + '\n')
		continue
	    if inexample == 2:
		if RE_EG_END.match(line):
		    inexample = 0
		    if line[0] == '<': line = line[1:]
		else:
		    out.append('' + cgi.escape(line) +
			    '\n')
		    continue
	    if RE_EG_START.match(line_tabs):
		inexample = 1
		line = line[0:-1]
	    if RE_SECTION.match(line_tabs):
		m = RE_SECTION.match(line)
		out.append(m.expand(r'\g<0>'))
		line = line[m.end():]
	    if filename == 'help.txt' and RE_LOCAL_ADD.match(line_tabs):
		faq_line = True
	    lastpos = 0
	    for match in RE_TAGWORD.finditer(line):
		pos = match.start()
		if pos > lastpos:
		    out.append(cgi.escape(line[lastpos:pos]))
		lastpos = match.end()
		pipeword, starword, opt, ctrl, special, title, note, \
			header, graphic, url, word = \
			match.group('pipe', 'star', 'opt', 'ctrl',
			'special', 'title', 'note', 'header', 'graphic', 'url', 'word')
		if pipeword is not None:
		    out.append(self.maplink(pipeword[1:-1], 'l'))
		elif starword is not None:
		    tag = starword[1:-1]
		    out.append('' + cgi.escape(tag) + '')
		elif opt is not None:
		    out.append(self.maplink(opt, 'o'))
		elif ctrl is not None:
		    out.append(self.maplink(ctrl, 'k'))
		elif special is not None:
		    out.append(self.maplink(special, 's'))
		elif title is not None:
		    out.append('' +
			    cgi.escape(title) + '')
		elif note is not None:
		    out.append('' +
			    cgi.escape(note) + '')
		elif header is not None:
		    out.append('' +
			    cgi.escape(header[:-1]) + '')
                elif graphic is not None:
                    out.append(cgi.escape(graphic[:-2]))
		elif url is not None:
		    out.append('' +
			    cgi.escape(url) + '')
		elif word is not None:
		    out.append(self.maplink(word))
	    if lastpos < len(line):
		out.append(cgi.escape(line[lastpos:]))
	    out.append('\n')
	    if inexample == 1: inexample = 2
	    if faq_line:
		out.append(VIM_FAQ_LINE)
		faq_line = False

        return (CONTENT_TYPE.replace('{charset}', encoding) if encoding else '') + \
	        HEADER1.replace('{filename}', filename) + \
		(START_HEADER if filename == 'help.txt' else '') + \
		SITENAVI + \
		(SITESEARCH if include_sitesearch else '') + \
		HEADER2 + \
		''.join(out) + \
		FOOTER + \
		SITENAVI + \
		FOOTER2

<|endoftext|>


<commit_before>from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import RequestContext
from django.db.models import Count
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from django.conf import settings

from poll.models import Poll, PollOption, Vote

from django_twilio.decorators import twilio_view
from twilio.twiml import Response

import json

def poll_default(request):
    try:
        poll = Poll.objects.get(is_active=True)
        return redirect(reverse('poll_results', args=[poll.pk, poll.slug]))
    except Poll.DoesNotExist:
        return redirect('no_active_polls')

def poll_results(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote'))

    max_vote = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')[0].vote_count

    # total_votes = Vote.objects.filter(poll_option__poll=poll).count()

    return render_to_response(
        'poll/result.html',
        context_instance=RequestContext(
            request,
            {
                'poll': poll, 'poll_options': poll_options,
                'max_vote': max_vote, 'sms_number': settings.VOTE_SMS_NUMBER
            }
            )
        )

def poll_refresh(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

    refresh_dict = {'max_votes': poll_options[0].vote_count}
    for po in poll_options:
        refresh_dict[po.title] = po.vote_count

    return HttpResponse(json.dumps(refresh_dict), content_type='application/json')

def sms_response_success(message):
    r = Response()
    r.message(u"%s\n\nFor more information reply to this SMS with POLLHELP." % message)
    return r

def sms_response_error(message):
    r = Response()
    r.message(u"%s\n\nFor more information reply to this SMS with POLLHELP." % message)
    return r

def sms_response_help(poll_title, poll_options):
    r = Response()
    r.message(
        u"%s\nOptions: %s.\n\nReply to this SMS with your vote!\n\nTo see the results text RESULTS.\n\nTo remove a vote text UNVOTE. \U0001F37A" % (
            poll_title, poll_options
            )
        )
    return r

# TWilio uses some words as commands to subscribe/unsubscribe people from the
# service, if one of these comes through we will want to ignore it.
# See here fore more info: https://www.twilio.com/help/faq/sms/does-twilio-support-stop-block-and-cancel-aka-sms-filtering
twilio_reserved_words = [
    'STOP', 'STOPALL', 'UNSUBSCRIBE', 'CANCEL', 'END', 'START', 'YES', 'HELP',
    'INFO',
    ]

@twilio_view
def sms_inbound(request):
    body = request.POST.get('Body', '')
    sender = request.POST.get('From', '')

    # Ignore any bodies with Twilio commands.
    if body.upper() in twilio_reserved_words:
        return Response()

    try:
        poll = Poll.objects.get(is_active=True)

        if body.lower() == 'pollhelp':

            poll_options = ', '.join(
                p.title for p in PollOption.objects.filter(poll=poll)
                )

            return sms_response_help(
                poll_title=poll.long_title, poll_options=poll_options
                )

        if body.lower() == 'results' or body.lower() == 'result':
            votes = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

            votes_text = '\n'.join(v.title + ': ' + unicode(v.vote_count) for v in votes)

            return sms_response_success(
                u"Results for: \"%s\"\n\n%s" % (poll.long_title, votes_text)
                )

        if body.lower() == 'unvote':
            try:
                vote = Vote.objects.get(
                    poll_option__poll=poll, sender_number=sender
                    )
                vote.delete()

                return sms_response_success(
                    u"Your vote has been removed! \U0001F631"
                    )

            except Vote.DoesNotExist:
                return sms_response_error(
                    u"Clearly you're drunk - you can't unvote if you haven't even voted yet! \U0001F632"
                    )

        try:
            poll_option = PollOption.objects.get(poll=poll, title__iexact=body)

            try:
                existing_vote = Vote.objects.get(
                    poll_option__poll=poll, sender_number=sender
                    )

                if existing_vote.poll_option.title.lower() == body.lower():
                    return sms_response_error(
                        u"You've already voted for %s. You're drunk! \U0001F632" % body
                        )
                else:
                    existing_vote.poll_option = poll_option
                    existing_vote.save()

                    return sms_response_success(
                        u"Your vote has been updated! Cheers. \U0001F37B"
                        )

            except Vote.DoesNotExist:
                vote = Vote()
                vote.poll_option = poll_option
                vote.sender_number = sender
                vote.save()

                return sms_response_success(
                    u"Thanks for your vote! Cheers. \U0001F37B"
                    )

        except PollOption.DoesNotExist:
            return sms_response_error(
                u"How drunk are you? That option does not exist! \U0001F632"
                )

    except Poll.DoesNotExist:
        return sms_response_error(
            u"You must be drunk, there are no open polls! \U0001F632"
            )

    return sms_response_error(
        u"What happened?! \U0001F632"
        )<commit_msg>Order poll options by title.<commit_after>from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import RequestContext
from django.db.models import Count
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from django.conf import settings

from poll.models import Poll, PollOption, Vote

from django_twilio.decorators import twilio_view
from twilio.twiml import Response

import json

def poll_default(request):
    try:
        poll = Poll.objects.get(is_active=True)
        return redirect(reverse('poll_results', args=[poll.pk, poll.slug]))
    except Poll.DoesNotExist:
        return redirect('no_active_polls')

def poll_results(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('title')

    max_vote = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')[0].vote_count

    # total_votes = Vote.objects.filter(poll_option__poll=poll).count()

    return render_to_response(
        'poll/result.html',
        context_instance=RequestContext(
            request,
            {
                'poll': poll, 'poll_options': poll_options,
                'max_vote': max_vote, 'sms_number': settings.VOTE_SMS_NUMBER
            }
            )
        )

def poll_refresh(request, poll_id, poll_slug):
    poll = get_object_or_404(Poll, pk=poll_id)

    poll_options = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

    refresh_dict = {'max_votes': poll_options[0].vote_count}
    for po in poll_options:
        refresh_dict[po.title] = po.vote_count

    return HttpResponse(json.dumps(refresh_dict), content_type='application/json')

def sms_response_success(message):
    r = Response()
    r.message(u"%s\n\nFor more information reply to this SMS with POLLHELP." % message)
    return r

def sms_response_error(message):
    r = Response()
    r.message(u"%s\n\nFor more information reply to this SMS with POLLHELP." % message)
    return r

def sms_response_help(poll_title, poll_options):
    r = Response()
    r.message(
        u"%s\nOptions: %s.\n\nReply to this SMS with your vote!\n\nTo see the results text RESULTS.\n\nTo remove a vote text UNVOTE. \U0001F37A" % (
            poll_title, poll_options
            )
        )
    return r

# TWilio uses some words as commands to subscribe/unsubscribe people from the
# service, if one of these comes through we will want to ignore it.
# See here fore more info: https://www.twilio.com/help/faq/sms/does-twilio-support-stop-block-and-cancel-aka-sms-filtering
twilio_reserved_words = [
    'STOP', 'STOPALL', 'UNSUBSCRIBE', 'CANCEL', 'END', 'START', 'YES', 'HELP',
    'INFO',
    ]

@twilio_view
def sms_inbound(request):
    body = request.POST.get('Body', '')
    sender = request.POST.get('From', '')

    # Ignore any bodies with Twilio commands.
    if body.upper() in twilio_reserved_words:
        return Response()

    try:
        poll = Poll.objects.get(is_active=True)

        if body.lower() == 'pollhelp':

            poll_options = ', '.join(
                p.title for p in PollOption.objects.filter(poll=poll)
                )

            return sms_response_help(
                poll_title=poll.long_title, poll_options=poll_options
                )

        if body.lower() == 'results' or body.lower() == 'result':
            votes = PollOption.objects.select_related().filter(poll=poll).annotate(vote_count=Count('vote')).order_by('-vote_count')

            votes_text = '\n'.join(v.title + ': ' + unicode(v.vote_count) for v in votes)

            return sms_response_success(
                u"Results for: \"%s\"\n\n%s" % (poll.long_title, votes_text)
                )

        if body.lower() == 'unvote':
            try:
                vote = Vote.objects.get(
                    poll_option__poll=poll, sender_number=sender
                    )
                vote.delete()

                return sms_response_success(
                    u"Your vote has been removed! \U0001F631"
                    )

            except Vote.DoesNotExist:
                return sms_response_error(
                    u"Clearly you're drunk - you can't unvote if you haven't even voted yet! \U0001F632"
                    )

        try:
            poll_option = PollOption.objects.get(poll=poll, title__iexact=body)

            try:
                existing_vote = Vote.objects.get(
                    poll_option__poll=poll, sender_number=sender
                    )

                if existing_vote.poll_option.title.lower() == body.lower():
                    return sms_response_error(
                        u"You've already voted for %s. You're drunk! \U0001F632" % body
                        )
                else:
                    existing_vote.poll_option = poll_option
                    existing_vote.save()

                    return sms_response_success(
                        u"Your vote has been updated! Cheers. \U0001F37B"
                        )

            except Vote.DoesNotExist:
                vote = Vote()
                vote.poll_option = poll_option
                vote.sender_number = sender
                vote.save()

                return sms_response_success(
                    u"Thanks for your vote! Cheers. \U0001F37B"
                    )

        except PollOption.DoesNotExist:
            return sms_response_error(
                u"How drunk are you? That option does not exist! \U0001F632"
                )

    except Poll.DoesNotExist:
        return sms_response_error(
            u"You must be drunk, there are no open polls! \U0001F632"
            )

    return sms_response_error(
        u"What happened?! \U0001F632"
        )<|endoftext|>


<commit_before>"""Plots spatially subset model evaluation."""

import os.path
import warnings
import argparse
import numpy
import matplotlib
matplotlib.use('agg')
from matplotlib import pyplot
from gewittergefahr.gg_utils import grids
from gewittergefahr.gg_utils import model_evaluation as model_eval
from gewittergefahr.gg_utils import file_system_utils

INPUT_DIR_ARG_NAME = 'input_dir_name'
OUTPUT_DIR_ARG_NAME = 'output_dir_name'

INPUT_DIR_HELP_STRING = (
    'Name of input directory.  Evaluation files therein will be found by '
    '`model_evaluation.find_file` and read by '
    '`model_evaluation.read_evaluation`.')

OUTPUT_DIR_HELP_STRING = (
    'Name of output directory.  Figures will be saved here.')

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + INPUT_DIR_ARG_NAME, type=str, required=True,
    help=INPUT_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_ARG_NAME, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)

def _run(evaluation_dir_name, output_dir_name):
    """Plots spatially subset model evaluation.

    This is effectively the main method.

    :param evaluation_dir_name: See documentation at top of file.
    :param output_dir_name: Same.
    """

    grid_metafile_name = grids.find_equidistant_metafile(
        directory_name=evaluation_dir_name, raise_error_if_missing=True)

    print('Reading grid metadata from: "{0:s}"...'.format(grid_metafile_name))
    grid_metadata_dict = grids.read_equidistant_metafile(grid_metafile_name)

    projection_object = grid_metadata_dict[grids.PROJECTION_KEY]
    print(dir(projection_object))
    print(projection_object.srs)

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    _run(
        evaluation_dir_name=getattr(INPUT_ARG_OBJECT, INPUT_DIR_ARG_NAME),
        output_dir_name=getattr(INPUT_ARG_OBJECT, OUTPUT_DIR_ARG_NAME)
    )
<commit_msg>Trying to fix stupid bug in plot_spatially_subset_eval.py.<commit_after>"""Plots spatially subset model evaluation."""

import os.path
import warnings
import argparse
import numpy
import matplotlib
matplotlib.use('agg')
from matplotlib import pyplot
from gewittergefahr.gg_utils import grids
from gewittergefahr.gg_utils import model_evaluation as model_eval
from gewittergefahr.gg_utils import file_system_utils

LAMBERT_CONFORMAL_STRING = 'lcc'

INPUT_DIR_ARG_NAME = 'input_dir_name'
OUTPUT_DIR_ARG_NAME = 'output_dir_name'

INPUT_DIR_HELP_STRING = (
    'Name of input directory.  Evaluation files therein will be found by '
    '`model_evaluation.find_file` and read by '
    '`model_evaluation.read_evaluation`.')

OUTPUT_DIR_HELP_STRING = (
    'Name of output directory.  Figures will be saved here.')

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + INPUT_DIR_ARG_NAME, type=str, required=True,
    help=INPUT_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_ARG_NAME, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)

def _get_lcc_params(projection_object):
    """Finds parameters for LCC (Lambert conformal conic) projection.

    :param projection_object: Instance of `pyproj.Proj`.
    :return: standard_latitudes_deg: length-2 numpy array of standard latitudes
        (deg N).
    :return: central_longitude_deg: Central longitude (deg E).
    :raises: ValueError: if projection is not LCC.
    """

    projection_string = projection_object.srs
    words = projection_string.split()

    property_names = [w.split('=')[0][1:] for w in words]
    property_values = [w.split('=')[1] for w in words]
    projection_dict = dict(list(
        zip(property_names, property_values)
    ))

    if projection_dict['proj'] != LAMBERT_CONFORMAL_STRING:
        error_string = 'Grid projection should be "{0:s}", not "{1:s}".'.format(
            LAMBERT_CONFORMAL_STRING, projection_dict['proj']
        )

        raise ValueError(error_string)

    central_longitude_deg = float(projection_dict['lon_0'])
    standard_latitudes_deg = numpy.array([
        float(projection_dict['lat_1']), float(projection_dict['lat_2'])
    ])

    return standard_latitudes_deg, central_longitude_deg

def _run(evaluation_dir_name, output_dir_name):
    """Plots spatially subset model evaluation.

    This is effectively the main method.

    :param evaluation_dir_name: See documentation at top of file.
    :param output_dir_name: Same.
    """

    grid_metafile_name = grids.find_equidistant_metafile(
        directory_name=evaluation_dir_name, raise_error_if_missing=True)

    print('Reading grid metadata from: "{0:s}"...'.format(grid_metafile_name))
    grid_metadata_dict = grids.read_equidistant_metafile(grid_metafile_name)

    standard_latitudes_deg, central_longitude_deg = _get_lcc_params(
        grid_metadata_dict[grids.PROJECTION_KEY]
    )

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    _run(
        evaluation_dir_name=getattr(INPUT_ARG_OBJECT, INPUT_DIR_ARG_NAME),
        output_dir_name=getattr(INPUT_ARG_OBJECT, OUTPUT_DIR_ARG_NAME)
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2011 Map Of Life
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from osgeo import gdal
import Queue
import datetime
import os
import shutil
import simplejson
import subprocess
import threading
import urllib
import urllib2
from urllib2 import URLError, HTTPError

GAE_URL = "http://localhost:8080/"

worker_q = Queue.Queue()

class BulkLoadTiles():
    """class for running the bulkloader to upload tilesets to GAE"""
    def __init__(self, id=None):
        self.id = id

    def uploadTiles(self):
        pass

class Layer():
    zoom = 1 #sets the maximum zoom we want to process
    info = {}
    errors = []
    converted = False
    tiled = False
    tileDir = "/ftp/tiles/" #/some/tmp/folder/for/tiles/
    ascDir = "/ftp/asc/" #/some/tmp/folder/for/asc/
    errDir = "/ftp/errors/"
    srcDir = "/ftp/newraster/"
    dstDir = "/ftp/grid/"
    idIsValid = False

    def __init__(self, fullpath=None):

        """raster: string filename of file to process"""
        print 'FULLPATH: %s' % fullpath
        (dirname, filename) = os.path.split(fullpath)
        (basename, fileext) = os.path.splitext(filename)
        if filename is not None:
            self.origRaster = fullpath
            self.id = basename
            self.tileFolder = self.tileDir + self.id
            self.ascName = self.ascDir + "%s.asc" % self.id
            self.nulfp = open(self.errDir + '%s.log' % self.id, 'w')
            print self.__dict__

    def verifyId(self):
        """Verifies the layer id using a web service on GAE that returns 200 for
        a valid id and a 404 for invalid id.
        """
        params = {'id': self.id}
        resource = "%sapi/validid?%s" % (GAE_URL, urllib.urlencode(params))
        try:
            urllib2.urlopen(resource)
            return True
        except HTTPError as e:
            print 'URLError: %s' % e.code
            return False

    def getInfo(self, fn):
        #use gdalinfo= to populate an info object
        layer = gdal.Open(fn)
        self.info['id'] = self.id
        self.info['zoom'] = self.zoom
        self.info['date'] = datetime.datetime.now()
        self.info['proj'] = layer.GetProjection()
        geog = layer.GetGeoTransform()
        #temp holder, should parse from the geog object above
        self.info['geog'] = {'maxLat': 1.4,
                             'minLat': 1.2,
                             'maxLon': 1.4,
                             'minLon': 1.2}
        return True

    def convertToASC(self):
        #create a geotiff
        self.translating = subprocess.Popen(
            ["gdal_translate",
            "-of",
            "AAIGrid",
            "-a_srs",
             "epsg:900913",
             self.origRaster,
             self.ascName
            ], stderr=self.nulfp)
        self.translating.wait()
        self.converted = True
        self.getInfo(self.ascName)

    def tile(self):
        if not self.converted:
            self.convertToASC()

        self.tiling = subprocess.Popen(
            ["java",
            "-mx300m",
            "-classpath",
            "/raster/classes:/raster/lib/maxent.jar",
            "-Djava.awt.headless=true",
            "raster/GridToGoogle",
            self.ascName,
            self.tileFolder,
            str(self.zoom + 1)
            ], stderr=self.nulfp)
        self.tiling.wait()

    def registerMetadata(self):
        #send metadata to GAE
        params = {'id': self.id,
                  'zoom': self.zoom,
                  'proj': self.info['proj'],
                  'date': str(datetime.datetime.now()),
                  'maxLat': str(self.info['geog']['maxLat']),
                  'minLat': str(self.info['geog']['minLat']),
                  'maxLon': self.info['geog']['maxLon'],
                  'minLon': self.info['geog']['minLon'],
                  'remoteLocation': 'http://mol.colorado.edu/tiles/%s/zoom/x/y.png' % self.id}
        resource = urllib2.Request("%sapi/layer/update" % GAE_URL, urllib.urlencode(params))
        response = urllib2.urlopen(resource)
        out = response.read()
        # TODO: Log out and response

    def storeTiles(self):
        #TODO: store tiles in couchdb
        return True

    def cleanup(self):
        files = [self.ascName,
                 self.ascName.replace('.asc', '.prj')]
        for file in files:
            try:
                os.remove(file)
            except:
                pass
        try:
            shutil.rmtree(self.dstDir + self.id)
        except:
            pass
        try:
            shutil.copytree(self.origRaster, self.dstDir + self.id)
        except:
            pass
        try:
            shutil.rmtree(self.srcDir + self.id)
        except:
            pass

class LayerProcessingThread(threading.Thread):
    def run(self):
        print 'Worker thread is running.'

        while True:
            data = worker_q.get()
            if data['jobtype'] == 'newraster':
                fullpath = data['fullpath']
                try:
                    # do task
                    layer = Layer(fullpath=fullpath)
                    layer.convertToASC()
                    layer.tile()
                    layer.registerMetadata()
                    layer.cleanup()
                    print 'We got %s tiles, do something with them!' % (fullpath)
                except Exception, e:
                    print 'Unable to process in worker thread: ' + str(e)
                worker_q.task_done()

            elif data['jobtype'] == 'bulkload-tiles':
                """run the BulkLoadTiles class above"""
                pass

def start_myworker():
    worker = LayerProcessingThread()
    worker.start()

<commit_msg>Modified to use new mol.service.Layer class<commit_after>#!/usr/bin/env python
#
# Copyright 2011 Map Of Life
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from mol.service import Layer, LayerError
import Queue
import threading

TILE_DIR = "/ftp/tiles/"
ASC_DIR = "/ftp/asc/"
ERR_DIR = "/ftp/errors/"
SRC_DIR = "/ftp/newraster/"
DST_DIR = "/ftp/grid/"

NEW_RASTER_JOB_TYPE = 'newraster'
BULKLOAD_TILES_JOB_TYPE = 'bulkload-tiles'
Q_ITEM_FULL_PATH = 'fullpath'
Q_ITEM_JOB_TYPE = 'jobtype'

worker_q = Queue.Queue()

class BulkLoadTiles():
    """class for running the bulkloader to upload tilesets to GAE"""
    def __init__(self, id=None):
        self.id = id

    def uploadTiles(self):
        raise NotImplementedError()

class LayerProcessingThread(threading.Thread):

    def run(self):
        """Pulls tasks from the queue and dispatches based on job type."""
        while True:
            task = worker_q.get()
            jobtype = task[Q_ITEM_JOB_TYPE]
            if jobtype == NEW_RASTER_JOB_TYPE:
                self.newraster(task)
            elif jobtype == BULKLOAD_TILES_JOB_TYPE:
                raise NotImplementedError()

    def newraster(self, task):
        """Tiles an asc file speicified in the task and registers metadata with
        GAE.

        Arguments:
            task - an item from the queue expected to have acs path
        """
        # Validates task:
        if task is None:
            # TODO
            return
        if not task.has_key(Q_ITEM_FULL_PATH):
            # TODO
            return
        fullpath = task[Q_ITEM_FULL_PATH]
        if fullpath is None or len(fullpath.strip()) == 0:
            # TODO
            return

        # Executes the job
        try:
            layer = Layer(fullpath, TILE_DIR, ASC_DIR, ERR_DIR, SRC_DIR, DST_DIR)
            layer.toasc()
            layer.totiles()
            layer.register()
            layer.cleanup()
        except LayerError as e:
            # TODO
            print 'LayerError: ' + e.msg
        except Exception as e:
            # TODO
            print 'Exception: ' + str(e)

        # Notifies queue that this formerly enqueued task is complete:
        worker_q.task_done()

def start_myworker():
    worker = LayerProcessingThread()
    worker.start()
<|endoftext|>


<commit_before>import html
import re
import sys
import logging

import plugin
from utils import url_parser, auto_requests

class Titlegiver(plugin.Plugin):

    TITLE_REGEX = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
    WHITESPACE_REGEX = re.compile(r'\s+')

    MAX_CONTENT_LENGTH = 64 * 1024
    USER_AGENT = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/43.0.2357.37 Safari/537.36")

    def __init__(self):
        plugin.Plugin.__init__(self, "titlegiver")

    @staticmethod
    def get_title_from_url(url):
        # Fetch page (no need to verfiy SSL certs for titles)
        response = auto_requests.get(url, verify=False, headers={"User-Agent": Titlegiver.USER_AGENT})
        content = response.text[:Titlegiver.MAX_CONTENT_LENGTH]

        # Avoid leaving dangling redirects when we've got the content
        response.connection.close()

        return Titlegiver.find_title_in_content(content).strip()

    @staticmethod
    def find_title_in_content(text):
        try:
            title = Titlegiver.WHITESPACE_REGEX.sub(" ", Titlegiver.TITLE_REGEX.search(text).group(1))
            return Titlegiver.unescape_entities(title)
        except:
            logging.exception("Regexp or unescape failed")
            return None

    @staticmethod
    def unescape_entities(text):
        def replace_entity(match):
            try:
                if match.group(1) in html.entities.name2codepoint:
                    return chr(html.entities.name2codepoint[match.group(1)])
                elif match.group(1).lower().startswith("#x"):
                    return chr(int(match.group(1)[2:], 16))
                elif match.group(1).startswith("#"):
                    return chr(int(match.group(1)[1:]))
            except (ValueError, KeyError):
                pass  # Fall through to default return
            return match.group(0)

        return re.sub(r'&([#a-zA-Z0-9]+);', replace_entity, text)

    def on_pubmsg(self, server, user, channel, message):
        for url in url_parser.find_urls(message):
            try:
                self.privmsg(server, channel, Titlegiver.get_title_from_url(url))
            except:
                logging.exception("Unable to find title for: %s", url)

if __name__ == "__main__":
    sys.exit(Titlegiver.run())
<commit_msg>Set accept-language header<commit_after>import html
import re
import sys
import logging

import plugin
from utils import url_parser, auto_requests

class Titlegiver(plugin.Plugin):

    TITLE_REGEX = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
    WHITESPACE_REGEX = re.compile(r'\s+')

    MAX_CONTENT_LENGTH = 64 * 1024
    USER_AGENT = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/43.0.2357.37 Safari/537.36")

    def __init__(self):
        plugin.Plugin.__init__(self, "titlegiver")

    @staticmethod
    def get_title_from_url(url):
        # Fetch page (no need to verfiy SSL certs for titles)
        response = auto_requests.get(url, verify=False, headers={"User-Agent": Titlegiver.USER_AGENT, "Accept-Language": "en_US"})
        content = response.text[:Titlegiver.MAX_CONTENT_LENGTH]

        # Avoid leaving dangling redirects when we've got the content
        response.connection.close()

        return Titlegiver.find_title_in_content(content).strip()

    @staticmethod
    def find_title_in_content(text):
        try:
            title = Titlegiver.WHITESPACE_REGEX.sub(" ", Titlegiver.TITLE_REGEX.search(text).group(1))
            return Titlegiver.unescape_entities(title)
        except:
            logging.exception("Regexp or unescape failed")
            return None

    @staticmethod
    def unescape_entities(text):
        def replace_entity(match):
            try:
                if match.group(1) in html.entities.name2codepoint:
                    return chr(html.entities.name2codepoint[match.group(1)])
                elif match.group(1).lower().startswith("#x"):
                    return chr(int(match.group(1)[2:], 16))
                elif match.group(1).startswith("#"):
                    return chr(int(match.group(1)[1:]))
            except (ValueError, KeyError):
                pass  # Fall through to default return
            return match.group(0)

        return re.sub(r'&([#a-zA-Z0-9]+);', replace_entity, text)

    def on_pubmsg(self, server, user, channel, message):
        for url in url_parser.find_urls(message):
            try:
                self.privmsg(server, channel, Titlegiver.get_title_from_url(url))
            except:
                logging.exception("Unable to find title for: %s", url)

if __name__ == "__main__":
    sys.exit(Titlegiver.run())
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

'''
Helper utilities related to the AWS API implementations
'''

import re
import itertools

def format_response(action, response):
    """
    Format response from engine into API format
    """
    return {'%sResponse' % action: {'%sResult' % action: response}}

def extract_param_pairs(params, prefix='', keyname='', valuename=''):
    """
    Extract a dictionary of user input parameters, from AWS style
    parameter-pair encoded list

    In the AWS API list items appear as two key-value
    pairs (passed as query parameters)  with keys of the form below:

    Prefix.member.1.keyname=somekey
    Prefix.member.1.keyvalue=somevalue
    Prefix.member.2.keyname=anotherkey
    Prefix.member.2.keyvalue=somevalue

    We reformat this into a dict here to match the heat
    engine API expected format
    """
    plist = extract_param_list(params, prefix)
    kvs = [(p[keyname], p[valuename]) for p in plist
                                      if keyname in p and valuename in p]

    return dict(kvs)

def extract_param_list(params, prefix=''):
    """
    Extract a list-of-dicts based on parameters containing AWS style list

    MetricData.member.1.MetricName=buffers
    MetricData.member.1.Unit=Bytes
    MetricData.member.1.Value=231434333
    MetricData.member.2.MetricName=buffers2
    MetricData.member.2.Unit=Bytes
    MetricData.member.2.Value=12345

    This can be extracted by passing prefix=MetricData, resulting in a
    list containing two dicts
    """

    key_re = re.compile(r"%s\.member\.([0-9]+)\.(.*)" % (prefix))

    def get_param_data(params):
        for param_name, value in params.items():
            match = key_re.match(param_name)
            if match:
                try:
                    index = int(match.group(1))
                except ValueError:
                    pass
                else:
                    key = match.group(2)

                    yield (index, (key, value))

    # Sort and group by index
    key_func = lambda d: d[0]
    data = sorted(get_param_data(params), key=key_func)
    members = itertools.groupby(data, key_func)

    return [dict(kv for di, kv in m) for mi, m in members]

def reformat_dict_keys(keymap={}, inputdict={}):
    '''
    Utility function for mapping one dict format to another
    '''
    return dict([(outk, inputdict[ink]) for ink, outk in keymap.items()])
<commit_msg>heat API : add get_param_value to API utils<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

'''
Helper utilities related to the AWS API implementations
'''

import re
import itertools
from heat.api.aws import exception

def format_response(action, response):
    """
    Format response from engine into API format
    """
    return {'%sResponse' % action: {'%sResult' % action: response}}

def extract_param_pairs(params, prefix='', keyname='', valuename=''):
    """
    Extract a dictionary of user input parameters, from AWS style
    parameter-pair encoded list

    In the AWS API list items appear as two key-value
    pairs (passed as query parameters)  with keys of the form below:

    Prefix.member.1.keyname=somekey
    Prefix.member.1.keyvalue=somevalue
    Prefix.member.2.keyname=anotherkey
    Prefix.member.2.keyvalue=somevalue

    We reformat this into a dict here to match the heat
    engine API expected format
    """
    plist = extract_param_list(params, prefix)
    kvs = [(p[keyname], p[valuename]) for p in plist
                                      if keyname in p and valuename in p]

    return dict(kvs)

def extract_param_list(params, prefix=''):
    """
    Extract a list-of-dicts based on parameters containing AWS style list

    MetricData.member.1.MetricName=buffers
    MetricData.member.1.Unit=Bytes
    MetricData.member.1.Value=231434333
    MetricData.member.2.MetricName=buffers2
    MetricData.member.2.Unit=Bytes
    MetricData.member.2.Value=12345

    This can be extracted by passing prefix=MetricData, resulting in a
    list containing two dicts
    """

    key_re = re.compile(r"%s\.member\.([0-9]+)\.(.*)" % (prefix))

    def get_param_data(params):
        for param_name, value in params.items():
            match = key_re.match(param_name)
            if match:
                try:
                    index = int(match.group(1))
                except ValueError:
                    pass
                else:
                    key = match.group(2)

                    yield (index, (key, value))

    # Sort and group by index
    key_func = lambda d: d[0]
    data = sorted(get_param_data(params), key=key_func)
    members = itertools.groupby(data, key_func)

    return [dict(kv for di, kv in m) for mi, m in members]

def get_param_value(params, key):
    """
    Helper function, looks up an expected parameter in a parsed
    params dict and returns the result.  If params does not contain
    the requested key we raise an exception of the appropriate type
    """
    try:
        return params[key]
    except KeyError:
        logger.error("Request does not contain %s parameter!" % key)
        raise exception.HeatMissingParameterError(key)

def reformat_dict_keys(keymap={}, inputdict={}):
    '''
    Utility function for mapping one dict format to another
    '''
    return dict([(outk, inputdict[ink]) for ink, outk in keymap.items()])
<|endoftext|>


<commit_before>from collections import OrderedDict
from django.db import models
from dynamic_rest.fields import DynamicRelationField
from rest_framework import serializers, fields, exceptions

class DynamicListSerializer(serializers.ListSerializer):

  def to_representation(self, data):
    iterable = data.all() if isinstance(data, models.Manager) else data
    return [self.child.to_representation(item) for item in iterable]

class DynamicModelSerializer(serializers.ModelSerializer):

  def __new__(cls, *args, **kwargs):
    """
    Custom constructor that sets the ListSerializer to DynamicListSerializer
    to avoid re-evaluating querysets.

    Addresses DRF 3.1.0 bug: https://github.com/tomchristie/django-rest-framework/issues/2704)
    """
    meta = getattr(cls, 'Meta', None)
    if not meta:
      meta = type('Meta', (), {})
      cls.Meta = meta
    if not hasattr(meta, 'list_serializer_class'):
      meta.list_serializer_class = DynamicListSerializer
    return super(DynamicModelSerializer, cls).__new__(cls, *args, **kwargs)

  def __init__(self, instance=None, data=fields.empty, include_fields=None, exclude_fields=None, only_fields=None,
               request_fields=None, dynamic=True, **kwargs):
    """
    Custom initializer that builds `request_fields` and
    sets a `ListSerializer` that doesn't re-evaluate querysets.

    Arguments:
      instance: instance for the serializer base
      include_fields: list of field names to include (adds to default field set)
      exclude_fields: list of field names to exclude (removes from default field set)
      only_fields: list of field names to render (overrides field set)
      request_fields: map of field names that supports inclusions, exclusions, and nested sideloads
      dynamic: if False, ignore deferred rules and revert to standard DRF behavior (default: True)
    """

    name = self.get_name()
    if data is not fields.empty and name in data and len(data) == 1:
      # support POST/PUT key'd by resource name
      data = data[name]

    if data is not fields.empty:
      # if a field is nullable but not required and the implementation
      # passes null as a value, remove the field from the data
      # this addresses the frontends that send
      # undefined resource fields as null on POST/PUT
      for field_name, field in self.get_all_fields().iteritems():
        if field.allow_null == False and field.required == False \
                and field_name in data and data[field_name] is None:
          data.pop(field_name)

    kwargs['instance'] = instance
    kwargs['data'] = data
    super(DynamicModelSerializer, self).__init__(**kwargs)

    self.dynamic = dynamic
    self.request_fields = request_fields or self._context.get('request_fields', {})
    self.only_fields = only_fields or self._context.get('only_fields', [])
    include_fields = include_fields or self._context.get('include_fields', [])
    exclude_fields = exclude_fields or self._context.get('exclude_fields', [])
    for name in include_fields:
      self.request_fields[name] = True
    for name in exclude_fields:
      self.request_fields[name] = False

  def get_name(self):
    """Returns the serializer name.

    The name must be defined on the Meta class.
    """
    return self.Meta.name

  def get_plural_name(self):
    """Returns the serializer's plural name.

    The plural name may be defined on the Meta class.
    If the plural name is not defined, the pluralized name will be returned.
    """
    return getattr(self.Meta, 'plural_name', self.get_name() + 's')

  def get_all_fields(self):
    """Returns the entire serializer field set.

    Does not respect dynamic field inclusions/exclusions.
    """
    if not hasattr(self, '_all_fields'):
      self._all_fields = super(DynamicModelSerializer, self).get_fields()
    return self._all_fields

  def get_fields(self):
    """Returns the serializer's field set.

    If `dynamic` is True, respects field inclusions/exlcusions, taking into account
    `field.deferred` (field-specific flag), `Meta.deferred_fields` (serializer-specific list),
    `only_fields` and/or `request_fields` (passed to the serializer by a viewset or parent serializer).
    """
    if self.dynamic == False:
      return self.get_all_fields()

    if self.id_only():
      return {}

    serializer_fields = self.get_all_fields()
    request_fields = self.request_fields
    only_fields = set(self.only_fields)

    # return only those fields specified, ignoring deferred rules
    if only_fields:
      return {k:v for k,v in serializer_fields.iteritems() if k in only_fields}

    # determine fields that are deferred by default
    meta_deferred = set(getattr(self.Meta, 'deferred_fields', []))
    deferred = set([name for name, field in serializer_fields.iteritems()
                    if getattr(field, 'deferred', None) == True or name in meta_deferred])

    # apply request overrides
    if request_fields:
      for name, include in request_fields.iteritems():
        if not name in serializer_fields:
          raise exceptions.ParseError(
              "'%s' is not a valid field name for '%s'" % (name, self.get_name()))
        if include != False and name in deferred:
          deferred.remove(name)
        elif include == False:
          deferred.add(name)

    # remove any deferred fields from the base list
    for name in deferred:
      serializer_fields.pop(name)

    # inject request_fields into sub-serializers
    for name, field in serializer_fields.iteritems():
      inject = None
      if isinstance(field, serializers.BaseSerializer):
        inject = field
      elif isinstance(field, DynamicRelationField):
        field.parent = self
        inject = field.serializer
      if isinstance(inject, serializers.ListSerializer):
        inject = field.child
      if inject:
        inject.request_fields = request_fields.get(name, True)

    return serializer_fields

  def to_representation(self, instance):
    if self.dynamic and self.id_only():
      return instance.pk
    else:
      representation = super(DynamicModelSerializer, self).to_representation(instance)
    # save the plural name and id
    # so that the DynamicRenderer can sideload in post-serialization
    representation['_name'] = self.get_plural_name()
    representation['_pk'] = instance.pk
    return representation

  def save(self, *args, **kwargs):
    update = getattr(self, 'instance', None) is not None
    instance = super(DynamicModelSerializer, self).save(*args, **kwargs)
    view = self._context.get('view')
    if update and view:
      # reload the object on update
      # to get around prefetched manager issues
      instance = view.get_object()
    return instance

  def id_only(self):
    """Whether or not the serializer should return an ID instead of an object.

    Returns:
      True iff `request_fields` == True
    """
    return self.request_fields == True
<commit_msg>set self.instance<commit_after>from collections import OrderedDict
from django.db import models
from dynamic_rest.fields import DynamicRelationField
from rest_framework import serializers, fields, exceptions

class DynamicListSerializer(serializers.ListSerializer):

  def to_representation(self, data):
    iterable = data.all() if isinstance(data, models.Manager) else data
    return [self.child.to_representation(item) for item in iterable]

class DynamicModelSerializer(serializers.ModelSerializer):

  def __new__(cls, *args, **kwargs):
    """
    Custom constructor that sets the ListSerializer to DynamicListSerializer
    to avoid re-evaluating querysets.

    Addresses DRF 3.1.0 bug: https://github.com/tomchristie/django-rest-framework/issues/2704)
    """
    meta = getattr(cls, 'Meta', None)
    if not meta:
      meta = type('Meta', (), {})
      cls.Meta = meta
    if not hasattr(meta, 'list_serializer_class'):
      meta.list_serializer_class = DynamicListSerializer
    return super(DynamicModelSerializer, cls).__new__(cls, *args, **kwargs)

  def __init__(self, instance=None, data=fields.empty, include_fields=None, exclude_fields=None, only_fields=None,
               request_fields=None, dynamic=True, **kwargs):
    """
    Custom initializer that builds `request_fields` and
    sets a `ListSerializer` that doesn't re-evaluate querysets.

    Arguments:
      instance: instance for the serializer base
      include_fields: list of field names to include (adds to default field set)
      exclude_fields: list of field names to exclude (removes from default field set)
      only_fields: list of field names to render (overrides field set)
      request_fields: map of field names that supports inclusions, exclusions, and nested sideloads
      dynamic: if False, ignore deferred rules and revert to standard DRF behavior (default: True)
    """

    name = self.get_name()
    if data is not fields.empty and name in data and len(data) == 1:
      # support POST/PUT key'd by resource name
      data = data[name]

    if data is not fields.empty:
      # if a field is nullable but not required and the implementation
      # passes null as a value, remove the field from the data
      # this addresses the frontends that send
      # undefined resource fields as null on POST/PUT
      for field_name, field in self.get_all_fields().iteritems():
        if field.allow_null == False and field.required == False \
                and field_name in data and data[field_name] is None:
          data.pop(field_name)

    kwargs['instance'] = instance
    kwargs['data'] = data
    super(DynamicModelSerializer, self).__init__(**kwargs)

    self.dynamic = dynamic
    self.request_fields = request_fields or self._context.get('request_fields', {})
    self.only_fields = only_fields or self._context.get('only_fields', [])
    include_fields = include_fields or self._context.get('include_fields', [])
    exclude_fields = exclude_fields or self._context.get('exclude_fields', [])
    for name in include_fields:
      self.request_fields[name] = True
    for name in exclude_fields:
      self.request_fields[name] = False

  def get_name(self):
    """Returns the serializer name.

    The name must be defined on the Meta class.
    """
    return self.Meta.name

  def get_plural_name(self):
    """Returns the serializer's plural name.

    The plural name may be defined on the Meta class.
    If the plural name is not defined, the pluralized name will be returned.
    """
    return getattr(self.Meta, 'plural_name', self.get_name() + 's')

  def get_all_fields(self):
    """Returns the entire serializer field set.

    Does not respect dynamic field inclusions/exclusions.
    """
    if not hasattr(self, '_all_fields'):
      self._all_fields = super(DynamicModelSerializer, self).get_fields()
    return self._all_fields

  def get_fields(self):
    """Returns the serializer's field set.

    If `dynamic` is True, respects field inclusions/exlcusions, taking into account
    `field.deferred` (field-specific flag), `Meta.deferred_fields` (serializer-specific list),
    `only_fields` and/or `request_fields` (passed to the serializer by a viewset or parent serializer).
    """
    if self.dynamic == False:
      return self.get_all_fields()

    if self.id_only():
      return {}

    serializer_fields = self.get_all_fields()
    request_fields = self.request_fields
    only_fields = set(self.only_fields)

    # return only those fields specified, ignoring deferred rules
    if only_fields:
      return {k:v for k,v in serializer_fields.iteritems() if k in only_fields}

    # determine fields that are deferred by default
    meta_deferred = set(getattr(self.Meta, 'deferred_fields', []))
    deferred = set([name for name, field in serializer_fields.iteritems()
                    if getattr(field, 'deferred', None) == True or name in meta_deferred])

    # apply request overrides
    if request_fields:
      for name, include in request_fields.iteritems():
        if not name in serializer_fields:
          raise exceptions.ParseError(
              "'%s' is not a valid field name for '%s'" % (name, self.get_name()))
        if include != False and name in deferred:
          deferred.remove(name)
        elif include == False:
          deferred.add(name)

    # remove any deferred fields from the base list
    for name in deferred:
      serializer_fields.pop(name)

    # inject request_fields into sub-serializers
    for name, field in serializer_fields.iteritems():
      inject = None
      if isinstance(field, serializers.BaseSerializer):
        inject = field
      elif isinstance(field, DynamicRelationField):
        field.parent = self
        inject = field.serializer
      if isinstance(inject, serializers.ListSerializer):
        inject = field.child
      if inject:
        inject.request_fields = request_fields.get(name, True)

    return serializer_fields

  def to_representation(self, instance):
    if self.dynamic and self.id_only():
      return instance.pk
    else:
      representation = super(DynamicModelSerializer, self).to_representation(instance)
    # save the plural name and id
    # so that the DynamicRenderer can sideload in post-serialization
    representation['_name'] = self.get_plural_name()
    representation['_pk'] = instance.pk
    return representation

  def save(self, *args, **kwargs):
    update = getattr(self, 'instance', None) is not None
    instance = super(DynamicModelSerializer, self).save(*args, **kwargs)
    view = self._context.get('view')
    if update and view:
      # reload the object on update
      # to get around prefetched manager issues
      instance = self.instance = view.get_object()
    return instance

  def id_only(self):
    """Whether or not the serializer should return an ID instead of an object.

    Returns:
      True iff `request_fields` == True
    """
    return self.request_fields == True
<|endoftext|>