<commit_before># -*- coding: utf-8 -*-
import os
import sys
import json
import errno
if sys.version_info > (3, 0):
    from http.server import HTTPServer, SimpleHTTPRequestHandler
else:
    from BaseHTTPServer import HTTPServer
    from SimpleHTTPServer import SimpleHTTPRequestHandler

__version__ = '0.1'
__author__ = 'Alberto Casero (@KasAppeal)'

API_ID_FIELD = 'id'
API_PATH = '/api/'
API_DATA_PATH = 'db'

def is_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False

class SparrestHandler(SimpleHTTPRequestHandler):
    """
    Manage the request received
    """
    server_version = "RestCampHTTPServer/" + __version__
    data = None
    content = None

    def is_api_request(self):
        """Checks if the request is referred to an API item"""
        return self.path[:len(API_PATH)] == API_PATH

    def get_content(self):
        """Reads the request body and returns it"""
        if not self.content:
            try:
                length = int(self.headers.get('content-length', '0'))
            except ValueError:
                length = 0
            self.content = self.rfile.read(length) if length > 0 else ''
        return self.content.decode('utf-8')

    def is_valid_content_type(self):
        """Checks if the set content type is valid"""
        return 'application/json' in self.headers.get('content-type', 'text/plain').lower()

    def is_valid_json(self):
        """Checks if the body content is a valid JSON"""
        data = self.get_data()
        return data is not None

    def get_data(self):
        """
        Returns the JSON data converted to a dict if the Content-Type header is application JSON and the JSON is valid.
        Otherwise, returns None
        """
        if self.data is None:
            try:
                self.data = json.loads(self.get_content())
            except ValueError:
                self.data = None
        return self.data

    def get_resource_parts(self):
        """
        Returns a list of resource parts: if URL is 'API_PATH/foo/bar' it returns ['foo', 'bar']
        If not is a valid API_REQUEST, returns an empty list
        """
        if not self.is_api_request():
            return []

        parts_list = list(filter(lambda x: x.replace(' ', '') != '', self.path.split(API_PATH)))
        if len(parts_list) <= 0:
            return []

        return list(filter(lambda x: x.replace(' ', '') != '', parts_list[0].split('/')))

    def write_response(self, data, code=200):
        """
        Writes the response to the request
        :param data: dict with data which will be converted to JSON
        :param code: optional integer with an HTTP response code
        """
        self.send_response(code)
        self.send_header("Content-Type", 'application/json')
        self.end_headers()
        self.wfile.write(json.dumps(data).encode())

    def process_non_api_request(self):
        """Process a non api request serving the static file requested"""
        f = self.send_head()
        if f:
            try:
                self.copyfile(f, self.wfile)
            finally:
                f.close()

    def process_get_list_resource_request(self, resource):
        """Look for the resource and returns its content or a 404 Not found HTTP response if it doesn't exist"""
        resource_path = os.path.join(API_DATA_PATH, resource)
        if not os.path.exists(resource_path):
            self.write_not_found_response(resource)
        else:
            response_items = []
            files = os.listdir(resource_path)
            for file_name in files:
                if not is_int(file_name):
                    continue
                file_path = os.path.join(resource_path, file_name)
                try:
                    fp = open(file_path, 'r')
                    item = json.load(fp)
                    fp.close()
                    response_items.append(item)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(file_path)
                    else:  # Not a permission error.
                        raise
            self.write_response(response_items, 200)

    def process_get_detail_resource_request(self, resource, resource_id):
        """Look for the resource and returns its content or a 404 Not found HTTP response if it doesn't exist"""
        resource_path = os.path.join(API_DATA_PATH, resource, resource_id)
        if not os.path.exists(resource_path):
            self.write_not_found_response(resource, resource_id)
        else:
            try:
                fp = open(resource_path, 'r')
                item = json.load(fp)
                fp.close()
                self.write_response(item, 200)
            except IOError as e:
                if e.errno == errno.EACCES:
                    self.write_no_access_permission_to_file_response(resource_path)
                else:  # Not a permission error.
                    raise

    def write_invalid_api_uri_format_response(self):
        """Writes the HTTP 406 Invalid API URI format error response"""
        self.write_response({
            406: 'Invalid API URI format. Expected format: {0}<resource> or {0}<resource>/'.format(API_PATH)
        }, 406)

    def write_method_not_allowed_response(self):
        """Writes the HTTP 405 Method not allowed error response"""
        self.write_response({405: 'Method not allowed'}, 405)

    def write_invalid_content_type_response(self):
        """Writes the HTTP 400 Bad request error response to set the content-type header to application/json"""
        self.write_response({400: 'Review your Content-Type header. I only speak application/json bro.'}, 400)

    def write_no_access_permission_to_file_response(self, file_path):
        """Writes the HTTP 403 Forbidden error response when trying to accessing a file without permission"""
        self.write_response({
            403: 'Server process owner has no access to {0}'.format(file_path)
        }, 403)

    def write_not_found_response(self, resource=None, resource_id=None):
        """Writes the HTTP 404 Not found"""
        if resource is not None and resource_id is not None:
            self.write_response({404: 'Resource {0}/{1} not found'.format(resource, resource_id)}, 404)
        elif resource is not None:
            self.write_response({404: 'Resource {0} not found'.format(resource)}, 404)
        else:
            self.write_response({404: 'Resource not found'}, 404)

    def do_GET(self):
        """Process a GET request"""
        if self.is_api_request():
            resource_parts = self.get_resource_parts()
            if resource_parts is None:
                self.write_method_not_allowed_response()
            elif len(resource_parts) == 1:
                self.process_get_list_resource_request(resource_parts[0])
            elif len(resource_parts) == 2:
                self.process_get_detail_resource_request(resource_parts[0], resource_parts[1])
            else:
                self.write_invalid_api_uri_format_response()
        else:
            self.process_non_api_request()

    def do_POST(self):
        """Process a POST request"""
        resource_parts = self.get_resource_parts()
        if resource_parts is None:
            self.write_method_not_allowed_response()
        elif not self.is_valid_content_type():
            self.write_invalid_content_type_response()
        elif not self.is_valid_json():
            self.write_invalid_content_type_response()
        elif len(resource_parts) != 1:
            self.write_invalid_api_uri_format_response()
        else:
            resource_path = os.path.join(API_DATA_PATH, resource_parts[0])
            if not os.path.exists(resource_path):
                try:
                    os.makedirs(resource_path)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(resource_path)
                    else:  # Not a permission error.
                        raise
            files = os.listdir(resource_path)
            max_id = 0
            for file_name in files:
                try:
                    int_value = int(file_name)
                    max_id = int_value if int_value > max_id else max_id
                except ValueError:
                    pass
            data = self.get_data()
            try:
                if type(data) == dict:
                    data = [data]
                for item in data:
                    max_id += 1
                    item[API_ID_FIELD] = max_id
                    new_file_name = os.path.join(resource_path, str(max_id))
                    fp = open(new_file_name, 'w')
                    json.dump(item, fp)
                    fp.close()
                self.write_response(data[0] if len(data) == 1 else data, 201)
            except IOError as e:
                if e.errno == errno.EACCES:
                    self.write_no_access_permission_to_file_response(resource_path)
                else:  # Not a permission error.
                    raise

    def do_PUT(self):
        """
        Process a PUT request
        """
        resource_parts = self.get_resource_parts()
        if len(resource_parts) != 2:
            self.write_invalid_api_uri_format_response()
        elif not self.is_valid_content_type():
            self.write_invalid_content_type_response()
        elif not self.is_valid_json():
            self.write_invalid_content_type_response()
        else:
            resource = resource_parts[0]
            resource_id = resource_parts[1]
            resource_path = os.path.join(API_DATA_PATH, resource, resource_id)
            if not os.path.exists(resource_path):
                self.write_not_found_response(resource, resource_id)
            else:
                try:
                    data = self.get_data()
                    data[API_ID_FIELD] = resource_id
                    fp = open(resource_path, 'w')
                    json.dump(data, fp)
                    fp.close()
                    self.write_response(data, 200)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(resource_path)
                    else:  # Not a permission error.
                        raise

    def do_DELETE(self):
        """Look for the resource and deletes it if exists"""
        resource_parts = self.get_resource_parts()
        if len(resource_parts) != 2:
            self.write_invalid_api_uri_format_response()
        else:
            resource = resource_parts[0]
            resource_id = resource_parts[1]
            resource_path = os.path.join(API_DATA_PATH, resource, resource_id)
            if not os.path.exists(resource_path):
                self.write_not_found_response(resource, resource_id)
            else:
                try:
                    os.remove(resource_path)
                    self.write_response({}, 204)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(resource_path)
                    else:  # Not a permission error.
                        raise

def run_on(ip, port):
    """
    Starts the HTTP server in the given port
    :param port: port to run the http server
    :return: void
    """
    print("Starting a server on port {0}. Use CNTRL+C to stop the server.".format(port))
    server_address = (ip, port)
    try:
        httpd = HTTPServer(server_address, SparrestHandler)
        httpd.serve_forever()
    except OSError as e:
        if e.errno == 48:  # port already in use
            print("ERROR: The port {0} is already used by another process.".format(port))
        else:
            raise OSError
    except KeyboardInterrupt as interrupt:
        print("Server stopped. Bye bye!")

if __name__ == "__main__":
    port = int(sys.argv[2]) if len(sys.argv) > 2 else 8000
    ip = str(sys.argv[1]) if len(sys.argv) > 1 else "127.0.0.1"
    run_on(ip, port)
<commit_msg>Added support for x-www-form-urlencoded and multipart/form-data content types<commit_after># -*- coding: utf-8 -*-
import cgi
import os
import sys
import json
import errno
if sys.version_info > (3, 0):
    from http.server import HTTPServer, SimpleHTTPRequestHandler
    from urllib.parse import parse_qs
else:
    from BaseHTTPServer import HTTPServer
    from SimpleHTTPServer import SimpleHTTPRequestHandler
    from urlparse import parse_qs

__version__ = '0.1'
__author__ = 'Alberto Casero (@KasAppeal)'

API_ID_FIELD = 'id'
API_PATH = '/api/'
API_DATA_PATH = 'db'

def is_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False

class SparrestHandler(SimpleHTTPRequestHandler):
    """
    Manage the request received
    """
    server_version = "SparRESTServer/" + __version__
    data = None
    content = None

    def is_api_request(self):
        """Checks if the request is referred to an API item"""
        return self.path[:len(API_PATH)] == API_PATH

    def get_content(self, decode=True):
        """Reads the request body and returns it"""
        if not self.content:
            try:
                length = int(self.headers.get('content-length', '0'))
            except ValueError:
                length = 0
            self.content = self.rfile.read(length) if length > 0 else ''
        return self.content.decode('utf-8') if decode else self.content

    def is_valid_content_type(self):
        """Checks if the set content type is valid"""
        return self.is_json_content_type() or self.is_form_urlencoded_data_content_type() or \
               self.is_multipart_form_data_content_type()

    def is_json_content_type(self):
        """Checks if the set content type is application/json"""
        return 'application/json' in self.headers.get('content-type', 'text/plain').lower()

    def is_form_urlencoded_data_content_type(self):
        """Checks if the set content type is form url encoded"""
        return 'application/x-www-form-urlencoded' in self.headers.get('content-type', 'text/plain').lower()

    def is_multipart_form_data_content_type(self):
        """Checks if the set content type is multipart/form-data"""
        return 'multipart/form-data' in self.headers.get('content-type', 'text/plain').lower()

    def is_valid_json(self):
        """Checks if the body content is a valid JSON"""
        data = self.get_data()
        return data is not None

    def get_multipart_boundary(self):
        """Returns the multipart boundary"""
        parts = self.headers.get('content-type', '').split('boundary=')
        return parts[1] if len(parts) > 1 else ''

    def get_data(self):
        """
        Returns the JSON data converted to a dict depending of the content-type sent. Only if data format is correct,
        returns the dict, otherwise, returns None
        """
        if self.data is None:
            if self.is_json_content_type():
                try:
                    self.data = json.loads(self.get_content())
                except ValueError:
                    self.data = None
            elif self.is_form_urlencoded_data_content_type():
                parsed_data = parse_qs(self.get_content(), keep_blank_values=True)
                self.data = dict(map(
                    lambda t: (t[0], t[1][0] if type(t[1]) == list and len(t[1]) == 1 else t[1]), parsed_data.items()
                ))
            elif self.is_multipart_form_data_content_type():
                ctype, pdict = cgi.parse_header(self.headers.get('content-type'))
                if 'boundary' in pdict:
                    pdict['boundary'] = pdict['boundary'].encode()
                parsed_data = cgi.parse_multipart(self.rfile, pdict)
                self.data = dict(map(
                    lambda t: (
                        t[0], t[1][0].decode('utf-8') if type(t[1]) == list and len(t[1]) == 1 else t[1].decode('utf-8')
                    ), parsed_data.items()
                ))
        return self.data

    def get_resource_parts(self):
        """
        Returns a list of resource parts: if URL is 'API_PATH/foo/bar' it returns ['foo', 'bar']
        If not is a valid API_REQUEST, returns an empty list
        """
        if not self.is_api_request():
            return []

        parts_list = list(filter(lambda x: x.replace(' ', '') != '', self.path.split(API_PATH)))
        if len(parts_list) <= 0:
            return []

        return list(filter(lambda x: x.replace(' ', '') != '', parts_list[0].split('/')))

    def write_response(self, data, code=200):
        """
        Writes the response to the request
        :param data: dict with data which will be converted to JSON
        :param code: optional integer with an HTTP response code
        """
        self.send_response(code)
        self.send_header("Content-Type", 'application/json')
        self.end_headers()
        self.wfile.write(json.dumps(data).encode())

    def process_non_api_request(self):
        """Process a non api request serving the static file requested"""
        f = self.send_head()
        if f:
            try:
                self.copyfile(f, self.wfile)
            finally:
                f.close()

    def process_get_list_resource_request(self, resource):
        """Look for the resource and returns its content or a 404 Not found HTTP response if it doesn't exist"""
        resource_path = os.path.join(API_DATA_PATH, resource)
        if not os.path.exists(resource_path):
            self.write_not_found_response(resource)
        else:
            response_items = []
            files = os.listdir(resource_path)
            for file_name in files:
                if not is_int(file_name):
                    continue
                file_path = os.path.join(resource_path, file_name)
                try:
                    fp = open(file_path, 'r')
                    item = json.load(fp)
                    fp.close()
                    response_items.append(item)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(file_path)
                    else:  # Not a permission error.
                        raise
            self.write_response(response_items, 200)

    def process_get_detail_resource_request(self, resource, resource_id):
        """Look for the resource and returns its content or a 404 Not found HTTP response if it doesn't exist"""
        resource_path = os.path.join(API_DATA_PATH, resource, resource_id)
        if not os.path.exists(resource_path):
            self.write_not_found_response(resource, resource_id)
        else:
            try:
                fp = open(resource_path, 'r')
                item = json.load(fp)
                fp.close()
                self.write_response(item, 200)
            except IOError as e:
                if e.errno == errno.EACCES:
                    self.write_no_access_permission_to_file_response(resource_path)
                else:  # Not a permission error.
                    raise

    def write_invalid_api_uri_format_response(self):
        """Writes the HTTP 406 Invalid API URI format error response"""
        self.write_response({
            406: 'Invalid API URI format. Expected format: {0}<resource> or {0}<resource>/'.format(API_PATH)
        }, 406)

    def write_method_not_allowed_response(self):
        """Writes the HTTP 405 Method not allowed error response"""
        self.write_response({405: 'Method not allowed'}, 405)

    def write_invalid_content_type_response(self):
        """Writes the HTTP 400 Bad request error response to set the content-type header to application/json"""
        self.write_response({400: 'Review your Content-Type header. I only speak application/json bro.'}, 400)

    def write_no_access_permission_to_file_response(self, file_path):
        """Writes the HTTP 403 Forbidden error response when trying to accessing a file without permission"""
        self.write_response({
            403: 'Server process owner has no access to {0}'.format(file_path)
        }, 403)

    def write_not_found_response(self, resource=None, resource_id=None):
        """Writes the HTTP 404 Not found"""
        if resource is not None and resource_id is not None:
            self.write_response({404: 'Resource {0}/{1} not found'.format(resource, resource_id)}, 404)
        elif resource is not None:
            self.write_response({404: 'Resource {0} not found'.format(resource)}, 404)
        else:
            self.write_response({404: 'Resource not found'}, 404)

    def do_GET(self):
        """Process a GET request"""
        if self.is_api_request():
            resource_parts = self.get_resource_parts()
            if resource_parts is None:
                self.write_method_not_allowed_response()
            elif len(resource_parts) == 1:
                self.process_get_list_resource_request(resource_parts[0])
            elif len(resource_parts) == 2:
                self.process_get_detail_resource_request(resource_parts[0], resource_parts[1])
            else:
                self.write_invalid_api_uri_format_response()
        else:
            self.process_non_api_request()

    def do_POST(self):
        """Process a POST request"""
        resource_parts = self.get_resource_parts()
        if resource_parts is None:
            self.write_method_not_allowed_response()
        elif not self.is_valid_content_type():
            self.write_invalid_content_type_response()
        elif self.is_json_content_type() and not self.is_valid_json():
            self.write_invalid_content_type_response()
        elif len(resource_parts) != 1:
            self.write_invalid_api_uri_format_response()
        else:
            resource_path = os.path.join(API_DATA_PATH, resource_parts[0])
            if not os.path.exists(resource_path):
                try:
                    os.makedirs(resource_path)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(resource_path)
                    else:  # Not a permission error.
                        raise
            files = os.listdir(resource_path)
            max_id = 0
            for file_name in files:
                try:
                    int_value = int(file_name)
                    max_id = int_value if int_value > max_id else max_id
                except ValueError:
                    pass
            data = self.get_data()
            try:
                if type(data) == dict:
                    data = [data]
                for item in data:
                    max_id += 1
                    item[API_ID_FIELD] = max_id
                    new_file_name = os.path.join(resource_path, str(max_id))
                    fp = open(new_file_name, 'w')
                    json.dump(item, fp)
                    fp.close()
                self.write_response(data[0] if len(data) == 1 else data, 201)
            except IOError as e:
                if e.errno == errno.EACCES:
                    self.write_no_access_permission_to_file_response(resource_path)
                else:  # Not a permission error.
                    raise

    def do_PUT(self):
        """
        Process a PUT request
        """
        resource_parts = self.get_resource_parts()
        if len(resource_parts) != 2:
            self.write_invalid_api_uri_format_response()
        elif not self.is_valid_content_type():
            self.write_invalid_content_type_response()
        elif not self.is_valid_json():
            self.write_invalid_content_type_response()
        else:
            resource = resource_parts[0]
            resource_id = resource_parts[1]
            resource_path = os.path.join(API_DATA_PATH, resource, resource_id)
            if not os.path.exists(resource_path):
                self.write_not_found_response(resource, resource_id)
            else:
                try:
                    data = self.get_data()
                    data[API_ID_FIELD] = resource_id
                    fp = open(resource_path, 'w')
                    json.dump(data, fp)
                    fp.close()
                    self.write_response(data, 200)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(resource_path)
                    else:  # Not a permission error.
                        raise

    def do_DELETE(self):
        """Look for the resource and deletes it if exists"""
        resource_parts = self.get_resource_parts()
        if len(resource_parts) != 2:
            self.write_invalid_api_uri_format_response()
        else:
            resource = resource_parts[0]
            resource_id = resource_parts[1]
            resource_path = os.path.join(API_DATA_PATH, resource, resource_id)
            if not os.path.exists(resource_path):
                self.write_not_found_response(resource, resource_id)
            else:
                try:
                    os.remove(resource_path)
                    self.write_response({}, 204)
                except IOError as e:
                    if e.errno == errno.EACCES:
                        self.write_no_access_permission_to_file_response(resource_path)
                    else:  # Not a permission error.
                        raise

def run_on(ip, port):
    """
    Starts the HTTP server in the given port
    :param port: port to run the http server
    :return: void
    """
    print("Starting a server on port {0}. Use CNTRL+C to stop the server.".format(port))
    server_address = (ip, port)
    try:
        httpd = HTTPServer(server_address, SparrestHandler)
        httpd.serve_forever()
    except OSError as e:
        if e.errno == 48:  # port already in use
            print("ERROR: The port {0} is already used by another process.".format(port))
        else:
            raise OSError
    except KeyboardInterrupt as interrupt:
        print("Server stopped. Bye bye!")

if __name__ == "__main__":
    port = int(sys.argv[2]) if len(sys.argv) > 2 else 8000
    ip = str(sys.argv[1]) if len(sys.argv) > 1 else "127.0.0.1"
    run_on(ip, port)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    Tests for old bugs
    ~~~~~~~~~~~~~~~~~~

    Unittest that test situations caused by various older bugs.

    :copyright: (c) 2009 by the Jinja Team.
    :license: BSD.
"""
from jinja2 import Template, Environment, DictLoader, TemplateSyntaxError

env = Environment()

from nose import SkipTest
from nose.tools import assert_raises

def test_keyword_folding():
    env = Environment()
    env.filters['testing'] = lambda value, some: value + some
    assert env.from_string("{{ 'test'|testing(some='stuff') }}") \
           .render() == 'teststuff'

def test_extends_output_bugs():
    env = Environment(loader=DictLoader({
        'parent.html': '(({% block title %}{% endblock %}))'
    }))

    t = env.from_string('{% if expr %}{% extends "parent.html" %}{% endif %}'
                        '[[{% block title %}title{% endblock %}]]'
                        '{% for item in [1, 2, 3] %}({{ item }}){% endfor %}')
    assert t.render(expr=False) == '[[title]](1)(2)(3)'
    assert t.render(expr=True) == '((title))'

def test_urlize_filter_escaping():
    tmpl = env.from_string('{{ "http://www.example.org/<foo"|urlize }}')
    assert tmpl.render() == 'http://www.example.org/<foo'

def test_loop_call_loop():
    tmpl = env.from_string('''

    {% macro test() %}
        {{ caller() }}
    {% endmacro %}

    {% for num1 in range(5) %}
        {% call test() %}
            {% for num2 in range(10) %}
                {{ loop.index }}
            {% endfor %}
        {% endcall %}
    {% endfor %}

    ''')

    assert tmpl.render().split() == map(unicode, range(1, 11)) * 5

def test_weird_inline_comment():
    env = Environment(line_statement_prefix='%')
    assert_raises(TemplateSyntaxError, env.from_string,
                  '% for item in seq {# missing #}\n...% endfor')

def test_old_macro_loop_scoping_bug():
    tmpl = env.from_string('{% for i in (1, 2) %}{{ i }}{% endfor %}'
                           '{% macro i() %}3{% endmacro %}{{ i() }}')
    assert tmpl.render() == '123'

def test_partial_conditional_assignments():
    tmpl = env.from_string('{% if b %}{% set a = 42 %}{% endif %}{{ a }}')
    assert tmpl.render(a=23) == '23'
    assert tmpl.render(b=True) == '42'

def test_stacked_locals_scoping_bug():
    env = Environment(line_statement_prefix='#')
    t = env.from_string('''\
# for j in [1, 2]:
#   set x = 1
#   for i in [1, 2]:
#     print x
#     if i % 2 == 0:
#       set x = x + 1
#     endif
#   endfor
# endfor
# if a
#   print 'A'
# elif b
#   print 'B'
# elif c == d
#   print 'C'
# else
#   print 'D'
# endif
''')
    assert t.render(a=0, b=False, c=42, d=42.0) == '1111C'

def test_call_with_args():
    t = Template("""{% macro dump_users(users) -%}

      {%- for user in users -%}

{{ user.username|e }}
{{ caller(user) }}

      {%- endfor -%}

    {%- endmacro -%}

    {% call(user) dump_users(list_of_user) -%}

Realname

{{ user.realname|e }}

Description

{{ user.description }}

    {% endcall %}""")

    assert [x.strip() for x in t.render(list_of_user=[{
        'username':'apo',
        'realname':'something else',
        'description':'test'
    }]).splitlines()] == [
        u'

apo

',
        u'
Realname
',
        u'
something else
',
        u'
Description
',
        u'
test
',
        u'
',
        u'

'
    ]

def test_empty_if_condition_fails():
    assert_raises(TemplateSyntaxError, Template, '{% if %}....{% endif %}')
    assert_raises(TemplateSyntaxError, Template, '{% if foo %}...{% elif %}...{% endif %}')
    assert_raises(TemplateSyntaxError, Template, '{% for x in %}..{% endfor %}')

def test_recursive_loop_bug():
    tpl1 = Template("""
    {% for p in foo recursive%}
        {{p.bar}}
        {% for f in p.fields recursive%}
            {{f.baz}}
            {{p.bar}}
            {% if f.rec %}
                {{ loop(f.sub) }}
            {% endif %}
        {% endfor %}
    {% endfor %}
    """)
    tpl1.render(ctx)

    tpl2 = Template("""
    {% for p in foo%}
        {{p.bar}}
        {% for f in p.fields recursive%}
            {{f.baz}}
            {{p.bar}}
            {% if f.rec %}
                {{ loop(f.sub) }}
            {% endif %}
        {% endfor %}
    {% endfor %}
    """)
    tpl2.render(ctx)
<commit_msg>Fixed bug in a testcase.<commit_after># -*- coding: utf-8 -*-
"""
    Tests for old bugs
    ~~~~~~~~~~~~~~~~~~

    Unittest that test situations caused by various older bugs.

    :copyright: (c) 2009 by the Jinja Team.
    :license: BSD.
"""
from jinja2 import Template, Environment, DictLoader, TemplateSyntaxError

env = Environment()

from nose import SkipTest
from nose.tools import assert_raises

def test_keyword_folding():
    env = Environment()
    env.filters['testing'] = lambda value, some: value + some
    assert env.from_string("{{ 'test'|testing(some='stuff') }}") \
           .render() == 'teststuff'

def test_extends_output_bugs():
    env = Environment(loader=DictLoader({
        'parent.html': '(({% block title %}{% endblock %}))'
    }))

    t = env.from_string('{% if expr %}{% extends "parent.html" %}{% endif %}'
                        '[[{% block title %}title{% endblock %}]]'
                        '{% for item in [1, 2, 3] %}({{ item }}){% endfor %}')
    assert t.render(expr=False) == '[[title]](1)(2)(3)'
    assert t.render(expr=True) == '((title))'

def test_urlize_filter_escaping():
    tmpl = env.from_string('{{ "http://www.example.org/<foo"|urlize }}')
    assert tmpl.render() == 'http://www.example.org/<foo'

def test_loop_call_loop():
    tmpl = env.from_string('''

    {% macro test() %}
        {{ caller() }}
    {% endmacro %}

    {% for num1 in range(5) %}
        {% call test() %}
            {% for num2 in range(10) %}
                {{ loop.index }}
            {% endfor %}
        {% endcall %}
    {% endfor %}

    ''')

    assert tmpl.render().split() == map(unicode, range(1, 11)) * 5

def test_weird_inline_comment():
    env = Environment(line_statement_prefix='%')
    assert_raises(TemplateSyntaxError, env.from_string,
                  '% for item in seq {# missing #}\n...% endfor')

def test_old_macro_loop_scoping_bug():
    tmpl = env.from_string('{% for i in (1, 2) %}{{ i }}{% endfor %}'
                           '{% macro i() %}3{% endmacro %}{{ i() }}')
    assert tmpl.render() == '123'

def test_partial_conditional_assignments():
    tmpl = env.from_string('{% if b %}{% set a = 42 %}{% endif %}{{ a }}')
    assert tmpl.render(a=23) == '23'
    assert tmpl.render(b=True) == '42'

def test_stacked_locals_scoping_bug():
    env = Environment(line_statement_prefix='#')
    t = env.from_string('''\
# for j in [1, 2]:
#   set x = 1
#   for i in [1, 2]:
#     print x
#     if i % 2 == 0:
#       set x = x + 1
#     endif
#   endfor
# endfor
# if a
#   print 'A'
# elif b
#   print 'B'
# elif c == d
#   print 'C'
# else
#   print 'D'
# endif
''')
    assert t.render(a=0, b=False, c=42, d=42.0) == '1111C'

def test_call_with_args():
    t = Template("""{% macro dump_users(users) -%}

      {%- for user in users -%}

{{ user.username|e }}
{{ caller(user) }}

      {%- endfor -%}

    {%- endmacro -%}

    {% call(user) dump_users(list_of_user) -%}

Realname

{{ user.realname|e }}

Description

{{ user.description }}

    {% endcall %}""")

    assert [x.strip() for x in t.render(list_of_user=[{
        'username':'apo',
        'realname':'something else',
        'description':'test'
    }]).splitlines()] == [
        u'

apo

',
        u'
Realname
',
        u'
something else
',
        u'
Description
',
        u'
test
',
        u'
',
        u'

'
    ]

def test_empty_if_condition_fails():
    assert_raises(TemplateSyntaxError, Template, '{% if %}....{% endif %}')
    assert_raises(TemplateSyntaxError, Template, '{% if foo %}...{% elif %}...{% endif %}')
    assert_raises(TemplateSyntaxError, Template, '{% for x in %}..{% endfor %}')

def test_recursive_loop_bug():
    tpl1 = Template("""
    {% for p in foo recursive%}
        {{p.bar}}
        {% for f in p.fields recursive%}
            {{f.baz}}
            {{p.bar}}
            {% if f.rec %}
                {{ loop(f.sub) }}
            {% endif %}
        {% endfor %}
    {% endfor %}
    """)

    tpl2 = Template("""
    {% for p in foo%}
        {{p.bar}}
        {% for f in p.fields recursive%}
            {{f.baz}}
            {{p.bar}}
            {% if f.rec %}
                {{ loop(f.sub) }}
            {% endif %}
        {% endfor %}
    {% endfor %}
    """)
<|endoftext|>


<commit_before>import blosc
import numpy as np
from math import ceil

class BandIndexer(object):

    def __init__(self, bands):
        self.bands = bands

    def __getitem__(self, key):
        if len(self.bands) == 1:
            if isinstance(key, np.ndarray):
                return self.bands[0][:,:][key]
            else:
                return self.bands[0][key]
        else:
            if isinstance(key, np.ndarray):
                return np.dstack([b[:,:][key] for b in self.bands])
            else:
                return np.dstack([b[key] for b in self.bands])

    def __setitem__(self, key, value):
        if len(self.bands) == 1:
            if isinstance(key, np.ndarray):
                self.bands[0][:,:][key] = value
            else:
                self.bands[0][key] = value
        else:
            if isinstance(key, np.ndarray):
                for b, v in zip(self.bands, value):
                    b[:,:][key] = v
            else:
                for b, v in zip(self.bands, value):
                    b[key] = v
        return

    def __iter__(self):
        for i in range(self.bands[0].size[0]):
            if len(self.bands) == 1:
                yield self.bands[0][i,:]
            else:
                yield np.vstack([b[i,:] for b in self.bands])

    @property
    def shape(self):
        if len(self.bands) == 0:
            raise ValueError("no bands")
        elif len(self.bands) == 1:
            return self.bands[0].size
        else:
            return (len(self.bands), self.bands[0].size[0], self.bands.size[1])

    @property
    def dtype(self):
        return self.bands[0].dtype

class SimpleBand(object):
    """ SimpleBand wraps a numpy.ndarray for storage. """

    def __init__(self, size, dtype, initval=None):
        self.size = size
        if initval is None:
            self.array = np.empty(size, dtype=dtype)
        else:
            self.array = initval * np.ones(size, dtype=dtype)
        self.dtype = dtype

    def __getitem__(self, key):
        return self.array[key]

    def __setitem__(self, key, value):
        self.array[key] = value
        return

class CompressedBand(object):
    """ CompressedBand is a chunked, blosc-compressed array. """
    CHUNKSET = 1
    CHUNKUNSET = 0

    def __init__(self, size, dtype, chunksize=(256, 256), initval=None):
        assert len(size) == 2
        self.size = size
        self.dtype = dtype
        self._chunksize = chunksize

        self.nchunkrows = int(ceil(float(size[0])/float(chunksize[0])))
        self.nchunkcols = int(ceil(float(size[1])/float(chunksize[1])))
        nchunks = self.nchunkrows * self.nchunkcols

        # Data store
        self._data = [None for i in range(nchunks)]

        # 0 => unset
        # 1 => set
        self.chunkstatus = np.zeros(nchunks, dtype=np.int8)

        if initval is not None:
            self[:,:] = initval*np.ones(size, dtype=dtype)
        return

    def __getitem__(self, key):

        if isinstance(key, int):
            irow = key // self.size[1]
            icol = key % self.size[1]
            return self._getblock(irow, icol, (1, 1))[0]

        elif isinstance(key, tuple):

            if len(key) != 2:
                raise IndexError("band can only be indexed along two dimensions")

            k0, k1 = key

            if isinstance(k0, int):
                yoff = k0
                ny = 1
                ystride = 1

            elif isinstance(k0, slice):
                ystart, ystop, ystride = k0.indices(self.size[0])
                yoff = min(ystart, ystop)
                ny = abs(ystop-ystart)
                if ystride < 0:
                    yoff += 1

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k0)))

            if isinstance(k1, int):
                xoff = k1
                nx = 1
                xstride = 1

            elif isinstance(k1, slice):
                xstart, xstop, xstride = k1.indices(self.size[1])
                xoff = min(xstart, xstop)
                nx = abs(xstop-xstart)
                if xstride < 0:
                    xoff += 1

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k1)))

            if nx == ny == 1:
                return self._getblock(yoff, xoff, (ny, nx))[0,0]
            else:
                return self._getblock(yoff, xoff, (ny, nx))[::ystride,::xstride]

        elif isinstance(key, slice):
            start, stop, stride = key.indices(self.size[0])
            yoff = min(start, stop)
            if stride < 0:
                yoff += 1
            ny = abs(stop-start)
            return self._getblock(yoff, 0, (ny, self.size[1]))[::stride]

        else:
            raise IndexError("indexing with instances of '{0}' not "
                             "supported".format(type(key)))

    def __setitem__(self, key, value):

        if isinstance(key, int):
            irow = key // self.size[1]
            icol = key % self.size[1]
            self._setblock(irow, icol, np.array(value, dtype=self.dtype))

        elif isinstance(key, tuple):

            if len(key) != 2:
                raise IndexError("band can only be indexed along two dimensions")

            k0, k1 = key

            if isinstance(k0, int):
                yoff = k0
                ny = 1
                sy = 1

            elif isinstance(k0, slice):
                if k0.start is None:
                    yoff = 0
                else:
                    yoff = k0.start
                if k0.stop is None:
                    ny = self.size[0]-yoff
                else:
                    ny = k0.stop-yoff
                if k0.step is None:
                    sy = 1
                else:
                    sy = k0.step

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k0)))

            if isinstance(k1, int):
                xoff = k1
                nx = 1
                sx = 1

            elif isinstance(k1, slice):
                if k1.start is None:
                    xoff = 0
                else:
                    xoff = k1.start
                if k1.stop is None:
                    nx = self.size[1]-xoff
                else:
                    nx = k1.stop-xoff
                if k1.step is None:
                    sx = 1
                else:
                    sx = k1.step

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k1)))

            vny, vnx = value.shape[:2]
            if (ny != vny) or (nx != vnx):
                raise IndexError("Cannot insert array with size ({vny}, {vnx}))"
                        " into slice with size ({ny}, {nx})".format(
                            vny=vny, vnx=vnx, ny=ny, nx=nx))

            if sy == sx == 1:
                self._setblock(yoff, xoff, value)

            else:
                temp = self._getblock(yoff, xoff, (ny, nx))
                temp[::sy,::sx] = value
                self._setblock(yoff, xoff, temp)

        else:
            raise IndexError("indexing with instances of '{0}' not "
                             "supported".format(type(key)))

        return

    def _store(self, array, index):
        self._data[index] = blosc.compress(array.tostring(),
                                           np.dtype(self.dtype).itemsize)
        self.chunkstatus[index] = self.CHUNKSET
        return

    def _retrieve(self, index):
        bytestr = blosc.decompress(self._data[index])
        return np.fromstring(bytestr, dtype=self.dtype).reshape(self._chunksize)

    def _getchunks(self, yoff, xoff, ny, nx):
        """ Return a generator returning tuples identifying chunks covered by a
        range. The tuples contain (chunk_number, ystart, yend, xstart, xend)
        for each chunk touched by a region defined by corner indices and region
        size. """
        chunksize = self._chunksize
        ystart = yoff // chunksize[0]
        yend = ceil(float(yoff+ny) / chunksize[0])
        xstart = xoff // chunksize[1]
        xend = ceil(float(xoff+nx) / chunksize[1])

        nxchunks = int(ceil(float(self.size[1])/float(chunksize[1])))

        i = ystart
        while i < yend:

            j = xstart

            while j < xend:
                chunk_number = i*nxchunks + j
                chunk_ystart = i*chunksize[0]
                chunk_xstart = j*chunksize[1]
                chunk_yend = min((i+1)*chunksize[0], self.size[0])
                chunk_xend = min((j+1)*chunksize[1], self.size[1])
                yield (chunk_number, chunk_ystart, chunk_yend,
                       chunk_xstart, chunk_xend)
                j += 1

            i+= 1

    def _setblock(self, yoff, xoff, array):
        """ Store block of values in *array* starting at offset *yoff*, *xoff*.
        """
        size = array.shape
        chunksize = self._chunksize

        for i, yst, yen, xst, xen in self._getchunks(yoff, xoff, *size):

            # Get from data store
            if self.chunkstatus[i] != self.CHUNKUNSET:
                chunkdata = self._retrieve(i)
            else:
                chunkdata = np.zeros(self._chunksize, dtype=self.dtype)

            # Compute region within chunk to place data in
            cy0 = max(0, yoff-yst)
            cy1 = min(chunksize[0], yoff+size[0]-yst)
            cx0 = max(0, xoff-xst)
            cx1 = min(chunksize[1], xoff+size[1]-xst)

            # Compute region to slice from data
            dy0 = max(0, yst-yoff)
            dy1 = min(size[0], yen-yoff)
            dx0 = max(0, xst-xoff)
            dx1 = min(size[1], xen-xoff)

            chunkdata[cy0:cy1, cx0:cx1] = array[dy0:dy1, dx0:dx1]

            # Return to data store
            self._store(chunkdata, i)
        return

    def _getblock(self, yoff, xoff, size):
        """ Retrieve values with dimensions *size*, starting at offset *yoff*,
        *xoff*.
        """
        result = np.empty(size, self.dtype)
        for i, yst, yen, xst, xen in self._getchunks(yoff, xoff, *size):

            # Compute the bounds in the output
            oy0 = max(0, yst-yoff)
            oy1 = min(size[0], yen-yoff)
            ox0 = max(0, xst-xoff)
            ox1 = min(size[1], xen-xoff)

            if self.chunkstatus[i] == self.CHUNKUNSET:
                result[oy0:oy1, ox0:ox1] = np.zeros((oy1-oy0, ox1-ox0),
                                                    dtype=self.dtype)

            else:
                # Compute the extents from the chunk to retain
                cy0 = max(yoff, yst) - yst
                cy1 = min(yoff+size[0], yen) - yst
                cx0 = max(xoff, xst) - xst
                cx1 = min(xoff+size[1], xen) - xst

                result[oy0:oy1, ox0:ox1] = self._retrieve(i)[cy0:cy1, cx0:cx1]

        return result

<commit_msg>fix size check false positive<commit_after>import blosc
import numpy as np
from math import ceil

class BandIndexer(object):

    def __init__(self, bands):
        self.bands = bands

    def __getitem__(self, key):
        if len(self.bands) == 1:
            if isinstance(key, np.ndarray):
                return self.bands[0][:,:][key]
            else:
                return self.bands[0][key]
        else:
            if isinstance(key, np.ndarray):
                return np.dstack([b[:,:][key] for b in self.bands])
            else:
                return np.dstack([b[key] for b in self.bands])

    def __setitem__(self, key, value):
        if len(self.bands) == 1:
            if isinstance(key, np.ndarray):
                self.bands[0][:,:][key] = value
            else:
                self.bands[0][key] = value
        else:
            if isinstance(key, np.ndarray):
                for b, v in zip(self.bands, value):
                    b[:,:][key] = v
            else:
                for b, v in zip(self.bands, value):
                    b[key] = v
        return

    def __iter__(self):
        for i in range(self.bands[0].size[0]):
            if len(self.bands) == 1:
                yield self.bands[0][i,:]
            else:
                yield np.vstack([b[i,:] for b in self.bands])

    @property
    def shape(self):
        if len(self.bands) == 0:
            raise ValueError("no bands")
        elif len(self.bands) == 1:
            return self.bands[0].size
        else:
            return (len(self.bands), self.bands[0].size[0], self.bands.size[1])

    @property
    def dtype(self):
        return self.bands[0].dtype

class SimpleBand(object):
    """ SimpleBand wraps a numpy.ndarray for storage. """

    def __init__(self, size, dtype, initval=None):
        self.size = size
        if initval is None:
            self.array = np.empty(size, dtype=dtype)
        else:
            self.array = initval * np.ones(size, dtype=dtype)
        self.dtype = dtype

    def __getitem__(self, key):
        return self.array[key]

    def __setitem__(self, key, value):
        self.array[key] = value
        return

class CompressedBand(object):
    """ CompressedBand is a chunked, blosc-compressed array. """
    CHUNKSET = 1
    CHUNKUNSET = 0

    def __init__(self, size, dtype, chunksize=(256, 256), initval=None):
        assert len(size) == 2
        self.size = size
        self.dtype = dtype
        self._chunksize = chunksize

        self.nchunkrows = int(ceil(float(size[0])/float(chunksize[0])))
        self.nchunkcols = int(ceil(float(size[1])/float(chunksize[1])))
        nchunks = self.nchunkrows * self.nchunkcols

        # Data store
        self._data = [None for i in range(nchunks)]

        # 0 => unset
        # 1 => set
        self.chunkstatus = np.zeros(nchunks, dtype=np.int8)

        if initval is not None:
            self[:,:] = initval*np.ones(size, dtype=dtype)
        return

    def __getitem__(self, key):

        if isinstance(key, int):
            irow = key // self.size[1]
            icol = key % self.size[1]
            return self._getblock(irow, icol, (1, 1))[0]

        elif isinstance(key, tuple):

            if len(key) != 2:
                raise IndexError("band can only be indexed along two dimensions")

            k0, k1 = key

            if isinstance(k0, int):
                yoff = k0
                ny = 1
                ystride = 1

            elif isinstance(k0, slice):
                ystart, ystop, ystride = k0.indices(self.size[0])
                yoff = min(ystart, ystop)
                ny = abs(ystop-ystart)
                if ystride < 0:
                    yoff += 1

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k0)))

            if isinstance(k1, int):
                xoff = k1
                nx = 1
                xstride = 1

            elif isinstance(k1, slice):
                xstart, xstop, xstride = k1.indices(self.size[1])
                xoff = min(xstart, xstop)
                nx = abs(xstop-xstart)
                if xstride < 0:
                    xoff += 1

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k1)))

            if nx == ny == 1:
                return self._getblock(yoff, xoff, (ny, nx))[0,0]
            else:
                return self._getblock(yoff, xoff, (ny, nx))[::ystride,::xstride]

        elif isinstance(key, slice):
            start, stop, stride = key.indices(self.size[0])
            yoff = min(start, stop)
            if stride < 0:
                yoff += 1
            ny = abs(stop-start)
            return self._getblock(yoff, 0, (ny, self.size[1]))[::stride]

        else:
            raise IndexError("indexing with instances of '{0}' not "
                             "supported".format(type(key)))

    def __setitem__(self, key, value):

        if isinstance(key, int):
            irow = key // self.size[1]
            icol = key % self.size[1]
            self._setblock(irow, icol, np.array(value, dtype=self.dtype))

        elif isinstance(key, tuple):

            if len(key) != 2:
                raise IndexError("band can only be indexed along two dimensions")

            k0, k1 = key

            if isinstance(k0, int):
                yoff = k0
                ny = 1
                sy = 1

            elif isinstance(k0, slice):
                if k0.start is None:
                    yoff = 0
                else:
                    yoff = k0.start
                if k0.stop is None:
                    ny = self.size[0]-yoff
                else:
                    ny = k0.stop-yoff
                if k0.step is None:
                    sy = 1
                else:
                    sy = k0.step

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k0)))

            if isinstance(k1, int):
                xoff = k1
                nx = 1
                sx = 1

            elif isinstance(k1, slice):
                if k1.start is None:
                    xoff = 0
                else:
                    xoff = k1.start
                if k1.stop is None:
                    nx = self.size[1]-xoff
                else:
                    nx = k1.stop-xoff
                if k1.step is None:
                    sx = 1
                else:
                    sx = k1.step

            else:
                raise IndexError("slicing with instances of '{0}' not "
                                 "supported".format(type(k1)))

            vny, vnx = value.shape[:2]
            if (ceil(float(ny)/sy) != vny) or (ceil(float(nx)/sx) != vnx):
                raise IndexError("Cannot insert array with size ({vny}, {vnx}))"
                        " into slice with size ({ny}, {nx})".format(
                            vny=vny, vnx=vnx, ny=ny, nx=nx))

            if sy == sx == 1:
                self._setblock(yoff, xoff, value)

            else:
                temp = self._getblock(yoff, xoff, (ny, nx))
                temp[::sy,::sx] = value
                self._setblock(yoff, xoff, temp)

        else:
            raise IndexError("indexing with instances of '{0}' not "
                             "supported".format(type(key)))

        return

    def _store(self, array, index):
        self._data[index] = blosc.compress(array.tostring(),
                                           np.dtype(self.dtype).itemsize)
        self.chunkstatus[index] = self.CHUNKSET
        return

    def _retrieve(self, index):
        bytestr = blosc.decompress(self._data[index])
        return np.fromstring(bytestr, dtype=self.dtype).reshape(self._chunksize)

    def _getchunks(self, yoff, xoff, ny, nx):
        """ Return a generator returning tuples identifying chunks covered by a
        range. The tuples contain (chunk_number, ystart, yend, xstart, xend)
        for each chunk touched by a region defined by corner indices and region
        size. """
        chunksize = self._chunksize
        ystart = yoff // chunksize[0]
        yend = ceil(float(yoff+ny) / chunksize[0])
        xstart = xoff // chunksize[1]
        xend = ceil(float(xoff+nx) / chunksize[1])

        nxchunks = int(ceil(float(self.size[1])/float(chunksize[1])))

        i = ystart
        while i < yend:

            j = xstart

            while j < xend:
                chunk_number = i*nxchunks + j
                chunk_ystart = i*chunksize[0]
                chunk_xstart = j*chunksize[1]
                chunk_yend = min((i+1)*chunksize[0], self.size[0])
                chunk_xend = min((j+1)*chunksize[1], self.size[1])
                yield (chunk_number, chunk_ystart, chunk_yend,
                       chunk_xstart, chunk_xend)
                j += 1

            i+= 1

    def _setblock(self, yoff, xoff, array):
        """ Store block of values in *array* starting at offset *yoff*, *xoff*.
        """
        size = array.shape
        chunksize = self._chunksize

        for i, yst, yen, xst, xen in self._getchunks(yoff, xoff, *size):

            # Get from data store
            if self.chunkstatus[i] != self.CHUNKUNSET:
                chunkdata = self._retrieve(i)
            else:
                chunkdata = np.zeros(self._chunksize, dtype=self.dtype)

            # Compute region within chunk to place data in
            cy0 = max(0, yoff-yst)
            cy1 = min(chunksize[0], yoff+size[0]-yst)
            cx0 = max(0, xoff-xst)
            cx1 = min(chunksize[1], xoff+size[1]-xst)

            # Compute region to slice from data
            dy0 = max(0, yst-yoff)
            dy1 = min(size[0], yen-yoff)
            dx0 = max(0, xst-xoff)
            dx1 = min(size[1], xen-xoff)

            chunkdata[cy0:cy1, cx0:cx1] = array[dy0:dy1, dx0:dx1]

            # Return to data store
            self._store(chunkdata, i)
        return

    def _getblock(self, yoff, xoff, size):
        """ Retrieve values with dimensions *size*, starting at offset *yoff*,
        *xoff*.
        """
        result = np.empty(size, self.dtype)
        for i, yst, yen, xst, xen in self._getchunks(yoff, xoff, *size):

            # Compute the bounds in the output
            oy0 = max(0, yst-yoff)
            oy1 = min(size[0], yen-yoff)
            ox0 = max(0, xst-xoff)
            ox1 = min(size[1], xen-xoff)

            if self.chunkstatus[i] == self.CHUNKUNSET:
                result[oy0:oy1, ox0:ox1] = np.zeros((oy1-oy0, ox1-ox0),
                                                    dtype=self.dtype)

            else:
                # Compute the extents from the chunk to retain
                cy0 = max(yoff, yst) - yst
                cy1 = min(yoff+size[0], yen) - yst
                cx0 = max(xoff, xst) - xst
                cx1 = min(xoff+size[1], xen) - xst

                result[oy0:oy1, ox0:ox1] = self._retrieve(i)[cy0:cy1, cx0:cx1]

        return result

<|endoftext|>


<commit_before>"""A thesaurus of words the player might use, so you can easily accept
synonyms of the word you want.

e.g. This will recognise take, pick, get or collect::

    if do[0] in weatbag.words.take:
        #...
"""
# Verbs
move = {'move', 'walk', 'go'}
give = {'give', 'feed', 'present'}
use = {'eat', 'use', 'wear'}
fight = {'fight', 'kill', 'hit', 'attack'}
drop = {'drop'}
take = {'pick', 'take', 'get', 'collect'}
look = {'look', 'inspect', 'examine'}
attack = {'attack', 'swing', 'hit', 'punch', 'fight'}
combine = {'combine', 'join', 'mix'}
talk = {'talk', 'speak', 'converse'}

# Nouns
inventory = {'inventory', 'possessions', 'belongings', 'bag'}
surroundings = {'surroundings', 'around', 'scenery'}

# Prepositions
prepositions = {'up', 'down', 'on', 'under', 'in', 'at', 'to', 'with', 'and'}

# Yes/No
yes = {'yes', 'y', 'yup', 'ye'}
no = {'no', 'n', 'nope'}

# Control
instructions = {'instructions', 'help', '?', 'tutorial'}
exit = {'exit', 'quit'}
<commit_msg>added verbs & nouns to words.py<commit_after>"""A thesaurus of words the player might use, so you can easily accept
synonyms of the word you want.

e.g. This will recognise take, pick, get or collect::

    if do[0] in weatbag.words.take:
        #...
"""
# Verbs
move = {'move', 'walk', 'go', 'travel', 'cross', 'pace', 'traverse'}
give = {'give', 'feed', 'present', 'impart', 'pass'}
use = {'eat', 'use', 'wear', 'utilise'}
fight = {'fight', 'kill', 'hit', 'attack', 'afflict'}
drop = {'drop', 'unload'}
take = {'pick', 'take', 'get', 'collect', 'occupy', 'acquire', 'adopt', 'withdraw'}
look = {'look', 'inspect', 'examine', 'search', 'check'}
attack = {'attack', 'swing', 'hit', 'punch', 'kick', 'fight'}
combine = {'combine', 'join', 'mix', 'unite', 'compound', 'aggregate', 'blend', 'coalesce', 'meld'}
talk = {'talk', 'speak', 'converse'}

# Nouns
inventory = {'inventory', 'possessions', 'belongings', 'bag'}
surroundings = {'surroundings', 'around', 'scenery'}

# Prepositions
prepositions = {'up', 'down', 'on', 'under', 'in', 'at', 'to', 'with', 'and'}

# Yes/No
yes = {'yes', 'y', 'yup', 'ye'}
no = {'no', 'n', 'nope'}

# Control
instructions = {'instructions', 'help', '?', 'tutorial'}
exit = {'exit', 'quit'}
<|endoftext|>


<commit_before><commit_msg>tools/bp_gdb.py: dump allocation addresses<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from subprocess import call
import os

import argparse
parser = argparse.ArgumentParser()

parser.add_argument("-i", "--input", help="FASTQ file (can be the location on S3)")

args = parser.parse_args()
fastq_file = args.input

print(fastq_file)

#mock up
# fastq_file = "../../../input/bam/test.recal.bam"

input_folder = "/home/ubuntu/projects/input/fastq"

base=os.path.basename(fastq_file)
print(base)

if fastq_file.startswith('s3://'):
    #download file to input folder
    command = "s3cmd get --force %s %s/" % (fastq_file, input_folder)
    output = call(command, shell=True)
    print(output)
    fastq_file = "%s/%s" % (input_folder, base)

# base_name = os.path.splitext(base)[0]
base_name = base.split('.')[0]
print(base_name)

memory_use = "15g"

#create one folder per sample
output_folder = "/home/ubuntu/projects/output/reports/fastq/%s" % (base_name)
if not os.path.exists(output_folder):
    os.makedirs(output_folder)

fastqc_dir = "/home/ubuntu/projects/programs/fastqc/FastQC/"

qualimap_dir = "/home/ubuntu/projects/programs/qualimap/qualimap_v2.2"

#s3
#if bam file start with s3 download from s3

#fastqc
command = "%s/fastqc -t 4 %s -o %s" % (fastqc_dir, fastq_file, output_folder)
output = call(command, shell=True)
print(output)
#done already!

#removefastq
os.remove(fastq_file)

# print('Running featureCounts')
# #featureCounts
# command = """/home/ubuntu/projects/programs/subread-1.5.1-Linux-x86_64/bin/featureCounts --donotsort -T 4 -p \
# -a /home/ubuntu/projects/input/gtf/Homo_sapiens.GRCh37.75.gtf \
# -o %s/%s.featureCounts.txt \
# %s""" % (output_folder, base_name, fastq_file)

# #bamtools
# print('Running bamtools')
# command = """/home/ubuntu/projects/programs/bamtools/bin/bamtools stats -in %s > %s/%s.bamtools.stats.txt""" % (fastq_file, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# print('Running DepthOfCoverage')
# #gatk DepthOfCoverage
# command = """
# java -Xmx15g -jar %s/GenomeAnalysisTK.jar -T DepthOfCoverage \
# -I %s \
# -R %s \
# -o %s/%s.DepthOfCoverage.txt \
# -nt 4 \
# --omitIntervalStatistics \
# -ct 5 -ct 10 -ct 20 -ct 30 \
# -log %s/%s.DepthofCoverage.log \
# """ % (gatk_dir, fastq_file, human_reference, output_folder, base_name, output_folder, base_name)
# output = call(command, shell=True)
# print(output)

# #qualimap BamQC
# print('Running qualimap BamQC')
# command = """%s/qualimap bamqc \
# --java-mem-size=15G \
# -bam %s \
# -outdir %s \
# -nt 4
# """ % (qualimap_dir, fastq_file, output_folder)
# # output = call(command, shell=True)
# # print(output)

# #samtools flagstat
# print('Running samtools flagstat')
# command = """%s/samtools flagstat %s > %s/%s.samtools.flagstat.txt
# """ % (samtools_dir, fastq_file, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # #picard
# # print('Running CollectAlignmentSummaryMetrics')
# # # #CollectAlignmentSummaryMetrics
# # command = """
# # java -jar -Xmx15g %s/picard.jar CollectAlignmentSummaryMetrics \
# # I=%s \
# # O=%s/%s.AlignmentSummaryMetrics.txt \
# # R=%s \
# # VALIDATION_STRINGENCY=SILENT""" % (picard_dir, fastq_file, output_folder, base_name, human_reference)
# # output = call(command, shell=True)
# # print(output)

# # print('Running CollectGcBiasMetrics')
# # # #CollectGcBiasMetrics
# # command = """
# # java -jar -Xmx15g %s/picard.jar CollectGcBiasMetrics \
# # I=%s \
# # O=%s/%s.gc_bias_metrics.txt \
# # R=%s \
# # CHART=%s/%s.gc_bias_metrics.pdf \
# # S=%s/%s.gc_bias_summary_metrics.txt \
# # VALIDATION_STRINGENCY=SILENT""" % (picard_dir, fastq_file, output_folder, base_name, human_reference, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running CollectInsertSizeMetrics')
# # #CollectInsertSizeMetrics
# # command = """java -Xmx%s -jar %s/picard.jar CollectInsertSizeMetrics \
# # I=%s \
# # O=%s/%s.insert_size_metrics.txt \
# # H=%s/%s.insert_size_histogram.pdf \
# # M=0.5 \
# # VALIDATION_STRINGENCY=SILENT""" % (memory_use, picard_dir, fastq_file, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # # #MeanQualityByCycle
# # print('Running MeanQualityByCycle')
# # command = """java -Xmx%s -jar %s/picard.jar MeanQualityByCycle \
# # I=%s \
# # O=%s/%s.mean_qual_by_cycle.txt \
# # CHART=%s/%s.mean_qual_by_cycle.pdf \
# # VALIDATION_STRINGENCY=SILENT """ % (memory_use, picard_dir, fastq_file, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running QualityScoreDistribution')
# # # #QualityScoreDistribution
# # command = """java -Xmx%s -jar %s/picard.jar QualityScoreDistribution \
# # I=%s \
# # O=%s/%s.qual_score_dist.txt \
# # CHART=%s/%s.qual_score_dist.pdf \
# # VALIDATION_STRINGENCY=SILENT """ % (memory_use, picard_dir, fastq_file, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running BamIndexStats')
# # # #BamIndexStats
# # command = """java -Xmx%s -jar %s/picard.jar BamIndexStats \
# # I=%s \
# # O=%s/%s.BamIndexStats.output.txt \
# # VALIDATION_STRINGENCY=SILENT""" % (memory_use, picard_dir, fastq_file, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running CollectHsMetrics')
# #CollectHsMetrics
# # command = """java -Xmx%s -jar %s/picard.jar CollectHsMetrics \
# # I=%s \
# # O=%s/%s.hs_metrics.txt \
# # R=%s \
# # BAIT_INTERVALS=%s \
# # TARGET_INTERVALS=%s \
# # VALIDATION_STRINGENCY=SILENT """ % (memory_use, picard_dir, fastq_file, output_folder, base_name, human_reference, target_file, target_file)
# # output = call(command, shell=True)
# # print(output)

<commit_msg>fastq genome stats<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from subprocess import call
import os

import argparse
parser = argparse.ArgumentParser()

parser.add_argument("-i", "--input", help="FASTQ file (can be the location on S3)")

args = parser.parse_args()
fastq_file = args.input

print(fastq_file)

#mock up
# fastq_file = "../../../input/bam/test.recal.bam"

input_folder = "/home/ubuntu/projects/input/fastq"

base=os.path.basename(fastq_file)
print(base)

if fastq_file.startswith('s3://'):
    #download file to input folder
    command = "s3cmd get --continue %s %s/" % (fastq_file, input_folder)
    output = call(command, shell=True)
    print(output)
    fastq_file = "%s/%s" % (input_folder, base)

# base_name = os.path.splitext(base)[0]
base_name = base.split('.')[0]
print(base_name)

memory_use = "15g"

#create one folder per sample
output_folder = "/home/ubuntu/projects/output/reports/fastq/%s" % (base_name)
if not os.path.exists(output_folder):
    os.makedirs(output_folder)

fastqc_dir = "/home/ubuntu/projects/programs/fastqc/FastQC/"

qualimap_dir = "/home/ubuntu/projects/programs/qualimap/qualimap_v2.2"

#s3
#if bam file start with s3 download from s3

#fastqc
command = "%s/fastqc -t 4 %s -o %s" % (fastqc_dir, fastq_file, output_folder)
output = call(command, shell=True)
print(output)
#done already!

#removefastq
os.remove(fastq_file)

# print('Running featureCounts')
# #featureCounts
# command = """/home/ubuntu/projects/programs/subread-1.5.1-Linux-x86_64/bin/featureCounts --donotsort -T 4 -p \
# -a /home/ubuntu/projects/input/gtf/Homo_sapiens.GRCh37.75.gtf \
# -o %s/%s.featureCounts.txt \
# %s""" % (output_folder, base_name, fastq_file)

# #bamtools
# print('Running bamtools')
# command = """/home/ubuntu/projects/programs/bamtools/bin/bamtools stats -in %s > %s/%s.bamtools.stats.txt""" % (fastq_file, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# print('Running DepthOfCoverage')
# #gatk DepthOfCoverage
# command = """
# java -Xmx15g -jar %s/GenomeAnalysisTK.jar -T DepthOfCoverage \
# -I %s \
# -R %s \
# -o %s/%s.DepthOfCoverage.txt \
# -nt 4 \
# --omitIntervalStatistics \
# -ct 5 -ct 10 -ct 20 -ct 30 \
# -log %s/%s.DepthofCoverage.log \
# """ % (gatk_dir, fastq_file, human_reference, output_folder, base_name, output_folder, base_name)
# output = call(command, shell=True)
# print(output)

# #qualimap BamQC
# print('Running qualimap BamQC')
# command = """%s/qualimap bamqc \
# --java-mem-size=15G \
# -bam %s \
# -outdir %s \
# -nt 4
# """ % (qualimap_dir, fastq_file, output_folder)
# # output = call(command, shell=True)
# # print(output)

# #samtools flagstat
# print('Running samtools flagstat')
# command = """%s/samtools flagstat %s > %s/%s.samtools.flagstat.txt
# """ % (samtools_dir, fastq_file, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # #picard
# # print('Running CollectAlignmentSummaryMetrics')
# # # #CollectAlignmentSummaryMetrics
# # command = """
# # java -jar -Xmx15g %s/picard.jar CollectAlignmentSummaryMetrics \
# # I=%s \
# # O=%s/%s.AlignmentSummaryMetrics.txt \
# # R=%s \
# # VALIDATION_STRINGENCY=SILENT""" % (picard_dir, fastq_file, output_folder, base_name, human_reference)
# # output = call(command, shell=True)
# # print(output)

# # print('Running CollectGcBiasMetrics')
# # # #CollectGcBiasMetrics
# # command = """
# # java -jar -Xmx15g %s/picard.jar CollectGcBiasMetrics \
# # I=%s \
# # O=%s/%s.gc_bias_metrics.txt \
# # R=%s \
# # CHART=%s/%s.gc_bias_metrics.pdf \
# # S=%s/%s.gc_bias_summary_metrics.txt \
# # VALIDATION_STRINGENCY=SILENT""" % (picard_dir, fastq_file, output_folder, base_name, human_reference, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running CollectInsertSizeMetrics')
# # #CollectInsertSizeMetrics
# # command = """java -Xmx%s -jar %s/picard.jar CollectInsertSizeMetrics \
# # I=%s \
# # O=%s/%s.insert_size_metrics.txt \
# # H=%s/%s.insert_size_histogram.pdf \
# # M=0.5 \
# # VALIDATION_STRINGENCY=SILENT""" % (memory_use, picard_dir, fastq_file, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # # #MeanQualityByCycle
# # print('Running MeanQualityByCycle')
# # command = """java -Xmx%s -jar %s/picard.jar MeanQualityByCycle \
# # I=%s \
# # O=%s/%s.mean_qual_by_cycle.txt \
# # CHART=%s/%s.mean_qual_by_cycle.pdf \
# # VALIDATION_STRINGENCY=SILENT """ % (memory_use, picard_dir, fastq_file, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running QualityScoreDistribution')
# # # #QualityScoreDistribution
# # command = """java -Xmx%s -jar %s/picard.jar QualityScoreDistribution \
# # I=%s \
# # O=%s/%s.qual_score_dist.txt \
# # CHART=%s/%s.qual_score_dist.pdf \
# # VALIDATION_STRINGENCY=SILENT """ % (memory_use, picard_dir, fastq_file, output_folder, base_name, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running BamIndexStats')
# # # #BamIndexStats
# # command = """java -Xmx%s -jar %s/picard.jar BamIndexStats \
# # I=%s \
# # O=%s/%s.BamIndexStats.output.txt \
# # VALIDATION_STRINGENCY=SILENT""" % (memory_use, picard_dir, fastq_file, output_folder, base_name)
# # output = call(command, shell=True)
# # print(output)

# # print('Running CollectHsMetrics')
# #CollectHsMetrics
# # command = """java -Xmx%s -jar %s/picard.jar CollectHsMetrics \
# # I=%s \
# # O=%s/%s.hs_metrics.txt \
# # R=%s \
# # BAIT_INTERVALS=%s \
# # TARGET_INTERVALS=%s \
# # VALIDATION_STRINGENCY=SILENT """ % (memory_use, picard_dir, fastq_file, output_folder, base_name, human_reference, target_file, target_file)
# # output = call(command, shell=True)
# # print(output)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst
#
# Astropy shared Sphinx settings.  These settings are shared between
# astropy itself and affiliated packages.
#
# Note that not all possible configuration values are present in this file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import os
import sys
import warnings

from os import path

import sphinx
from distutils.version import LooseVersion

# -- General configuration ----------------------------------------------------

# The version check in Sphinx itself can only compare the major and
# minor parts of the version number, not the micro.  To do a more
# specific version check, call check_sphinx_version("x.y.z.") from
# your project's conf.py
needs_sphinx = '1.3'

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

def check_sphinx_version(expected_version):
    sphinx_version = LooseVersion(sphinx.__version__)
    expected_version = LooseVersion(expected_version)
    if sphinx_version < expected_version:
        raise RuntimeError(
            "At least Sphinx version {0} is required to build this "
            "documentation.  Found {1}.".format(
                expected_version, sphinx_version))

# Configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'python': ('https://docs.python.org/3/',
               (None, 'http://data.astropy.org/intersphinx/python3.inv')),
    'pythonloc': ('http://docs.python.org/',
                  path.abspath(path.join(path.dirname(__file__),
                                         'local/python3_local_links.inv'))),
    'numpy': ('https://docs.scipy.org/doc/numpy/',
              (None, 'http://data.astropy.org/intersphinx/numpy.inv')),
    'scipy': ('https://docs.scipy.org/doc/scipy/reference/',
              (None, 'http://data.astropy.org/intersphinx/scipy.inv')),
    'matplotlib': ('http://matplotlib.org/',
                   (None, 'http://data.astropy.org/intersphinx/matplotlib.inv')),
    'astropy': ('http://docs.astropy.org/en/stable/', None),
    'h5py': ('http://docs.h5py.org/en/latest/', None)}

if sys.version_info[0] == 2:
    intersphinx_mapping['python'] = (
        'https://docs.python.org/2/',
        (None, 'http://data.astropy.org/intersphinx/python2.inv'))
    intersphinx_mapping['pythonloc'] = (
        'http://docs.python.org/',
        path.abspath(path.join(path.dirname(__file__),
                               'local/python2_local_links.inv')))

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# Add any paths that contain templates here, relative to this directory.
# templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# The reST default role (used for this markup: `text`) to use for all
# documents. Set to the "smart" one.
default_role = 'obj'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# This is added to the end of RST files - a good place to put substitutions to
# be used globally.
rst_epilog = """
.. _Astropy: http://astropy.org
"""

# A list of warning types to suppress arbitrary warning messages. We mean to
# override directives in astropy_helpers.sphinx.ext.autodoc_enhancements,
# thus need to ignore those warning. This can be removed once the patch gets
# released in upstream Sphinx (https://github.com/sphinx-doc/sphinx/pull/1843).
# Suppress the warnings requires Sphinx v1.4.2
suppress_warnings = ['app.add_directive', ]

# -- Project information ------------------------------------------------------

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
#pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Settings for extensions and extension options ----------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.inheritance_diagram',
    'sphinx.ext.viewcode',
    'astropy_helpers.extern.numpydoc',
    'astropy_helpers.extern.automodapi.automodapi',
    'astropy_helpers.extern.automodapi.smart_resolver',
    'astropy_helpers.sphinx.ext.tocdepthfix',
    'astropy_helpers.sphinx.ext.doctest',
    'astropy_helpers.sphinx.ext.changelog_links']

if not on_rtd and LooseVersion(sphinx.__version__) < LooseVersion('1.4'):
    extensions.append('sphinx.ext.pngmath')
else:
    extensions.append('sphinx.ext.mathjax')

try:
    import matplotlib.sphinxext.plot_directive
    extensions += [matplotlib.sphinxext.plot_directive.__name__]
# AttributeError is checked here in case matplotlib is installed but
# Sphinx isn't.  Note that this module is imported by the config file
# generator, even if we're not building the docs.
except (ImportError, AttributeError):
    warnings.warn(
        "matplotlib's plot_directive could not be imported. " +
        "Inline plots will not be included in the output")

# Don't show summaries of the members in each class along with the
# class' docstring
numpydoc_show_class_members = False

autosummary_generate = True

automodapi_toctreedirnm = 'api'

# Class documentation should contain *both* the class docstring and
# the __init__ docstring
autoclass_content = "both"

# Render inheritance diagrams in SVG
graphviz_output_format = "svg"

graphviz_dot_args = [
    '-Nfontsize=10',
    '-Nfontname=Helvetica Neue, Helvetica, Arial, sans-serif',
    '-Efontsize=10',
    '-Efontname=Helvetica Neue, Helvetica, Arial, sans-serif',
    '-Gfontsize=10',
    '-Gfontname=Helvetica Neue, Helvetica, Arial, sans-serif'
]

# -- Options for HTML output -------------------------------------------------

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = [path.abspath(path.join(path.dirname(__file__), 'themes'))]

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'bootstrap-astropy'

# Custom sidebar templates, maps document names to template names.
html_sidebars = {
    '**': ['localtoc.html'],
    'search': [],
    'genindex': [],
    'py-modindex': [],
}

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.

# included in the bootstrap-astropy theme
html_favicon = path.join(html_theme_path[0], html_theme, 'static',
                         'astropy_logo.ico')

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%d %b %Y'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# -- Options for LaTeX output ------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
latex_toplevel_sectioning = 'part'

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

latex_elements = {}

# Additional stuff for the LaTeX preamble.
latex_elements['preamble'] = r"""
% Use a more modern-looking monospace font
\usepackage{inconsolata}

% The enumitem package provides unlimited nesting of lists and enums.
% Sphinx may use this in the future, in which case this can be removed.
% See https://bitbucket.org/birkenfeld/sphinx/issue/777/latex-output-too-deeply-nested
\usepackage{enumitem}
\setlistdepth{15}

% In the parameters section, place a newline after the Parameters
% header.  (This is stolen directly from Numpy's conf.py, since it
% affects Numpy-style docstrings).
\usepackage{expdlist}
\let\latexdescription=\description
\def\description{\latexdescription{}{} \breaklabel}

% Support the superscript Unicode numbers used by the "unicode" units
% formatter
\DeclareUnicodeCharacter{2070}{\ensuremath{^0}}
\DeclareUnicodeCharacter{00B9}{\ensuremath{^1}}
\DeclareUnicodeCharacter{00B2}{\ensuremath{^2}}
\DeclareUnicodeCharacter{00B3}{\ensuremath{^3}}
\DeclareUnicodeCharacter{2074}{\ensuremath{^4}}
\DeclareUnicodeCharacter{2075}{\ensuremath{^5}}
\DeclareUnicodeCharacter{2076}{\ensuremath{^6}}
\DeclareUnicodeCharacter{2077}{\ensuremath{^7}}
\DeclareUnicodeCharacter{2078}{\ensuremath{^8}}
\DeclareUnicodeCharacter{2079}{\ensuremath{^9}}
\DeclareUnicodeCharacter{207B}{\ensuremath{^-}}
\DeclareUnicodeCharacter{00B0}{\ensuremath{^{\circ}}}
\DeclareUnicodeCharacter{2032}{\ensuremath{^{\prime}}}
\DeclareUnicodeCharacter{2033}{\ensuremath{^{\prime\prime}}}

% Make the "warning" and "notes" sections use a sans-serif font to
% make them stand out more.
\renewenvironment{notice}[2]{
  \def\py@noticetype{#1}
  \csname py@noticestart@#1\endcsname
  \textsf{\textbf{#2}}
}{\csname py@noticeend@\py@noticetype\endcsname}
"""

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# -- Options for the linkcheck builder ----------------------------------------

# A timeout value, in seconds, for the linkcheck builder
linkcheck_timeout = 60
<commit_msg>Removing py2 intersphinx from conf, too<commit_after># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst
#
# Astropy shared Sphinx settings.  These settings are shared between
# astropy itself and affiliated packages.
#
# Note that not all possible configuration values are present in this file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import os
import sys
import warnings

from os import path

import sphinx
from distutils.version import LooseVersion

# -- General configuration ----------------------------------------------------

# The version check in Sphinx itself can only compare the major and
# minor parts of the version number, not the micro.  To do a more
# specific version check, call check_sphinx_version("x.y.z.") from
# your project's conf.py
needs_sphinx = '1.3'

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

def check_sphinx_version(expected_version):
    sphinx_version = LooseVersion(sphinx.__version__)
    expected_version = LooseVersion(expected_version)
    if sphinx_version < expected_version:
        raise RuntimeError(
            "At least Sphinx version {0} is required to build this "
            "documentation.  Found {1}.".format(
                expected_version, sphinx_version))

# Configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'python': ('https://docs.python.org/3/',
               (None, 'http://data.astropy.org/intersphinx/python3.inv')),
    'pythonloc': ('http://docs.python.org/',
                  path.abspath(path.join(path.dirname(__file__),
                                         'local/python3_local_links.inv'))),
    'numpy': ('https://docs.scipy.org/doc/numpy/',
              (None, 'http://data.astropy.org/intersphinx/numpy.inv')),
    'scipy': ('https://docs.scipy.org/doc/scipy/reference/',
              (None, 'http://data.astropy.org/intersphinx/scipy.inv')),
    'matplotlib': ('http://matplotlib.org/',
                   (None, 'http://data.astropy.org/intersphinx/matplotlib.inv')),
    'astropy': ('http://docs.astropy.org/en/stable/', None),
    'h5py': ('http://docs.h5py.org/en/latest/', None)}

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# Add any paths that contain templates here, relative to this directory.
# templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# The reST default role (used for this markup: `text`) to use for all
# documents. Set to the "smart" one.
default_role = 'obj'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# This is added to the end of RST files - a good place to put substitutions to
# be used globally.
rst_epilog = """
.. _Astropy: http://astropy.org
"""

# A list of warning types to suppress arbitrary warning messages. We mean to
# override directives in astropy_helpers.sphinx.ext.autodoc_enhancements,
# thus need to ignore those warning. This can be removed once the patch gets
# released in upstream Sphinx (https://github.com/sphinx-doc/sphinx/pull/1843).
# Suppress the warnings requires Sphinx v1.4.2
suppress_warnings = ['app.add_directive', ]

# -- Project information ------------------------------------------------------

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
#pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Settings for extensions and extension options ----------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.inheritance_diagram',
    'sphinx.ext.viewcode',
    'astropy_helpers.extern.numpydoc',
    'astropy_helpers.extern.automodapi.automodapi',
    'astropy_helpers.extern.automodapi.smart_resolver',
    'astropy_helpers.sphinx.ext.tocdepthfix',
    'astropy_helpers.sphinx.ext.doctest',
    'astropy_helpers.sphinx.ext.changelog_links']

if not on_rtd and LooseVersion(sphinx.__version__) < LooseVersion('1.4'):
    extensions.append('sphinx.ext.pngmath')
else:
    extensions.append('sphinx.ext.mathjax')

try:
    import matplotlib.sphinxext.plot_directive
    extensions += [matplotlib.sphinxext.plot_directive.__name__]
# AttributeError is checked here in case matplotlib is installed but
# Sphinx isn't.  Note that this module is imported by the config file
# generator, even if we're not building the docs.
except (ImportError, AttributeError):
    warnings.warn(
        "matplotlib's plot_directive could not be imported. " +
        "Inline plots will not be included in the output")

# Don't show summaries of the members in each class along with the
# class' docstring
numpydoc_show_class_members = False

autosummary_generate = True

automodapi_toctreedirnm = 'api'

# Class documentation should contain *both* the class docstring and
# the __init__ docstring
autoclass_content = "both"

# Render inheritance diagrams in SVG
graphviz_output_format = "svg"

graphviz_dot_args = [
    '-Nfontsize=10',
    '-Nfontname=Helvetica Neue, Helvetica, Arial, sans-serif',
    '-Efontsize=10',
    '-Efontname=Helvetica Neue, Helvetica, Arial, sans-serif',
    '-Gfontsize=10',
    '-Gfontname=Helvetica Neue, Helvetica, Arial, sans-serif'
]

# -- Options for HTML output -------------------------------------------------

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = [path.abspath(path.join(path.dirname(__file__), 'themes'))]

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'bootstrap-astropy'

# Custom sidebar templates, maps document names to template names.
html_sidebars = {
    '**': ['localtoc.html'],
    'search': [],
    'genindex': [],
    'py-modindex': [],
}

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.

# included in the bootstrap-astropy theme
html_favicon = path.join(html_theme_path[0], html_theme, 'static',
                         'astropy_logo.ico')

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%d %b %Y'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# -- Options for LaTeX output ------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
latex_toplevel_sectioning = 'part'

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

latex_elements = {}

# Additional stuff for the LaTeX preamble.
latex_elements['preamble'] = r"""
% Use a more modern-looking monospace font
\usepackage{inconsolata}

% The enumitem package provides unlimited nesting of lists and enums.
% Sphinx may use this in the future, in which case this can be removed.
% See https://bitbucket.org/birkenfeld/sphinx/issue/777/latex-output-too-deeply-nested
\usepackage{enumitem}
\setlistdepth{15}

% In the parameters section, place a newline after the Parameters
% header.  (This is stolen directly from Numpy's conf.py, since it
% affects Numpy-style docstrings).
\usepackage{expdlist}
\let\latexdescription=\description
\def\description{\latexdescription{}{} \breaklabel}

% Support the superscript Unicode numbers used by the "unicode" units
% formatter
\DeclareUnicodeCharacter{2070}{\ensuremath{^0}}
\DeclareUnicodeCharacter{00B9}{\ensuremath{^1}}
\DeclareUnicodeCharacter{00B2}{\ensuremath{^2}}
\DeclareUnicodeCharacter{00B3}{\ensuremath{^3}}
\DeclareUnicodeCharacter{2074}{\ensuremath{^4}}
\DeclareUnicodeCharacter{2075}{\ensuremath{^5}}
\DeclareUnicodeCharacter{2076}{\ensuremath{^6}}
\DeclareUnicodeCharacter{2077}{\ensuremath{^7}}
\DeclareUnicodeCharacter{2078}{\ensuremath{^8}}
\DeclareUnicodeCharacter{2079}{\ensuremath{^9}}
\DeclareUnicodeCharacter{207B}{\ensuremath{^-}}
\DeclareUnicodeCharacter{00B0}{\ensuremath{^{\circ}}}
\DeclareUnicodeCharacter{2032}{\ensuremath{^{\prime}}}
\DeclareUnicodeCharacter{2033}{\ensuremath{^{\prime\prime}}}

% Make the "warning" and "notes" sections use a sans-serif font to
% make them stand out more.
\renewenvironment{notice}[2]{
  \def\py@noticetype{#1}
  \csname py@noticestart@#1\endcsname
  \textsf{\textbf{#2}}
}{\csname py@noticeend@\py@noticetype\endcsname}
"""

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# -- Options for the linkcheck builder ----------------------------------------

# A timeout value, in seconds, for the linkcheck builder
linkcheck_timeout = 60
<|endoftext|>