<commit_before>#!/usr/bin/env python
#
## Licensed to the .NET Foundation under one or more agreements.
## The .NET Foundation licenses this file to you under the MIT license.
## See the LICENSE file in the project root for more information.
#
##
# Title               :format.py
#
################################################################################
# Script to install and run jit-format over jit source for all configurations.
################################################################################

import urllib
import argparse
import os
import sys
import tarfile
import zipfile
import subprocess
import urllib2
import shutil

def expandPath(path):
    return os.path.abspath(os.path.expanduser(path))

def del_rw(action, name, exc):
    os.chmod(name, 0651)
    os.remove(name)

def main(argv):
    parser = argparse.ArgumentParser()
    required = parser.add_argument_group('required arguments')
    required.add_argument('-a', '--arch', type=str,
            default=None, help='architecture to run jit-format on')
    required.add_argument('-o', '--os', type=str,
            default=None, help='operating system')
    required.add_argument('-c', '--coreclr', type=str,
            default=None, help='full path to coreclr')

    args, unknown = parser.parse_known_args(argv)

    if unknown:
        print('Ignorning argument(s): ', ','.join(unknown))

    if args.coreclr is None:
        print('Specify --coreclr')
        return -1
    if args.os is None:
        print('Specifiy --os')
        return -1
    if args.arch is None:
        print('Specify --arch')
        return -1

    if not os.path.isdir(expandPath(args.coreclr)):
        print('Bad path to coreclr')
        return -1

    coreclr = args.coreclr
    platform = args.os
    arch = args.arch

    my_env = os.environ

    # Download .Net CLI

    dotnetcliUrl = ""
    dotnetcliFilename = ""

    # build.cmd removes the Tools directory, so we need to put our version of jitutils
    # outside of the Tools directory

    dotnetcliPath = os.path.join(coreclr, 'dotnetcli-jitutils')

    # Try to make the dotnetcli-jitutils directory if it doesn't exist

    try:
        os.makedirs(dotnetcliPath)
    except OSError:
        if not os.path.isdir(dotnetcliPath):
            raise

    print("Downloading .Net CLI")
    if platform == 'Linux':
        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809129"
        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz')
    elif platform == 'OSX':
        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809128"
        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz')
    elif platform == 'Windows_NT':
        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809126"
        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.zip')
    else:
        print('Unknown os ', os)
        return -1

    response = urllib2.urlopen(dotnetcliUrl)
    request_url = response.geturl()
    testfile = urllib.URLopener()
    testfile.retrieve(request_url, dotnetcliFilename)

    if not os.path.isfile(dotnetcliFilename):
        print("Did not download .Net CLI!")
        return -1

    # Install .Net CLI

    if platform == 'Linux' or platform == 'OSX':
        tar = tarfile.open(dotnetcliFilename)
        tar.extractall(dotnetcliPath)
        tar.close()
    elif platform == 'Windows_NT':
        with zipfile.ZipFile(dotnetcliFilename, "r") as z:
            z.extractall(dotnetcliPath)

    dotnet = ""
    if platform == 'Linux' or platform == 'OSX':
        dotnet = "dotnet"
    elif platform == 'Windows_NT':
        dotnet = "dotnet.exe"

    if not os.path.isfile(os.path.join(dotnetcliPath, dotnet)):
        print("Did not extract .Net CLI from download")
        return -1

    # Download bootstrap

    bootstrapFilename = ""

    jitUtilsPath = os.path.join(coreclr, "jitutils")

    if os.path.isdir(jitUtilsPath):
        print("Deleting " + jitUtilsPath)
        shutil.rmtree(jitUtilsPath, onerror=del_rw)

    if platform == 'Linux' or platform == 'OSX':
        bootstrapFilename = "bootstrap.sh"
    elif platform == 'Windows_NT':
        bootstrapFilename = "bootstrap.cmd"

    bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename

    bootstrapPath = os.path.join(coreclr, bootstrapFilename)
    testfile.retrieve(bootstrapUrl, bootstrapPath)

    if not os.path.isfile(bootstrapPath):
        print("Did not download bootstrap!")
        return -1

    # On *nix platforms, we need to make the bootstrap file executable

    if platform == 'Linux' or platform == 'OSX':
        print("Making bootstrap executable")
        os.chmod(bootstrapPath, 0751)

    print(bootstrapPath)

    # Run bootstrap

    my_env["PATH"] += os.pathsep + dotnetcliPath
    if platform == 'Linux' or platform == 'OSX':
        print("Running bootstrap")
        proc = subprocess.Popen(['bash', bootstrapPath], env=my_env)
        output,error = proc.communicate()
    elif platform == 'Windows_NT':
        proc = subprocess.Popen([bootstrapPath], env=my_env)
        output,error = proc.communicate()

    # Run jit-format

    returncode = 0
    jitutilsBin = os.path.join(coreclr, "jitutils", "bin")
    my_env["PATH"] += os.pathsep + jitutilsBin
    current_dir = os.getcwd()

    if not os.path.isdir(jitutilsBin):
        print("Jitutils not built!")
        return -1

    jitformat = jitutilsBin

    if platform == 'Linux' or platform == 'OSX':
        jitformat = os.path.join(jitformat, "jit-format")
    elif platform == 'Windows_NT':
        jitformat = os.path.join(jitformat,"jit-format.cmd")
    errorMessage = ""

    builds = ["Checked", "Debug", "Release"]
    projects = ["dll", "standalone", "crossgen"]

    for build in builds:
        for project in projects:
            proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env)
            output,error = proc.communicate()
            errorcode = proc.returncode

            if errorcode != 0:
                errorMessage += "\tjit-format -a " + arch + " -b " + build + " -o " + platform
                errorMessage += " -c <absolute-path-to-coreclr> --verbose --fix --projects " + project +"\n"
                returncode = errorcode

    os.chdir(current_dir)

    if os.path.isdir(jitUtilsPath):
        print("Deleting " + jitUtilsPath)
        shutil.rmtree(jitUtilsPath, onerror=del_rw)

    if os.path.isdir(dotnetcliPath):
        print("Deleting " + dotnetcliPath)
        shutil.rmtree(dotnetcliPath, onerror=del_rw)

    if os.path.isfile(bootstrapPath):
        print("Deleting " + bootstrapPath)
        os.remove(bootstrapPath)

    if returncode != 0:
        print("There were errors in formatting. Please run jit-format locally with: \n")
        print(errorMessage)

    return returncode

if __name__ == '__main__':
    return_code = main(sys.argv[1:])
    sys.exit(return_code)
<commit_msg>Fix format.py to look for .bat on Windows<commit_after>#!/usr/bin/env python
#
## Licensed to the .NET Foundation under one or more agreements.
## The .NET Foundation licenses this file to you under the MIT license.
## See the LICENSE file in the project root for more information.
#
##
# Title               :format.py
#
################################################################################
# Script to install and run jit-format over jit source for all configurations.
################################################################################

import urllib
import argparse
import os
import sys
import tarfile
import zipfile
import subprocess
import urllib2
import shutil

def expandPath(path):
    return os.path.abspath(os.path.expanduser(path))

def del_rw(action, name, exc):
    os.chmod(name, 0651)
    os.remove(name)

def main(argv):
    parser = argparse.ArgumentParser()
    required = parser.add_argument_group('required arguments')
    required.add_argument('-a', '--arch', type=str,
            default=None, help='architecture to run jit-format on')
    required.add_argument('-o', '--os', type=str,
            default=None, help='operating system')
    required.add_argument('-c', '--coreclr', type=str,
            default=None, help='full path to coreclr')

    args, unknown = parser.parse_known_args(argv)

    if unknown:
        print('Ignorning argument(s): ', ','.join(unknown))

    if args.coreclr is None:
        print('Specify --coreclr')
        return -1
    if args.os is None:
        print('Specifiy --os')
        return -1
    if args.arch is None:
        print('Specify --arch')
        return -1

    if not os.path.isdir(expandPath(args.coreclr)):
        print('Bad path to coreclr')
        return -1

    coreclr = args.coreclr
    platform = args.os
    arch = args.arch

    my_env = os.environ

    # Download .Net CLI

    dotnetcliUrl = ""
    dotnetcliFilename = ""

    # build.cmd removes the Tools directory, so we need to put our version of jitutils
    # outside of the Tools directory

    dotnetcliPath = os.path.join(coreclr, 'dotnetcli-jitutils')

    # Try to make the dotnetcli-jitutils directory if it doesn't exist

    try:
        os.makedirs(dotnetcliPath)
    except OSError:
        if not os.path.isdir(dotnetcliPath):
            raise

    print("Downloading .Net CLI")
    if platform == 'Linux':
        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809129"
        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz')
    elif platform == 'OSX':
        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809128"
        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz')
    elif platform == 'Windows_NT':
        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809126"
        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.zip')
    else:
        print('Unknown os ', os)
        return -1

    response = urllib2.urlopen(dotnetcliUrl)
    request_url = response.geturl()
    testfile = urllib.URLopener()
    testfile.retrieve(request_url, dotnetcliFilename)

    if not os.path.isfile(dotnetcliFilename):
        print("Did not download .Net CLI!")
        return -1

    # Install .Net CLI

    if platform == 'Linux' or platform == 'OSX':
        tar = tarfile.open(dotnetcliFilename)
        tar.extractall(dotnetcliPath)
        tar.close()
    elif platform == 'Windows_NT':
        with zipfile.ZipFile(dotnetcliFilename, "r") as z:
            z.extractall(dotnetcliPath)

    dotnet = ""
    if platform == 'Linux' or platform == 'OSX':
        dotnet = "dotnet"
    elif platform == 'Windows_NT':
        dotnet = "dotnet.exe"

    if not os.path.isfile(os.path.join(dotnetcliPath, dotnet)):
        print("Did not extract .Net CLI from download")
        return -1

    # Download bootstrap

    bootstrapFilename = ""

    jitUtilsPath = os.path.join(coreclr, "jitutils")

    if os.path.isdir(jitUtilsPath):
        print("Deleting " + jitUtilsPath)
        shutil.rmtree(jitUtilsPath, onerror=del_rw)

    if platform == 'Linux' or platform == 'OSX':
        bootstrapFilename = "bootstrap.sh"
    elif platform == 'Windows_NT':
        bootstrapFilename = "bootstrap.cmd"

    bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename

    bootstrapPath = os.path.join(coreclr, bootstrapFilename)
    testfile.retrieve(bootstrapUrl, bootstrapPath)

    if not os.path.isfile(bootstrapPath):
        print("Did not download bootstrap!")
        return -1

    # On *nix platforms, we need to make the bootstrap file executable

    if platform == 'Linux' or platform == 'OSX':
        print("Making bootstrap executable")
        os.chmod(bootstrapPath, 0751)

    print(bootstrapPath)

    # Run bootstrap

    my_env["PATH"] += os.pathsep + dotnetcliPath
    if platform == 'Linux' or platform == 'OSX':
        print("Running bootstrap")
        proc = subprocess.Popen(['bash', bootstrapPath], env=my_env)
        output,error = proc.communicate()
    elif platform == 'Windows_NT':
        proc = subprocess.Popen([bootstrapPath], env=my_env)
        output,error = proc.communicate()

    # Run jit-format

    returncode = 0
    jitutilsBin = os.path.join(coreclr, "jitutils", "bin")
    my_env["PATH"] += os.pathsep + jitutilsBin
    current_dir = os.getcwd()

    if not os.path.isdir(jitutilsBin):
        print("Jitutils not built!")
        return -1

    jitformat = jitutilsBin

    if platform == 'Linux' or platform == 'OSX':
        jitformat = os.path.join(jitformat, "jit-format")
    elif platform == 'Windows_NT':
        jitformat = os.path.join(jitformat,"jit-format.bat")
    errorMessage = ""

    builds = ["Checked", "Debug", "Release"]
    projects = ["dll", "standalone", "crossgen"]

    for build in builds:
        for project in projects:
            proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env)
            output,error = proc.communicate()
            errorcode = proc.returncode

            if errorcode != 0:
                errorMessage += "\tjit-format -a " + arch + " -b " + build + " -o " + platform
                errorMessage += " -c <absolute-path-to-coreclr> --verbose --fix --projects " + project +"\n"
                returncode = errorcode

    os.chdir(current_dir)

    if os.path.isdir(jitUtilsPath):
        print("Deleting " + jitUtilsPath)
        shutil.rmtree(jitUtilsPath, onerror=del_rw)

    if os.path.isdir(dotnetcliPath):
        print("Deleting " + dotnetcliPath)
        shutil.rmtree(dotnetcliPath, onerror=del_rw)

    if os.path.isfile(bootstrapPath):
        print("Deleting " + bootstrapPath)
        os.remove(bootstrapPath)

    if returncode != 0:
        print("There were errors in formatting. Please run jit-format locally with: \n")
        print(errorMessage)

    return returncode

if __name__ == '__main__':
    return_code = main(sys.argv[1:])
    sys.exit(return_code)
<|endoftext|>


<commit_before>import flask
import logging
import blinker
from collections import defaultdict
#from google.appengine.ext import ndb
from google.appengine.ext.ndb import polymodel

from . import base as base_base
from . import edit as base_edit
from . import list as base_list

logger = logging.getLogger(__name__)

class PolymodelMeta(base_base.KibbleMeta):
    """
    Custom metaclass to store a map of `_class_key()` to
    dispatch requests to the right type.
    """
    cls_map = defaultdict(dict)

    def __new__(mcls, name, bases, attrs):
        cls = super(base_base.KibbleMeta, mcls)\
            .__new__(mcls, name, bases, attrs)

        if not getattr(cls, 'model', None) or \
                not issubclass(cls.model, polymodel.PolyModel):
            return cls

        if blinker and cls.model:
            cls.pre_signal = blinker.signal("{}-pre-{}".format(
                cls.kind(), cls.action))

            cls.post_signal = blinker.signal("{}-post-{}".format(
                cls.kind(), cls.action))
        else:
            cls.pre_signal = base_base.StubSignal()
            cls.post_signal = base_base.StubSignal()

        mcls.cls_map[tuple(cls.model._class_key())][cls.action] = cls

        return cls

class PolymodelMixin(object):
    __metaclass__ = PolymodelMeta

class Create(PolymodelMixin, base_edit.Create):
    pass

class Edit(PolymodelMixin, base_edit.Edit):
    pass

def _dispatch_polyrequest(action, poly_kind, base_kind, **kwargs):
    base_key = tuple(base_kind._class_key())

    for cls_key, views in PolymodelMeta.cls_map.items():
        if action not in views:
            continue

        if cls_key[-1].lower() == poly_kind.lower() and \
                cls_key[:len(base_key)] == base_key:
            v = views[action]()
            return v.dispatch_request(**kwargs)

    logger.debug("No %r view for PolyModel subclass %r found",
                    action, poly_kind)
    flask.abort(404)

class PolymodelList(base_list.List):
    @classmethod
    def kind(cls):
        return cls.model._class_name()

class PolymodelEdit(base_base.KibbleView):
    """
    Dispatch class for Edit views.
    """
    action = 'edit'

    _url_patterns = [
        ("/{key}/", {}),
    ]

    @classmethod
    def kind(cls):
        return cls.model._class_name()

    def dispatch_request(self, key):
        instance = key.get()
        if instance is None:
            logger.debug("Unable to find instance with key %r", key)
            flask.abort(404)

        return _dispatch_polyrequest(
            'edit',
            instance._class_name(),
            self.model,
            key=key)

class PolymodelCreate(base_base.KibbleView):
    """
    Class to dispatch create requests based on the polymodel subclass
    type.
    """
    action = 'new'

    _url_patterns = [
        ('/{kind_lower}/new/<poly_kind>/', {'ancestor_key': None}),
        ('/{ancestor_key}/{kind_lower}/new/<poly_kind>/', {}),
        ('/{kind_lower}/new/', {'ancestor_key': None, 'poly_kind': None}),
        ('/{ancestor_key}/{kind_lower}/new/', {'poly_kind': None}),
    ]
    _methods = ['GET', 'POST']
    _requires_instance = False

    @classmethod
    def kind(cls):
        return cls.model._class_name()

    @property
    def templates(self):
        kwargs = {
            'action': self.action,
            'path': self.path().lower(),
            'kind': self.kind().lower(),
        }
        return [
            'kibble/polymodel/{path}/{action}.html'.format(**kwargs),
            'kibble/polymodel/{kind}/{action}.html'.format(**kwargs),
            'kibble/polymodel/{action}.html'.format(**kwargs),
        ]

    def _sub_kinds(cls):
        root_key = tuple(cls._class_key())
        l = len(root_key)

    def dispatch_request(self, ancestor_key, poly_kind):
        if poly_kind:
            return _dispatch_polyrequest(
                'create',
                poly_kind,
                self.model,
                ancestor_key=ancestor_key)
        return self.select_class(ancestor_key)

    def select_class(self, ancestor_key=None):
        classes = []

        for cls_key, views in PolymodelMeta.cls_map.items():
            if 'create' not in views:
                continue

            base_path = tuple(self.model._class_key())

            if cls_key[:len(base_path)] == base_path:
                model = views['create'].model
                classes.append((model, self.url_for(
                    ancestor_key=ancestor_key,
                    poly_kind=cls_key[-1].lower())))

        ctx = self.base_context()
        ctx['classes'] = classes
        return flask.render_template(self.templates, **ctx)

<commit_msg>Added some notes to polymodels<commit_after>"""
Basic PolyModel support for Kibble.

As Poly classes all have the same Kind, things don't entirely work.

In order to use PolyModels you need to do two things :

    1. Subclass `kibble.polymodel.Create` and `kibble.polymodel.Edit` views
       for each subclass of PolyModel you want to create/edit.

    2. Create a `kibble.polymodel.PolymodelList`,
       `kibble.polymodel.PolymodelCreate` and `kibble.polymodel.PolymodelEdit`
       views for the 'root' class you want.

The views for the parent classes dispatch requests to the subclass views based
on the kind of the instance, or requested url parameter.

Notes:
    * Permissions are /probably/ all tied around Polymodel, not the actual
      classes
    * You'll probably get some funky shit WRT to ancestors.
"""

import flask
import logging
import blinker
from collections import defaultdict
#from google.appengine.ext import ndb
from google.appengine.ext.ndb import polymodel

from . import base as base_base
from . import edit as base_edit
from . import list as base_list

logger = logging.getLogger(__name__)

class PolymodelMeta(base_base.KibbleMeta):
    """
    Custom metaclass to store a map of `_class_key()` to
    dispatch requests to the right type.
    """
    cls_map = defaultdict(dict)

    def __new__(mcls, name, bases, attrs):
        cls = super(base_base.KibbleMeta, mcls)\
            .__new__(mcls, name, bases, attrs)

        if not getattr(cls, 'model', None) or \
                not issubclass(cls.model, polymodel.PolyModel):
            return cls

        if blinker and cls.model:
            cls.pre_signal = blinker.signal("{}-pre-{}".format(
                cls.kind(), cls.action))

            cls.post_signal = blinker.signal("{}-post-{}".format(
                cls.kind(), cls.action))
        else:
            cls.pre_signal = base_base.StubSignal()
            cls.post_signal = base_base.StubSignal()

        mcls.cls_map[tuple(cls.model._class_key())][cls.action] = cls

        return cls

class PolymodelMixin(object):
    __metaclass__ = PolymodelMeta

class Create(PolymodelMixin, base_edit.Create):
    pass

class Edit(PolymodelMixin, base_edit.Edit):
    pass

def _dispatch_polyrequest(action, poly_kind, base_kind, **kwargs):
    base_key = tuple(base_kind._class_key())

    for cls_key, views in PolymodelMeta.cls_map.items():
        if action not in views:
            continue

        if cls_key[-1].lower() == poly_kind.lower() and \
                cls_key[:len(base_key)] == base_key:
            v = views[action]()
            return v.dispatch_request(**kwargs)

    logger.debug("No %r view for PolyModel subclass %r found",
                    action, poly_kind)
    flask.abort(404)

class PolymodelList(base_list.List):
    @classmethod
    def kind(cls):
        return cls.model._class_name()

class PolymodelEdit(base_base.KibbleView):
    """
    Dispatch class for Edit views.
    """
    action = 'edit'

    button_icon = 'pencil'

    _url_patterns = [
        ("/{key}/", {}),
    ]

    @classmethod
    def kind(cls):
        return cls.model._class_name()

    def dispatch_request(self, key):
        instance = key.get()
        if instance is None:
            logger.debug("Unable to find instance with key %r", key)
            flask.abort(404)

        return _dispatch_polyrequest(
            'edit',
            instance._class_name(),
            self.model,
            key=key)

class PolymodelCreate(base_base.KibbleView):
    """
    Class to dispatch create requests based on the polymodel subclass
    type.
    """
    action = 'new'

    button_icon = 'plus-sign'

    _url_patterns = [
        ('/{kind_lower}/new/<poly_kind>/', {'ancestor_key': None}),
        ('/{ancestor_key}/{kind_lower}/new/<poly_kind>/', {}),
        ('/{kind_lower}/new/', {'ancestor_key': None, 'poly_kind': None}),
        ('/{ancestor_key}/{kind_lower}/new/', {'poly_kind': None}),
    ]
    _methods = ['GET', 'POST']
    _requires_instance = False

    @classmethod
    def kind(cls):
        return cls.model._class_name()

    @property
    def templates(self):
        kwargs = {
            'action': self.action,
            'path': self.path().lower(),
            'kind': self.kind().lower(),
        }
        return [
            'kibble/polymodel/{path}/{action}.html'.format(**kwargs),
            'kibble/polymodel/{kind}/{action}.html'.format(**kwargs),
            'kibble/polymodel/{action}.html'.format(**kwargs),
        ]

    def _sub_kinds(cls):
        root_key = tuple(cls._class_key())
        l = len(root_key)

    def dispatch_request(self, ancestor_key, poly_kind):
        if poly_kind:
            return _dispatch_polyrequest(
                'create',
                poly_kind,
                self.model,
                ancestor_key=ancestor_key)
        return self.select_class(ancestor_key)

    def select_class(self, ancestor_key=None):
        classes = []

        for cls_key, views in PolymodelMeta.cls_map.items():
            if 'create' not in views:
                continue

            base_path = tuple(self.model._class_key())

            if cls_key[:len(base_path)] == base_path:
                model = views['create'].model
                classes.append((model, self.url_for(
                    ancestor_key=ancestor_key,
                    poly_kind=cls_key[-1].lower())))

        ctx = self.base_context()
        ctx['classes'] = classes
        return flask.render_template(self.templates, **ctx)

<|endoftext|>


<commit_before>import cgen as c

from devito.dimension import IncrDimension
from devito.ir.iet import (Expression, Iteration, List, ntags, FindAdjacent,
                           FindNodes, IsPerfectIteration, Transformer,
                           compose_nodes, retrieve_iteration_tree)
from devito.symbolics import as_symbol, xreplace_indices
from devito.tools import as_tuple, flatten

__all__ = ['BlockDimension', 'fold_blockable_tree', 'unfold_blocked_tree']

def fold_blockable_tree(node, exclude_innermost=False):
    """
    Create :class:`IterationFold`s from sequences of nested :class:`Iteration`.
    """
    found = FindAdjacent(Iteration).visit(node)

    mapper = {}
    for k, v in found.items():
        for i in v:
            # Pre-condition: they all must be perfect iterations
            assert len(i) > 1
            if any(not IsPerfectIteration().visit(j) for j in i):
                continue
            # Only retain consecutive trees having same depth
            trees = [retrieve_iteration_tree(j)[0] for j in i]
            handle = []
            for j in trees:
                if len(j) != len(trees[0]):
                    break
                handle.append(j)
            trees = handle
            if not trees:
                continue
            # Check foldability
            pairwise_folds = list(zip(*reversed(trees)))
            if any(not is_foldable(j) for j in pairwise_folds):
                continue
            # Maybe heuristically exclude innermost Iteration
            if exclude_innermost is True:
                pairwise_folds = pairwise_folds[:-1]
            # Perhaps there's nothing to fold
            if len(pairwise_folds) == 1:
                continue
            # TODO: we do not currently support blocking if any of the foldable
            # iterations writes to user data (need min/max loop bounds?)
            exprs = flatten(FindNodes(Expression).visit(j.root) for j in trees[:-1])
            if any(j.write.is_Input for j in exprs):
                continue
            # Perform folding
            for j in pairwise_folds:
                root, remainder = j[0], j[1:]
                folds = [(tuple(y-x for x, y in zip(i.offsets, root.offsets)), i.nodes)
                         for i in remainder]
                mapper[root] = IterationFold(folds=folds, **root.args)
                for k in remainder:
                    mapper[k] = None

    # Insert the IterationFolds in the Iteration/Expression tree
    processed = Transformer(mapper, nested=True).visit(node)

    return processed

def unfold_blocked_tree(node):
    """
    Unfold nested :class:`IterationFold`.

    :Example:

    Given a section of Iteration/Expression tree as below: ::

        for i = 1 to N-1  // folded
          for j = 1 to N-1  // folded
            foo1()

    Assuming a fold with offset 1 in both /i/ and /j/ and body ``foo2()``, create: ::

        for i = 1 to N-1
          for j = 1 to N-1
            foo1()
        for i = 2 to N-2
          for j = 2 to N-2
            foo2()
    """
    # Search the unfolding candidates
    candidates = []
    for tree in retrieve_iteration_tree(node):
        handle = tuple(i for i in tree if i.is_IterationFold)
        if handle:
            # Sanity check
            assert IsPerfectIteration().visit(handle[0])
            candidates.append(handle)

    # Perform unfolding
    tag = ntags()
    mapper = {}
    for tree in candidates:
        trees = list(zip(*[i.unfold() for i in tree]))
        # Update tag
        for i, _tree in enumerate(list(trees)):
            trees[i] = tuple(j.retag(tag + i) for j in _tree)
        trees = optimize_unfolded_tree(trees[:-1], trees[-1])
        mapper[tree[0]] = List(body=trees)

    # Insert the unfolded Iterations in the Iteration/Expression tree
    processed = Transformer(mapper).visit(node)

    return processed

def is_foldable(nodes):
    """
    Return True if the iterable ``nodes`` consists of foldable :class:`Iteration`s,
    False otherwise.
    """
    nodes = as_tuple(nodes)
    if len(nodes) <= 1 or any(not i.is_Iteration for i in nodes):
        return False
    main = nodes[0]
    return all(i.dim == main.dim and i.limits == main.limits and i.index == main.index
               and i.properties == main.properties for i in nodes)

def optimize_unfolded_tree(unfolded, root):
    """
    Transform folded trees to reduce the memory footprint.

    Examples
    ========
    Given:

        .. code-block::
            for i = 1 to N - 1  # Folded tree
              for j = 1 to N - 1
                tmp[i,j] = ...
            for i = 2 to N - 2  # Root
              for j = 2 to N - 2
                ... = ... tmp[i,j] ...

    The temporary ``tmp`` has shape ``(N-1, N-1)``. However, as soon as the
    iteration space is blocked, with blocks of shape ``(i_bs, j_bs)``, the
    ``tmp`` shape can be shrunk to ``(i_bs-1, j_bs-1)``. The resulting
    iteration tree becomes:

        .. code-block::
            for i = 1 to i_bs + 1  # Folded tree
              for j = 1 to j_bs + 1
                i' = i + i_block - 2
                j' = j + j_block - 2
                tmp[i,j] = ... # use i' and j'
            for i = i_block to i_block + i_bs  # Root
              for j = j_block to j_block + j_bs
                i' = i - x_block
                j' = j - j_block
                ... = ... tmp[i',j'] ...
    """
    processed = []
    for i, tree in enumerate(unfolded):
        assert len(tree) == len(root)

        # We can optimize the folded trees only if they compute temporary
        # arrays, but not if they compute input data
        exprs = FindNodes(Expression).visit(tree[-1])
        writes = [j.write for j in exprs if j.is_tensor]
        if not all(j.is_Array for j in writes):
            processed.append(compose_nodes(tree))
            root = compose_nodes(root)
            continue

        modified_tree = []
        modified_root = []
        mapper = {}

        # "Shrink" the iteration space
        for t1, t2 in zip(tree, root):
            t1_udim = IncrDimension(t1.dim, t1.limits[0], 1, "%ss%d" % (t1.index, i))
            limits = (0, t1.limits[1] - t1.limits[0], t1.symbolic_incr)
            modified_tree.append(t1._rebuild(limits=limits,
                                             uindices=t1.uindices + (t1_udim,)))

            t2_udim = IncrDimension(t1.dim, -t1.limits[0], 1, "%ss%d" % (t1.index, i))
            modified_root.append(t2._rebuild(uindices=t2.uindices + (t2_udim,)))

            mapper[t1.dim] = t1_udim

        # Temporary arrays can now be moved onto the stack
        if all(not j.is_Remainder for j in modified_tree):
            dimensions = tuple(j.limits[0] for j in modified_root)
            for j in writes:
                if j.is_Array:
                    j_dimensions = dimensions + j.dimensions[len(modified_root):]
                    j_shape = tuple(k.symbolic_size for k in j_dimensions)
                    j.update(shape=j_shape, dimensions=j_dimensions, scope='stack')

        # Substitute iteration variables within the folded trees
        modified_tree = compose_nodes(modified_tree)
        replaced = xreplace_indices([j.expr for j in exprs], mapper, only_rhs=True)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        processed.append(Transformer(dict(zip(exprs, subs))).visit(modified_tree))

        # Introduce the new iteration variables within /root/
        modified_root = compose_nodes(modified_root)
        exprs = FindNodes(Expression).visit(modified_root)
        candidates = [as_symbol(j.output) for j in subs]
        replaced = xreplace_indices([j.expr for j in exprs], mapper, candidates)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        root = Transformer(dict(zip(exprs, subs))).visit(modified_root)

    return processed + [root]

class IterationFold(Iteration):

    """
    An IterationFold is a special :class:`Iteration` object that represents
    a sequence of consecutive (in program order) Iterations. In an IterationFold,
    all Iterations of the sequence but the so called ``root`` are "hidden"; that is,
    they cannot be visited by an Iteration/Expression tree visitor.

    The Iterations in the sequence represented by the IterationFold all have same
    dimension and properties. However, their extent is relative to that of the ``root``.
    """

    is_IterationFold = True

    def __init__(self, *args, **kwargs):
        self.folds = kwargs.pop('folds', None)
        super(IterationFold, self).__init__(*args, **kwargs)

    def __repr__(self):
        properties = ""
        if self.properties:
            properties = [str(i) for i in self.properties]
            properties = "WithProperties[%s]::" % ",".join(properties)
        index = self.index
        if self.uindices:
            index += '[%s]' % ','.join(i.name for i in self.uindices)
        length = "Length %d" % len(self.folds)
        return "<%sIterationFold %s; %s; %s>" % (properties, index, self.limits, length)

    @property
    def ccode(self):
        comment = c.Comment('This IterationFold is "hiding" one or more Iterations')
        code = super(IterationFold, self).ccode
        return c.Module([comment, code])

    def unfold(self):
        """
        Return the corresponding :class:`Iteration` objects from each fold in ``self``.
        """
        args = self.args
        args.pop('folds')

        # Construct the root Iteration
        root = Iteration(**args)

        # Construct the folds
        args.pop('nodes')
        ofs = args.pop('offsets')
        try:
            start, end, incr = args.pop('limits')
        except TypeError:
            start, end, incr = self.limits
        folds = tuple(Iteration(nodes, limits=(start, end, incr),
                                offsets=tuple(i-j for i, j in zip(ofs, shift)), **args)
                      for shift, nodes in self.folds)

        return folds + as_tuple(root)

class BlockDimension(IncrDimension):

    def _arg_defaults(self, **kwargs):
        # TODO: need a heuristic to pick a default block size
        return {self.step.name: 8}

    def _arg_values(self, args, interval, grid, **kwargs):
        if self.step.name in kwargs:
            return {self.step.name: kwargs.pop(self.step)}
        else:
            blocksize = self._arg_defaults()[self.step.name]
            if args[self.root.min_name] < blocksize < args[self.root.max_name]:
                return {self.step.name: blocksize}
            else:
                # Avoid OOB
                return {self.step.name: 1}
<commit_msg>dle: Fix BlockDimension<commit_after>import cgen as c

from devito.dimension import IncrDimension
from devito.ir.iet import (Expression, Iteration, List, ntags, FindAdjacent,
                           FindNodes, IsPerfectIteration, Transformer,
                           compose_nodes, retrieve_iteration_tree)
from devito.symbolics import as_symbol, xreplace_indices
from devito.tools import as_tuple, flatten

__all__ = ['BlockDimension', 'fold_blockable_tree', 'unfold_blocked_tree']

def fold_blockable_tree(node, exclude_innermost=False):
    """
    Create :class:`IterationFold`s from sequences of nested :class:`Iteration`.
    """
    found = FindAdjacent(Iteration).visit(node)

    mapper = {}
    for k, v in found.items():
        for i in v:
            # Pre-condition: they all must be perfect iterations
            assert len(i) > 1
            if any(not IsPerfectIteration().visit(j) for j in i):
                continue
            # Only retain consecutive trees having same depth
            trees = [retrieve_iteration_tree(j)[0] for j in i]
            handle = []
            for j in trees:
                if len(j) != len(trees[0]):
                    break
                handle.append(j)
            trees = handle
            if not trees:
                continue
            # Check foldability
            pairwise_folds = list(zip(*reversed(trees)))
            if any(not is_foldable(j) for j in pairwise_folds):
                continue
            # Maybe heuristically exclude innermost Iteration
            if exclude_innermost is True:
                pairwise_folds = pairwise_folds[:-1]
            # Perhaps there's nothing to fold
            if len(pairwise_folds) == 1:
                continue
            # TODO: we do not currently support blocking if any of the foldable
            # iterations writes to user data (need min/max loop bounds?)
            exprs = flatten(FindNodes(Expression).visit(j.root) for j in trees[:-1])
            if any(j.write.is_Input for j in exprs):
                continue
            # Perform folding
            for j in pairwise_folds:
                root, remainder = j[0], j[1:]
                folds = [(tuple(y-x for x, y in zip(i.offsets, root.offsets)), i.nodes)
                         for i in remainder]
                mapper[root] = IterationFold(folds=folds, **root.args)
                for k in remainder:
                    mapper[k] = None

    # Insert the IterationFolds in the Iteration/Expression tree
    processed = Transformer(mapper, nested=True).visit(node)

    return processed

def unfold_blocked_tree(node):
    """
    Unfold nested :class:`IterationFold`.

    :Example:

    Given a section of Iteration/Expression tree as below: ::

        for i = 1 to N-1  // folded
          for j = 1 to N-1  // folded
            foo1()

    Assuming a fold with offset 1 in both /i/ and /j/ and body ``foo2()``, create: ::

        for i = 1 to N-1
          for j = 1 to N-1
            foo1()
        for i = 2 to N-2
          for j = 2 to N-2
            foo2()
    """
    # Search the unfolding candidates
    candidates = []
    for tree in retrieve_iteration_tree(node):
        handle = tuple(i for i in tree if i.is_IterationFold)
        if handle:
            # Sanity check
            assert IsPerfectIteration().visit(handle[0])
            candidates.append(handle)

    # Perform unfolding
    tag = ntags()
    mapper = {}
    for tree in candidates:
        trees = list(zip(*[i.unfold() for i in tree]))
        # Update tag
        for i, _tree in enumerate(list(trees)):
            trees[i] = tuple(j.retag(tag + i) for j in _tree)
        trees = optimize_unfolded_tree(trees[:-1], trees[-1])
        mapper[tree[0]] = List(body=trees)

    # Insert the unfolded Iterations in the Iteration/Expression tree
    processed = Transformer(mapper).visit(node)

    return processed

def is_foldable(nodes):
    """
    Return True if the iterable ``nodes`` consists of foldable :class:`Iteration`s,
    False otherwise.
    """
    nodes = as_tuple(nodes)
    if len(nodes) <= 1 or any(not i.is_Iteration for i in nodes):
        return False
    main = nodes[0]
    return all(i.dim == main.dim and i.limits == main.limits and i.index == main.index
               and i.properties == main.properties for i in nodes)

def optimize_unfolded_tree(unfolded, root):
    """
    Transform folded trees to reduce the memory footprint.

    Examples
    ========
    Given:

        .. code-block::
            for i = 1 to N - 1  # Folded tree
              for j = 1 to N - 1
                tmp[i,j] = ...
            for i = 2 to N - 2  # Root
              for j = 2 to N - 2
                ... = ... tmp[i,j] ...

    The temporary ``tmp`` has shape ``(N-1, N-1)``. However, as soon as the
    iteration space is blocked, with blocks of shape ``(i_bs, j_bs)``, the
    ``tmp`` shape can be shrunk to ``(i_bs-1, j_bs-1)``. The resulting
    iteration tree becomes:

        .. code-block::
            for i = 1 to i_bs + 1  # Folded tree
              for j = 1 to j_bs + 1
                i' = i + i_block - 2
                j' = j + j_block - 2
                tmp[i,j] = ... # use i' and j'
            for i = i_block to i_block + i_bs  # Root
              for j = j_block to j_block + j_bs
                i' = i - x_block
                j' = j - j_block
                ... = ... tmp[i',j'] ...
    """
    processed = []
    for i, tree in enumerate(unfolded):
        assert len(tree) == len(root)

        # We can optimize the folded trees only if they compute temporary
        # arrays, but not if they compute input data
        exprs = FindNodes(Expression).visit(tree[-1])
        writes = [j.write for j in exprs if j.is_tensor]
        if not all(j.is_Array for j in writes):
            processed.append(compose_nodes(tree))
            root = compose_nodes(root)
            continue

        modified_tree = []
        modified_root = []
        mapper = {}

        # "Shrink" the iteration space
        for t1, t2 in zip(tree, root):
            t1_udim = IncrDimension(t1.dim, t1.limits[0], 1, "%ss%d" % (t1.index, i))
            limits = (0, t1.limits[1] - t1.limits[0], t1.symbolic_incr)
            modified_tree.append(t1._rebuild(limits=limits,
                                             uindices=t1.uindices + (t1_udim,)))

            t2_udim = IncrDimension(t1.dim, -t1.limits[0], 1, "%ss%d" % (t1.index, i))
            modified_root.append(t2._rebuild(uindices=t2.uindices + (t2_udim,)))

            mapper[t1.dim] = t1_udim

        # Temporary arrays can now be moved onto the stack
        if all(not j.is_Remainder for j in modified_tree):
            dimensions = tuple(j.limits[0] for j in modified_root)
            for j in writes:
                if j.is_Array:
                    j_dimensions = dimensions + j.dimensions[len(modified_root):]
                    j_shape = tuple(k.symbolic_size for k in j_dimensions)
                    j.update(shape=j_shape, dimensions=j_dimensions, scope='stack')

        # Substitute iteration variables within the folded trees
        modified_tree = compose_nodes(modified_tree)
        replaced = xreplace_indices([j.expr for j in exprs], mapper, only_rhs=True)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        processed.append(Transformer(dict(zip(exprs, subs))).visit(modified_tree))

        # Introduce the new iteration variables within /root/
        modified_root = compose_nodes(modified_root)
        exprs = FindNodes(Expression).visit(modified_root)
        candidates = [as_symbol(j.output) for j in subs]
        replaced = xreplace_indices([j.expr for j in exprs], mapper, candidates)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        root = Transformer(dict(zip(exprs, subs))).visit(modified_root)

    return processed + [root]

class IterationFold(Iteration):

    """
    An IterationFold is a special :class:`Iteration` object that represents
    a sequence of consecutive (in program order) Iterations. In an IterationFold,
    all Iterations of the sequence but the so called ``root`` are "hidden"; that is,
    they cannot be visited by an Iteration/Expression tree visitor.

    The Iterations in the sequence represented by the IterationFold all have same
    dimension and properties. However, their extent is relative to that of the ``root``.
    """

    is_IterationFold = True

    def __init__(self, *args, **kwargs):
        self.folds = kwargs.pop('folds', None)
        super(IterationFold, self).__init__(*args, **kwargs)

    def __repr__(self):
        properties = ""
        if self.properties:
            properties = [str(i) for i in self.properties]
            properties = "WithProperties[%s]::" % ",".join(properties)
        index = self.index
        if self.uindices:
            index += '[%s]' % ','.join(i.name for i in self.uindices)
        length = "Length %d" % len(self.folds)
        return "<%sIterationFold %s; %s; %s>" % (properties, index, self.limits, length)

    @property
    def ccode(self):
        comment = c.Comment('This IterationFold is "hiding" one or more Iterations')
        code = super(IterationFold, self).ccode
        return c.Module([comment, code])

    def unfold(self):
        """
        Return the corresponding :class:`Iteration` objects from each fold in ``self``.
        """
        args = self.args
        args.pop('folds')

        # Construct the root Iteration
        root = Iteration(**args)

        # Construct the folds
        args.pop('nodes')
        ofs = args.pop('offsets')
        try:
            start, end, incr = args.pop('limits')
        except TypeError:
            start, end, incr = self.limits
        folds = tuple(Iteration(nodes, limits=(start, end, incr),
                                offsets=tuple(i-j for i, j in zip(ofs, shift)), **args)
                      for shift, nodes in self.folds)

        return folds + as_tuple(root)

class BlockDimension(IncrDimension):

    @property
    def _arg_names(self):
        return (self.step.name,) + self.parent._arg_names

    def _arg_defaults(self, **kwargs):
        # TODO: need a heuristic to pick a default block size
        return {self.step.name: 8}

    def _arg_values(self, args, interval, grid, **kwargs):
        if self.step.name in kwargs:
            return {self.step.name: kwargs.pop(self.step.name)}
        else:
            blocksize = self._arg_defaults()[self.step.name]
            if args[self.root.min_name] < blocksize < args[self.root.max_name]:
                return {self.step.name: blocksize}
            else:
                # Avoid OOB
                return {self.step.name: 1}
<|endoftext|>


<commit_before>from flask import request
from werkzeug.datastructures import MultiDict
import flask_restful

class Namespace(dict):
    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)
    def __setattr__(self, name, value):
        self[name] = value

class Argument(object):

    def __init__(self, name, default=None, dest=None, required=False,
                 ignore=False, type=unicode, location=('values',),
                 choices=(), action='store', help=None, operators=('=',),
                 case_sensitive=True):
        """
        :param name: Either a name or a list of option strings, e.g. foo or
                        -f, --foo.
        :param default: The value produced if the argument is absent from the
            request.
        :param dest: The name of the attribute to be added to the object
            returned by parse_args(req).
        :param required: Whether or not the argument may be omitted (optionals
            only).
        :param action: The basic type of action to be taken when this argument
            is encountered in the request.
        :param ignore: Whether to ignore cases where the argument fails type
            conversion
        :param type: The type to which the request argument should be
            converted. If a type raises a ValidationError, the message in the
            error will be returned in the response.
        :param location: Where to source the arguments from the Flask request
            (ex: headers, args, etc.), can be an iterator
        :param choices: A container of the allowable values for the argument.
        :param help: A brief description of the argument, returned in the
            response when the argument is invalid. This takes precedence over
            the message passed to a ValidationError raised by a type converter.
        :param case_sensitive: Whether the arguments in the request are case
            sensitive or not
        """

        self.name = name
        self.default = default
        self.dest = dest
        self.required = required
        self.ignore = ignore
        self.location = location
        self.type = type
        self.choices = choices
        self.action = action
        self.help = help
        self.case_sensitive = case_sensitive
        self.operators = operators

    def source(self, request):
        """Pulls values off the request in the provided location
        :param request: The flask request object to parse arguments from
        """
        if isinstance(self.location, basestring):
            value = getattr(request, self.location, MultiDict())
            if value is not None:
                return value
        else:
            for l in self.location:
                value = getattr(request, l, None)
                if value is not None:
                    return value

        return MultiDict()

    def convert(self, value, op):
        try:
            return self.type(value, self.name, op)
        except TypeError:
            try:
                return self.type(value, self.name)
            except TypeError:
                return self.type(value)

    def handle_validation_error(self, error):
        """Called when an error is raised while parsing. Aborts the request
        with a 400 status and an error message

        :param error: the error that was raised
        """
        msg = self.help if self.help is not None else str(error)
        flask_restful.abort(400, message=msg)

    def parse(self, request):
        """Parses argument value(s) from the request, converting according to
        the argument's type.

        :param request: The flask request object to parse arguments from
        """
        source = self.source(request)

        results = []

        for operator in self.operators:
            name = self.name + operator.replace("=", "", 1)
            if name in source:
                # Account for MultiDict and regular dict
                if hasattr(source, "getlist"):
                    values = source.getlist(name)
                else:
                    values = [source.get(name)]

                for value in values:
                    if not self.case_sensitive:
                        value = value.lower()
                    if self.choices and value not in self.choices:
                        self.handle_validation_error(ValueError(
                            u"{0} is not a valid choice".format(value)))
                    try:
                        value = self.convert(value, operator)
                    except Exception as error:
                        if self.ignore:
                            continue

                        self.handle_validation_error(error)

                    results.append(value)

        if not results and self.required:
            if isinstance(self.location, basestring):
                error_msg = u"{0} is required in {1}".format(
                    self.name,
                    self.location
                )
            else:
                error_msg = u"{0} is required in {1}".format(
                    self.name,
                    ' or '.join(self.location)
                )
            self.handle_validation_error(ValueError(error_msg))

        if not results:
            return self.default

        if self.action == 'append':
            return results

        if self.action == 'store' or len(results) == 1:
            return results[0]
        return results

class RequestParser(object):
    """Enables adding and parsing of multiple arguments in the context of a
    single request. Ex::

        from flask import request

        parser = RequestParser()
        parser.add_argument('foo')
        parser.add_argument('int_bar', type=int)
        args = parser.parse_args()
    """

    def __init__(self, argument_class=Argument, namespace_class=Namespace):
        self.args = []
        self.argument_class = argument_class
        self.namespace_class = namespace_class

    def add_argument(self, *args, **kwargs):
        """Adds an argument to be parsed. See Argument's constructor for
        documentation on the available options.
        """

        self.args.append(self.argument_class(*args, **kwargs))
        return self

    def parse_args(self, req=None):
        """Parse all arguments from the provided request and return the results
        as a Namespace
        """
        if req is None:
            req = request

        namespace = self.namespace_class()

        for arg in self.args:
            namespace[arg.dest or arg.name] = arg.parse(req)

        return namespace
<commit_msg>fix up doc link<commit_after>from flask import request
from werkzeug.datastructures import MultiDict
import flask_restful

class Namespace(dict):
    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)
    def __setattr__(self, name, value):
        self[name] = value

class Argument(object):

    def __init__(self, name, default=None, dest=None, required=False,
                 ignore=False, type=unicode, location=('values',),
                 choices=(), action='store', help=None, operators=('=',),
                 case_sensitive=True):
        """
        :param name: Either a name or a list of option strings, e.g. foo or
                        -f, --foo.
        :param default: The value produced if the argument is absent from the
            request.
        :param dest: The name of the attribute to be added to the object
            returned by parse_args(req).
        :param required: Whether or not the argument may be omitted (optionals
            only).
        :param action: The basic type of action to be taken when this argument
            is encountered in the request.
        :param ignore: Whether to ignore cases where the argument fails type
            conversion
        :param type: The type to which the request argument should be
            converted. If a type raises a ValidationError, the message in the
            error will be returned in the response.
        :param location: Where to source the arguments from the Flask request
            (ex: headers, args, etc.), can be an iterator
        :param choices: A container of the allowable values for the argument.
        :param help: A brief description of the argument, returned in the
            response when the argument is invalid. This takes precedence over
            the message passed to a ValidationError raised by a type converter.
        :param case_sensitive: Whether the arguments in the request are case
            sensitive or not
        """

        self.name = name
        self.default = default
        self.dest = dest
        self.required = required
        self.ignore = ignore
        self.location = location
        self.type = type
        self.choices = choices
        self.action = action
        self.help = help
        self.case_sensitive = case_sensitive
        self.operators = operators

    def source(self, request):
        """Pulls values off the request in the provided location
        :param request: The flask request object to parse arguments from
        """
        if isinstance(self.location, basestring):
            value = getattr(request, self.location, MultiDict())
            if value is not None:
                return value
        else:
            for l in self.location:
                value = getattr(request, l, None)
                if value is not None:
                    return value

        return MultiDict()

    def convert(self, value, op):
        try:
            return self.type(value, self.name, op)
        except TypeError:
            try:
                return self.type(value, self.name)
            except TypeError:
                return self.type(value)

    def handle_validation_error(self, error):
        """Called when an error is raised while parsing. Aborts the request
        with a 400 status and an error message

        :param error: the error that was raised
        """
        msg = self.help if self.help is not None else str(error)
        flask_restful.abort(400, message=msg)

    def parse(self, request):
        """Parses argument value(s) from the request, converting according to
        the argument's type.

        :param request: The flask request object to parse arguments from
        """
        source = self.source(request)

        results = []

        for operator in self.operators:
            name = self.name + operator.replace("=", "", 1)
            if name in source:
                # Account for MultiDict and regular dict
                if hasattr(source, "getlist"):
                    values = source.getlist(name)
                else:
                    values = [source.get(name)]

                for value in values:
                    if not self.case_sensitive:
                        value = value.lower()
                    if self.choices and value not in self.choices:
                        self.handle_validation_error(ValueError(
                            u"{0} is not a valid choice".format(value)))
                    try:
                        value = self.convert(value, operator)
                    except Exception as error:
                        if self.ignore:
                            continue

                        self.handle_validation_error(error)

                    results.append(value)

        if not results and self.required:
            if isinstance(self.location, basestring):
                error_msg = u"{0} is required in {1}".format(
                    self.name,
                    self.location
                )
            else:
                error_msg = u"{0} is required in {1}".format(
                    self.name,
                    ' or '.join(self.location)
                )
            self.handle_validation_error(ValueError(error_msg))

        if not results:
            return self.default

        if self.action == 'append':
            return results

        if self.action == 'store' or len(results) == 1:
            return results[0]
        return results

class RequestParser(object):
    """Enables adding and parsing of multiple arguments in the context of a
    single request. Ex::

        from flask import request

        parser = RequestParser()
        parser.add_argument('foo')
        parser.add_argument('int_bar', type=int)
        args = parser.parse_args()
    """

    def __init__(self, argument_class=Argument, namespace_class=Namespace):
        self.args = []
        self.argument_class = argument_class
        self.namespace_class = namespace_class

    def add_argument(self, *args, **kwargs):
        """Adds an argument to be parsed. See :class:`Argument`'s constructor
        for documentation on the available options.
        """

        self.args.append(self.argument_class(*args, **kwargs))
        return self

    def parse_args(self, req=None):
        """Parse all arguments from the provided request and return the results
        as a Namespace
        """
        if req is None:
            req = request

        namespace = self.namespace_class()

        for arg in self.args:
            namespace[arg.dest or arg.name] = arg.parse(req)

        return namespace
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import shell as sh
from devstack.components import nova

LOG = logging.getLogger("devstack.components.novnc")

#id
TYPE = settings.NOVNC

UTIL_DIR = 'utils'

# FIXME, need to get actual location of nova.API_CONF
APP_OPTIONS = {
    'nova-novncproxy': ['--flagfile-file', sh.joinpths('%ROOT%', "bin", nova.API_CONF), '--web'],
}

class NoVNCUninstaller(comp.PkgUninstallComponent):
    def __init__(self, *args, **kargs):
        comp.PkgUninstallComponent.__init__(self, TYPE, *args, **kargs)

class NoVNCInstaller(comp.PkgInstallComponent):
    def __init__(self, *args, **kargs):
        comp.PkgInstallComponent.__init__(self, TYPE, *args, **kargs)
        self.git_loc = self.cfg.get("git", "novnc_repo")
        self.git_branch = self.cfg.get("git", "novnc_branch")

    def _get_download_locations(self):
        places = comp.PkgInstallComponent._get_download_locations(self)
        places.append({
            'uri': self.git_loc,
            'branch': self.git_branch,
        })
        return places

class NoVNCRuntime(comp.ProgramRuntime):
    def __init__(self, *args, **kargs):
        comp.ProgramRuntime.__init__(self, TYPE, *args, **kargs)

    def _get_apps_to_start(self):
        apps = list()
        for app_name in APP_OPTIONS.keys():
            apps.append({
                'name': app_name,
                'path': sh.joinpths(self.appdir, UTIL_DIR, app_name),
            })
        return apps

    def _get_app_options(self, app):
        return APP_OPTIONS.get(app)
<commit_msg>Updated to point to the nova conf correctly.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import shell as sh
from devstack.components import nova

LOG = logging.getLogger("devstack.components.novnc")

#id
TYPE = settings.NOVNC

UTIL_DIR = 'utils'

# FIXME, need to get actual location of nova.API_CONF
VNC_PROXY_APP = 'nova-novncproxy'
APP_OPTIONS = {
    VNC_PROXY_APP: ['--flagfile-file=%NOVA_CONF%', '--web'],
}

class NoVNCUninstaller(comp.PkgUninstallComponent):
    def __init__(self, *args, **kargs):
        comp.PkgUninstallComponent.__init__(self, TYPE, *args, **kargs)

class NoVNCInstaller(comp.PkgInstallComponent):
    def __init__(self, *args, **kargs):
        comp.PkgInstallComponent.__init__(self, TYPE, *args, **kargs)
        self.git_loc = self.cfg.get("git", "novnc_repo")
        self.git_branch = self.cfg.get("git", "novnc_branch")

    def _get_download_locations(self):
        places = comp.PkgInstallComponent._get_download_locations(self)
        places.append({
            'uri': self.git_loc,
            'branch': self.git_branch,
        })
        return places

class NoVNCRuntime(comp.ProgramRuntime):
    def __init__(self, *args, **kargs):
        comp.ProgramRuntime.__init__(self, TYPE, *args, **kargs)

    def _get_apps_to_start(self):
        apps = list()
        for app_name in APP_OPTIONS.keys():
            apps.append({
                'name': app_name,
                'path': sh.joinpths(self.appdir, UTIL_DIR, app_name),
            })
        return apps

    def _get_param_map(self, app_name):
        root_params = comp.ProgramRuntime._get_param_map(self, app_name)
        if(app_name == VNC_PROXY_APP and settings.NOVA in self.instances):
            nova_runtime = self.instances.get(settings.NOVA)
            root_params['NOVA_CONF'] = sh.joinpths(nova_runtime.cfgdir, nova.API_CONF)
        return root_params

    def _get_app_options(self, app):
        return APP_OPTIONS.get(app)
<|endoftext|>


<commit_before>"""Flask application that creates a RESTful API from SQLAlchemy models."""

from flask import Blueprint, jsonify

from flask_sandboy.service import WriteService, ReadService
from flask_sandboy.models import SerializableModel
from flask_sandboy.exception import (
    BadRequestException,
    ForbiddenException,
    NotAcceptableException,
    NotFoundException,
    ConflictException,
    ServerErrorException,
    NotImplementedException,
    ServiceUnavailableException)

__version__ = '0.0.4'

class Sandboy(object):
    """Main object for injecting RESTful HTTP endpoint."""
    def __init__(self, app, db, models, url_prefix=None,
            readonly=False, before_request=[], decorators=[]):
        """Initialize and register the given *models*."""
        self.app = app
        self.db = db
        app.extensions['sandboy'] = self
        self.blueprint = None
        self.url_prefix = url_prefix
        self.readonly = readonly
        self.before_request = before_request
        self.decorators = decorators
        self.init_app(app, models)

    def init_app(self, app, models):
        """Initialize and register error handlers."""

        # pylint: disable=unused-variable
        self.blueprint = Blueprint('sandboy', __name__, url_prefix=self.url_prefix)
        for br in self.before_request:
            self.blueprint.before_request(br)

        @self.blueprint.errorhandler(BadRequestException)
        @self.blueprint.errorhandler(ForbiddenException)
        @self.blueprint.errorhandler(NotAcceptableException)
        @self.blueprint.errorhandler(NotFoundException)
        @self.blueprint.errorhandler(ConflictException)
        @self.blueprint.errorhandler(ServerErrorException)
        @self.blueprint.errorhandler(NotImplementedException)
        @self.blueprint.errorhandler(ServiceUnavailableException)
        @self.blueprint.errorhandler(Exception)
        def handle_application_error(error):
            """Handler used to send JSON error messages rather than default
            HTML ones."""
            if hasattr(error, 'to_dict'):
                response = jsonify(error.to_dict())
            else:
                response = jsonify({'msg':str(error)})
            if hasattr(error, 'code'):
                response.status_code = error.code
            else:
                response.status_code = 500
            return response

        self.register(models)
        app.register_blueprint(self.blueprint)

    def register(self, cls_list):
        """Register a class to be given a REST API."""
        for cls in cls_list:
            serializable_model = type(
                cls.__name__ + 'Serializable',
                (cls, SerializableModel),
                {})

            service = ReadService if self.readonly else WriteService

            service.decorators = self.decorators

            new_endpoint = type(
                cls.__name__ + 'Endpoint',
                (service,),
                {'__model__': serializable_model,
                 '__db__': self.db})
            view_func = new_endpoint.as_view(
                new_endpoint.__model__.__tablename__)
            self.blueprint.add_url_rule(
                '/' + new_endpoint.__model__.__tablename__,
                defaults={'resource_id': None},
                view_func=view_func)
            self.blueprint.add_url_rule(
                '/{resource}/<resource_id>'.format(
                    resource=new_endpoint.__model__.__tablename__),
                view_func=view_func)
<commit_msg>respond with error token instead of potentially sensitive information<commit_after>"""Flask application that creates a RESTful API from SQLAlchemy models."""

from flask import (
    Blueprint,
    jsonify,
    current_app,
)
import uuid

from flask_sandboy.service import WriteService, ReadService
from flask_sandboy.models import SerializableModel
from flask_sandboy.exception import (
    BadRequestException,
    ForbiddenException,
    NotAcceptableException,
    NotFoundException,
    ConflictException,
    ServerErrorException,
    NotImplementedException,
    ServiceUnavailableException)

__version__ = '0.0.4'

class Sandboy(object):
    """Main object for injecting RESTful HTTP endpoint."""
    def __init__(self, app, db, models, url_prefix=None,
            readonly=False, before_request=[], decorators=[]):
        """Initialize and register the given *models*."""
        self.app = app
        self.db = db
        app.extensions['sandboy'] = self
        self.blueprint = None
        self.url_prefix = url_prefix
        self.readonly = readonly
        self.before_request = before_request
        self.decorators = decorators
        self.init_app(app, models)

    def init_app(self, app, models):
        """Initialize and register error handlers."""

        # pylint: disable=unused-variable
        self.blueprint = Blueprint('sandboy', __name__, url_prefix=self.url_prefix)
        for br in self.before_request:
            self.blueprint.before_request(br)

        @self.blueprint.errorhandler(BadRequestException)
        @self.blueprint.errorhandler(ForbiddenException)
        @self.blueprint.errorhandler(NotAcceptableException)
        @self.blueprint.errorhandler(NotFoundException)
        @self.blueprint.errorhandler(ConflictException)
        @self.blueprint.errorhandler(ServerErrorException)
        @self.blueprint.errorhandler(NotImplementedException)
        @self.blueprint.errorhandler(ServiceUnavailableException)
        @self.blueprint.errorhandler(Exception)
        def handle_application_error(error):
            """Handler used to send JSON error messages rather than default
            HTML ones."""
            error_token = str(uuid.uuid4())
            current_app.logger.error('error_token=%s' % error_token)
            current_app.logger.exception(error)
            if hasattr(error, 'to_dict'):
                error_dict = error.to_dict()
                error_dict.update({'error_token': error_token})
                response = jsonify(error_dict)
            else:
                response = jsonify({'msg':'An error occurred.', 'error_token': error_token})
            if hasattr(error, 'code'):
                response.status_code = error.code
            else:
                response.status_code = 500
            return response

        self.register(models)
        app.register_blueprint(self.blueprint)

    def register(self, cls_list):
        """Register a class to be given a REST API."""
        for cls in cls_list:
            serializable_model = type(
                cls.__name__ + 'Serializable',
                (cls, SerializableModel),
                {})

            service = ReadService if self.readonly else WriteService

            service.decorators = self.decorators

            new_endpoint = type(
                cls.__name__ + 'Endpoint',
                (service,),
                {'__model__': serializable_model,
                 '__db__': self.db})
            view_func = new_endpoint.as_view(
                new_endpoint.__model__.__tablename__)
            self.blueprint.add_url_rule(
                '/' + new_endpoint.__model__.__tablename__,
                defaults={'resource_id': None},
                view_func=view_func)
            self.blueprint.add_url_rule(
                '/{resource}/<resource_id>'.format(
                    resource=new_endpoint.__model__.__tablename__),
                view_func=view_func)
<|endoftext|>


<commit_before>import json
from urllib2 import urlopen, Request
import ifcb
from ifcb.io import Timestamped
from ifcb.io.path import Resolver
from ifcb.io.file import TARGET_INFO
import re

import urllib2 as urllib
from PIL import Image
from cStringIO import StringIO

def get_json(pid):
    print 'fetching ' + pid + '.json'
    return json.loads(''.join(urlopen(Request(pid + '.json'))))

class Target(object):
    info = None
    pid = None
    bin = None

    def __init__(self, pid, bin=None, info=None):
        self.pid = pid
        self.bin = bin
        self.info = info

    def __getattribute__(self,name):
        if name in TARGET_INFO:
            return self.info[name]
        elif name == 'info':
            if object.__getattribute__(self,'info') is None:
                self.info = get_json(object.__getattribute__(self,'pid'))
            return object.__getattribute__(self,'info')
        elif name == 'bin':
            if object.__getattribute__(self,'bin') is None:
                self.bin = Bin(self.binID)
            return self.bin
        else:
            return object.__getattribute__(self,name)

    def image(self):
        img_file = urllib.urlopen(self.pid+'.png')
        return Image.open(StringIO(img_file.read()))

class Bin(Timestamped):
    bin_info = None # dict containing information about the bin

    def __init__(self, pid, info=None):
        self.pid = pid
        self.instrument = self.info()['instrument']

    def __repr__(self):
        return '{Bin ' + self.pid + '}'

    def info(self):
        if self.bin_info is None:
            self.bin_info = get_json(self.pid+'/full')
        return self.bin_info

    def properties(self,include_pid=False):
        props = self.info().copy()
        del props['targets']
        if not include_pid:
            del props['pid']
        return props

    def headers(self):
        return self.properties()

    # generate all targets
    def __iter__(self):
        for t in self.info()['targets']:
            yield Target(t['pid'], self, t)

    def all_targets(self):
        return list(self)

    # retrieve the nth target (1-based!)
    # more efficient than subscripting the result of all_targets
    def target(self,n):
        for target in self:
            if n == target.targetNumber:
                return target

    # return number of targets
    def length(self):
        return length(self.all_targets())

    ## image access
    def all_images(self):
        for target in self:
            yield target.image()

    # convenience method for getting a specific image
    def image(self,n):
        return self.target(n).image()

    # compute the pid of a target, given its index
    def target_pid(self,target_number):
        return self.target(n).targetNumber

class Client(Resolver):
    def latest_bins(self,n=10):
        pass

    def latest_bin(self):
        """Return latest bin"""
        return self.latest_bins(1)[0]

    def resolve(self,pid):
        lid = ifcb.lid(pid)
        if re.match(r'^IFCB\d+_\d{4}_\d{3}_\d{6}$',lid): # bin pattern # FIXME tile
            return Bin(pid)
        elif re.match(r'^IFCB\d+_\d{4}_\d{3}_\d{6}_\d+$',lid): # target pattern # FIXME tile
            return Target(pid)
        # FIXME add day dir
        raise KeyError('unrecognized pid '+pid)<commit_msg>big performance improvement on bin.instrument<commit_after>import json
from urllib2 import urlopen, Request
import ifcb
from ifcb.io import Timestamped
from ifcb.io.path import Resolver
from ifcb.io.file import TARGET_INFO
import re

import urllib2 as urllib
from PIL import Image
from cStringIO import StringIO

def get_json(pid):
    print 'fetching ' + pid + '.json'
    return json.loads(''.join(urlopen(Request(pid + '.json'))))

class Target(object):
    info = None
    pid = None
    bin = None

    def __init__(self, pid, bin=None, info=None):
        self.pid = pid
        self.bin = bin
        self.info = info

    def __getattribute__(self,name):
        if name in TARGET_INFO:
            return self.info[name]
        elif name == 'info':
            if object.__getattribute__(self,'info') is None:
                self.info = get_json(object.__getattribute__(self,'pid'))
            return object.__getattribute__(self,'info')
        elif name == 'bin':
            if object.__getattribute__(self,'bin') is None:
                self.bin = Bin(self.binID)
            return self.bin
        else:
            return object.__getattribute__(self,name)

    def image(self):
        img_file = urllib.urlopen(self.pid+'.png')
        return Image.open(StringIO(img_file.read()))

class Bin(Timestamped):
    bin_info = None # dict containing information about the bin

    def __init__(self, pid, info=None):
        self.pid = pid
        #self.instrument = self.info()['instrument']

    def __repr__(self):
        return '{Bin ' + self.pid + '}'

    def info(self):
        if self.bin_info is None:
            self.bin_info = get_json(self.pid+'/full')
        return self.bin_info

    def properties(self,include_pid=False):
        props = self.info().copy()
        del props['targets']
        if not include_pid:
            del props['pid']
        return props

    def headers(self):
        return self.properties()

    # generate all targets
    def __iter__(self):
        for t in self.info()['targets']:
            yield Target(t['pid'], self, t)

    def all_targets(self):
        return list(self)

    # retrieve the nth target (1-based!)
    # more efficient than subscripting the result of all_targets
    def target(self,n):
        for target in self:
            if n == target.targetNumber:
                return target

    # return number of targets
    def length(self):
        return length(self.all_targets())

    ## image access
    def all_images(self):
        for target in self:
            yield target.image()

    # convenience method for getting a specific image
    def image(self,n):
        return self.target(n).image()

    # compute the pid of a target, given its index
    def target_pid(self,target_number):
        return self.target(n).targetNumber

class Client(Resolver):
    def latest_bins(self,n=10):
        feed = get_json('http://ifcb-data.whoi.edu/feed')
        length = min(len(feed),n)
        return [Bin(info['pid']) for info in feed[:length]]

    def latest_bin(self):
        """Return latest bin"""
        return self.latest_bins(1)[0]

    def resolve(self,pid):
        lid = ifcb.lid(pid)
        if re.match(r'^IFCB\d+_\d{4}_\d{3}_\d{6}$',lid): # bin pattern # FIXME tile
            return Bin(pid)
        elif re.match(r'^IFCB\d+_\d{4}_\d{3}_\d{6}_\d+$',lid): # target pattern # FIXME tile
            return Target(pid)
        # FIXME add day dir
        raise KeyError('unrecognized pid '+pid)<|endoftext|>