<commit_before># -*- coding: utf-8 -*-
# <markment - markdown-based documentation generator for python>
# Copyright (C) <2013>  Gabriel Falcão <gabriel@nacaolivre.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals

import os
import re
import time
import shutil

from fnmatch import fnmatch
from functools import partial
from os.path import (
    abspath,
    join,
    dirname,
    exists,
    split,
    relpath,
    expanduser,
    basename,
)
from os.path import isfile as isfile_base
from os.path import isdir as isdir_base

LOCAL_FILE = lambda *path: join(abspath(dirname(__file__)), *path)

def isfile(path):
    if exists(path):
        return isfile_base(path)

    return '.' in split(path)[-1]

def isdir(path):
    if exists(path):
        return isdir_base(path)

    return '.' not in split(path)[-1]

class DotDict(dict):
    def __getattr__(self, attr):
        try:
            return super(DotDict, self).__getattribute(attr)
        except AttributeError:
            return self[attr]

STAT_LABELS = ["mode", "ino", "dev", "nlink", "uid", "gid", "size", "atime", "mtime", "ctime"]

DOTDOTSLASH = '..{0}'.format(os.sep)

class Node(object):
    def __init__(self, path):
        self.path = abspath(expanduser(path)).rstrip('/')
        self.path_regex = '^{0}'.format(re.escape(self.path))
        self.exists = exists(self.path)
        try:
            stats = os.stat(self.path)
        except OSError:
            stats = [0] * len(STAT_LABELS)

        self.metadata = DotDict(zip(STAT_LABELS, stats))
        self.is_file = isfile(self.path)
        self.is_dir = isdir(self.path)

    @property
    def basename(self):
        return basename(self.path)

    @property
    def dir(self):
        if not self.is_dir:
            return self.parent
        else:
            return self

    @property
    def parent(self):
        return self.__class__(dirname(self.path))

    def reduce_to_base_path(self, path):
        levels = []
        while path.startswith(DOTDOTSLASH):
            levels.append('..')
            path = path.replace(DOTDOTSLASH, '', 1)

        way_back = os.sep.join(levels)
        return self.cd(way_back).dir.path, path, way_back

    def could_be_updated_by(self, other):
        return self.metadata.mtime < other.metadata.mtime

    def create_tree_if_not_exists(self):
        if not self.dir.exists:
            os.makedirs(self.dir.path)

    def relative(self, path):
        """##### `Node#relative(path)`

        returns a given path subtracted by the node.path [python`unicode`]

        ```python
        rel = Node('/Users/gabrielfalcao/').relative('/Users/gabrielfalcao/profile-picture.png')
        assert rel == 'profile-picture.png'
        ```
        """
        return re.sub(self.path_regex, '', path).lstrip(os.sep)

    def trip_at(self, path, lazy=False):
        """ ##### `Node#trip_at(path)`

        does a os.walk at the given path and yields the absolute path
        to each file

        ```python
        for filename in node.trip_at('/etc/smb'):
            print filename
        ```
        """
        def iterator():
            for root, folders, filenames in os.walk(path):
                for filename in filenames:
                    yield join(root, filename)

        return lazy and iterator() or list(iterator())

    def walk(self):
        return self.trip_at(self.path)

    def glob(self, pattern, lazy=False):
        """ ##### `Node#glob(pattern)`

        searches for globs recursively in all the children node of the
        current node returning a respective [python`Node`] instance
        for that given.

        ```python
        for node in Node('/Users/gabrifalcao').glob('*.png'):
            print node.path  # will print the absolute
                             # path of the found file
        ```
        """
        def iterator():
            for filename in self.walk():
                if fnmatch(filename, pattern):
                    yield self.__class__(filename)

        return lazy and iterator() or list(iterator())

    def grep(self, pattern, flags=0, lazy=False):
        """ ##### `Node#grep(pattern)`

        searches recursively for children that match the given regex
        returning a respective [python`Node`] instance for that given.

        ```python
        for node in Node('/Users/gabrifalcao').glob('*.png'):
            print node.path  # will print the absolute
                             # path of the found file
        ```
        """

        def iterator():
            for filename in self.walk():
                if re.search(pattern, filename, flags):
                    yield self.__class__(filename)

        return lazy and iterator() or list(iterator())

    def find(self, relative_path):
        """ ##### `Node#find(relative_path)`

        finds a file given the relative_path, it glob and grep using
        the given relative_path to match as a glob and then tries
        to return the first found node.

        If nothing is found, returns None

        ```python

        logo = Node('~/projects/personal/markment').find('logo.png')
        assert logo.path == os.path.expanduser('~/projects/personal/markment')
        ```
        """
        found = list(self.glob(relative_path)) + list(self.grep(relative_path))
        if found:
            return found[0]

        return None

    def depth_of(self, path):
        """Returns the level of depth of the given path inside of the
        instance's path.

        Only really works with paths that are relative to the class.

        ```python
        level = Node('/foo/bar').depth_of('/foo/bar/another/dir/file.py')
        assert level == 2
        ```
        """
        new_path = self.relative(path)
        if isfile(self.join(new_path)):
            new_path = dirname(new_path)

        new_path = new_path.rstrip('/')
        new_path = "{0}/".format(new_path)
        return new_path.count(os.sep)

    def path_to_related(self, path):
        """Returns the path to a related file. (is under a subtree the
        same tree as the node).

        It's useful to know how to go back to the root of this node
        instance.

        ```python
        way_back = Node('/foo/bar').depth_of('/foo/bar/another/dir/file.py')
        assert way_back == '../../'

        way_back = Node('/foo/bar/docs/static/file.css').depth_of('/foo/bar/docs/intro/index.md')
        assert way_back == '../static/file.css'
        ```
        """
        # self.path = "...functional/fixtures/img/logo.png"
        # path = "...functional/fixtures/docs/index.md"
        current = self.dir

        while not path.startswith(current.dir.path):
            current = current.dir.parent.dir

        remaining = current.relative(self.path)

        level = current.relative(path).count(os.sep)

        way_back = os.sep.join(['..'] * level) or '.'
        result = "{0}/{1}".format(way_back, remaining)

        return result

    def cd(self, path):
        return self.__class__(self.join(path))

    def contains(self, path):
        return exists(self.join(path))

    def join(self, path):
        return abspath(join(self.path, path))

    def open(self, path, *args, **kw):
        return open(self.join(path), *args, **kw)

    def __repr__(self):
        return '<markment.fs.Node (path={0})>'.format(self.path)

class TreeMaker(object):
    def __init__(self, path):
        self.path = abspath(path.rstrip('/'))
        self.path_regex = '^{0}'.format(re.escape(self.path))
        self.node = Node(path)

    def __repr__(self):
        return '<markment.fs.TreeMaker(path={0})>'.format(self.path)

    def relative(self, path):
        return re.sub(self.path_regex, '', path).lstrip(os.sep)

    def find_all_markdown_files(self):
        dirs = []
        for fullpath in self.node.walk():
            folder = dirname(fullpath)
            if fnmatch(fullpath, '*.md'):
                if folder != self.path and folder not in dirs:
                    dirs.append(folder)
                    yield {
                        'path': folder,
                        'relative_path': self.relative(folder),
                        'type': 'tree',
                    }

                yield {
                    'path': fullpath,
                    'relative_path': self.relative(fullpath),
                    'type': 'blob',
                }

class Generator(object):
    regex = re.compile(r'[.](md|markdown)$', re.I)

    def __init__(self, project, theme):
        self.files_to_copy = []
        self.project = project
        self.theme = theme

    def rename_markdown_filename(self, path):
        return self.regex.sub('.html', path)

    def relative_link_callback(self, original_link, current_document_info, destination_root):
        fixed_link, levels = self.get_levels(original_link)

        is_markdown = self.regex.search(original_link)

        found = self.project.node.find(fixed_link)
        if not found:
            return original_link

        found_base = self.project.node.dir

        relative_path = found_base.relative(found.path)

        current_document_path = self.rename_markdown_filename(current_document_info['relative_path'])

        item_destination = destination_root.cd(relative_path)
        destination_document = destination_root.cd(current_document_path)

        prefix = item_destination.path_to_related(destination_document.path)

        if found.path not in self.files_to_copy and not is_markdown:
            self.files_to_copy.append(found.path)

        if prefix == './':
            prefix = prefix + fixed_link

        if is_markdown:
            prefix = self.rename_markdown_filename(prefix)

        return prefix

    def get_levels(self, link):
        levels = []
        while link.startswith(DOTDOTSLASH):
            levels.append('..')
            link = link.replace(DOTDOTSLASH, '', 1)

        return link, levels

    def static_url_callback(self, link, current_document_info, destination_root):
        is_markdown = self.regex.search(link)
        prefix = link

        link, levels = self.get_levels(link)
        in_theme = self.theme.node.find(link)
        in_local = self.project.node.find(link)

        if levels:
            prefix = link

        if in_local:
            found = in_local
            found_base = self.project.node.dir
            link = found_base.relative(found.path)

        elif in_theme:
            found = in_theme
            found_base = self.theme.node.dir
            found_relative = found_base.relative(found.path)
            asset_destination = destination_root.cd(found_relative)
            current_document_destination = destination_root.cd(current_document_info['relative_path'])
            if not levels:
                prefix = asset_destination.path_to_related(current_document_destination.path)

        else:
            raise IOError("BOOM, could not find {0} anywhere".format(link))

        if is_markdown:
            prefix = self.rename_markdown_filename(prefix)

        elif found and found.path not in self.files_to_copy:
            self.files_to_copy.append(found.path)

        return prefix

    def persist(self, destination_path, gently=False):
        destination = Node(destination_path)

        master_index = self.project.generate(
            self.theme,
            static_url_cb=partial(self.static_url_callback, destination_root=destination),
            link_cb=partial(self.relative_link_callback, destination_root=destination),
        )

        if not exists(destination_path):
            os.makedirs(destination_path)

        ret = []

        for item in master_index:
            if item['type'] != 'blob':
                continue

            dest_filename = self.rename_markdown_filename(item['relative_path'])
            destiny = destination.join(dest_filename)
            relative_destiny = split(destiny)[0]

            if relative_destiny and not exists(relative_destiny):
                os.makedirs(relative_destiny)

            with destination.open(destiny, 'w') as f:
                f.write(item['html'])

            ret.append(destiny)

        missed_files = []
        for src in self.files_to_copy:

            src = relpath(src)
            in_theme = self.theme.node.find(src)
            in_local = self.project.node.find(src)

            if in_local:
                found = in_local
                found_base = self.project.node.dir

            elif in_theme:
                found = in_theme
                found_base = self.theme.node.dir

            else:
                missed_files.append(src)
                continue

            source = found.path
            relative_source = found_base.relative(found.path)
            destiny = destination.join(relative_source)
            ret.append(destiny)
            destiny_folder = dirname(destiny)
            if not exists(destiny_folder):
                os.makedirs(destiny_folder)

            already_exists = exists(destiny)
            should_update = already_exists and Node(destiny).could_be_updated_by(Node(source))

            if not already_exists or should_update:
                shutil.copy2(source, destiny)

        cloner = AssetsCloner(self.theme.index['static_path'])

        cloner.clone_to(destination_path)

        if missed_files and not gently:
            raise IOError("The documentation refers to {0} "
                          "but they doesn't exist anythere".format(
                              ", ".join(missed_files)))

        return ret

class AssetsCloner(object):
    def __init__(self, path):
        self.assets_path = path
        self.root = dirname(path)
        self.node = Node(self.root)
        self.basename = split(path)[-1]

    def clone_to(self, root):
        ret = []

        dest_node = Node(root)

        for source_path in self.node.trip_at(self.assets_path):
            relative_path = self.node.relative(source_path)
            destination_path = dest_node.join(relative_path)
            destination_dir = dirname(destination_path)

            if not dest_node.contains(destination_dir):
                os.makedirs(destination_dir)

            shutil.copy2(source_path, destination_path)
            ret.append(destination_path)

        return ret
<commit_msg>fixing api docs<commit_after># -*- coding: utf-8 -*-
# <markment - markdown-based documentation generator for python>
# Copyright (C) <2013>  Gabriel Falcão <gabriel@nacaolivre.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals

import os
import re
import time
import shutil

from fnmatch import fnmatch
from functools import partial
from os.path import (
    abspath,
    join,
    dirname,
    exists,
    split,
    relpath,
    expanduser,
    basename,
)
from os.path import isfile as isfile_base
from os.path import isdir as isdir_base

LOCAL_FILE = lambda *path: join(abspath(dirname(__file__)), *path)

def isfile(path):
    if exists(path):
        return isfile_base(path)

    return '.' in split(path)[-1]

def isdir(path):
    if exists(path):
        return isdir_base(path)

    return '.' not in split(path)[-1]

class DotDict(dict):
    def __getattr__(self, attr):
        try:
            return super(DotDict, self).__getattribute(attr)
        except AttributeError:
            return self[attr]

STAT_LABELS = ["mode", "ino", "dev", "nlink", "uid", "gid", "size", "atime", "mtime", "ctime"]

DOTDOTSLASH = '..{0}'.format(os.sep)

class Node(object):
    def __init__(self, path):
        self.path = abspath(expanduser(path)).rstrip('/')
        self.path_regex = '^{0}'.format(re.escape(self.path))
        self.exists = exists(self.path)
        try:
            stats = os.stat(self.path)
        except OSError:
            stats = [0] * len(STAT_LABELS)

        self.metadata = DotDict(zip(STAT_LABELS, stats))
        self.is_file = isfile(self.path)
        self.is_dir = isdir(self.path)

    @property
    def basename(self):
        return basename(self.path)

    @property
    def dir(self):
        if not self.is_dir:
            return self.parent
        else:
            return self

    @property
    def parent(self):
        return self.__class__(dirname(self.path))

    def reduce_to_base_path(self, path):
        levels = []
        while path.startswith(DOTDOTSLASH):
            levels.append('..')
            path = path.replace(DOTDOTSLASH, '', 1)

        way_back = os.sep.join(levels)
        return self.cd(way_back).dir.path, path, way_back

    def could_be_updated_by(self, other):
        return self.metadata.mtime < other.metadata.mtime

    def create_tree_if_not_exists(self):
        if not self.dir.exists:
            os.makedirs(self.dir.path)

    def relative(self, path):
        """##### `Node#relative(path)`

        returns a given path subtracted by the node.path [python`unicode`]

        ```python
        rel = Node('/Users/gabrielfalcao/').relative('/Users/gabrielfalcao/profile-picture.png')
        assert rel == 'profile-picture.png'
        ```
        """
        return re.sub(self.path_regex, '', path).lstrip(os.sep)

    def trip_at(self, path, lazy=False):
        """ ##### `Node#trip_at(path)`

        does a os.walk at the given path and yields the absolute path
        to each file

        ```python
        for filename in node.trip_at('/etc/smb'):
            print filename
        ```
        """
        def iterator():
            for root, folders, filenames in os.walk(path):
                for filename in filenames:
                    yield join(root, filename)

        return lazy and iterator() or list(iterator())

    def walk(self):
        return self.trip_at(self.path)

    def glob(self, pattern, lazy=False):
        """ ##### `Node#glob(pattern)`

        searches for globs recursively in all the children node of the
        current node returning a respective [python`Node`] instance
        for that given.

        ```python
        for node in Node('/Users/gabrifalcao').glob('*.png'):
            print node.path  # will print the absolute
                             # path of the found file
        ```
        """
        def iterator():
            for filename in self.walk():
                if fnmatch(filename, pattern):
                    yield self.__class__(filename)

        return lazy and iterator() or list(iterator())

    def grep(self, pattern, flags=0, lazy=False):
        """ ##### `Node#grep(pattern)`

        searches recursively for children that match the given regex
        returning a respective [python`Node`] instance for that given.

        ```python
        for node in Node('/Users/gabrifalcao').glob('*.png'):
            print node.path  # will print the absolute
                             # path of the found file
        ```
        """

        def iterator():
            for filename in self.walk():
                if re.search(pattern, filename, flags):
                    yield self.__class__(filename)

        return lazy and iterator() or list(iterator())

    def find(self, relative_path):
        """ ##### `Node#find(relative_path)`

        finds a file given the relative_path, it glob and grep using
        the given relative_path to match as a glob and then tries
        to return the first found node.

        If nothing is found, returns None

        ```python

        logo = Node('~/projects/personal/markment').find('logo.png')
        assert logo.path == os.path.expanduser('~/projects/personal/markment')
        ```
        """
        found = list(self.glob(relative_path)) + list(self.grep(relative_path))
        if found:
            return found[0]

        return None

    def depth_of(self, path):
        """Returns the level of depth of the given path inside of the
        instance's path.

        Only really works with paths that are relative to the class.

        ```python
        level = Node('/foo/bar').depth_of('/foo/bar/another/dir/file.py')
        assert level == 2
        ```
        """
        new_path = self.relative(path)
        if isfile(self.join(new_path)):
            new_path = dirname(new_path)

        new_path = new_path.rstrip('/')
        new_path = "{0}/".format(new_path)
        return new_path.count(os.sep)

    def path_to_related(self, path):
        """Returns the path to a related file. (is under a subtree the
        same tree as the node).

        It's useful to know how to go back to the root of this node
        instance.

        ```python
        way_back = Node('/foo/bar').path_to_related('/foo/bar/another/dir/file.py')
        assert way_back == '../../'

        way_back = Node('/foo/bar/docs/static/file.css').path_to_related('/foo/bar/docs/intro/index.md')
        assert way_back == '../static/file.css'
        ```
        """
        # self.path = "...functional/fixtures/img/logo.png"
        # path = "...functional/fixtures/docs/index.md"
        current = self.dir

        while not path.startswith(current.dir.path):
            current = current.dir.parent.dir

        remaining = current.relative(self.path)

        level = current.relative(path).count(os.sep)

        way_back = os.sep.join(['..'] * level) or '.'
        result = "{0}/{1}".format(way_back, remaining)

        return result

    def cd(self, path):
        return self.__class__(self.join(path))

    def contains(self, path):
        return exists(self.join(path))

    def join(self, path):
        return abspath(join(self.path, path))

    def open(self, path, *args, **kw):
        return open(self.join(path), *args, **kw)

    def __repr__(self):
        return '<markment.fs.Node (path={0})>'.format(self.path)

class TreeMaker(object):
    def __init__(self, path):
        self.path = abspath(path.rstrip('/'))
        self.path_regex = '^{0}'.format(re.escape(self.path))
        self.node = Node(path)

    def __repr__(self):
        return '<markment.fs.TreeMaker(path={0})>'.format(self.path)

    def relative(self, path):
        return re.sub(self.path_regex, '', path).lstrip(os.sep)

    def find_all_markdown_files(self):
        dirs = []
        for fullpath in self.node.walk():
            folder = dirname(fullpath)
            if fnmatch(fullpath, '*.md'):
                if folder != self.path and folder not in dirs:
                    dirs.append(folder)
                    yield {
                        'path': folder,
                        'relative_path': self.relative(folder),
                        'type': 'tree',
                    }

                yield {
                    'path': fullpath,
                    'relative_path': self.relative(fullpath),
                    'type': 'blob',
                }

class Generator(object):
    regex = re.compile(r'[.](md|markdown)$', re.I)

    def __init__(self, project, theme):
        self.files_to_copy = []
        self.project = project
        self.theme = theme

    def rename_markdown_filename(self, path):
        return self.regex.sub('.html', path)

    def relative_link_callback(self, original_link, current_document_info, destination_root):
        fixed_link, levels = self.get_levels(original_link)

        is_markdown = self.regex.search(original_link)

        found = self.project.node.find(fixed_link)
        if not found:
            return original_link

        found_base = self.project.node.dir

        relative_path = found_base.relative(found.path)

        current_document_path = self.rename_markdown_filename(current_document_info['relative_path'])

        item_destination = destination_root.cd(relative_path)
        destination_document = destination_root.cd(current_document_path)

        prefix = item_destination.path_to_related(destination_document.path)

        if found.path not in self.files_to_copy and not is_markdown:
            self.files_to_copy.append(found.path)

        if prefix == './':
            prefix = prefix + fixed_link

        if is_markdown:
            prefix = self.rename_markdown_filename(prefix)

        return prefix

    def get_levels(self, link):
        levels = []
        while link.startswith(DOTDOTSLASH):
            levels.append('..')
            link = link.replace(DOTDOTSLASH, '', 1)

        return link, levels

    def static_url_callback(self, link, current_document_info, destination_root):
        is_markdown = self.regex.search(link)
        prefix = link

        link, levels = self.get_levels(link)
        in_theme = self.theme.node.find(link)
        in_local = self.project.node.find(link)

        if levels:
            prefix = link

        if in_local:
            found = in_local
            found_base = self.project.node.dir
            link = found_base.relative(found.path)

        elif in_theme:
            found = in_theme
            found_base = self.theme.node.dir
            found_relative = found_base.relative(found.path)
            asset_destination = destination_root.cd(found_relative)
            current_document_destination = destination_root.cd(current_document_info['relative_path'])
            if not levels:
                prefix = asset_destination.path_to_related(current_document_destination.path)

        else:
            raise IOError("BOOM, could not find {0} anywhere".format(link))

        if is_markdown:
            prefix = self.rename_markdown_filename(prefix)

        elif found and found.path not in self.files_to_copy:
            self.files_to_copy.append(found.path)

        return prefix

    def persist(self, destination_path, gently=False):
        destination = Node(destination_path)

        master_index = self.project.generate(
            self.theme,
            static_url_cb=partial(self.static_url_callback, destination_root=destination),
            link_cb=partial(self.relative_link_callback, destination_root=destination),
        )

        if not exists(destination_path):
            os.makedirs(destination_path)

        ret = []

        for item in master_index:
            if item['type'] != 'blob':
                continue

            dest_filename = self.rename_markdown_filename(item['relative_path'])
            destiny = destination.join(dest_filename)
            relative_destiny = split(destiny)[0]

            if relative_destiny and not exists(relative_destiny):
                os.makedirs(relative_destiny)

            with destination.open(destiny, 'w') as f:
                f.write(item['html'])

            ret.append(destiny)

        missed_files = []
        for src in self.files_to_copy:

            src = relpath(src)
            in_theme = self.theme.node.find(src)
            in_local = self.project.node.find(src)

            if in_local:
                found = in_local
                found_base = self.project.node.dir

            elif in_theme:
                found = in_theme
                found_base = self.theme.node.dir

            else:
                missed_files.append(src)
                continue

            source = found.path
            relative_source = found_base.relative(found.path)
            destiny = destination.join(relative_source)
            ret.append(destiny)
            destiny_folder = dirname(destiny)
            if not exists(destiny_folder):
                os.makedirs(destiny_folder)

            already_exists = exists(destiny)
            should_update = already_exists and Node(destiny).could_be_updated_by(Node(source))

            if not already_exists or should_update:
                shutil.copy2(source, destiny)

        cloner = AssetsCloner(self.theme.index['static_path'])

        cloner.clone_to(destination_path)

        if missed_files and not gently:
            raise IOError("The documentation refers to {0} "
                          "but they doesn't exist anythere".format(
                              ", ".join(missed_files)))

        return ret

class AssetsCloner(object):
    def __init__(self, path):
        self.assets_path = path
        self.root = dirname(path)
        self.node = Node(self.root)
        self.basename = split(path)[-1]

    def clone_to(self, root):
        ret = []

        dest_node = Node(root)

        for source_path in self.node.trip_at(self.assets_path):
            relative_path = self.node.relative(source_path)
            destination_path = dest_node.join(relative_path)
            destination_dir = dirname(destination_path)

            if not dest_node.contains(destination_dir):
                os.makedirs(destination_dir)

            shutil.copy2(source_path, destination_path)
            ret.append(destination_path)

        return ret
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    jinja2.exceptions
    ~~~~~~~~~~~~~~~~~

    Jinja exceptions.

    :copyright: (c) 2010 by the Jinja Team.
    :license: BSD, see LICENSE for more details.
"""
import six
from six.moves import map

class TemplateError(Exception):
    """Baseclass for all template errors."""

    def __init__(self, message=None):
        if message is not None:
            message = six.text_type(message).encode('utf-8')
        Exception.__init__(self, message)

    @property
    def message(self):
        if self.args:
            message = self.args[0]
            if message is not None:
                return message.decode('utf-8', 'replace')

class TemplateNotFound(IOError, LookupError, TemplateError):
    """Raised if a template does not exist."""

    # looks weird, but removes the warning descriptor that just
    # bogusly warns us about message being deprecated
    message = None

    def __init__(self, name, message=None):
        IOError.__init__(self)
        if message is None:
            message = name
        self.message = message
        self.name = name
        self.templates = [name]

    def __str__(self):
        return self.message.encode('utf-8')

    # unicode goes after __str__ because we configured 2to3 to rename
    # __unicode__ to __str__.  because the 2to3 tree is not designed to
    # remove nodes from it, we leave the above __str__ around and let
    # it override at runtime.
    def __unicode__(self):
        return self.message

class TemplatesNotFound(TemplateNotFound):
    """Like :class:`TemplateNotFound` but raised if multiple templates
    are selected.  This is a subclass of :class:`TemplateNotFound`
    exception, so just catching the base exception will catch both.

    .. versionadded:: 2.2
    """

    def __init__(self, names=(), message=None):
        if message is None:
            message = u'none of the templates given were found: ' + \
                      u', '.join(map(six.text_type, names))
        TemplateNotFound.__init__(self, names and names[-1] or None, message)
        self.templates = list(names)

class TemplateSyntaxError(TemplateError):
    """Raised to tell the user that there is a problem with the template."""

    def __init__(self, message, lineno, name=None, filename=None):
        TemplateError.__init__(self, message)
        self.lineno = lineno
        self.name = name
        self.filename = filename
        self.source = None

        # this is set to True if the debug.translate_syntax_error
        # function translated the syntax error into a new traceback
        self.translated = False

    def __str__(self):
        s = self.__unicode__()
        return s if six.PY3 else s.encode('utf-8')

    def __unicode__(self):
        # for translated errors we only return the message
        if self.translated:
            return self.message

        # otherwise attach some stuff
        location = 'line %d' % self.lineno
        name = self.filename or self.name
        if name:
            location = 'File "%s", %s' % (name, location)
        lines = [self.message, '  ' + location]

        # if the source is set, add the line to the output
        if self.source is not None:
            try:
                line = self.source.splitlines()[self.lineno - 1]
            except IndexError:
                line = None
            if line:
                lines.append('    ' + line.strip())

        return u'\n'.join(lines)

class TemplateAssertionError(TemplateSyntaxError):
    """Like a template syntax error, but covers cases where something in the
    template caused an error at compile time that wasn't necessarily caused
    by a syntax error.  However it's a direct subclass of
    :exc:`TemplateSyntaxError` and has the same attributes.
    """

class TemplateRuntimeError(TemplateError):
    """A generic runtime error in the template engine.  Under some situations
    Jinja may raise this exception.
    """

class UndefinedError(TemplateRuntimeError):
    """Raised if a template tries to operate on :class:`Undefined`."""

class SecurityError(TemplateRuntimeError):
    """Raised if a template tries to do something insecure if the
    sandbox is enabled.
    """

class FilterArgumentError(TemplateRuntimeError):
    """This error is raised if a filter was called with inappropriate
    arguments
    """
<commit_msg>Python 3 exceptions can handle unicode messages<commit_after># -*- coding: utf-8 -*-
"""
    jinja2.exceptions
    ~~~~~~~~~~~~~~~~~

    Jinja exceptions.

    :copyright: (c) 2010 by the Jinja Team.
    :license: BSD, see LICENSE for more details.
"""
import sys
import six
from six.moves import map

class TemplateError(Exception):
    """Baseclass for all template errors."""

    if sys.version_info[0] < 3:
        def __init__(self, message=None):
            if message is not None:
                message = six.text_type(message).encode('utf-8')
            Exception.__init__(self, message)

        @property
        def message(self):
            if self.args:
                message = self.args[0]
                if message is not None:
                    return message.decode('utf-8', 'replace')

    else:
        @property
        def message(self):
            if self.args:
                message = self.args[0]
                if message is not None:
                    return message

class TemplateNotFound(IOError, LookupError, TemplateError):
    """Raised if a template does not exist."""

    # looks weird, but removes the warning descriptor that just
    # bogusly warns us about message being deprecated
    message = None

    def __init__(self, name, message=None):
        IOError.__init__(self)
        if message is None:
            message = name
        self.message = message
        self.name = name
        self.templates = [name]

    def __str__(self):
        return self.message.encode('utf-8')

    # unicode goes after __str__ because we configured 2to3 to rename
    # __unicode__ to __str__.  because the 2to3 tree is not designed to
    # remove nodes from it, we leave the above __str__ around and let
    # it override at runtime.
    def __unicode__(self):
        return self.message

class TemplatesNotFound(TemplateNotFound):
    """Like :class:`TemplateNotFound` but raised if multiple templates
    are selected.  This is a subclass of :class:`TemplateNotFound`
    exception, so just catching the base exception will catch both.

    .. versionadded:: 2.2
    """

    def __init__(self, names=(), message=None):
        if message is None:
            message = u'none of the templates given were found: ' + \
                      u', '.join(map(six.text_type, names))
        TemplateNotFound.__init__(self, names and names[-1] or None, message)
        self.templates = list(names)

class TemplateSyntaxError(TemplateError):
    """Raised to tell the user that there is a problem with the template."""

    def __init__(self, message, lineno, name=None, filename=None):
        TemplateError.__init__(self, message)
        self.lineno = lineno
        self.name = name
        self.filename = filename
        self.source = None

        # this is set to True if the debug.translate_syntax_error
        # function translated the syntax error into a new traceback
        self.translated = False

    def __str__(self):
        s = self.__unicode__()
        return s if six.PY3 else s.encode('utf-8')

    def __unicode__(self):
        # for translated errors we only return the message
        if self.translated:
            return self.message

        # otherwise attach some stuff
        location = 'line %d' % self.lineno
        name = self.filename or self.name
        if name:
            location = 'File "%s", %s' % (name, location)
        lines = [self.message, '  ' + location]

        # if the source is set, add the line to the output
        if self.source is not None:
            try:
                line = self.source.splitlines()[self.lineno - 1]
            except IndexError:
                line = None
            if line:
                lines.append('    ' + line.strip())

        return u'\n'.join(lines)

class TemplateAssertionError(TemplateSyntaxError):
    """Like a template syntax error, but covers cases where something in the
    template caused an error at compile time that wasn't necessarily caused
    by a syntax error.  However it's a direct subclass of
    :exc:`TemplateSyntaxError` and has the same attributes.
    """

class TemplateRuntimeError(TemplateError):
    """A generic runtime error in the template engine.  Under some situations
    Jinja may raise this exception.
    """

class UndefinedError(TemplateRuntimeError):
    """Raised if a template tries to operate on :class:`Undefined`."""

class SecurityError(TemplateRuntimeError):
    """Raised if a template tries to do something insecure if the
    sandbox is enabled.
    """

class FilterArgumentError(TemplateRuntimeError):
    """This error is raised if a filter was called with inappropriate
    arguments
    """
<|endoftext|>


<commit_before>"""
    Jinja2 based template renderer.
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Compiles and render Jinja2-based template files.

    :copyright: (c) 2012 by Satoru SATOH <ssato@redhat.com>
    :license: BSD-3

 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

   * Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
   * Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.
   * Neither the name of the author nor the names of its contributors may
     be used to endorse or promote products derived from this software
     without specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 Requirements: python-jinja2, python-simplejson (if python < 2.6) and PyYAML
 References: http://jinja.pocoo.org
"""
import codecs
import glob
import itertools
import jinja2
import jinja2.meta
import locale
import logging
import optparse
import os.path
import os
import sys

from logging import DEBUG, INFO

try:
    chain_from_iterable = itertools.chain.from_iterable
except AttributeError:
    # Borrowed from library doc, 9.7.1 Itertools functions:
    def _from_iterable(iterables):
        for it in iterables:
            for element in it:
                yield element

    chain_from_iterable = _from_iterable

# Data loaders: Key=file_extension, Value=load_func
_LOADERS = {}
_ENCODING = locale.getdefaultlocale()[1]

sys.stderr = codecs.getwriter(_ENCODING)(sys.stderr)

# Override the default implementation:
def open(path, flag='r', enc=_ENCODING):
    return codecs.open(path, flag, enc)

try:
    import json
    _LOADERS["json"] = _LOADERS["jsn"] = json.loads
except ImportError:
    try:
        import simplejson as json
        _LOADERS["json"] = _LOADERS["jsn"] = json.loads
    except ImportError:
        sys.stderr.write(u"JSON support is disabled as module not found.\n")

try:
    import yaml
    _LOADERS["yaml"] = _LOADERS["yml"] = yaml.load
except ImportError:
    sys.stderr.write(u"YAML support is disabled as module not found.\n")

def is_dict(x):
    return isinstance(x, (MyDict, dict))

def is_iterable(x):
    return isinstance(x, (list, tuple)) or getattr(x, "next", False)

class MyDict(dict):

    @classmethod
    def createFromDict(cls, dic={}):
        md = MyDict()

        for k, v in dic.iteritems():
            md[k] = cls.createFromDict(v) if is_dict(v) else v

        return md

    def update(self, other, merge_lists=False):
        """Merge `self` and `other` recursively.

        :param merge_lists: Merge not only dicts but also lists,
            e.g. [1, 2], [3, 4] ==> [1, 2, 3, 4]
        """
        if is_dict(other):
            for k, v in other.iteritems():
                if k in self and is_dict(v) and is_dict(self[k]):
                    self[k].update(v, merge_lists)  # update recursively.
                else:
                    if merge_lists and is_iterable(v):
                        self[k] = self[k] + list(v)  # append v :: list
                    else:
                        self[k] = v  # replace self[k] w/ v or set.

def get_fileext(filepath):
    """
    >>> get_fileext("a.json")
    'json'
    >>> get_fileext("abc")
    ''
    """
    return os.path.splitext(filepath)[1][1:]

def get_loader(filepath=None, filetype=None, loaders=_LOADERS):
    if filepath is None and filetype is None:
        logging.error("Could not determine loader type")
        return None

    if filepath is None or filetype is not None:
        ext = filetype
    else:
        ext = get_fileext(filepath)

    return loaders.get(ext, None)

def load_context(filepath, filetype=None, enc=_ENCODING, werror=False):
    """Load context data from given file at `filepath`.

    :param filepath: Context data file path :: str
    :param filetype: Forced context file type
    :param enc: Character encoding of context file
    :param werror: raise exception if any error occured like gcc's -Werr
    """
    default = MyDict.createFromDict()

    loader = get_loader(filepath, filetype)
    if loader is None:
        m = "Couldn't get loader: path=%s, type=%s" % (filepath, filetype)
        if werror:
            raise RuntimeError(m)

        logging.warn(m)
        return default

    logging.debug("Loader found: path=%s, type=%s" % (filepath, filetype))
    data = open(filepath, enc=enc).read()
    try:
        x = loader(data)
        if not is_dict(x):
            logging.warn("Top-level object is not a dict: " + filepath)
            return default

        return MyDict.createFromDict(x)

    except Exception, e:
        if werror:
            raise RuntimeError(str(e))

        logging.warn(str(e))
        return default

def load_contexts(pathspecs, enc, werror=False):
    """Load context data from given files.

    :param paths: Context data file path list :: [str]
    """
    d = MyDict.createFromDict()
    for path, filetype in pathspecs:
        diff = load_context(path, filetype, enc, werror)
        if diff:
            d.update(diff)

    return d

def uniq(xs):
    """Remove duplicates in given list with its order kept.

    >>> uniq([])
    []
    >>> uniq([1, 4, 5, 1, 2, 3, 5, 10])
    [1, 4, 5, 2, 3, 10]
    """
    acc = xs[:1]
    for x in xs[1:]:
        if x not in acc:
            acc += [x]

    return acc

def mk_template_paths(filepath, template_paths=[]):
    """
    :param filepath: (Base) filepath of template file
    :param template_paths: Template search paths
    """
    tmpldir = os.path.abspath(os.path.dirname(filepath))
    if template_paths:
        return uniq(template_paths + [tmpldir])
    else:
        # default:
        return [os.curdir, tmpldir]

def render(filepath, context, paths):
    """
    Compile and render template, and returns the result.

    see also: http://jinja.pocoo.org/docs/api/#basics

    :param filepath: (Base) filepath of template file
    :param context: Context dict needed to instantiate templates
    :param paths: Template search paths
    """
    filename = os.path.basename(filepath)
    env = jinja2.Environment(loader=jinja2.FileSystemLoader(paths))
    return env.get_template(filename).render(**context)

def template_path(filepath, paths):
    """
    Return resolved path of given template file

    :param filepath: (Base) filepath of template file
    :param paths: Template search paths
    """
    for p in paths:
        candidate = os.path.join(p, filepath)
        if os.path.exists(candidate):
            return candidate

    logging.warn("Could not find template=%s in paths=%s" % (filepath, paths))
    return None

def get_ast(filepath, paths):
    """Parse template (`filepath`) and return an abstract syntax tree.

    see also: http://jinja.pocoo.org/docs/api/#the-meta-api

    :param filepath: (Base) filepath of template file
    :param paths: Template search paths
    """
    env = jinja2.Environment(loader=jinja2.FileSystemLoader(paths))
    try:
        return env.parse(open(filepath).read())
    except:
        return None

def find_templates(filepath, paths, acc=[]):
    """
    Find and return template paths including ones refered in given template
    recursively.

    :param filepath: Maybe base filepath of template file
    :param paths: Template search paths
    """
    filepath = template_path(filepath, paths)
    ast = get_ast(filepath, paths)

    if ast:
        if filepath not in acc:
            acc.append(filepath)  # Add self.

        ref_templates = [
            template_path(f, paths) for f in
                jinja2.meta.find_referenced_templates(ast) if f
        ]

        for f in ref_templates:
            if f not in acc:
                acc.append(f)

            acc += [t for t in find_templates(f, paths, acc) if t not in acc]

    return acc

def flip(xy):
    (x, y) = xy
    return (y, x)

def concat(xss):
    """
    >>> concat([[]])
    []
    >>> concat((()))
    []
    >>> concat([[1,2,3],[4,5]])
    [1, 2, 3, 4, 5]
    >>> concat([[1,2,3],[4,5,[6,7]]])
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat((i, i*2) for i in range(3))
    [0, 0, 1, 2, 2, 4]
    """
    return list(chain_from_iterable(xs for xs in xss))

# TODO:
#def __is_glob(s, gpat='*'):
#    return gpat in s and ...

def parse_filespec(fspec, sep=':', gpat='*'):
    """
    Parse given filespec `fspec` and return [(filetype, filepath)].

    :param fspec: filespec
    :param sep: a char separating filetype and filepath in filespec
    :param gpat: a char for glob pattern

    >>> parse_filespec("base.json")
    [('base.json', 'json')]
    >>> parse_filespec("yaml:foo.yaml")
    [('foo.yaml', 'yaml')]
    >>> parse_filespec("yaml:foo.dat")
    [('foo.dat', 'yaml')]

    # FIXME: How to test it?
    # >>> parse_filespec("yaml:bar/*.conf")
    # [('bar/a.conf', 'yaml'), ('bar/b.conf', 'yaml')]

    TODO: Allow '*' (glob pattern) in filepath when escaped with '\\', etc.
    """
    tp = (ft, fp) = tuple(fspec.split(sep)) if sep in fspec else \
        (get_fileext(fspec), fspec)

    return [(fs, ft) for fs in glob.glob(fp)] if gpat in fspec else [flip(tp)]

def parse_and_load_contexts(contexts, enc, werr):
    """
    :param contexts: list of context file specs
    :param enc: Input encoding of context files
    :param werr: Exit immediately if True and any errors occurrs
        while loading context files
    """
    if contexts:
        ctx = load_contexts(
            concat(parse_filespec(f) for f in contexts), enc, werr
        )
    else:
        ctx = MyDict.createFromDict()

    return ctx

def parse_template_paths(tmpl, paths, sep=":"):
    """
    Parse template_paths option string and return [template_path].

    :param tmpl: Template file to render
    :param paths: str to specify template path list separated by `sep`
    :param sep: template path list separator
    """
    if paths:
        try:
            paths = mk_template_paths(tmpl, paths.split(sep))
            assert paths
        except:
            sys.stderr.write(u"Ignored as invalid form: '%s'\n" % paths)
            paths = mk_template_paths(tmpl, [])
    else:
        paths = mk_template_paths(tmpl, [])

    logging.debug("Template search paths: " + str(paths))
    return paths

def option_parser(argv=sys.argv):
    defaults = dict(
        template_paths=None, output=None, contexts=[], debug=False,
        encoding=_ENCODING, werror=False,
    )

    p = optparse.OptionParser(
        "%prog [OPTION ...] TEMPLATE_FILE", prog=argv[0],
    )
    p.set_defaults(**defaults)

    p.add_option("-T", "--template-paths",
        help="Colon ':' separated template search paths. " + \
            "Please note that dir in which given template exists " + \
            "is always included in the search paths (at the end of " + \
            "the path list) regardless of this option. " + \
            "[., dir in which given template file exists]")
    p.add_option("-C", "--contexts", action="append",
        help="Specify file path and optionally its filetype, to provides "
            " context data to instantiate templates. "
            " The option argument's format is "
            " [type:]<file_name_or_path_or_glob_pattern>"
            " ex. -C json:common.json -C ./specific.yaml -C yaml:test.dat, "
            "     -C yaml:/etc/foo.d/*.conf"
    )
    p.add_option("-o", "--output", help="Output filename [stdout]")
    p.add_option("-E", "--encoding", help="I/O encoding [%default]")
    p.add_option("-D", "--debug", action="store_true", help="Debug mode")
    p.add_option("-W", "--werror", action="store_true",
        help="Exit on warnings if True such like -Werror optoin for gcc"
    )

    return p

def write_to_output(output=None, encoding="utf-8", content=""):
    if output and not output == '-':
        outdir = os.path.dirname(output)
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        open(output, "w", encoding).write(content)
    else:
        codecs.getwriter(encoding)(sys.stdout).write(content)

def main(argv):
    p = option_parser(argv)
    (options, args) = p.parse_args(argv[1:])

    if not args:
        p.print_help()
        sys.exit(0)

    logging.getLogger().setLevel(DEBUG if options.debug else INFO)

    tmpl = args[0]
    ctx = parse_and_load_contexts(
        options.contexts, options.encoding, options.werror
    )
    paths = parse_template_paths(tmpl, options.template_paths)
    result = render(tmpl, ctx, paths)

    write_to_output(options.output, options.encoding, result)

if __name__ == '__main__':
    main(sys.argv)

# vim:sw=4:ts=4:et:
<commit_msg>[render] allow '-' (sys.stdin) as template input<commit_after>"""
    Jinja2 based template renderer.
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Compiles and render Jinja2-based template files.

    :copyright: (c) 2012 by Satoru SATOH <ssato@redhat.com>
    :license: BSD-3

 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

   * Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
   * Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.
   * Neither the name of the author nor the names of its contributors may
     be used to endorse or promote products derived from this software
     without specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 Requirements: python-jinja2, python-simplejson (if python < 2.6) and PyYAML
 References: http://jinja.pocoo.org,
    especially http://jinja.pocoo.org/docs/api/#basics
"""
import codecs
import glob
import itertools
import jinja2
import jinja2.meta
import locale
import logging
import optparse
import os.path
import os
import sys

from logging import DEBUG, INFO

try:
    chain_from_iterable = itertools.chain.from_iterable
except AttributeError:
    # Borrowed from library doc, 9.7.1 Itertools functions:
    def _from_iterable(iterables):
        for it in iterables:
            for element in it:
                yield element

    chain_from_iterable = _from_iterable

# Data loaders: Key=file_extension, Value=load_func
_LOADERS = {}
_ENCODING = locale.getdefaultlocale()[1]

sys.stderr = codecs.getwriter(_ENCODING)(sys.stderr)

# Override the default implementation:
def open(path, flag='r', enc=_ENCODING):
    return codecs.open(path, flag, enc)

try:
    import json
    _LOADERS["json"] = _LOADERS["jsn"] = json.loads
except ImportError:
    try:
        import simplejson as json
        _LOADERS["json"] = _LOADERS["jsn"] = json.loads
    except ImportError:
        sys.stderr.write(u"JSON support is disabled as module not found.\n")

try:
    import yaml
    _LOADERS["yaml"] = _LOADERS["yml"] = yaml.load
except ImportError:
    sys.stderr.write(u"YAML support is disabled as module not found.\n")

def is_dict(x):
    return isinstance(x, (MyDict, dict))

def is_iterable(x):
    return isinstance(x, (list, tuple)) or getattr(x, "next", False)

class MyDict(dict):

    @classmethod
    def createFromDict(cls, dic={}):
        md = MyDict()

        for k, v in dic.iteritems():
            md[k] = cls.createFromDict(v) if is_dict(v) else v

        return md

    def update(self, other, merge_lists=False):
        """Merge `self` and `other` recursively.

        :param merge_lists: Merge not only dicts but also lists,
            e.g. [1, 2], [3, 4] ==> [1, 2, 3, 4]
        """
        if is_dict(other):
            for k, v in other.iteritems():
                if k in self and is_dict(v) and is_dict(self[k]):
                    self[k].update(v, merge_lists)  # update recursively.
                else:
                    if merge_lists and is_iterable(v):
                        self[k] = self[k] + list(v)  # append v :: list
                    else:
                        self[k] = v  # replace self[k] w/ v or set.

def get_fileext(filepath):
    """
    >>> get_fileext("a.json")
    'json'
    >>> get_fileext("abc")
    ''
    """
    return os.path.splitext(filepath)[1][1:]

def get_loader(filepath=None, filetype=None, loaders=_LOADERS):
    if filepath is None and filetype is None:
        logging.error("Could not determine loader type")
        return None

    if filepath is None or filetype is not None:
        ext = filetype
    else:
        ext = get_fileext(filepath)

    return loaders.get(ext, None)

def load_context(filepath, filetype=None, enc=_ENCODING, werror=False):
    """Load context data from given file at `filepath`.

    :param filepath: Context data file path :: str
    :param filetype: Forced context file type
    :param enc: Character encoding of context file
    :param werror: raise exception if any error occured like gcc's -Werr
    """
    default = MyDict.createFromDict()

    loader = get_loader(filepath, filetype)
    if loader is None:
        m = "Couldn't get loader: path=%s, type=%s" % (filepath, filetype)
        if werror:
            raise RuntimeError(m)

        logging.warn(m)
        return default

    logging.debug("Loader found: path=%s, type=%s" % (filepath, filetype))
    data = open(filepath, enc=enc).read()
    try:
        x = loader(data)
        if not is_dict(x):
            logging.warn("Top-level object is not a dict: " + filepath)
            return default

        return MyDict.createFromDict(x)

    except Exception, e:
        if werror:
            raise RuntimeError(str(e))

        logging.warn(str(e))
        return default

def load_contexts(pathspecs, enc, werror=False):
    """Load context data from given files.

    :param paths: Context data file path list :: [str]
    """
    d = MyDict.createFromDict()
    for path, filetype in pathspecs:
        diff = load_context(path, filetype, enc, werror)
        if diff:
            d.update(diff)

    return d

def uniq(xs):
    """Remove duplicates in given list with its order kept.

    >>> uniq([])
    []
    >>> uniq([1, 4, 5, 1, 2, 3, 5, 10])
    [1, 4, 5, 2, 3, 10]
    """
    acc = xs[:1]
    for x in xs[1:]:
        if x not in acc:
            acc += [x]

    return acc

def mk_template_paths(filepath, template_paths=[]):
    """
    :param filepath: (Base) filepath of template file
    :param template_paths: Template search paths
    """
    tmpldir = os.path.abspath(os.path.dirname(filepath))
    if template_paths:
        return uniq(template_paths + [tmpldir])
    else:
        # default:
        return [os.curdir, tmpldir]

def tmpl_env(paths):
    """
    :param paths: Template search paths
    """
    return jinja2.Environment(loader=jinja2.FileSystemLoader(paths))

def render_s(tmpl_s, ctx, paths=[os.curdir]):
    """
    Compile and render given template string `tmpl_s` with context `context`.

    :param tmpl_s: Template string
    :param ctx: Context dict needed to instantiate templates
    :param paths: Template search paths

    >>> render_s('a = {{ a }}, b = "{{ b }}"', {'a': 1, 'b': 'bbb'})
    'a = 1, b = "bbb"'
    """
    return tmpl_env(paths).from_string(tmpl_s).render(**ctx)

def render(filepath, ctx, paths):
    """
    Compile and render template, and return the result.

    Similar to the above but template is given as a file path `filepath` or
    sys.stdin if `filepath` is '-'.

    :param filepath: (Base) filepath of template file or '-' (stdin)
    :param ctx: Context dict needed to instantiate templates
    :param paths: Template search paths
    """
    if filepath == '-':
        return render_s(sys.stdin.read(), ctx, paths)
    else:
        env = tmpl_env(paths)
        return env.get_template(os.path.basename(filepath)).render(**ctx)

def template_path(filepath, paths):
    """
    Return resolved path of given template file

    :param filepath: (Base) filepath of template file
    :param paths: Template search paths
    """
    for p in paths:
        candidate = os.path.join(p, filepath)
        if os.path.exists(candidate):
            return candidate

    logging.warn("Could not find template=%s in paths=%s" % (filepath, paths))
    return None

def get_ast(filepath, paths):
    """Parse template (`filepath`) and return an abstract syntax tree.

    see also: http://jinja.pocoo.org/docs/api/#the-meta-api

    :param filepath: (Base) filepath of template file
    :param paths: Template search paths
    """
    env = jinja2.Environment(loader=jinja2.FileSystemLoader(paths))
    try:
        return env.parse(open(filepath).read())
    except:
        return None

def find_templates(filepath, paths, acc=[]):
    """
    Find and return template paths including ones refered in given template
    recursively.

    :param filepath: Maybe base filepath of template file
    :param paths: Template search paths
    """
    filepath = template_path(filepath, paths)
    ast = get_ast(filepath, paths)

    if ast:
        if filepath not in acc:
            acc.append(filepath)  # Add self.

        ref_templates = [
            template_path(f, paths) for f in
                jinja2.meta.find_referenced_templates(ast) if f
        ]

        for f in ref_templates:
            if f not in acc:
                acc.append(f)

            acc += [t for t in find_templates(f, paths, acc) if t not in acc]

    return acc

def flip(xy):
    (x, y) = xy
    return (y, x)

def concat(xss):
    """
    >>> concat([[]])
    []
    >>> concat((()))
    []
    >>> concat([[1,2,3],[4,5]])
    [1, 2, 3, 4, 5]
    >>> concat([[1,2,3],[4,5,[6,7]]])
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat((i, i*2) for i in range(3))
    [0, 0, 1, 2, 2, 4]
    """
    return list(chain_from_iterable(xs for xs in xss))

# TODO:
#def __is_glob(s, gpat='*'):
#    return gpat in s and ...

def parse_filespec(fspec, sep=':', gpat='*'):
    """
    Parse given filespec `fspec` and return [(filetype, filepath)].

    :param fspec: filespec
    :param sep: a char separating filetype and filepath in filespec
    :param gpat: a char for glob pattern

    >>> parse_filespec("base.json")
    [('base.json', 'json')]
    >>> parse_filespec("yaml:foo.yaml")
    [('foo.yaml', 'yaml')]
    >>> parse_filespec("yaml:foo.dat")
    [('foo.dat', 'yaml')]

    # FIXME: How to test it?
    # >>> parse_filespec("yaml:bar/*.conf")
    # [('bar/a.conf', 'yaml'), ('bar/b.conf', 'yaml')]

    TODO: Allow '*' (glob pattern) in filepath when escaped with '\\', etc.
    """
    tp = (ft, fp) = tuple(fspec.split(sep)) if sep in fspec else \
        (get_fileext(fspec), fspec)

    return [(fs, ft) for fs in glob.glob(fp)] if gpat in fspec else [flip(tp)]

def parse_and_load_contexts(contexts, enc, werr):
    """
    :param contexts: list of context file specs
    :param enc: Input encoding of context files
    :param werr: Exit immediately if True and any errors occurrs
        while loading context files
    """
    if contexts:
        ctx = load_contexts(
            concat(parse_filespec(f) for f in contexts), enc, werr
        )
    else:
        ctx = MyDict.createFromDict()

    return ctx

def parse_template_paths(tmpl, paths, sep=":"):
    """
    Parse template_paths option string and return [template_path].

    :param tmpl: Template file to render
    :param paths: str to specify template path list separated by `sep`
    :param sep: template path list separator
    """
    if paths:
        try:
            paths = mk_template_paths(tmpl, paths.split(sep))
            assert paths
        except:
            sys.stderr.write(u"Ignored as invalid form: '%s'\n" % paths)
            paths = mk_template_paths(tmpl, [])
    else:
        paths = mk_template_paths(tmpl, [])

    logging.debug("Template search paths: " + str(paths))
    return paths

def option_parser(argv=sys.argv):
    defaults = dict(
        template_paths=None, output=None, contexts=[], debug=False,
        encoding=_ENCODING, werror=False,
    )

    p = optparse.OptionParser(
        "%prog [OPTION ...] TEMPLATE_FILE", prog=argv[0],
    )
    p.set_defaults(**defaults)

    p.add_option("-T", "--template-paths",
        help="Colon ':' separated template search paths. " + \
            "Please note that dir in which given template exists " + \
            "is always included in the search paths (at the end of " + \
            "the path list) regardless of this option. " + \
            "[., dir in which given template file exists]")
    p.add_option("-C", "--contexts", action="append",
        help="Specify file path and optionally its filetype, to provides "
            " context data to instantiate templates. "
            " The option argument's format is "
            " [type:]<file_name_or_path_or_glob_pattern>"
            " ex. -C json:common.json -C ./specific.yaml -C yaml:test.dat, "
            "     -C yaml:/etc/foo.d/*.conf"
    )
    p.add_option("-o", "--output", help="Output filename [stdout]")
    p.add_option("-E", "--encoding", help="I/O encoding [%default]")
    p.add_option("-D", "--debug", action="store_true", help="Debug mode")
    p.add_option("-W", "--werror", action="store_true",
        help="Exit on warnings if True such like -Werror optoin for gcc"
    )

    return p

def write_to_output(output=None, encoding="utf-8", content=""):
    if output and not output == '-':
        outdir = os.path.dirname(output)
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        open(output, "w", encoding).write(content)
    else:
        codecs.getwriter(encoding)(sys.stdout).write(content)

def main(argv):
    p = option_parser(argv)
    (options, args) = p.parse_args(argv[1:])

    if not args:
        p.print_help()
        sys.exit(0)

    logging.getLogger().setLevel(DEBUG if options.debug else INFO)

    tmpl = args[0]
    ctx = parse_and_load_contexts(
        options.contexts, options.encoding, options.werror
    )
    paths = parse_template_paths(tmpl, options.template_paths)
    result = render(tmpl, ctx, paths)

    write_to_output(options.output, options.encoding, result)

if __name__ == '__main__':
    main(sys.argv)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.auth.models import User
from django import forms
from django.core.validators import RegexValidator
from django.core.exceptions import FieldError
import re

# Create your models here.

# assumes NetID only allows alphanumeric characters and underscores
netid_validator = RegexValidator(
    regex=r'^[a-zA-Z0-9_]*$', message='NetID must be valid')

class Platform(models.Model):
    name = models.CharField(max_length=15)
    app_store = models.CharField(max_length=30)

    def __str__(self):
        return self.app_store

class Sponsor(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(max_length=16, validators=[netid_validator])
    email = models.EmailField(max_length=256)
    title = models.CharField(max_length=50)
    department = models.CharField(max_length=30)
    unit = models.CharField(max_length=30)

class Manager(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(
        max_length=16,
        validators=[netid_validator],
        error_messages={'required':'Please enter a valid NetID'}
    )
    email = models.EmailField(max_length=256)

class App(models.Model):
    name = models.CharField(max_length=50)
    primary_language = models.CharField(max_length=20)
    platform = models.ManyToManyField(Platform)
    app_manager = models.ForeignKey(Manager, on_delete=models.CASCADE)
    app_sponsor = models.ForeignKey(Sponsor, on_delete=models.CASCADE)
    requestor = models.ForeignKey(User, on_delete=models.CASCADE)
    request_date = models.DateTimeField(auto_now_add=True)

class Agreement(models.Model):
    app = models.ForeignKey(App, on_delete=models.CASCADE)
    agree = models.BooleanField(default=True)
    agree_time = models.DateTimeField(auto_now_add=True)

class SponsorForm(forms.ModelForm):
    class Meta:
        model = Sponsor
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class ManagerForm(forms.ModelForm):
    class Meta:
        model = Manager
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class AppForm(forms.ModelForm):
    class Meta:
        model = App
        fields = ["name", "primary_language", "platform"]
        labels = {
            "name": "Application Name",
            "platform": "Distribution type"
        }

    def __init__(self, *args, **kwargs):
        super(AppForm, self).__init__(*args, **kwargs)
        self.fields["primary_language"] \
            .widget.attrs["placeholder"] = "e.g. English"
        self.fields["platform"].widget = forms.CheckboxSelectMultiple()
        self.fields["platform"].queryset = Platform.objects.all()
<commit_msg>fixes pystyle whitespace<commit_after>from django.db import models
from django.contrib.auth.models import User
from django import forms
from django.core.validators import RegexValidator
from django.core.exceptions import FieldError
import re

# Create your models here.

# assumes NetID only allows alphanumeric characters and underscores
netid_validator = RegexValidator(
    regex=r'^[a-zA-Z0-9_]*$', message='NetID must be valid')

class Platform(models.Model):
    name = models.CharField(max_length=15)
    app_store = models.CharField(max_length=30)

    def __str__(self):
        return self.app_store

class Sponsor(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(max_length=16, validators=[netid_validator])
    email = models.EmailField(max_length=256)
    title = models.CharField(max_length=50)
    department = models.CharField(max_length=30)
    unit = models.CharField(max_length=30)

class Manager(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(
        max_length=16,
        validators=[netid_validator],
        error_messages={'required': 'Please enter a valid NetID'}
    )
    email = models.EmailField(max_length=256)

class App(models.Model):
    name = models.CharField(max_length=50)
    primary_language = models.CharField(max_length=20)
    platform = models.ManyToManyField(Platform)
    app_manager = models.ForeignKey(Manager, on_delete=models.CASCADE)
    app_sponsor = models.ForeignKey(Sponsor, on_delete=models.CASCADE)
    requestor = models.ForeignKey(User, on_delete=models.CASCADE)
    request_date = models.DateTimeField(auto_now_add=True)

class Agreement(models.Model):
    app = models.ForeignKey(App, on_delete=models.CASCADE)
    agree = models.BooleanField(default=True)
    agree_time = models.DateTimeField(auto_now_add=True)

class SponsorForm(forms.ModelForm):
    class Meta:
        model = Sponsor
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class ManagerForm(forms.ModelForm):
    class Meta:
        model = Manager
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class AppForm(forms.ModelForm):
    class Meta:
        model = App
        fields = ["name", "primary_language", "platform"]
        labels = {
            "name": "Application Name",
            "platform": "Distribution type"
        }

    def __init__(self, *args, **kwargs):
        super(AppForm, self).__init__(*args, **kwargs)
        self.fields["primary_language"] \
            .widget.attrs["placeholder"] = "e.g. English"
        self.fields["platform"].widget = forms.CheckboxSelectMultiple()
        self.fields["platform"].queryset = Platform.objects.all()
<|endoftext|>


<commit_before>import logging
from datetime import date, timedelta

import requests
from celery import Celery

from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp

app = Celery()
app.config_from_object(settings)

database.setup()
logger = logging.getLogger(__name__)

@app.task
@events.creates_task(events.HARVESTER_RUN)
def run_harvester(harvester_name, start_date=None, end_date=None):
    logger.info('Running harvester "{}"'.format(harvester_name))

    start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date or date.today()

    normalization = begin_normalization.s(harvester_name)
    start_harvest = harvest.si(harvester_name, timestamp(), start_date=start_date, end_date=end_date)

    # Form and start a celery chain
    (start_harvest | normalization).apply_async()

@app.task
@events.logged(events.HARVESTER_RUN)
def harvest(harvester_name, job_created, start_date=None, end_date=None):
    harvest_started = timestamp()
    harvester = registry[harvester_name]

    start_date = start_date if start_date else date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date if end_date else date.today()

    logger.info('Harvester "{}" has begun harvesting'.format(harvester_name))

    result = harvester.harvest(start_date=start_date, end_date=end_date)

    # result is a list of all of the RawDocuments harvested
    return result, {
        'harvestFinished': timestamp(),
        'harvestTaskCreated': job_created,
        'harvestStarted': harvest_started,
    }

@app.task
def begin_normalization((raw_docs, timestamps), harvester_name):
    '''harvest_ret is harvest return value:
        a tuple contaiing list of rawDocuments and
        a dictionary of timestamps
    '''
    logger.info('Normalizing {} documents for harvester "{}"'
                .format(len(raw_docs), harvester_name))
    # raw is a single raw document
    for raw in raw_docs:
        spawn_tasks(raw, timestamps, harvester_name)

@events.creates_task(events.PROCESSING)
@events.creates_task(events.NORMALIZATION)
def spawn_tasks(raw, timestamps, harvester_name):
        raw['timestamps'] = timestamps
        raw['timestamps']['normalizeTaskCreated'] = timestamp()
        chain = (normalize.si(raw, harvester_name) | process_normalized.s(raw))

        chain.apply_async()
        process_raw.delay(raw)

@app.task
@events.logged(events.PROCESSING, 'raw')
def process_raw(raw_doc, **kwargs):
    processing.process_raw(raw_doc, kwargs)

@app.task
@events.logged(events.NORMALIZATION)
def normalize(raw_doc, harvester_name):
    normalized_started = timestamp()
    harvester = registry[harvester_name]

    normalized = harvester.normalize(raw_doc)

    if not normalized:
        raise events.Skip('Did not normalize document with id {}'.format(raw_doc['docID']))

    normalized['timestamps'] = util.stamp_from_raw(raw_doc, normalizeStarted=normalized_started)

    return normalized  # returns a single normalized document

@app.task
@events.logged(events.PROCESSING, 'normalized')
def process_normalized(normalized_doc, raw_doc, **kwargs):
    if not normalized_doc:
        raise events.Skip('Not processing document with id {}'.format(raw_doc['docID']))
    processing.process_normalized(raw_doc, normalized_doc, kwargs)

@app.task
def update_pubsubhubbub():
    payload = {'hub.mode': 'publish', 'hub.url': '{url}rss/'.format(url=settings.OSF_APP_URL)}
    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
    return requests.post('https://pubsubhubbub.appspot.com', headers=headers, params=payload)
<commit_msg>use date parser<commit_after>import logging
from datetime import date, timedelta

import requests
from celery import Celery

from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp

app = Celery()
app.config_from_object(settings)

database.setup()
logger = logging.getLogger(__name__)

@app.task
@events.creates_task(events.HARVESTER_RUN)
def run_harvester(harvester_name, start_date=None, end_date=None):
    logger.info('Running harvester "{}"'.format(harvester_name))

    start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date or date.today()

    normalization = begin_normalization.s(harvester_name)
    start_harvest = harvest.si(harvester_name, timestamp(), start_date=start_date, end_date=end_date)

    # Form and start a celery chain
    (start_harvest | normalization).apply_async()

@app.task
@events.logged(events.HARVESTER_RUN)
def harvest(harvester_name, job_created, start_date=None, end_date=None):
    harvest_started = timestamp()
    harvester = registry[harvester_name]

    start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date or date.today()

    logger.info('Harvester "{}" has begun harvesting'.format(harvester_name))

    result = harvester.harvest(start_date=start_date, end_date=end_date)

    # result is a list of all of the RawDocuments harvested
    return result, {
        'harvestFinished': timestamp(),
        'harvestTaskCreated': job_created,
        'harvestStarted': harvest_started,
    }

@app.task
def begin_normalization((raw_docs, timestamps), harvester_name):
    '''harvest_ret is harvest return value:
        a tuple contaiing list of rawDocuments and
        a dictionary of timestamps
    '''
    logger.info('Normalizing {} documents for harvester "{}"'
                .format(len(raw_docs), harvester_name))
    # raw is a single raw document
    for raw in raw_docs:
        spawn_tasks(raw, timestamps, harvester_name)

@events.creates_task(events.PROCESSING)
@events.creates_task(events.NORMALIZATION)
def spawn_tasks(raw, timestamps, harvester_name):
        raw['timestamps'] = timestamps
        raw['timestamps']['normalizeTaskCreated'] = timestamp()
        chain = (normalize.si(raw, harvester_name) | process_normalized.s(raw))

        chain.apply_async()
        process_raw.delay(raw)

@app.task
@events.logged(events.PROCESSING, 'raw')
def process_raw(raw_doc, **kwargs):
    processing.process_raw(raw_doc, kwargs)

@app.task
@events.logged(events.NORMALIZATION)
def normalize(raw_doc, harvester_name):
    normalized_started = timestamp()
    harvester = registry[harvester_name]

    normalized = harvester.normalize(raw_doc)

    if not normalized:
        raise events.Skip('Did not normalize document with id {}'.format(raw_doc['docID']))

    normalized['timestamps'] = util.stamp_from_raw(raw_doc, normalizeStarted=normalized_started)

    return normalized  # returns a single normalized document

@app.task
@events.logged(events.PROCESSING, 'normalized')
def process_normalized(normalized_doc, raw_doc, **kwargs):
    if not normalized_doc:
        raise events.Skip('Not processing document with id {}'.format(raw_doc['docID']))
    processing.process_normalized(raw_doc, normalized_doc, kwargs)

@app.task
def update_pubsubhubbub():
    payload = {'hub.mode': 'publish', 'hub.url': '{url}rss/'.format(url=settings.OSF_APP_URL)}
    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
    return requests.post('https://pubsubhubbub.appspot.com', headers=headers, params=payload)
<|endoftext|>


<commit_before># coding=utf-8
import json

import content
from bottle import Bottle, response, request

api = Bottle(autojson=False)

class ApiException:
    def __init__(self, code, message):
        self.code = code
        self.message = message

class ApiPlugin(object):
    def apply(self, callback, route):
        def decorator(*args, **kwargs):
            response.headers['Content-Type'] = 'application/json; charset=utf-8'

            try:
                result = callback(*args, **kwargs)
            except ApiException, e:
                response.status = e.code
                result = {
                    'error': e.message
                }

            return json.dumps(result, encoding='utf-8', ensure_ascii=False, indent=True)

        return decorator

api.install(ApiPlugin())

DEBUG = True

@api.get('/exercises')
def get_exercises():
    columns = [
        'id',
        'created_time',
        'updated_time',
        'scope',
        'scope_letters',
        'definition',
        'notes',
        'rating',
        'practice_time'
    ]

    result = []

    cursor = content.open_connection()
    for row in cursor.execute('SELECT %s FROM exercises' % ', '.join(columns)):
        result.append(dict(zip(columns, row)))

    return result

@api.post('/exercises')
def add_exercise():
    exercise = request.json

    columns = []
    placeholders = []
    params = []
    for column, value in exercise.items():
        columns.append(column)
        placeholders.append('?')
        params.append(value)

    connection = content.open_connection()
    cursor = connection.cursor()
    cursor.execute(
        ''.join(['INSERT INTO exercises (', ', '.join(columns), ') VALUES (', ', '.join(placeholders), ')']),
        params
    )

    return {
        'id': cursor.lastrowid
    }

@api.route('/exercises/<exercise_id:int>', method='PATCH')
def update_exercise(exercise_id):
    exercise = request.json

    setters = []
    params = []
    for column, value in exercise.items():
        setters.append('%s = ?' % column)
        params.append(value)

    params.append(exercise_id)
    params.append(exercise['updated_time'])

    connection = content.open_connection()
    cursor = connection.cursor()
    cursor.execute(
        ''.join(['UPDATE exercises SET ', ', '.join(setters), ' WHERE id = ? AND updated_time < CAST(? AS INTEGER)']),
        params
    )

    return {
        'updated': cursor.rowcount
    }
<commit_msg>ETag in the /api/v1/exercises response<commit_after># coding=utf-8
import json
import hashlib

import content
from bottle import Bottle, response, request

api = Bottle(autojson=False)

def json_response(callback):
    def wrapper(*args, **kwargs):
        response.headers['Content-Type'] = 'application/json; charset=utf-8'

        try:
            result = callback(*args, **kwargs)
        except Exception, e:
            response.status = 500
            result = {
                'error': str(e)
            }

        return json.dumps(result, encoding='utf-8', ensure_ascii=False, indent=True)

    return wrapper

def cached_response(callback):
    def wrapper(*args, **kwargs):
        result = callback(*args, **kwargs)

        etag = '"%s"' % hashlib.md5(result.encode('utf-8')).hexdigest()
        response.set_header('ETag', etag)

        if request.get_header('If-None-Match') == etag:
            response.status = 304
            result = ''

        return result

    return wrapper

DEBUG = True

@api.get('/exercises', apply=[cached_response, json_response])
def get_exercises():
    columns = [
        'id',
        'created_time',
        'updated_time',
        'scope',
        'scope_letters',
        'definition',
        'notes',
        'rating',
        'practice_time'
    ]

    result = []

    cursor = content.open_connection()
    for row in cursor.execute('SELECT %s FROM exercises' % ', '.join(columns)):
        result.append(dict(zip(columns, row)))

    return result

@api.post('/exercises', apply=[json_response])
def add_exercise():
    exercise = request.json

    columns = []
    placeholders = []
    params = []
    for column, value in exercise.items():
        columns.append(column)
        placeholders.append('?')
        params.append(value)

    connection = content.open_connection()
    cursor = connection.cursor()
    cursor.execute(
        ''.join(['INSERT INTO exercises (', ', '.join(columns), ') VALUES (', ', '.join(placeholders), ')']),
        params
    )

    return {
        'id': cursor.lastrowid
    }

@api.route('/exercises/<exercise_id:int>', method='PATCH', apply=[json_response])
def update_exercise(exercise_id):
    exercise = request.json

    setters = []
    params = []
    for column, value in exercise.items():
        setters.append('%s = ?' % column)
        params.append(value)

    params.append(exercise_id)
    params.append(exercise['updated_time'])

    connection = content.open_connection()
    cursor = connection.cursor()
    cursor.execute(
        ''.join(['UPDATE exercises SET ', ', '.join(setters), ' WHERE id = ? AND updated_time < CAST(? AS INTEGER)']),
        params
    )

    return {
        'updated': cursor.rowcount
    }
<|endoftext|>


<commit_before># --------------------------------------------------------------------------------------------------
#     __  ______  ____  __    ________________
#    / / / / __ \/ __ \/ /   /  _/_  __/ ____/
#   / /_/ / / / / /_/ / /    / /  / / / __/
#  / __  / /_/ / ____/ /____/ /  / / / /___
# /_/ /_/\____/_/   /_____/___/ /_/ /_____/   (v1.0 . Achilles)
#
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
# --------------------------------------------------------------------------------------------------
"""
Base class for all Hoplite classes
"""
import os
import sys
import logging
# -------------------------
sys.dont_write_bytecode = True
# -------------------------

# By setting the environment variable HOPLITE_DEBUG, debugging is enabled in the log.
# LOGGING_LEVEL = logging.DEBUG if os.getenv('HOPLITE_DEBUG', False) else logging.INFO
# TODO: Uncomment line above, remove line below once development is done
LOGGING_LEVEL = logging.DEBUG

class HopliteError(Exception):
    """ Custom exception for Hoplite.
    """

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class HopliteBase(object):
    """ Base class for HOPLITE objects. Defines several functionalities and functions
    generic across all modules. Mainly logging functionalities for now.
    """

    def __init__(self, name, work_path=None, log=None):
        # Use current directory unless specified otherwise
        self._work_path = work_path if work_path else os.getcwd()
        if not os.path.exists(self._work_path):
            os.makedirs(self._work_path)
        # Use provided log if any, otherwise configure from scratch.
        self._log = log if log else name
        self._configure_log(self._log)
        self._logger = logging.getLogger(self._log)

    def info(self, message, *args):
        """Shortcut for printing info messages.
        """
        self._logger.info(message, *args)

    def warning(self, message, *args):
        """Shortcut for printing warning messages.
        """
        self._logger.warning(message, *args)

    def error(self, message, *args):
        """Shortcut for printing error messages.
        """
        self._logger.error(message, *args)

    def debug(self, message, *args):
        """Shortcut for printing debug messages.
        """
        self._logger.debug(message, *args)

    def fatal(self, message, *args):
        """Shortcut for printing error messages. Additionaly, raises an exception.
        """
        self._logger.error(message, *args)
        raise HopliteError(message % args)

    def _configure_log(self, name):
        """Configure the log and the output format, both for file-based and shell output.
        """
        destination = os.path.join(self._work_path, '%s.log' % self._log)
        logger = logging.getLogger(name)
        logger.handlers = []
        logger.setLevel(logging.DEBUG)
        # Configure command line logging
        handler = logging.StreamHandler()
        level = logging.INFO
        log_format = '%(name)s - %(message)s'
        self._configure_logger(logger, handler, level, log_format)
        # Configure file logging
        handler = logging.FileHandler(destination, 'w', encoding=None, delay='true')
        level = LOGGING_LEVEL
        log_format = '[%(asctime)s] %(levelname)s - %(message)s'
        date_format = '%Y/%m/%d %H:%M:%S'
        self._configure_logger(logger, handler, level, log_format, date_format)

    @staticmethod
    def _configure_logger(logger, handler, level, log_format, date_format=None):
        """ Configure the given logger with the provided formats.

        Args:
            log (logging.Logger): Logger that is being modified.
            handler (logging.[File|Stream]Handler): Handler to be configured.
            level (logging.LEVEL): Level of verbosity for the logger.
            log_format (str): Format in which the log messages will be printed.
            date_format (str): Format in which the date will be printed in the log.
        """
        handler.setLevel(level)
        formatter = logging.Formatter(log_format, date_format)
        handler.setFormatter(formatter)
        logger.addHandler(handler)

# --------------------------------------------------------------------------------------------------
#   0    1    1    2    2    3    3    4    4    5    5    6    6    7    7    8    8    9    9    0
#   5    0    5    0    5    0    5    0    5    0    5    0    5    0    5    0    5    0    5    0
<commit_msg>HopliteBase searches for the config file automatically.<commit_after># --------------------------------------------------------------------------------------------------
#     __  ______  ____  __    ________________
#    / / / / __ \/ __ \/ /   /  _/_  __/ ____/
#   / /_/ / / / / /_/ / /    / /  / / / __/
#  / __  / /_/ / ____/ /____/ /  / / / /___
# /_/ /_/\____/_/   /_____/___/ /_/ /_____/   (v1.0 . Achilles)
#
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
# --------------------------------------------------------------------------------------------------
"""
Base class for all Hoplite classes
"""
import os
import sys
import logging
import yaml
# -------------------------
sys.dont_write_bytecode = True
# -------------------------

# By setting the environment variable HOPLITE_DEBUG, debugging is enabled in the log.
# LOGGING_LEVEL = logging.DEBUG if os.getenv('HOPLITE_DEBUG', False) else logging.INFO
# TODO: Uncomment line above, remove line below once development is done
LOGGING_LEVEL = logging.DEBUG

class HopliteError(Exception):
    """ Custom exception for Hoplite.
    """

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class HopliteBase(object):
    """ Base class for HOPLITE objects. Defines several functionalities and functions
    generic across all modules. Mainly logging functionalities for now.
    """

    def __init__(self, name, parent=None, work_path=None, log=None):
        # Use current directory unless specified otherwise
        self._work_path = work_path if work_path else os.getcwd()
        if not os.path.exists(self._work_path):
            os.makedirs(self._work_path)
        # Use provided log if any, otherwise configure from scratch.
        self._log = log if log else name
        self._configure_log(self._log)
        self._logger = logging.getLogger(self._log)
        self._parent = parent
        self._config = self._load_config(name)

    def info(self, message, *args):
        """Shortcut for printing info messages.
        """
        self._logger.info(message, *args)

    def warning(self, message, *args):
        """Shortcut for printing warning messages.
        """
        self._logger.warning(message, *args)

    def error(self, message, *args):
        """Shortcut for printing error messages.
        """
        self._logger.error(message, *args)

    def debug(self, message, *args):
        """Shortcut for printing debug messages.
        """
        self._logger.debug(message, *args)

    def fatal(self, message, *args):
        """Shortcut for printing error messages. Additionaly, raises an exception.
        """
        self._logger.error(message, *args)
        raise HopliteError(message % args)

    def _configure_log(self, name):
        """Configure the log and the output format, both for file-based and shell output.
        """
        destination = os.path.join(self._work_path, '%s.log' % self._log)
        logger = logging.getLogger(name)
        logger.handlers = []
        logger.setLevel(logging.DEBUG)
        # Configure command line logging
        handler = logging.StreamHandler()
        level = logging.INFO
        log_format = '%(name)s - %(message)s'
        self._configure_logger(logger, handler, level, log_format)
        # Configure file logging
        handler = logging.FileHandler(destination, 'w', encoding=None, delay='true')
        level = LOGGING_LEVEL
        log_format = '[%(asctime)s] %(levelname)s - %(message)s'
        date_format = '%Y/%m/%d %H:%M:%S'
        self._configure_logger(logger, handler, level, log_format, date_format)

    @staticmethod
    def _configure_logger(logger, handler, level, log_format, date_format=None):
        """ Configure the given logger with the provided formats.
        Args:
            log (logging.Logger): Logger that is being modified.
            handler (logging.[File|Stream]Handler): Handler to be configured.
            level (logging.LEVEL): Level of verbosity for the logger.
            log_format (str): Format in which the log messages will be printed.
            date_format (str): Format in which the date will be printed in the log.
        """
        handler.setLevel(level)
        formatter = logging.Formatter(log_format, date_format)
        handler.setFormatter(formatter)
        logger.addHandler(handler)

    def _load_config(self, name):
        """ Loads the config file of the class.
        """
        # The .hop file must be on the same folder as the source .py file. Look for it and parse
        # its contents as a YAML file.
        config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '%s.hop' % name)
        if not os.path.exists(config_path):
            self.fatal('Configuration file for class %s not found', name)
        with open(config_path, 'r') as stream:
            # Beware, if something is wrong with the file (for example bad YAML formatting) this
            # will explode big time.
            config = yaml.load(stream)
        return config

# --------------------------------------------------------------------------------------------------
#   0    1    1    2    2    3    3    4    4    5    5    6    6    7    7    8    8    9    9    0
#   5    0    5    0    5    0    5    0    5    0    5    0    5    0    5    0    5    0    5    0
<|endoftext|>


<commit_before>#! /usr/bin/env python

import string, traceback, os, sys, string
VERSION = '0.1'
SH_COMMAND_NOT_FOUND = 32512

from csshex import UserAlias, UserImports

class csShell:
	UserGlobals = {}
	UserLocals = {}
	Prompt='[cssh%s ' % VERSION

	def GetPrompt(self):
		return self.Prompt + '%s] ' % os.getcwd()

	def GetCommand(self):
		Input = None
		while not Input:
			Input = string.rstrip(raw_input(self.GetPrompt()))
		return Input

	def Shell(self, Command):
		if os.system(Command) == SH_COMMAND_NOT_FOUND:
			traceback.print_exc()

#TODO Add support for parameters using the $1 perl notation or equiv
	def Alias(self, Command):
		if UserAlias.has_key(Command):
			return UserAlias[Command]
		return Command

	def LoadImports(self):
		print 'Loading',
		for x in UserImports:
			print x,
			if(x[0]=='*'):
			  exec 'from '+x[1:]+' import *' in self.UserGlobals, self.UserLocals
			else:
			  exec 'import '+x in self.UserGlobals, self.UserLocals
		print

	def Handle(self, Command):
		Exec=1
		if Command[0] == '#':
			Command=Command[1:]
		else:
			Command=self.Alias(Command)

		if Command[0] == ';':
			Command=Command[1:]
			self.Shell(Command)
			Exec=0

		else:
			try:
				if(Exec):
					exec Command in self.UserGlobals, self.UserLocals
			except SystemExit: #Allow script to exit
				raise SystemExit
			except:
				traceback.print_exc()

	def Loop(self):
		while 1:
			try:
				Command = self.GetCommand()
				self.Handle(Command)
			except EOFError:
				print
				raise SystemExit, 0

	def __init__(self):
		self.LoadImports()

shell=csShell()
shell.Loop()

<commit_msg>Fixed for console mode<commit_after>#! /usr/bin/env python

import string, traceback, os, sys, string
VERSION = '0.1'
SH_COMMAND_NOT_FOUND = 32512

from csshex import UserAlias, UserImports

class csShell:
	UserGlobals = {}
	UserLocals = {}
	Prompt='[cssh%s ' % VERSION

	def GetPrompt(self):
		return self.Prompt + '%s] ' % os.getcwd()

	def GetCommand(self):
		Input = None
		while not Input:
			Input = string.rstrip(raw_input(self.GetPrompt()))
		return Input

	def Shell(self, Command):
		if os.system(Command) == SH_COMMAND_NOT_FOUND:
			traceback.print_exc()

#TODO Add support for parameters using the $1 perl notation or equiv
	def Alias(self, Command):
		if UserAlias.has_key(Command):
			return UserAlias[Command]
		return Command

	def LoadImports(self):
		print 'Loading',
		for x in UserImports:
			print x,
			try:
				if(x[0]=='*'):
					exec 'from '+x[1:]+' import *' in self.UserGlobals, self.UserLocals
				else:
					exec 'import '+x in self.UserGlobals, self.UserLocals
			except ImportError, e:
				print
				print e
		print

	def Handle(self, Command):
		Exec=1
		if Command[0] == '#':
			Command=Command[1:]
		else:
			Command=self.Alias(Command)

		if Command[0] == ';':
			Command=Command[1:]
			self.Shell(Command)
			Exec=0

		else:
			try:
				if(Exec):
					exec Command in self.UserGlobals, self.UserLocals
			except SystemExit: #Allow script to exit
				raise SystemExit
			except:
				traceback.print_exc()

	def Loop(self):
		while 1:
			try:
				Command = self.GetCommand()
				self.Handle(Command)
			except EOFError:
				print
				raise SystemExit, 0

	def __init__(self):
		self.LoadImports()

shell=csShell()
shell.Loop()

<|endoftext|>