<commit_before># -*- coding: utf-8 -*-

"""
hierarchy -- provides hierarchical organization of pages.
"""

import os
import logging
from copy import copy
from collections import OrderedDict
import functools

from pelican import signals, contents, generators, utils

__version__ = (0, 0, 9)

logger = logging.getLogger(__name__)

# todo: allow more extensions to be defined in the settings
STATIC_EXTENSIONS = ('png', 'jpeg', 'jpg', 'gif', 'tif', 'tiff',
                     'doc', 'docx', 'xls',
                     'pdf',
                     'zip', 'tar', 'gz')

def ascii_tree(tree, prefix=[], last_prefix="", print_item=repr):
    s = "".join(prefix[:-1]) + last_prefix + print_item(tree) + "\n"
    try:
        last_i = len(tree) - 1
        # if tree is tree.sub_pages[0]:
            # return s  # avoid infinit recursion
        for i, item in enumerate(tree):
            if i == last_i:
                s += ascii_tree(item,
                                prefix=prefix + ["    "],
                                last_prefix="└── ",
                                print_item=print_item)
            else:
                s += ascii_tree(item,
                                prefix=prefix + ["│   "],
                                last_prefix="├── ",
                                print_item=print_item)
    except TypeError:
        pass
    return s

class CopyStaticAssetsGenerator(generators.Generator):

    """Copy files (with certain extensions) to the `output_path`."""

    def generate_output(self, writer):
        self._generate_output_for(writer, 'ARTICLE')
        self._generate_output_for(writer, 'PAGE')

    def _generate_output_for(self, writer, kind):
        for f in self.get_files(
                self.settings[kind + '_PATHS'],
                exclude=self.settings[kind + '_EXCLUDES'],
                extensions=STATIC_EXTENSIONS):
            # Remove PAGE_PATHS for output
            #dest = copy(f)
            #if kind == 'PAGE':
            #    print("removing ", f)
            #    dest = os.path.join(*f.split(os.path.sep)[1:])
            #print("f", f)
            #print("self.path", self.path)
            #print("output_path", self.output_path)
            #print("dest", os.path.dirname(dest))
            utils.copy(os.path.join(self.path, f),
                       os.path.join(self.output_path, f))

# todo: Add the filenames of the assests to the url list in context
#       so that _update_content can fix relative URLs
# look for dirs and extract XXX_ in front for ordering
# overwrite with Title if a page exists

# If there is dir in PAGE_DIR, then create an output dir
# and add an index.hmtl (unless there is an index.md)

# Todo: Submit a PR to Pelican to allow cleanly replaceing PagesGenerator.
#       This is currently not possible, because in `__init__.py` in the method
#       `run` there is only a local variable `generators` that is not
#       available to any signal.

class HiPage(contents.Page):

    """
    HiPage inherits from `Page` and implements a hierarchical organization.

    There are the properties `parent` and `level` to access the hierarchy.
    A HiPage page re-uses the page template.

    A HiPage is iterable to go over all (direct) sub pages. It also supports
    the `__contains__` method based on the slug, so that `if "foobar" in page`
    works by checking if a page with slug "foobar" is among the sub_pages of
    this page.
    Under the hood, the hierarchy is implemented as an OrderedDict
    in the property `sub_pages`, where the slugs are the keys to the mapping.
    If you need to add sub pages, you'll have to work directly with the
    `sub_pages` OrderedDict.
    """

    def __init__(self, content, metadata=None, settings=None,
                 source_path=None, context=None, slug=None, title=None,
                 parent=None, **kws):
        self.parent = parent
        title = title or slug  # fall back to use slug for title

        if source_path is None:
            assert slug is not None,\
                "If no `source_path` given, a slug is needed."
            assert title is not None,\
                "If no `source_path` given, a title is needed."

        super(HiPage, self).__init__(content,
                                     metadata=metadata,
                                     settings=settings,
                                     source_path=source_path,
                                     context=context,
                                     **kws)
        # Only set slug/title if they were explicitly provided as keyword args,
        # otherwise let the `Page` constructor do its work.
        if slug is not None:
            self.slug = slug
        if title is not None:
            self.title = title
        self.sub_pages = OrderedDict()

    def __iter__(self):
        # The "iter" around is necessary because Jinja cannot yet handle
        # a ValuesView :-/
        return iter(self.sub_pages.values())

    def __contains__(self, item):
        if isinstance(item, contents.Page):
            item = item.slug
        return True if item in self.sub_pages else False

    def __getitem__(self, key):
        return self.sub_pages[key]

    def __len__(self):
        return len(self.sub_pages)

    @property
    def has_children(self):
        return bool(len(self))

    @property
    def hierarchy(self):
        """Generator giving pages, from self.parent upwards to root."""
        up = self.parent
        while up is not None:
            yield up
            up = up.parent

    @property
    def level(self):
        """The level (depth) in the hierarchy. The virtual root page has 0."""
        return len(list(self.hierarchy))

    @property
    def breadcrumps(self):
        """Returning tuples `(url, title)` from root to this page."""
        bc = [(self.url, self.title)]
        for p in self.hierarchy:
            bc.append((p.url, p.title))
        return reversed(bc)

    def get_url_setting(self, key, separator="/"):
        """
        Helper building the path used for the properties `url` and `save_as`.

        `key` is either "usr" or "save_as".

        The get_url_setting method of Page usually returns something like
        "pages/spam.html" as it is defined by Pelican and can be configured
        by setting PAGE_LANG_URL etc.. We split this at the `separator`, so
        we can insert our hierachical layout, e.g.
        "pages/projects/fun/spam.thml"
        """
        # The root HiPage is not wanted here, and we ignore it by checking if
        # it has a parent.
        url = []
        slugs = reversed(list(p.slug for p in self.hierarchy if p.parent))
        orig = contents.Page.get_url_setting(self, key).split(separator)
        url.extend(orig[:-1])
        url.extend(slugs)
        url.extend(orig[-1:])
        return separator.join(url)

    # We have to override these properties, so that we can inject
    # our hierarchical layout.
    url = property(functools.partial(get_url_setting, key='url'))
    save_as = property(functools.partial(get_url_setting, key='save_as',
                                         separator=os.sep))

    # Override this method to avoid having the user to tell to use HIPAGE_...
    # settings instead of the default PAGE_... settings.
    def _expand_settings(self, key):
        """Expant to full qualified settings, but prefixed with "PAGE_"."""
        fq_key = ('%s_%s' % ("Page", key)).upper()
        return self.settings[fq_key].format(**self.url_format)

    def __hash__(self):
        """Ignoring the self.sub_pages and just make HiPage hashable."""
        return object.__hash__(self)

    def __repr__(self):
        return "<{cl} {url}>".format(cl=self.__class__.__name__, url=self.url)

    def print_tree(self):
        print(ascii_tree(self, print_item=lambda x: x.slug))

class HiPagesGenerator(generators.PagesGenerator):

    """
    Generate hierarchically arranged pages (of class `HiPage`).

    This class can replace `generator.PagesGenerator`.
    """

    def __init__(self, *args, **kws):
        super(HiPagesGenerator, self).__init__(*args, **kws)

    def generate_context(self):
        """
        Overwrites `PagesGenerator.generate_context` and builds up `HiPage`es.
        """
        root = HiPage("", slug='../index', title='Home', parent=None)
        all_pages = []
        hidden_pages = []

        path = os.path.join(self.path, self.settings['PAGE_PATHS'][0])
        all_pages, hidden_pages = self._scan_dir_r(
            path,
            exclude=self.settings['PAGE_EXCLUDES'],
            parent=root)

        self.pages, self.translations = utils.process_translations(all_pages)
        self.hidden_pages, self.hidden_translations = (
            utils.process_translations(hidden_pages))

        self._update_context(('pages', ))
        self.context['PAGES'] = self.pages
        self.context['PAGES_TREE'] = root
        # logger.info(ascii_tree(root, print_item=lambda x: x.slug))

        signals.page_generator_finalized.send(self)

    def _scan_dir_r(self, path, exclude=None, parent=None, extensions=None):
        """
        Recursively scan `path` for source files to convert to `HiPages`.

        Symlinks are followed.

        `path`: Absolute directory path (`str`) to scan through.
        `exclude`: Directory names to exclude.
        `parent`: The `HiPage` that is used as the parent for items in `path`.
        """
        pages = []
        hidden_pages = []
        page_dir = os.path.join(self.path, self.settings['PAGE_PATHS'][0]) #XXX the [0] part is a hack!

        for item in sorted(os.listdir(path)):
            # Get the path to the `item` relative to the "PAGE_PATHS[0]" which
            # is "pages" in the default settings.
            rel_path = os.path.join(os.path.relpath(path, page_dir), item)

            if os.path.isfile(os.path.join(path, item)):
                # Using the include logic from parent class:
                if not self._include_path(rel_path, extensions):
                    logger.debug("Skipping {}".format(rel_path))
                    continue

                try:
                    page = self.readers.read_file(
                        base_path=page_dir,
                        path=rel_path,
                        content_class=HiPage,
                        context=self.context,
                        preread_signal=signals.page_generator_preread,
                        preread_sender=self,
                        context_signal=signals.page_generator_context,
                        context_sender=self)
                except Exception as e:
                    logger.warning('Could not process {}\n{}'
                                   .format(rel_path, e))
                    continue

                if not contents.is_valid_content(page, rel_path):
                    continue

                self.add_source_path(page)

                # XXX
                if page.slug in parent.sub_pages.keys():
                    # Instead of the virtual page use the parents's one
                    print("replace a virtual HiPage by this " + repr(page))
                    for sub in parent.sub_pages[page.slug]:
                        page.sub_pages[sub.slug] = sub
                    pages.remove(parent.sub_pages[page.slug])
                    del parent.sub_pages[page.slug]
                    print("so now parent.sub_pages is " + repr(parent.sub_pages))

                if page.status == "published":
                    pages.append(page)
                    page.parent = parent
                    if page.in_default_lang:
                        # Always adding the page if it is in the default lang,
                        # possibly overwriting an older assignment from a
                        # translated page that happend to be processed before:
                        parent.sub_pages[page.slug] = page
                    elif page.slug not in parent:
                        # Only add a translation to the parent, if there is
                        # not already a page with the same slug in parent:
                        parent.sub_pages[page.slug] = page
                    else:
                        logger.debug(parent.slug + " has already another "
                                     "entry for " + page.slug)

                elif page.status == "hidden":
                    # Don't add hidden pages to the parent. They are still
                    # there but not in the `sub_pages` dict of parent.
                    hidden_pages.append(page)
                else:
                    logger.warning("Unknown status '%s' for file %s, "
                                   "skipping it." %
                                   (repr(page.status), repr(rel_path)))

            elif os.path.isdir(os.path.join(path, item)):

                if item in exclude:
                    continue

                # Virtual page, that is used when there is not already a page
                # with the same slug under the current `parent`.
                page = HiPage("", source_path=os.path.join(page_dir, rel_path),
                              slug=utils.slugify(item),
                              parent=parent)

                # Because we allow to override the HiPage for a directory
                # `foo` by providing a `foo.md` file, we check now whether the
                # file already exists, or an empty `HiPage` should be created
                # for this directory.
                print(page.slug)
                if page.slug in parent.sub_pages.keys():
                    # Instead of the virtual page use the parents's one
                    print("don't use virtual page... " + repr(parent.sub_pages))
                    page = parent.sub_pages[page.slug]
                else:
                    # Add the virtual page (created above) to the parent
                    print("adding virt. page: " + page.slug)
                    parent.sub_pages[page.slug] = page

                # Recursively descent into the directory
                p, h = self._scan_dir_r(path=os.path.join(page_dir, rel_path),
                                        exclude=exclude,
                                        parent=page)
                self.add_source_path(page)
                pages.append(page)
                pages.extend(p)
                hidden_pages.extend(h)

            else:
                raise Exception("not a dir and not a file")  # possible at all?

        return pages, hidden_pages

def remove_normal_pages(self):
    """This is a hack to keep Pelican from generating the normal Pages."""
    if self.__class__ == generators.PagesGenerator:
        def noop(self, *args, **kws):
            logger.debug(self.__class__.__name__ +
                         ": hierarchy plugin: Dummy `noop` called instead of "
                         "`get_files` to replace `Page` by `HiPage`.")
            return []
        # The easiest way, I found, was just to make `get_files` a noop.
        self.get_files = noop
        logger.warn("hierarchy plugin: `get_files()` of "
                    + self.__class__.__name__ +
                    " disabled in favor of `HiPagesGenerator`.")

def add_hi_pages_generator(self):
    return HiPagesGenerator

def add_copy_statics_generator(self):
    return CopyStaticAssetsGenerator

# A pelican plugin has to have this function. It's the entry point.
def register():
    signals.get_generators.connect(add_hi_pages_generator)
    signals.get_generators.connect(add_copy_statics_generator)
    signals.page_generator_init.connect(remove_normal_pages)
<commit_msg>hierarchy: logging the hierachical structure<commit_after># -*- coding: utf-8 -*-

"""
hierarchy -- provides hierarchical organization of pages.
"""

import os
import logging
from copy import copy
from collections import OrderedDict
import functools

from pelican import signals, contents, generators, utils

__version__ = (0, 1, 0)

logger = logging.getLogger(__name__)

# todo: allow more extensions to be defined in the settings
STATIC_EXTENSIONS = ('png', 'jpeg', 'jpg', 'gif', 'tif', 'tiff',
                     'doc', 'docx', 'xls',
                     'pdf',
                     'zip', 'tar', 'gz',
                     'js')

def ascii_tree(tree, prefix=[], last_prefix="", print_item=repr):
    s = "".join(prefix[:-1]) + last_prefix + print_item(tree) + "\n"
    try:
        last_i = len(tree) - 1
        # if tree is tree.sub_pages[0]:
            # return s  # avoid infinit recursion
        for i, item in enumerate(tree):
            if i == last_i:
                s += ascii_tree(item,
                                prefix=prefix + ["    "],
                                last_prefix="└── ",
                                print_item=print_item)
            else:
                s += ascii_tree(item,
                                prefix=prefix + ["│   "],
                                last_prefix="├── ",
                                print_item=print_item)
    except TypeError:
        pass
    return s

class CopyStaticAssetsGenerator(generators.Generator):

    """Copy files (with certain extensions) to the `output_path`."""

    def generate_output(self, writer):
        self._generate_output_for(writer, 'ARTICLE')
        self._generate_output_for(writer, 'PAGE')

    def _generate_output_for(self, writer, kind):
        for f in self.get_files(
                self.settings[kind + '_PATHS'],
                exclude=self.settings[kind + '_EXCLUDES'],
                extensions=STATIC_EXTENSIONS):
            # Remove PAGE_PATHS for output
            #dest = copy(f)
            #if kind == 'PAGE':
            #    print("removing ", f)
            #    dest = os.path.join(*f.split(os.path.sep)[1:])
            #print("f", f)
            #print("self.path", self.path)
            #print("output_path", self.output_path)
            #print("dest", os.path.dirname(dest))
            utils.copy(os.path.join(self.path, f),
                       os.path.join(self.output_path, f))

# todo: Add the filenames of the assests to the url list in context
#       so that _update_content can fix relative URLs
# look for dirs and extract XXX_ in front for ordering
# overwrite with Title if a page exists

# If there is dir in PAGE_DIR, then create an output dir
# and add an index.hmtl (unless there is an index.md)

# Todo: Submit a PR to Pelican to allow cleanly replaceing PagesGenerator.
#       This is currently not possible, because in `__init__.py` in the method
#       `run` there is only a local variable `generators` that is not
#       available to any signal.

class HiPage(contents.Page):

    """
    HiPage inherits from `Page` and implements a hierarchical organization.

    There are the properties `parent` and `level` to access the hierarchy.
    A HiPage page re-uses the page template.

    A HiPage is iterable to go over all (direct) sub pages. It also supports
    the `__contains__` method based on the slug, so that `if "foobar" in page`
    works by checking if a page with slug "foobar" is among the sub_pages of
    this page.
    Under the hood, the hierarchy is implemented as an OrderedDict
    in the property `sub_pages`, where the slugs are the keys to the mapping.
    If you need to add sub pages, you'll have to work directly with the
    `sub_pages` OrderedDict.
    """

    def __init__(self, content, metadata=None, settings=None,
                 source_path=None, context=None, slug=None, title=None,
                 parent=None, **kws):
        self.parent = parent
        title = title or slug  # fall back to use slug for title

        if source_path is None:
            assert slug is not None,\
                "If no `source_path` given, a slug is needed."
            assert title is not None,\
                "If no `source_path` given, a title is needed."

        super(HiPage, self).__init__(content,
                                     metadata=metadata,
                                     settings=settings,
                                     source_path=source_path,
                                     context=context,
                                     **kws)
        # Only set slug/title if they were explicitly provided as keyword args,
        # otherwise let the `Page` constructor do its work.
        if slug is not None:
            self.slug = slug
        if title is not None:
            self.title = title
        self.sub_pages = OrderedDict()

    def __iter__(self):
        # The "iter" around is necessary because Jinja cannot yet handle
        # a ValuesView :-/
        return iter(self.sub_pages.values())

    def __contains__(self, item):
        if isinstance(item, contents.Page):
            item = item.slug
        return True if item in self.sub_pages else False

    def __getitem__(self, key):
        return self.sub_pages[key]

    def __len__(self):
        return len(self.sub_pages)

    @property
    def has_children(self):
        return bool(len(self))

    @property
    def hierarchy(self):
        """Generator giving pages, from self.parent upwards to root."""
        up = self.parent
        while up is not None:
            yield up
            up = up.parent

    @property
    def level(self):
        """The level (depth) in the hierarchy. The virtual root page has 0."""
        return len(list(self.hierarchy))

    @property
    def breadcrumps(self):
        """Returning tuples `(url, title)` from root to this page."""
        bc = [(self.url, self.title)]
        for p in self.hierarchy:
            bc.append((p.url, p.title))
        return reversed(bc)

    def get_url_setting(self, key, separator="/"):
        """
        Helper building the path used for the properties `url` and `save_as`.

        `key` is either "usr" or "save_as".

        The get_url_setting method of Page usually returns something like
        "pages/spam.html" as it is defined by Pelican and can be configured
        by setting PAGE_LANG_URL etc.. We split this at the `separator`, so
        we can insert our hierachical layout, e.g.
        "pages/projects/fun/spam.thml"
        """
        # The root HiPage is not wanted here, and we ignore it by checking if
        # it has a parent.
        url = []
        slugs = reversed(list(p.slug for p in self.hierarchy if p.parent))
        orig = contents.Page.get_url_setting(self, key).split(separator)
        url.extend(orig[:-1])
        url.extend(slugs)
        url.extend(orig[-1:])
        return separator.join(url)

    # We have to override these properties, so that we can inject
    # our hierarchical layout.
    url = property(functools.partial(get_url_setting, key='url'))
    save_as = property(functools.partial(get_url_setting, key='save_as',
                                         separator=os.sep))

    # Override this method to avoid having the user to tell to use HIPAGE_...
    # settings instead of the default PAGE_... settings.
    def _expand_settings(self, key):
        """Expant to full qualified settings, but prefixed with "PAGE_"."""
        fq_key = ('%s_%s' % ("Page", key)).upper()
        return self.settings[fq_key].format(**self.url_format)

    def __hash__(self):
        """Ignoring the self.sub_pages and just make HiPage hashable."""
        return object.__hash__(self)

    def __repr__(self):
        return "<{cl} {url}>".format(cl=self.__class__.__name__, url=self.url)

    def print_tree(self):
        print(ascii_tree(self, print_item=lambda x: x.slug))

class HiPagesGenerator(generators.PagesGenerator):

    """
    Generate hierarchically arranged pages (of class `HiPage`).

    This class can replace `generator.PagesGenerator`.
    """

    def __init__(self, *args, **kws):
        super(HiPagesGenerator, self).__init__(*args, **kws)

    def generate_context(self):
        """
        Overwrites `PagesGenerator.generate_context` and builds up `HiPage`es.
        """
        root = HiPage("", slug='../index', title='Home', parent=None)
        all_pages = []
        hidden_pages = []

        path = os.path.join(self.path, self.settings['PAGE_PATHS'][0])
        all_pages, hidden_pages = self._scan_dir_r(
            path,
            exclude=self.settings['PAGE_EXCLUDES'],
            parent=root)

        self.pages, self.translations = utils.process_translations(all_pages)
        self.hidden_pages, self.hidden_translations = (
            utils.process_translations(hidden_pages))

        self._update_context(('pages', ))
        self.context['PAGES'] = self.pages
        self.context['PAGES_TREE'] = root
        logger.info(ascii_tree(root, print_item=lambda x: x.slug))

        signals.page_generator_finalized.send(self)

    def _scan_dir_r(self, path, exclude=None, parent=None, extensions=None):
        """
        Recursively scan `path` for source files to convert to `HiPages`.

        Symlinks are followed.

        `path`: Absolute directory path (`str`) to scan through.
        `exclude`: Directory names to exclude.
        `parent`: The `HiPage` that is used as the parent for items in `path`.
        """
        pages = []
        hidden_pages = []
        page_dir = os.path.join(self.path, self.settings['PAGE_PATHS'][0]) #XXX the [0] part is a hack!

        for item in sorted(os.listdir(path)):
            # Get the path to the `item` relative to the "PAGE_PATHS[0]" which
            # is "pages" in the default settings.
            rel_path = os.path.join(os.path.relpath(path, page_dir), item)

            if os.path.isfile(os.path.join(path, item)):
                # Using the include logic from parent class:
                if not self._include_path(rel_path, extensions):
                    logger.debug("Skipping {}".format(rel_path))
                    continue

                try:
                    page = self.readers.read_file(
                        base_path=page_dir,
                        path=rel_path,
                        content_class=HiPage,
                        context=self.context,
                        preread_signal=signals.page_generator_preread,
                        preread_sender=self,
                        context_signal=signals.page_generator_context,
                        context_sender=self)
                except Exception as e:
                    logger.warning('Could not process {}\n{}'
                                   .format(rel_path, e))
                    continue

                if not contents.is_valid_content(page, rel_path):
                    continue

                self.add_source_path(page)

                # XXX
                if page.slug in parent.sub_pages.keys():
                    # Instead of the virtual page use the parents's one
                    print("replace a virtual HiPage by this " + repr(page))
                    for sub in parent.sub_pages[page.slug]:
                        page.sub_pages[sub.slug] = sub
                    pages.remove(parent.sub_pages[page.slug])
                    del parent.sub_pages[page.slug]
                    print("so now parent.sub_pages is " + repr(parent.sub_pages))

                if page.status == "published":
                    pages.append(page)
                    page.parent = parent
                    if page.in_default_lang:
                        # Always adding the page if it is in the default lang,
                        # possibly overwriting an older assignment from a
                        # translated page that happend to be processed before:
                        parent.sub_pages[page.slug] = page
                    elif page.slug not in parent:
                        # Only add a translation to the parent, if there is
                        # not already a page with the same slug in parent:
                        parent.sub_pages[page.slug] = page
                    else:
                        logger.debug(parent.slug + " has already another "
                                     "entry for " + page.slug)

                elif page.status == "hidden":
                    # Don't add hidden pages to the parent. They are still
                    # there but not in the `sub_pages` dict of parent.
                    hidden_pages.append(page)
                else:
                    logger.warning("Unknown status '%s' for file %s, "
                                   "skipping it." %
                                   (repr(page.status), repr(rel_path)))

            elif os.path.isdir(os.path.join(path, item)):

                if item in exclude:
                    continue

                # Virtual page, that is used when there is not already a page
                # with the same slug under the current `parent`.
                page = HiPage("", source_path=os.path.join(page_dir, rel_path),
                              slug=utils.slugify(item),
                              parent=parent)

                # Because we allow to override the HiPage for a directory
                # `foo` by providing a `foo.md` file, we check now whether the
                # file already exists, or an empty `HiPage` should be created
                # for this directory.
                print(page.slug)
                if page.slug in parent.sub_pages.keys():
                    # Instead of the virtual page use the parents's one
                    print("don't use virtual page... " + repr(parent.sub_pages))
                    page = parent.sub_pages[page.slug]
                else:
                    # Add the virtual page (created above) to the parent
                    print("adding virt. page: " + page.slug)
                    parent.sub_pages[page.slug] = page

                # Recursively descent into the directory
                p, h = self._scan_dir_r(path=os.path.join(page_dir, rel_path),
                                        exclude=exclude,
                                        parent=page)
                self.add_source_path(page)
                pages.append(page)
                pages.extend(p)
                hidden_pages.extend(h)

            else:
                raise Exception("not a dir and not a file")  # possible at all?

        return pages, hidden_pages

def remove_normal_pages(self):
    """This is a hack to keep Pelican from generating the normal Pages."""
    if self.__class__ == generators.PagesGenerator:
        def noop(self, *args, **kws):
            logger.debug(self.__class__.__name__ +
                         ": hierarchy plugin: Dummy `noop` called instead of "
                         "`get_files` to replace `Page` by `HiPage`.")
            return []
        # The easiest way, I found, was just to make `get_files` a noop.
        self.get_files = noop
        logger.warn("hierarchy plugin: `get_files()` of "
                    + self.__class__.__name__ +
                    " disabled in favor of `HiPagesGenerator`.")

def add_hi_pages_generator(self):
    return HiPagesGenerator

def add_copy_statics_generator(self):
    return CopyStaticAssetsGenerator

# A pelican plugin has to have this function. It's the entry point.
def register():
    signals.get_generators.connect(add_hi_pages_generator)
    signals.get_generators.connect(add_copy_statics_generator)
    signals.page_generator_init.connect(remove_normal_pages)
<|endoftext|>


<commit_before># Higgins - A multi-media server
# Copyright (c) 2007-2009  Michael Frank <msfrank@syntaxjockey.com>
#
# This program is free software; for license information see
# the COPYING file.

import random, string, urllib
from twisted.internet.defer import maybeDeferred
from xml.etree.ElementTree import Element, SubElement
from higgins.service import Service
from higgins.conf import conf
from higgins.upnp.device_service import UPNPDeviceService
from higgins.upnp.prettyprint import prettyprint
from higgins.upnp.logger import logger

class DeviceDeclarativeParser(type):
    def __new__(cls, name, bases, attrs):
        # TODO: verify required service attributes
        # create UDN if needed
        udn_conf_key = "UPNP_" + name + "_UDN"
        udn = conf.get(udn_conf_key)
        if udn == None:
            udn = ''.join(map(lambda x: random.choice(string.letters), xrange(20)))
            conf[udn_conf_key] = udn
        attrs['upnp_UDN'] = udn
        logger.log_debug("UDN for %s is %s" % (name, udn))
        # load services
        services = {}
        for key,svc in attrs.items():
            if isinstance(svc, UPNPDeviceService):
                services[svc.upnp_service_id] = svc
        for base in bases:
            if hasattr(base, '_upnp_services'):
                base._upnp_services.update(services)
                services = base._upnp_services
        attrs['_upnp_services'] = services
        return super(DeviceDeclarativeParser,cls).__new__(cls, name, bases, attrs)

class UPNPDevice(Service):
    __metaclass__ = DeviceDeclarativeParser

    upnp_manufacturer = "Higgins Project"
    upnp_manufacturer_url = "http://syntaxjockey.com/higgins"
    upnp_model_name = "Higgins UPnP Device"
    upnp_model_description = "Higgins UPnP Device"
    upnp_model_url = "http://syntaxjockey.com/higgins"
    upnp_model_number = ""
    upnp_serial_number = ""
    upnp_device_name = None
    upnp_device_type = None
    upnp_friendly_name = None
    upnp_UDN = None

    def startService(self):
        Service.startService(self)

    def stopService(self):
        return maybeDeferred(Service.stopService, self)

    def get_description(self, host=''):
        root = Element("root")
        root.attrib['xmlns'] = 'urn:schemas-upnp-org:device-1-0'
        version = SubElement(root, "specVersion")
        SubElement(version, "major").text = "1"
        SubElement(version, "minor").text = "0"
        device = SubElement(root, "device")
        SubElement(device, "deviceType").text = self.upnp_device_type
        SubElement(device, "friendlyName").text = self.upnp_friendly_name
        SubElement(device, "manufacturer").text = self.upnp_manufacturer
        SubElement(device, "manufacturerURL").text = self.upnp_manufacturer_url
        SubElement(device, "modelName").text = self.upnp_model_name
        SubElement(device, "modelDescription").text = self.upnp_model_description
        SubElement(device, "modelURL").text = self.upnp_model_url
        SubElement(device, "modelNumber").text = self.upnp_model_number
        SubElement(device, "serialNumber").text = self.upnp_serial_number
        SubElement(device, "UDN").text = "uuid:%s" % self.upnp_UDN
        SubElement(device, "URLBase").text = "%s/%s/" % (host, self.upnp_UDN)
        svc_list = SubElement(device, "serviceList")
        for svc in self._upnp_services.values():
            service = SubElement(svc_list, "service")
            SubElement(service, "serviceType").text = svc.upnp_service_type
            SubElement(service, "serviceId").text = svc.upnp_service_id
            SubElement(service, "SCPDURL").text = "%s" % (svc.upnp_service_id.replace(':', '_'))
            SubElement(service, "controlURL").text = "%s/control" % (svc.upnp_service_id.replace(':', '_'))
            SubElement(service, "eventSubURL").text = "%s/event" % (svc.upnp_service_id.replace(':', '_'))
        return prettyprint(root)

    def __str__(self):
        return self.upnp_UDN

# Define the public API
__all__ = ['UPNPDevice',]
<commit_msg>added relativeUrls param to Device.getDescription<commit_after># Higgins - A multi-media server
# Copyright (c) 2007-2009  Michael Frank <msfrank@syntaxjockey.com>
#
# This program is free software; for license information see
# the COPYING file.

import random, string, urllib
from twisted.internet.defer import maybeDeferred
from xml.etree.ElementTree import Element, SubElement
from higgins.service import Service
from higgins.conf import conf
from higgins.upnp.device_service import UPNPDeviceService
from higgins.upnp.prettyprint import prettyprint
from higgins.upnp.logger import logger

class DeviceDeclarativeParser(type):
    def __new__(cls, name, bases, attrs):
        # TODO: verify required service attributes
        # create UDN if needed
        udn_conf_key = "UPNP_" + name + "_UDN"
        udn = conf.get(udn_conf_key)
        if udn == None:
            udn = ''.join(map(lambda x: random.choice(string.letters), xrange(20)))
            conf[udn_conf_key] = udn
        attrs['upnp_UDN'] = udn
        logger.log_debug("UDN for %s is %s" % (name, udn))
        # load services
        services = {}
        for key,svc in attrs.items():
            if isinstance(svc, UPNPDeviceService):
                services[svc.upnp_service_id] = svc
        for base in bases:
            if hasattr(base, '_upnp_services'):
                base._upnp_services.update(services)
                services = base._upnp_services
        attrs['_upnp_services'] = services
        return super(DeviceDeclarativeParser,cls).__new__(cls, name, bases, attrs)

class UPNPDevice(Service):
    __metaclass__ = DeviceDeclarativeParser

    upnp_manufacturer = "Higgins Project"
    upnp_manufacturer_url = "http://syntaxjockey.com/higgins"
    upnp_model_name = "Higgins UPnP Device"
    upnp_model_description = "Higgins UPnP Device"
    upnp_model_url = "http://syntaxjockey.com/higgins"
    upnp_model_number = ""
    upnp_serial_number = ""
    upnp_device_name = None
    upnp_device_type = None
    upnp_friendly_name = None
    upnp_UDN = None

    def startService(self):
        Service.startService(self)

    def stopService(self):
        return maybeDeferred(Service.stopService, self)

    def get_description(self, host, relativeUrls=False):
        root = Element("root")
        root.attrib['xmlns'] = 'urn:schemas-upnp-org:device-1-0'
        version = SubElement(root, "specVersion")
        SubElement(version, "major").text = "1"
        SubElement(version, "minor").text = "0"
        device = SubElement(root, "device")
        SubElement(device, "deviceType").text = self.upnp_device_type
        SubElement(device, "friendlyName").text = self.upnp_friendly_name
        SubElement(device, "manufacturer").text = self.upnp_manufacturer
        SubElement(device, "manufacturerURL").text = self.upnp_manufacturer_url
        SubElement(device, "modelName").text = self.upnp_model_name
        SubElement(device, "modelDescription").text = self.upnp_model_description
        SubElement(device, "modelURL").text = self.upnp_model_url
        SubElement(device, "modelNumber").text = self.upnp_model_number
        SubElement(device, "serialNumber").text = self.upnp_serial_number
        SubElement(device, "UDN").text = "uuid:%s" % self.upnp_UDN
        if relativeUrls:
            urlbase = ''
            SubElement(device, "URLBase").text = "%s/%s/" % (host, self.upnp_UDN)
        else:
            urlbase = "http://%s/%s/" % (host, self.upnp_UDN)
        svc_list = SubElement(device, "serviceList")
        for svc in self._upnp_services.values():
            service = SubElement(svc_list, "service")
            SubElement(service, "serviceType").text = svc.upnp_service_type
            SubElement(service, "serviceId").text = svc.upnp_service_id
            SubElement(service, "SCPDURL").text = "%s%s" % (
                urlbase,
                svc.upnp_service_id.replace(':', '_')
                )
            SubElement(service, "controlURL").text = "%s%s/control" % (
                urlbase,
                svc.upnp_service_id.replace(':', '_')
                )
            SubElement(service, "eventSubURL").text = "%s%s/event" % (
                urlbase,
                svc.upnp_service_id.replace(':', '_')
                )
        return prettyprint(root)

    def __str__(self):
        return self.upnp_UDN

# Define the public API
__all__ = ['UPNPDevice',]
<|endoftext|>


<commit_before>#
# Collective Knowledge (Grigori's misc research functions)
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
#

cfg={}  # Will be updated by CK (meta description of this module)
work={} # Will be updated by CK (temporal data)
ck=None # Will be updated by CK (initialized CK kernel)

# Local settings

##############################################################################
# Initialize module

def init(i):
    """

    Input:  {}

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """
    return {'return':0}

##############################################################################
# Replace string in file

def replace_string_in_file(i):
    """
    Input:  {
              file
              (file_out)  - if !='', use this file for output, otherwise overwrite original one!
              string      - string to replace
              replacement - replacement string
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0

              (updated)    - if 'yes', files was updated
            }

    """

    import copy

    o=i.get('out','')

    fin=i['file']
    s1=i['string']
    s2=i['replacement']

    fout=i.get('file_out','')
    if fout=='': fout=fin

    rx=ck.load_text_file({'text_file':fin})
    if rx['return']>0: return rx

    s=rx['string']
    sx=s.replace(s1,s2)

    r={'return':0, 'updated':'no'}

    if s!=sx or fin!=fout:
       r=ck.save_text_file({'text_file':fout, 'string':sx})
       r['updated']='yes'

    return r

##############################################################################
# updating json file

def refresh_json(i):
    """
    Input:  {
              json_file     - file with json
              (output_file) - if !='' use this file for output instead of rewriting original file
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    jf=i['json_file']

    of=i.get('output_file','')
    if of=='': of=jf

    r=ck.load_json_file({'json_file':jf})
    if r['return']>0: return r

    d=r['dict']

    return ck.save_json_to_file({'json_file':of, 'dict':d, 'sort_keys':'yes'})

##############################################################################
# process all files recursively using some action

def process_all_files_recursively(i):
    """
    Input:  {
               (path)     - starting path (or current)
               (pattern)  - file pattern
               (cmd)      - perform action with a file

               (ck)       - call CK access
               (file_key) - substitute this key in 'ck' with file name with full path
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import os

    o=i.get('out','')

    p=i.get('path','')
    if p=='':
       p=os.getcwd()

    cka=i.get('ck',{})
    fk=i.get('file_key','')

    pat=i.get('pattern','')

    if o=='con':
       ck.out('')
       x=''
       if pat!='': x=' ('+pat+')'
       ck.out('Obtaining list of all files'+x+'. May take some time ...')

    r=ck.list_all_files({'path':p, 'pattern':pat, 'all':'yes'})
    if r['return']>0: return r

    lst=r['list']

    for qq in lst:
        if p=='': q=qq
        else:     q=os.path.join(p,qq)

        if len(cka)>0:
           if fk!='':
              cka[fk]=q

           if o=='con':
              ck.out('  '+q)

           r=ck.access(cka)
           if r['return']>0: return r

    return {'return':0, 'list':lst}

##############################################################################
# merge dictionaries in 2 files

def merge_dicts(i):
    """
    Input:  {
              file1   - dict1
              file2   - dict2
              (file3) - output to this file. If empty use file1
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    f1=i.get('file1','')
    f2=i.get('file2','')

    if f1=='' or f2=='':
       return {'return':1, 'error':'--file1 and --file2 should be specified'}

    fo=i.get('file3','')
    if fo=='': fo=f1

    r=ck.load_json_file({'json_file':f1})
    if r['return']>0: return r
    d1=r['dict']

    r=ck.load_json_file({'json_file':f2})
    if r['return']>0: return r
    d2=r['dict']

    if type(d1)==list and type(d2)==list:
       for q in d2:
           d1.append(q)
    else:
       r=ck.merge_dicts({'dict1':d1, 'dict2':d2})
       if r['return']>0: return r
       d1=r['dict1']

    r=ck.save_json_to_file({'json_file':fo, 'dict':d1})
    if r['return']>0: return r

    return {'return':0}

##############################################################################
# sort JSON file

def sort_json_file(i):
    """
    Input:  {
              json_file     - file with json
              (output_file) - if !='' use this file for output instead of rewriting original file
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    return refresh_json(i)

##############################################################################
# add key to meta/files of different entries

def add_key(i):
    """
    Input:  {
              data            - CID of entries to update (can be wild cards)
              (tags)          - prune entries by tags

              key             - key in flat format
              value           - value

              (ignore_update) - ignore update info in entries
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    o=i.get('out','')

    data=i.get('data','')
    if data=='':
       return {'return':1, 'error':'"data" is not defined'}

    tags=i.get('tags','')

    key=i.get('key','')
    if key=='':
       return {'return':1, 'error':'"key" is not defined'}

    value=i.get('value','')
    if value=='':
       return {'return':1, 'error':'"value" is not defined'}

    iu=i.get('ignore_update','')

    # Search entries
    r=ck.access({'action':'search',
                 'cid':data,
                 'tags':tags})
    if r['return']>0: return r
    lst=r['lst']

    llst=len(lst)

    if llst>0 and o=='con':
       ck.out('Updating '+str(llst)+' entries ...')
       ck.out('')

    # Iterate over entries
    for l in lst:
        ruid=l['repo_uid']
        ruoa=l['repo_uoa']
        muid=l['module_uid']
        muoa=l['module_uoa']
        duid=l['data_uid']
        duoa=l['data_uoa']

        if o=='con':
           ck.out('* '+ruoa+':'+muoa+':'+duoa)

        # Load meta
        r=ck.access({'action':'load',
                     'repo_uoa':ruid,
                     'module_uoa':muid,
                     'data_uoa':duid})
        if r['return']>0: return r
        d=r['dict']

        # Updating dict
        r=ck.set_by_flat_key({'dict':d,
                              'key':key,
                              'value':value})
        if r['return']>0: return r

        # Store meta
        r=ck.access({'action':'update',
                     'repo_uoa':ruid,
                     'module_uoa':muid,
                     'data_uoa':duid,
                     'dict':d,
                     'substitute':'yes',
                     'ignore_update':iu})
        if r['return']>0: return r

    return {'return':0}

##############################################################################
# internal file to ignore files/directories from templates

def prepare_entry_template_ignore_files(dr, files):
    lst=['.cm','tmp']
    return lst

##############################################################################
# prepare template for a given entry
# (trying to unify templates for CK when adding program, soft, package, program, dataset, etc)

def prepare_entry_template(i):
    """
    Input:  {
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import os
    import shutil

    o=i.get('out','')
    oo=''
    if o=='con': oo=o

    omuoa=i.get('original_module_uoa','')
    ruoa=i.get('repo_uoa','')
    duoa=i.get('data_uoa','')

    # Search templates
    tuoa=i.get('template','')
    truoa=i.get('template_repo','')

    if tuoa=='':
       lst=[{'data_uid':'', 'data_uoa':'', 'repo_uid':'', 'info':{'data_name':'Empty entry'}, 'meta':{'sort':9999}}] # Add it to the end

       if tuoa=='':
          ii={'action':'search',
              'module_uoa':omuoa,
              'data_uoa':tuoa,
              'search_dict':{'template':'yes'}}
          r=ck.access(ii)
          if r['return']>0: return r

          lst+=r['lst']

       if len(lst)==1:
          c=lst[0]

       # Make selection
       if oo=='con' and len(lst)>1:
          r=select_uoa({'text':'Select template for the new entry',
                        'choices':lst})
          if r['return']>0: return r

          c=r['choice']

       tuoa=c['data_uid']
       truoa=c['repo_uid']

    d=i.get('dict',{})

    if tuoa!='':
       # Attempt to load entry
       ii={'action':'load',
           'module_uoa':omuoa,
           'data_uoa':tuoa,
           'repo_uoa':truoa}
       r=ck.access(ii)
       if r['return']>0: return r

       p=os.path.realpath(r['path'])

       d.update(r['dict'])

    i['action']='add'
    i['module_uoa']=omuoa

    i['common_func']='yes'
    i['sort_keys']='yes'

    i['dict']=d

    r=ck.access(i)
    if r['return']>0: return r

    # Copy files to a new entry if template
    if tuoa!='':
       pnew=r['path']

       d=os.listdir(p)
       for f in d:
           if f!='.cm' and not f.endswith('.pyc') and 'tmp' not in f:
              porig2=os.path.join(p,f)
              pnew2=os.path.join(pnew,f)

              try:

                 if os.path.isdir(porig2):
                    shutil.copytree(porig2, pnew2, ignore=shutil.ignore_patterns('.cm', '*tmp*', '*.pyc'))
                 else:
                    shutil.copyfile(porig2, pnew2)

              except IOError as e:
                 return {'return':1, 'error':'problem copying files from template ('+str(e)+')'}

    return r

##############################################################################
# Universal UOA selector (improved version forked
# from ck-autotuning:module:choice and ck.kernel)

def select_uoa(i):
    """
    Input:  {
              choices      - list from search function
              (text)       - selection text
              (skip_enter) - if 'yes', do not select 0 when user presses Enter
              (skip_sort)  - if 'yes', do not sort array
            }

    Output: {
              return  - return code =  0, if successful
                                    >  0, if error
              (error) - error text if return > 0
              choice  - {dict of selection from lst}
            }

    """

    se=i.get('skip_enter','')

    lst=i.get('choices',[])

    # Prepare data_name and then data_uoa

    if i.get('skip_sort','')!='yes':
       slst=sorted(lst, key=lambda v: (v.get('meta',{}).get('sort',0), v.get('info',{}).get('data_name',''), v['data_uoa']))
    else:
       slst=lst

    array={}
    n=0

    for x in slst:
        sn=str(n)
        array[sn]=x

        duoa=x['data_uoa']
        name=x.get('info',{}).get('data_name','')
        if name=='': name=duoa

        s=sn+') '+name
        if duoa!='': s+=' (--template='+duoa+')'

        ck.out(s)

        n+=1

    ck.out('')

    text=i.get('text','')
    if text=='': text='Select UOA'

    s=text
    if se!='yes': s+=' (or press Enter for 0)'
    s+=': '

    rx=ck.inp({'text':s})
    y=rx['string'].strip()

    if y=='' and se!='yes': y='0'

    if y not in array:
       return {'return':1, 'error':'number is not recognized'}

    return {'return':0, 'choice':array[y]}
<commit_msg>fixing universal template<commit_after>#
# Collective Knowledge (Grigori's misc research functions)
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
#

cfg={}  # Will be updated by CK (meta description of this module)
work={} # Will be updated by CK (temporal data)
ck=None # Will be updated by CK (initialized CK kernel)

# Local settings

##############################################################################
# Initialize module

def init(i):
    """

    Input:  {}

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """
    return {'return':0}

##############################################################################
# Replace string in file

def replace_string_in_file(i):
    """
    Input:  {
              file
              (file_out)  - if !='', use this file for output, otherwise overwrite original one!
              string      - string to replace
              replacement - replacement string
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0

              (updated)    - if 'yes', files was updated
            }

    """

    import copy

    o=i.get('out','')

    fin=i['file']
    s1=i['string']
    s2=i['replacement']

    fout=i.get('file_out','')
    if fout=='': fout=fin

    rx=ck.load_text_file({'text_file':fin})
    if rx['return']>0: return rx

    s=rx['string']
    sx=s.replace(s1,s2)

    r={'return':0, 'updated':'no'}

    if s!=sx or fin!=fout:
       r=ck.save_text_file({'text_file':fout, 'string':sx})
       r['updated']='yes'

    return r

##############################################################################
# updating json file

def refresh_json(i):
    """
    Input:  {
              json_file     - file with json
              (output_file) - if !='' use this file for output instead of rewriting original file
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    jf=i['json_file']

    of=i.get('output_file','')
    if of=='': of=jf

    r=ck.load_json_file({'json_file':jf})
    if r['return']>0: return r

    d=r['dict']

    return ck.save_json_to_file({'json_file':of, 'dict':d, 'sort_keys':'yes'})

##############################################################################
# process all files recursively using some action

def process_all_files_recursively(i):
    """
    Input:  {
               (path)     - starting path (or current)
               (pattern)  - file pattern
               (cmd)      - perform action with a file

               (ck)       - call CK access
               (file_key) - substitute this key in 'ck' with file name with full path
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import os

    o=i.get('out','')

    p=i.get('path','')
    if p=='':
       p=os.getcwd()

    cka=i.get('ck',{})
    fk=i.get('file_key','')

    pat=i.get('pattern','')

    if o=='con':
       ck.out('')
       x=''
       if pat!='': x=' ('+pat+')'
       ck.out('Obtaining list of all files'+x+'. May take some time ...')

    r=ck.list_all_files({'path':p, 'pattern':pat, 'all':'yes'})
    if r['return']>0: return r

    lst=r['list']

    for qq in lst:
        if p=='': q=qq
        else:     q=os.path.join(p,qq)

        if len(cka)>0:
           if fk!='':
              cka[fk]=q

           if o=='con':
              ck.out('  '+q)

           r=ck.access(cka)
           if r['return']>0: return r

    return {'return':0, 'list':lst}

##############################################################################
# merge dictionaries in 2 files

def merge_dicts(i):
    """
    Input:  {
              file1   - dict1
              file2   - dict2
              (file3) - output to this file. If empty use file1
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    f1=i.get('file1','')
    f2=i.get('file2','')

    if f1=='' or f2=='':
       return {'return':1, 'error':'--file1 and --file2 should be specified'}

    fo=i.get('file3','')
    if fo=='': fo=f1

    r=ck.load_json_file({'json_file':f1})
    if r['return']>0: return r
    d1=r['dict']

    r=ck.load_json_file({'json_file':f2})
    if r['return']>0: return r
    d2=r['dict']

    if type(d1)==list and type(d2)==list:
       for q in d2:
           d1.append(q)
    else:
       r=ck.merge_dicts({'dict1':d1, 'dict2':d2})
       if r['return']>0: return r
       d1=r['dict1']

    r=ck.save_json_to_file({'json_file':fo, 'dict':d1})
    if r['return']>0: return r

    return {'return':0}

##############################################################################
# sort JSON file

def sort_json_file(i):
    """
    Input:  {
              json_file     - file with json
              (output_file) - if !='' use this file for output instead of rewriting original file
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    return refresh_json(i)

##############################################################################
# add key to meta/files of different entries

def add_key(i):
    """
    Input:  {
              data            - CID of entries to update (can be wild cards)
              (tags)          - prune entries by tags

              key             - key in flat format
              value           - value

              (ignore_update) - ignore update info in entries
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    o=i.get('out','')

    data=i.get('data','')
    if data=='':
       return {'return':1, 'error':'"data" is not defined'}

    tags=i.get('tags','')

    key=i.get('key','')
    if key=='':
       return {'return':1, 'error':'"key" is not defined'}

    value=i.get('value','')
    if value=='':
       return {'return':1, 'error':'"value" is not defined'}

    iu=i.get('ignore_update','')

    # Search entries
    r=ck.access({'action':'search',
                 'cid':data,
                 'tags':tags})
    if r['return']>0: return r
    lst=r['lst']

    llst=len(lst)

    if llst>0 and o=='con':
       ck.out('Updating '+str(llst)+' entries ...')
       ck.out('')

    # Iterate over entries
    for l in lst:
        ruid=l['repo_uid']
        ruoa=l['repo_uoa']
        muid=l['module_uid']
        muoa=l['module_uoa']
        duid=l['data_uid']
        duoa=l['data_uoa']

        if o=='con':
           ck.out('* '+ruoa+':'+muoa+':'+duoa)

        # Load meta
        r=ck.access({'action':'load',
                     'repo_uoa':ruid,
                     'module_uoa':muid,
                     'data_uoa':duid})
        if r['return']>0: return r
        d=r['dict']

        # Updating dict
        r=ck.set_by_flat_key({'dict':d,
                              'key':key,
                              'value':value})
        if r['return']>0: return r

        # Store meta
        r=ck.access({'action':'update',
                     'repo_uoa':ruid,
                     'module_uoa':muid,
                     'data_uoa':duid,
                     'dict':d,
                     'substitute':'yes',
                     'ignore_update':iu})
        if r['return']>0: return r

    return {'return':0}

##############################################################################
# internal file to ignore files/directories from templates

def prepare_entry_template_ignore_files(dr, files):
    lst=['.cm','tmp']
    return lst

##############################################################################
# prepare template for a given entry
# (trying to unify templates for CK when adding program, soft, package, program, dataset, etc)

def prepare_entry_template(i):
    """
    Input:  {
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import os
    import shutil

    o=i.get('out','')
    oo=''
    if o=='con': oo=o

    omuoa=i.get('original_module_uoa','')
    ruoa=i.get('repo_uoa','')
    duoa=i.get('data_uoa','')

    # Search templates
    tuoa=i.get('template','')
    truoa=i.get('template_repo','')

    if tuoa=='':
       lst=[{'data_uid':'', 'data_uoa':'', 'repo_uid':'', 'info':{'data_name':'Empty entry'}, 'meta':{'sort':9999}}] # Add it to the end

       if tuoa=='':
          ii={'action':'search',
              'module_uoa':omuoa,
              'data_uoa':tuoa,
              'add_info':'yes',
              'search_dict':{'template':'yes'}}
          r=ck.access(ii)
          if r['return']>0: return r

          lst+=r['lst']

       if len(lst)==1:
          c=lst[0]

       # Make selection
       if oo=='con' and len(lst)>1:
          r=select_uoa({'text':'Select template for the new entry',
                        'choices':lst})
          if r['return']>0: return r

          c=r['choice']

       tuoa=c['data_uid']
       truoa=c['repo_uid']

    d=i.get('dict',{})

    if tuoa!='':
       # Attempt to load entry
       ii={'action':'load',
           'module_uoa':omuoa,
           'data_uoa':tuoa,
           'repo_uoa':truoa}
       r=ck.access(ii)
       if r['return']>0: return r

       p=os.path.realpath(r['path'])

       d.update(r['dict'])

    i['action']='add'
    i['module_uoa']=omuoa

    i['common_func']='yes'
    i['sort_keys']='yes'

    i['dict']=d

    r=ck.access(i)
    if r['return']>0: return r

    # Copy files to a new entry if template
    if tuoa!='':
       pnew=r['path']

       d=os.listdir(p)
       for f in d:
           if f!='.cm' and not f.endswith('.pyc') and 'tmp' not in f:
              porig2=os.path.join(p,f)
              pnew2=os.path.join(pnew,f)

              try:

                 if os.path.isdir(porig2):
                    shutil.copytree(porig2, pnew2, ignore=shutil.ignore_patterns('.cm', '*tmp*', '*.pyc'))
                 else:
                    shutil.copyfile(porig2, pnew2)

              except IOError as e:
                 return {'return':1, 'error':'problem copying files from template ('+str(e)+')'}

    return r

##############################################################################
# Universal UOA selector (improved version forked
# from ck-autotuning:module:choice and ck.kernel)

def select_uoa(i):
    """
    Input:  {
              choices      - list from search function
              (text)       - selection text
              (skip_enter) - if 'yes', do not select 0 when user presses Enter
              (skip_sort)  - if 'yes', do not sort array
            }

    Output: {
              return  - return code =  0, if successful
                                    >  0, if error
              (error) - error text if return > 0
              choice  - {dict of selection from lst}
            }

    """

    se=i.get('skip_enter','')

    lst=i.get('choices',[])

    # Prepare data_name and then data_uoa

    if i.get('skip_sort','')!='yes':
       slst=sorted(lst, key=lambda v: (v.get('meta',{}).get('sort',0), v.get('info',{}).get('data_name',''), v['data_uoa']))
    else:
       slst=lst

    array={}
    n=0

    for x in slst:
        sn=str(n)
        array[sn]=x

        duoa=x['data_uoa']
        name=x.get('info',{}).get('data_name','')
        if name=='': name=duoa

        s=sn+') '+name
        if duoa!='': s+=' (--template='+duoa+')'

        ck.out(s)

        n+=1

    ck.out('')

    text=i.get('text','')
    if text=='': text='Select UOA'

    s=text
    if se!='yes': s+=' (or press Enter for 0)'
    s+=': '

    rx=ck.inp({'text':s})
    y=rx['string'].strip()

    if y=='' and se!='yes': y='0'

    if y not in array:
       return {'return':1, 'error':'number is not recognized'}

    return {'return':0, 'choice':array[y]}
<|endoftext|>


<commit_before>#!/usr/pkg/bin/env python
"""
Get information for hilbmetric
"""

__description__ = "Python program for the Hilbert projective metric"
__version__ = "1.0-pre01"
__url__ = "https://github.com/kiaderouiche/hilbmetric"<commit_msg>n file<commit_after><|endoftext|>


<commit_before># $Id: module_base.py,v 1.20 2002/09/18 15:42:59 cpbotha Exp $

# ----------------------------------------------------------------------------

"""Module containing class-related utilities and code for dscas3 modules.

In this module you'll find a base class as well as several useful mixins.
Please see the documentation.

author: Charl P. Botha <cpbotha@ieee.org>
"""

class module_base:
    """Base class for all modules.

    Any module wishing to take part in the dscas3 party will have to offer all
    of these methods.
    """

    def __init__(self, module_manager):
        """We need to know where the module manager is so we can query
        it about the module path, e.g.
        """
        self._module_manager = module_manager

    def close(self):
	"""Idempotent method for de-initialising module as far as possible.

        We can't guarantee the calling of __del__, as Python does garbage
        collection and the object might destruct a while after we've removed
        all references to it.

        In addition, with python garbage collection, __del__ can cause
        uncollectable objects, so try to avoid it as far as possible.
        """
	raise NotImplementedError

    def get_input_descriptions(self):
	"""Returns tuple of input descriptions, mostly used by the graph editor
	to make a nice glyph for this module."""
	raise NotImplementedError

    def set_input(self, idx, input_stream):
	"""Attaches input_stream (which is e.g. the output of a previous
        module) to this module's input at position idx.
        """
	raise NotImplementedError

    def get_output_descriptions(self):
	"""Returns a tuple of output descriptions.

        Mostly used by the graph editor to make a nice glyph for this module.
        These are also clues to the user as to which glyphs can be connected.
        """
	raise NotImplementedError

    def get_output(self, idx):
	"""Get the n-th output.

        This will be used for connecting this output to the input of another
        module.
        """
	raise NotImplementedError

    def sync_config(self):
        """Synchronise module view with configuration of underlying system.
        """
	raise NotImplementedError

    def apply_config(self):
        """Apply changes that have been made to view (dialogue) to the
        underlying system.
        """
	raise NotImplementedError

    def execute_module(self):
	raise NotImplementedError

    def view(self):
	"""Pop up a dialog with all config possibilities, including optional
        use of the pipeline browser.
        """
	raise NotImplementedError

# ----------------------------------------------------------------------------

from external.vtkPipeline.ConfigVtkObj import ConfigVtkObj
from external.vtkPipeline.vtkPipeline import vtkPipelineBrowser

class module_mixin_vtk_pipeline_config:
    """Mixin to use for modules that want to make use of the vtkPipeline
    functionality.

    Modules that use this as mixin can make use of the vtk_object_configure
    and vtk_pipeline_configure methods to use ConfigVtkObj and
    vtkPipelineBrowser, respectively.  These methods will make sure that you
    use only one instance of a browser/config class per object.
    """

    def vtk_object_configure(self, parent, renwin, vtk_obj):
        """This will instantiate and show only one object config frame per
        unique vtk_obj (per module instance).

        If it is called multiple times for the same object, it will merely
        bring the pertinent window to the top (by show()ing).

        parent: parent wxWindow derivative.  It's important to pass a parent,
        else the here-created window might never be destroyed.
        renwin: render window (optionally None) which will be
        render()ed when changes are made to the vtk object which is being
        configured.
        vtk_obj: the object you want to config.
        """
        if not hasattr(self, '_vtk_obj_cfs'):
            self._vtk_obj_cfs = {}
        if not self._vtk_obj_cfs.has_key(vtk_obj):
            self._vtk_obj_cfs[vtk_obj] = ConfigVtkObj(parent, renwin, vtk_obj)
        self._vtk_obj_cfs[vtk_obj].show()

    def close_vtk_object_configure(self):
        """Explicitly close() all ConfigVtkObj's that vtk_objct_configure has
        created.

        Usually, the ConfigVtkObj windows will be children of some frame, and
        when that frame gets destroyed, they will be too.  However, when this
        is not the case, you can make use of this method.
        """
        if hasattr(self, '_vtk_obj_cfs'):
            for key in self._vtk_obj_cfs.keys():
                self._vtk_obj_cfs[key].close()
            self._vtk_obj_cfs.clear()

    def vtk_pipeline_configure(self, parent, renwin, objects=None):
        """This will instantiate and show only one pipeline config per
        specified renwin and objects.

        parent: parent wxWindow derivative.  It's important to pass a parent,
        else the here-created window might never be destroy()ed.
        renwin: render window (optionally None) which will be render()ed
        when changes are made AND if objects is None, will be used to determine
        the pipeline.
        objects: if you don't want the pipeline to be extracted from the
        renderwindow, you can specify a sequence of objects to be used as the
        multiple roots of a partial pipeline.

        NOTE: renwin and objects can't BOTH be None/empty.
        """
        if not hasattr(self, '_vtk_pipeline_cfs'):
            self._vtk_pipeline_cfs = {}

        # create a dictionary key: a tuple containing renwin + objects
        # (if objects != None)
        this_key = (renwin,)
        if objects:
            this_key = this_key + objects

        # see if we have this pipeline lying around or not
        # if not, create it and store
        if not self._vtk_pipeline_cfs.has_key(this_key):
            self._vtk_pipeline_cfs[this_key] = vtkPipelineBrowser(
                parent, renwin, objects)

        # yay display
        self._vtk_pipeline_cfs[this_key].show()

    def close_pipeline_configure(self):
        """Explicitly close() the pipeline browser of this module.

        This should happen automatically if a valid 'parent' was passed to
        vtk_pipeline_configure(), i.e. when the parent dies, the pipeline
        browser will die too.  However, you can use this method to take
        care of it explicitly.
        """
        if hasattr(self, '_vtk_pipeline_cfs'):
            for key in self._vtk_pipeline_cfs:
                self._vtk_pipeline_cfs[key].close()

        self._vtk_pipeline_cfs.clear()

# ----------------------------------------------------------------------------

from wxPython.wx import wxFileDialog, wxDirDialog, wxOPEN, wxID_OK

class module_mixin_fo_dialog:
    """Module mixin to make use of file open dialog."""

    def fn_browse(self, parent, message, wildcard, style=wxOPEN):
        """Utility method to make use of wxFileDialog.

        This function will open up exactly one dialog per 'message' and this
        dialog won't be destroyed.  This persistence makes sure that the dialog
        retains its previous settings and also that there is less overhead for
        subsequent creations.  The dialog will be a child of 'parent', so when
        parent is destroyed, this dialog will be too.
        """
        if not hasattr(self, '_fo_dlgs'):
            self._fo_dlgs = {}
        if not self._fo_dlgs.has_key(message):
            self._fo_dlgs[message] = wxFileDialog(parent,
                                                  message, "", "",
                                                  wildcard, style)
        if self._fo_dlgs[message].ShowModal() == wxID_OK:
            return self._fo_dlgs[message].GetPath()
        else:
            return None

    def close_fn_browse(self):
        """Use this method to close all created dialogs explicitly.

        This should be taken care of automatically if you've passed in a valid
        'parent'.  Use this method in cases where this was not possible.
        """
        if hasattr(self, '_fo_dlgs'):
            for key in self._fo_dlgs.keys():
                self._fo_dlgs[key].Destroy()
            self._fo_dlgs.clear()

    def dn_browse(self, parent, message, default_path=""):
        """Utility method to make use of wxDirDialog.

        This function will open up exactly one dialog per 'message' and this
        dialog won't be destroyed.  This function is more or less identical
        to fn_browse().
        """
        if not hasattr(self, '_do_dlgs'):
            self._do_dlgs = {}

        if not self._do_dlgs.has_key(message):
            self._do_dlgs[message] = wxDirDialog(parent, message, default_path)

        if self._do_dlgs[message].ShowModal() == wxID_OK:
            return self._do_dlgs[message].GetPath()
        else:
            return None

<commit_msg>Small fixes.<commit_after># $Id: module_base.py,v 1.21 2002/09/30 15:30:30 cpbotha Exp $

# ----------------------------------------------------------------------------

"""Module containing class-related utilities and code for dscas3 modules.

In this module you'll find a base class as well as several useful mixins.
Please see the documentation.

author: Charl P. Botha <cpbotha@ieee.org>
"""

class module_base:
    """Base class for all modules.

    Any module wishing to take part in the dscas3 party will have to offer all
    of these methods.
    """

    def __init__(self, module_manager):
        """We need to know where the module manager is so we can query
        it about the module path, e.g.
        """
        self._module_manager = module_manager

    def close(self):
	"""Idempotent method for de-initialising module as far as possible.

        We can't guarantee the calling of __del__, as Python does garbage
        collection and the object might destruct a while after we've removed
        all references to it.

        In addition, with python garbage collection, __del__ can cause
        uncollectable objects, so try to avoid it as far as possible.
        """
	raise NotImplementedError

    def get_input_descriptions(self):
	"""Returns tuple of input descriptions, mostly used by the graph editor
	to make a nice glyph for this module."""
	raise NotImplementedError

    def set_input(self, idx, input_stream):
	"""Attaches input_stream (which is e.g. the output of a previous
        module) to this module's input at position idx.
        """
	raise NotImplementedError

    def get_output_descriptions(self):
	"""Returns a tuple of output descriptions.

        Mostly used by the graph editor to make a nice glyph for this module.
        These are also clues to the user as to which glyphs can be connected.
        """
	raise NotImplementedError

    def get_output(self, idx):
	"""Get the n-th output.

        This will be used for connecting this output to the input of another
        module.  Whatever is returned by this object MUST have an Update()
        method.  However you choose to implement it, the Update() should make
        sure that the whole chain of logic resulting in the data object has
        executed so that the data object is up to date.
        """
	raise NotImplementedError

    def sync_config(self):
        """Synchronise module view with configuration of underlying system.
        """
	raise NotImplementedError

    def apply_config(self):
        """Apply changes that have been made to view (dialogue) to the
        underlying system.
        """
	raise NotImplementedError

    def execute_module(self):
        """This should make the model do whatever processing it was designed
        to do.

        It's important that when this method is called, the module should be
        able to cause ALL of the modules preceding it in a glyph chain to
        execute (if necessary).  If the whole chain consists of VTK objects,
        this is easy.

        If not, extra measures need to be taken.  According to API,
        each output/input data object MUST have an Update() method
        that can ensure that the logic responsible for that object has
        executed thus making the data object current.

        In short, execute_module() should call Update() on all of this modules
        input objects, directly or indirectly.
        """
	raise NotImplementedError

    def view(self):
	"""Pop up a dialog with all config possibilities, including optional
        use of the pipeline browser.
        """
	raise NotImplementedError

# ----------------------------------------------------------------------------

from external.vtkPipeline.ConfigVtkObj import ConfigVtkObj
from external.vtkPipeline.vtkPipeline import vtkPipelineBrowser

class module_mixin_vtk_pipeline_config:
    """Mixin to use for modules that want to make use of the vtkPipeline
    functionality.

    Modules that use this as mixin can make use of the vtk_object_configure
    and vtk_pipeline_configure methods to use ConfigVtkObj and
    vtkPipelineBrowser, respectively.  These methods will make sure that you
    use only one instance of a browser/config class per object.
    """

    def vtk_object_configure(self, parent, renwin, vtk_obj):
        """This will instantiate and show only one object config frame per
        unique vtk_obj (per module instance).

        If it is called multiple times for the same object, it will merely
        bring the pertinent window to the top (by show()ing).

        parent: parent wxWindow derivative.  It's important to pass a parent,
        else the here-created window might never be destroyed.
        renwin: render window (optionally None) which will be
        render()ed when changes are made to the vtk object which is being
        configured.
        vtk_obj: the object you want to config.
        """
        if not hasattr(self, '_vtk_obj_cfs'):
            self._vtk_obj_cfs = {}
        if not self._vtk_obj_cfs.has_key(vtk_obj):
            self._vtk_obj_cfs[vtk_obj] = ConfigVtkObj(parent, renwin, vtk_obj)
        self._vtk_obj_cfs[vtk_obj].show()

    def close_vtk_object_configure(self):
        """Explicitly close() all ConfigVtkObj's that vtk_objct_configure has
        created.

        Usually, the ConfigVtkObj windows will be children of some frame, and
        when that frame gets destroyed, they will be too.  However, when this
        is not the case, you can make use of this method.
        """
        if hasattr(self, '_vtk_obj_cfs'):
            for key in self._vtk_obj_cfs.keys():
                self._vtk_obj_cfs[key].close()
            self._vtk_obj_cfs.clear()

    def vtk_pipeline_configure(self, parent, renwin, objects=None):
        """This will instantiate and show only one pipeline config per
        specified renwin and objects.

        parent: parent wxWindow derivative.  It's important to pass a parent,
        else the here-created window might never be destroy()ed.
        renwin: render window (optionally None) which will be render()ed
        when changes are made AND if objects is None, will be used to determine
        the pipeline.
        objects: if you don't want the pipeline to be extracted from the
        renderwindow, you can specify a sequence of objects to be used as the
        multiple roots of a partial pipeline.

        NOTE: renwin and objects can't BOTH be None/empty.
        """
        if not hasattr(self, '_vtk_pipeline_cfs'):
            self._vtk_pipeline_cfs = {}

        # create a dictionary key: a tuple containing renwin + objects
        # (if objects != None)
        this_key = (renwin,)
        if objects:
            this_key = this_key + objects

        # see if we have this pipeline lying around or not
        # if not, create it and store
        if not self._vtk_pipeline_cfs.has_key(this_key):
            self._vtk_pipeline_cfs[this_key] = vtkPipelineBrowser(
                parent, renwin, objects)

        # yay display
        self._vtk_pipeline_cfs[this_key].show()

    def close_pipeline_configure(self):
        """Explicitly close() the pipeline browser of this module.

        This should happen automatically if a valid 'parent' was passed to
        vtk_pipeline_configure(), i.e. when the parent dies, the pipeline
        browser will die too.  However, you can use this method to take
        care of it explicitly.
        """
        if hasattr(self, '_vtk_pipeline_cfs'):
            for key in self._vtk_pipeline_cfs:
                self._vtk_pipeline_cfs[key].close()

        self._vtk_pipeline_cfs.clear()

# ----------------------------------------------------------------------------

from wxPython.wx import wxFileDialog, wxDirDialog, wxOPEN, wxID_OK

class module_mixin_fo_dialog:
    """Module mixin to make use of file open dialog."""

    def fn_browse(self, parent, message, wildcard, style=wxOPEN):
        """Utility method to make use of wxFileDialog.

        This function will open up exactly one dialog per 'message' and this
        dialog won't be destroyed.  This persistence makes sure that the dialog
        retains its previous settings and also that there is less overhead for
        subsequent creations.  The dialog will be a child of 'parent', so when
        parent is destroyed, this dialog will be too.
        """
        if not hasattr(self, '_fo_dlgs'):
            self._fo_dlgs = {}
        if not self._fo_dlgs.has_key(message):
            self._fo_dlgs[message] = wxFileDialog(parent,
                                                  message, "", "",
                                                  wildcard, style)
        if self._fo_dlgs[message].ShowModal() == wxID_OK:
            return self._fo_dlgs[message].GetPath()
        else:
            return None

    def close_fn_browse(self):
        """Use this method to close all created dialogs explicitly.

        This should be taken care of automatically if you've passed in a valid
        'parent'.  Use this method in cases where this was not possible.
        """
        if hasattr(self, '_fo_dlgs'):
            for key in self._fo_dlgs.keys():
                self._fo_dlgs[key].Destroy()
            self._fo_dlgs.clear()

    def dn_browse(self, parent, message, default_path=""):
        """Utility method to make use of wxDirDialog.

        This function will open up exactly one dialog per 'message' and this
        dialog won't be destroyed.  This function is more or less identical
        to fn_browse().
        """
        if not hasattr(self, '_do_dlgs'):
            self._do_dlgs = {}

        if not self._do_dlgs.has_key(message):
            self._do_dlgs[message] = wxDirDialog(parent, message, default_path)

        if self._do_dlgs[message].ShowModal() == wxID_OK:
            return self._do_dlgs[message].GetPath()
        else:
            return None

<|endoftext|>