<commit_before>import html
import logging
import re
from copy import deepcopy
from datetime import datetime
from textwrap import TextWrapper
from urllib.parse import quote_plus as urlquote_plus
from urllib.parse import urljoin

import dateparser
import lxml
from dateutil.parser import parse as dateutil_parse
from dateutil.tz import gettz
from lxml.cssselect import CSSSelector
from lxml.html.clean import Cleaner
from scrapy.loader import ItemLoader
from scrapy.loader.processors import Compose, Identity, Join, MapCompose, TakeFirst
from w3lib.html import remove_tags

from feeds.items import FeedEntryItem, FeedItem
from feeds.settings import get_feeds_settings

logger = logging.getLogger(__name__)

_lxml_cleaner = Cleaner(
    scripts=True, javascript=True, comments=True, style=True, inline_style=True
)

def parse_datetime(date_time, loader_context):
    if isinstance(date_time, datetime):
        return date_time
    elif isinstance(date_time, str):
        try:
            return dateutil_parse(
                date_time.strip(),
                dayfirst=loader_context.get("dayfirst", False),
                yearfirst=loader_context.get("yearfirst", True),
                ignoretz=loader_context.get("ignoretz", False),
            )
        except ValueError:
            # If dateutil can't parse it, it might be a human-readable date.
            return dateparser.parse(date_time)
    else:
        raise ValueError("date_time must be datetime or a str.")

def apply_timezone(date_time, loader_context):
    if not date_time.tzinfo:
        # If date_time object is not aware, apply timezone from loader_context.
        # In case a timezone is not supplied, just assume UTC.
        date_time = date_time.replace(
            tzinfo=gettz(loader_context.get("timezone", "UTC"))
        )
    return date_time

def replace_regex(text, loader_context):
    for pattern, repl in loader_context.get("replace_regex", {}).items():
        text = re.sub(pattern, repl, text)

    return text

def build_tree(text, loader_context=None):
    base_url = loader_context.get("base_url", None) if loader_context else None
    tree = lxml.html.fragment_fromstring(text, create_parent="div", base_url=base_url)

    if base_url:
        # Workaround for https://bugs.launchpad.net/lxml/+bug/1576598.
        # FIXME: Remove this when a workaround is released.
        tree.getroottree().docinfo.URL = base_url

    # Scrapy expects an iterator which it unpacks and feeds to the next
    # function in the pipeline. trees are iterators but we don't want to them
    # to get unpacked so we wrap the tree in another iterator.
    return [tree]

def serialize_tree(tree, in_make_links=False):
    return lxml.html.tostring(tree, encoding="unicode")

def make_links_absolute(tree):
    if tree.base_url:
        # Make references in tags like  and  absolute.
        tree.make_links_absolute(handle_failures="ignore")
    return [tree]

def pullup_elems(tree, loader_context):
    for elem_child, parent_dist in loader_context.get("pullup_elems", {}).items():
        selector = CSSSelector(elem_child)
        for elem in selector(tree):
            parent = elem
            for _ in range(parent_dist):
                parent = parent.getparent()
            if parent is not None and parent.getparent() is not None:
                elem.tail = parent.tail
                parent.getparent().replace(parent, elem)
            else:
                logger.error(
                    'Could not find parent with distance {} for selector "{}".'.format(
                        parent_dist, elem_child
                    )
                )

    return [tree]

def replace_elems(tree, loader_context):
    for elem_sel, elem_repl in loader_context.get("replace_elems", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            # If elem_repl is callable, call it to create a new element (or just modify
            # the old one).
            if callable(elem_repl):
                elem_new = elem_repl(elem)
            else:
                elem_new = elem_repl

            # The new element is None, just remove the old one.
            if elem_new is None:
                elem.drop_tree()
            else:
                if isinstance(elem_new, str):
                    # The new element is a string, create a proper element out of it.
                    elem_new = lxml.html.fragment_fromstring(elem_new)
                else:
                    # Create a copy of elem_new in case the element should be used as a
                    # replacement more than once.
                    elem_new = deepcopy(elem_new)
                # Take care to preserve the tail of the old element.
                elem_new.tail = elem.tail
                elem.getparent().replace(elem, elem_new)

    return [tree]

def remove_elems(tree, loader_context):
    remove_elems = []

    settings = get_feeds_settings()
    remove_images = settings.getbool("FEEDS_CONFIG_REMOVE_IMAGES")
    if remove_images:
        remove_elems += ["img"]

    # Remove tags.
    for elem_sel in loader_context.get("remove_elems", []) + remove_elems:
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.drop_tree()

    for elem_sel in loader_context.get("remove_elems_xpath", []):
        for elem in tree.xpath(elem_sel):
            elem.drop_tree()

    return [tree]

def change_attribs(tree, loader_context):
    # Change attrib names.
    for elem_sel, attribs in loader_context.get("change_attribs", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            for attrib in elem.attrib.keys():
                if attrib in attribs:
                    old_attrib_value = elem.attrib.pop(attrib)
                    if attribs[attrib] is not None:
                        # If attribs[attrib] is None, attrib is removed instead of
                        # renamed.
                        elem.attrib[attribs[attrib]] = old_attrib_value
    return [tree]

def change_tags(tree, loader_context):
    # Change tag names.
    for elem_sel, elem_tag in loader_context.get("change_tags", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.tag = elem_tag

    return [tree]

def cleanup_html(tree, loader_context):
    # tree.iter() iterates over the tree including the root node.
    for elem in tree.iter():
        # Remove class and id attribute from all elements which are not needed
        # in the feed.
        elem.attrib.pop("class", None)
        elem.attrib.pop("id", None)
        # Delete data- attributes that have no general meaning.
        for attrib in list(elem.attrib.keys()):
            if attrib.startswith("data-"):
                elem.attrib.pop(attrib)

    return [tree]

def lxml_cleaner(tree):
    _lxml_cleaner(tree)
    return [tree]

def convert_footnotes(tree, loader_context):
    # Convert footnotes.
    for elem_sel in loader_context.get("convert_footnotes", []):
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            if elem.text is not None:
                elem.tag = "small"
                elem.text = " ({})".format(elem.text.strip())

    return [tree]

def convert_iframes(tree, loader_context):
    """Convert iframes to divs with links to its src.

    convert_iframes() is called after remove_elems() so that unwanted iframes can be
    eliminated first.
    """
    base_url = loader_context.get("base_url", None) if loader_context else None
    selector = CSSSelector("iframe")
    for elem in selector(tree):
        if "src" not in elem.attrib:
            continue
        url = urljoin(base_url, elem.attrib.pop("src"))
        elem_new = lxml.html.fragment_fromstring(
            '
{url}
'.format(url=url)
        )
        elem_new.tail = elem.tail
        elem.getparent().replace(elem, elem_new)

    return [tree]

def skip_empty_tree(tree):
    if tree.text:
        # Has a text.
        return [tree]

    if len(tree):
        # Has children.
        return [tree]

    return None

def skip_none(value):
    """Skip values that are None immediately."""
    if value is not None:
        return value

    return None

def skip_false(value):
    """
    Skip values that evaluate to False.

    Scrapy only skips values that are None by default. In feeds we want to
    tighten that policy and also skip empty strings, False and everything else
    that evaluates to False.
    """
    if value:
        return value

    return None

def truncate_tree(tree, limit, drop=False):
    """Truncates the tree in-place so that its text will not exceed limit characters.

    If the limit is exceeded, words are dropped and "..." is added to the end. The
    difference between the actual length of the text and the limit is returned.
    If drop is True, the tree itself will be removed from its parent.
    """

    def _truncate(text, width):
        shorten = len(text) > width
        if shorten:
            # shorten() removes trailing spaces, so we use TextWrapper directly.
            text = TextWrapper(
                width=width, max_lines=1, placeholder="...", drop_whitespace=False
            ).fill(text)
        return (text, width - len(text), shorten)

    remaining = limit

    if remaining < 10 or drop:
        # This will effectively remove all children and the tail as well.
        tree.getparent().remove(tree)
        truncated = True
    else:
        truncated = False

        # Try to truncate the text of the tree.
        if tree.text:
            tree.text, remaining, truncated = _truncate(tree.text, remaining)

        # If there was not enough text but there are children, try to truncate the
        # children.
        for child in tree:
            # drop children if truncation was already done.
            remaining, truncated = truncate_tree(child, remaining, drop=truncated)

        # If there is a tail either truncate it or remove it.
        if tree.tail:
            if remaining >= 10:
                tree.tail, remaining, truncated = _truncate(tree.tail, remaining)
            else:
                tree.tail = None

    return remaining, truncated

def truncate_text(text):
    settings = get_feeds_settings()
    truncate_words = settings.getint("FEEDS_CONFIG_TRUNCATE_WORDS")
    if truncate_words and truncate_words > 0:
        tree = build_tree(text)[0]
        # Assume that a word has 5 characters on average.
        truncate_tree(tree, truncate_words * 5)
        text = serialize_tree(tree)
    return text

class BaseItemLoader(ItemLoader):
    # Defaults
    # Unescape twice to get rid of &&xxx; encoding errors.
    default_input_processor = MapCompose(
        skip_none, str.strip, skip_false, html.unescape, html.unescape
    )
    default_output_processor = TakeFirst()

    # Join first two elements on ": " and the rest on " - ".
    title_out = Compose(lambda t: [": ".join(t[:2])] + t[2:], Join(" - "))

    updated_in = MapCompose(skip_false, parse_datetime, apply_timezone)

    author_name_out = Join(", ")

    # Optional
    path_in = MapCompose(urlquote_plus)
    path_out = Identity()

class FeedItemLoader(BaseItemLoader):
    default_item_class = FeedItem

class FeedEntryItemLoader(BaseItemLoader):
    default_item_class = FeedEntryItem

    # Field specific
    content_text_in = MapCompose(skip_false, str.strip, remove_tags)
    content_text_out = Join("\n")

    content_html_in = MapCompose(
        skip_false,
        replace_regex,
        build_tree,
        convert_footnotes,
        pullup_elems,
        replace_elems,
        remove_elems,
        change_attribs,
        change_tags,
        cleanup_html,
        convert_iframes,
        lxml_cleaner,
        skip_empty_tree,
        make_links_absolute,
        serialize_tree,
    )
    content_html_out = Compose(Join(), truncate_text)

    # Use sorted to keep the output stable.
    category_out = Compose(set, sorted)

    enclosure_in = Identity()
    enclosure_out = Identity()
<commit_msg>Flatten the resulting tree by removing depth and useless elements.<commit_after>import html
import logging
import re
from copy import deepcopy
from datetime import datetime
from textwrap import TextWrapper
from urllib.parse import quote_plus as urlquote_plus
from urllib.parse import urljoin

import dateparser
import lxml
from dateutil.parser import parse as dateutil_parse
from dateutil.tz import gettz
from lxml.cssselect import CSSSelector
from lxml.html.clean import Cleaner
from scrapy.loader import ItemLoader
from scrapy.loader.processors import Compose, Identity, Join, MapCompose, TakeFirst
from w3lib.html import remove_tags

from feeds.items import FeedEntryItem, FeedItem
from feeds.settings import get_feeds_settings

logger = logging.getLogger(__name__)

_lxml_cleaner = Cleaner(
    scripts=True, javascript=True, comments=True, style=True, inline_style=True
)

# List of so-called empty elements in HTML.
# Source: https://developer.mozilla.org/en-US/docs/Glossary/Empty_element
EMPTY_ELEMENTS = [
    "area",
    "base",
    "br",
    "col",
    "embed",
    "hr",
    "img",
    "input",
    "keygen",
    "link",
    "meta",
    "param",
    "source",
    "track",
    "wbr",
]

def parse_datetime(date_time, loader_context):
    if isinstance(date_time, datetime):
        return date_time
    elif isinstance(date_time, str):
        try:
            return dateutil_parse(
                date_time.strip(),
                dayfirst=loader_context.get("dayfirst", False),
                yearfirst=loader_context.get("yearfirst", True),
                ignoretz=loader_context.get("ignoretz", False),
            )
        except ValueError:
            # If dateutil can't parse it, it might be a human-readable date.
            return dateparser.parse(date_time)
    else:
        raise ValueError("date_time must be datetime or a str.")

def apply_timezone(date_time, loader_context):
    if not date_time.tzinfo:
        # If date_time object is not aware, apply timezone from loader_context.
        # In case a timezone is not supplied, just assume UTC.
        date_time = date_time.replace(
            tzinfo=gettz(loader_context.get("timezone", "UTC"))
        )
    return date_time

def replace_regex(text, loader_context):
    for pattern, repl in loader_context.get("replace_regex", {}).items():
        text = re.sub(pattern, repl, text)

    return text

def build_tree(text, loader_context=None):
    base_url = loader_context.get("base_url", None) if loader_context else None
    tree = lxml.html.fragment_fromstring(text, create_parent="div", base_url=base_url)

    if base_url:
        # Workaround for https://bugs.launchpad.net/lxml/+bug/1576598.
        # FIXME: Remove this when a workaround is released.
        tree.getroottree().docinfo.URL = base_url

    # Scrapy expects an iterator which it unpacks and feeds to the next
    # function in the pipeline. trees are iterators but we don't want to them
    # to get unpacked so we wrap the tree in another iterator.
    return [tree]

def serialize_tree(tree, in_make_links=False):
    return lxml.html.tostring(tree, encoding="unicode")

def make_links_absolute(tree):
    if tree.base_url:
        # Make references in tags like  and  absolute.
        tree.make_links_absolute(handle_failures="ignore")
    return [tree]

def pullup_elems(tree, loader_context):
    for elem_child, parent_dist in loader_context.get("pullup_elems", {}).items():
        selector = CSSSelector(elem_child)
        for elem in selector(tree):
            parent = elem
            for _ in range(parent_dist):
                parent = parent.getparent()
            if parent is not None and parent.getparent() is not None:
                elem.tail = parent.tail
                parent.getparent().replace(parent, elem)
            else:
                logger.error(
                    'Could not find parent with distance {} for selector "{}".'.format(
                        parent_dist, elem_child
                    )
                )

    return [tree]

def replace_elems(tree, loader_context):
    for elem_sel, elem_repl in loader_context.get("replace_elems", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            # If elem_repl is callable, call it to create a new element (or just modify
            # the old one).
            if callable(elem_repl):
                elem_new = elem_repl(elem)
            else:
                elem_new = elem_repl

            # The new element is None, just remove the old one.
            if elem_new is None:
                elem.drop_tree()
            else:
                if isinstance(elem_new, str):
                    # The new element is a string, create a proper element out of it.
                    elem_new = lxml.html.fragment_fromstring(elem_new)
                else:
                    # Create a copy of elem_new in case the element should be used as a
                    # replacement more than once.
                    elem_new = deepcopy(elem_new)
                # Take care to preserve the tail of the old element.
                elem_new.tail = elem.tail
                elem.getparent().replace(elem, elem_new)

    return [tree]

def remove_elems(tree, loader_context):
    remove_elems = []

    settings = get_feeds_settings()
    remove_images = settings.getbool("FEEDS_CONFIG_REMOVE_IMAGES")
    if remove_images:
        remove_elems += ["img"]

    # Remove tags.
    for elem_sel in loader_context.get("remove_elems", []) + remove_elems:
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.drop_tree()

    for elem_sel in loader_context.get("remove_elems_xpath", []):
        for elem in tree.xpath(elem_sel):
            elem.drop_tree()

    return [tree]

def change_attribs(tree, loader_context):
    # Change attrib names.
    for elem_sel, attribs in loader_context.get("change_attribs", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            for attrib in elem.attrib.keys():
                if attrib in attribs:
                    old_attrib_value = elem.attrib.pop(attrib)
                    if attribs[attrib] is not None:
                        # If attribs[attrib] is None, attrib is removed instead of
                        # renamed.
                        elem.attrib[attribs[attrib]] = old_attrib_value
    return [tree]

def change_tags(tree, loader_context):
    # Change tag names.
    for elem_sel, elem_tag in loader_context.get("change_tags", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.tag = elem_tag

    return [tree]

def cleanup_html(tree, loader_context):
    # tree.iter() iterates over the tree including the root node.
    for elem in tree.iter():
        # Remove class and id attribute from all elements which are not needed
        # in the feed.
        elem.attrib.pop("class", None)
        elem.attrib.pop("id", None)
        # Delete data- attributes that have no general meaning.
        for attrib in list(elem.attrib.keys()):
            if attrib.startswith("data-"):
                elem.attrib.pop(attrib)

    return [tree]

def lxml_cleaner(tree):
    _lxml_cleaner(tree)
    return [tree]

def convert_footnotes(tree, loader_context):
    # Convert footnotes.
    for elem_sel in loader_context.get("convert_footnotes", []):
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            if elem.text is not None:
                elem.tag = "small"
                elem.text = " ({})".format(elem.text.strip())

    return [tree]

def convert_iframes(tree, loader_context):
    """Convert iframes to divs with links to its src.

    convert_iframes() is called after remove_elems() so that unwanted iframes can be
    eliminated first.
    """
    base_url = loader_context.get("base_url", None) if loader_context else None
    selector = CSSSelector("iframe")
    for elem in selector(tree):
        if "src" not in elem.attrib:
            continue
        url = urljoin(base_url, elem.attrib.pop("src"))
        elem_new = lxml.html.fragment_fromstring(
            '
{url}
'.format(url=url)
        )
        elem_new.tail = elem.tail
        elem.getparent().replace(elem, elem_new)

    return [tree]

def flatten_tree(tree):
    # Post-order traversal.
    for child in tree.iterchildren():
        flatten_tree(child)

    # Points to the first child if tree has a child and it's the only child or None
    # otherwise.
    only_child = list(tree)[0] if len(tree) == 1 else None
    if (
        tree.tag not in EMPTY_ELEMENTS
        and (tree.text is None or tree.text.strip() == "")
        and len(tree) == 0
        and tree.getparent() is not None
    ):
        # Remove elements which don't have a text and are not supposed to be empty.
        tree.drop_tree()
        return None
    elif only_child and only_child.tag == tree.tag and tree.getparent() is not None:
        # Replace tree with child if there is only one child and it has the same tag.
        only_child.tail = tree.tail
        tree.getparent().replace(tree, only_child)

    return [tree]

def skip_empty_tree(tree):
    if tree.text:
        # Has a text.
        return [tree]

    if len(tree):
        # Has children.
        return [tree]

    return None

def skip_none(value):
    """Skip values that are None immediately."""
    if value is not None:
        return value

    return None

def skip_false(value):
    """
    Skip values that evaluate to False.

    Scrapy only skips values that are None by default. In feeds we want to
    tighten that policy and also skip empty strings, False and everything else
    that evaluates to False.
    """
    if value:
        return value

    return None

def truncate_tree(tree, limit, drop=False):
    """Truncates the tree in-place so that its text will not exceed limit characters.

    If the limit is exceeded, words are dropped and "..." is added to the end. The
    difference between the actual length of the text and the limit is returned.
    If drop is True, the tree itself will be removed from its parent.
    """

    def _truncate(text, width):
        shorten = len(text) > width
        if shorten:
            # shorten() removes trailing spaces, so we use TextWrapper directly.
            text = TextWrapper(
                width=width, max_lines=1, placeholder="...", drop_whitespace=False
            ).fill(text)
        return (text, width - len(text), shorten)

    remaining = limit

    if remaining < 10 or drop:
        # This will effectively remove all children and the tail as well.
        tree.getparent().remove(tree)
        truncated = True
    else:
        truncated = False

        # Try to truncate the text of the tree.
        if tree.text:
            tree.text, remaining, truncated = _truncate(tree.text, remaining)

        # If there was not enough text but there are children, try to truncate the
        # children.
        for child in tree:
            # drop children if truncation was already done.
            remaining, truncated = truncate_tree(child, remaining, drop=truncated)

        # If there is a tail either truncate it or remove it.
        if tree.tail:
            if remaining >= 10:
                tree.tail, remaining, truncated = _truncate(tree.tail, remaining)
            else:
                tree.tail = None

    return remaining, truncated

def truncate_text(text):
    settings = get_feeds_settings()
    truncate_words = settings.getint("FEEDS_CONFIG_TRUNCATE_WORDS")
    if truncate_words and truncate_words > 0:
        tree = build_tree(text)[0]
        # Assume that a word has 5 characters on average.
        truncate_tree(tree, truncate_words * 5)
        text = serialize_tree(tree)
    return text

class BaseItemLoader(ItemLoader):
    # Defaults
    # Unescape twice to get rid of &&xxx; encoding errors.
    default_input_processor = MapCompose(
        skip_none, str.strip, skip_false, html.unescape, html.unescape
    )
    default_output_processor = TakeFirst()

    # Join first two elements on ": " and the rest on " - ".
    title_out = Compose(lambda t: [": ".join(t[:2])] + t[2:], Join(" - "))

    updated_in = MapCompose(skip_false, parse_datetime, apply_timezone)

    author_name_out = Join(", ")

    # Optional
    path_in = MapCompose(urlquote_plus)
    path_out = Identity()

class FeedItemLoader(BaseItemLoader):
    default_item_class = FeedItem

class FeedEntryItemLoader(BaseItemLoader):
    default_item_class = FeedEntryItem

    # Field specific
    content_text_in = MapCompose(skip_false, str.strip, remove_tags)
    content_text_out = Join("\n")

    content_html_in = MapCompose(
        skip_false,
        replace_regex,
        build_tree,
        convert_footnotes,
        pullup_elems,
        replace_elems,
        remove_elems,
        change_attribs,
        change_tags,
        cleanup_html,
        convert_iframes,
        lxml_cleaner,
        flatten_tree,
        skip_empty_tree,
        make_links_absolute,
        serialize_tree,
    )
    content_html_out = Compose(Join(), truncate_text)

    # Use sorted to keep the output stable.
    category_out = Compose(set, sorted)

    enclosure_in = Identity()
    enclosure_out = Identity()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from gimpfu import *
import os

# unique_filename from http://stackoverflow.com/questions/183480/is-this-the-best-way-to-get-unique-version-of-filename-w-python
def unique_filename(file_name):
    counter = 1
    file_name_parts = os.path.splitext(file_name) # returns ('/path/file', '.ext')
    while os.path.isfile(file_name):
        file_name = file_name_parts[0] + '_' + str(counter) + file_name_parts[1]
        counter += 1
    return file_name

# MAIN PART
def python_export_all_layers(image, path):
    print "python_save_all_layers START: \n"
    #first choose the path to safe the images:
    #(path is input parameter "path" given from gui function PF_DIRNAME)
    print "save layers in \n %s \n" %path

    # 2.: Choose File-Format (later - for now its png)
    print "file format selection not yet implemented \n"

    # 3.: get layers
    print "image has following layers:\n", image.layers, "\n"

    # 4.: loop through layers
    # we don't want the image to be called "image.jpg.png", so...
    false_file_extensions = [".jpg", ".JPG", ".jpeg", ".bmp", ".BMP", ".gif", ".GIF", ".tif", ".TIF", ".tiff", ".TIFF", ".psd", ".PSD"]
    for layer in image.layers:
        layer_name = layer.name
        print "current layer: %s \n" %layer_name

        # 5.: ...check for right file extension (assume, that there is no "image.png-weird.name"
        if not ".png" in layer_name:
            #if file does not end with .png, check for false extension...
            print "add .png file extension to layer %s...\n" %layer_name
            for e in false_file_extensions:
                if e in layer_name:
                    print "replace", e
                    #...and cut the string plus concatenate .png
                    layer_name = layer_name[:layer_name.find(e)]
                    print "new name:", layer_name
                    break
            layer_name = layer_name + ".png"

        # 6.: build the full path and ensure that it is unique
        file_path = unique_filename(path + "/" + layer_name)
        print "file path/name for that layer:\n %s \n" %file_path

        # 7.: and FINALLY save that layer!
        pdb.file_png_save_defaults(image, layer, file_path, file_path)

    print "all layers saved \n ------END------ \n"
    return

register(
        "python-fu-export-all-layers-rl",
         "Export all layers based on their names as .png",
         """-Filenames are based on each layers names \n
         -if there are file-extensions (.jpg, .jpeg, .bmp, .gif, .psd, .tif, .tiff), they will be replaced with .png
         -saves images with default preferences for PNG-Format""",
         "Breaker222 <lexon222@gmx.de",
         "GNU",
         "2014",
         "export all layers as .png",
         "*",
         [
            (PF_IMAGE, "image", "takes current image", None),
            (PF_DIRNAME, "directory", "the desired directory where to save", os.getcwd())
         ],
         [],
         python_export_all_layers,
         menu="<Image>/File",
         #domain=("gimp20-python", gimp.locale_directory)
        )
main()<commit_msg>added "JPEG" to filter<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from gimpfu import *
import os

# unique_filename from http://stackoverflow.com/questions/183480/is-this-the-best-way-to-get-unique-version-of-filename-w-python
def unique_filename(file_name):
    counter = 1
    file_name_parts = os.path.splitext(file_name) # returns ('/path/file', '.ext')
    while os.path.isfile(file_name):
        file_name = file_name_parts[0] + '_' + str(counter) + file_name_parts[1]
        counter += 1
    return file_name

# MAIN PART
def python_export_all_layers(image, path):
    print "python_save_all_layers START: \n"
    #first choose the path to safe the images:
    #(path is input parameter "path" given from gui function PF_DIRNAME)
    print "save layers in \n %s \n" %path

    # 2.: Choose File-Format (later - for now its png)
    print "file format selection not yet implemented \n"

    # 3.: get layers
    print "image has following layers:\n", image.layers, "\n"

    # 4.: loop through layers
    # we don't want the image to be called "image.jpg.png", so...
    false_file_extensions = [".jpg", ".JPG", ".jpeg", "JPEG", ".bmp", ".BMP", ".gif", ".GIF", ".tif", ".TIF", ".tiff", ".TIFF", ".psd", ".PSD"]
    for layer in image.layers:
        layer_name = layer.name
        print "current layer: %s \n" %layer_name

        # 5.: ...check for right file extension (assume, that there is no "image.png-weird.name"
        if not ".png" in layer_name:
            #if file does not end with .png, check for false extension...
            print "add .png file extension to layer %s...\n" %layer_name
            for e in false_file_extensions:
                if e in layer_name:
                    print "replace", e
                    #...and cut the string plus concatenate .png
                    layer_name = layer_name[:layer_name.find(e)]
                    print "new name:", layer_name
                    break
            layer_name = layer_name + ".png"

        # 6.: build the full path and ensure that it is unique
        file_path = unique_filename(path + "/" + layer_name)
        print "file path/name for that layer:\n %s \n" %file_path

        # 7.: and FINALLY save that layer!
        pdb.file_png_save_defaults(image, layer, file_path, file_path)

    print "all layers saved \n ------END------ \n"
    return

register(
        "python-fu-export-all-layers-rl",
         "Export all layers based on their names as .png",
         """-Filenames are based on each layers names \n
         -if there are file-extensions (.jpg, .jpeg, .bmp, .gif, .psd, .tif, .tiff), they will be replaced with .png
         -saves images with default preferences for PNG-Format""",
         "Breaker222 <lexon222@gmx.de",
         "GNU",
         "2014",
         "export all layers as .png",
         "*",
         [
            (PF_IMAGE, "image", "takes current image", None),
            (PF_DIRNAME, "directory", "the desired directory where to save", os.getcwd())
         ],
         [],
         python_export_all_layers,
         menu="<Image>/File",
         #domain=("gimp20-python", gimp.locale_directory)
        )
main()<|endoftext|>


<commit_before>#pipe2py modules package
#Author: Greg Gaughan

#Note: each module name must match the name used internally by Yahoo, preceded by pipe

__all__ = ['pipeforever',

           'pipefetch',
           'pipefetchdata',
           'pipefetchpage',
           'pipeyql',
           'pipecsv',
           'pipefeedautodiscovery',
           'pipefetchsitefeed',
           'pipexpathfetchpage',

           'pipedatebuilder',
           'pipeurlbuilder',
           'pipeitembuilder',
           'piperssitembuilder',
           'pipetextinput',
           'pipeurlinput',
           'pipestrconcat',
           'pipedateformat',
           'pipestrregex',
           'pipenumberinput',
           'pipeprivateinput',

           'pipefilter',
           'piperename',
           'piperegex',
           'pipeunion',
           'pipeloop',
           'pipesort',
           'pipecount',
           'pipetruncate',
           'pipereverse',
           'pipeuniq',
           'pipesubstr',
           'pipestrreplace',
           'pipesubelement',
           'pipetail',
           'pipecreaterss',
           'pipesplit',

           'pipesimplemath',

           'pipeoutput',

           'pipestringtokenizer',
          ]
<commit_msg>Fix comment spacing<commit_after># pipe2py modules package
# Author: Greg Gaughan

# Note: each module name must match the name used internally by Yahoo, preceded by pipe

__all__ = ['pipeforever',

           'pipefetch',
           'pipefetchdata',
           'pipefetchpage',
           'pipeyql',
           'pipecsv',
           'pipefeedautodiscovery',
           'pipefetchsitefeed',
           'pipexpathfetchpage',

           'pipedatebuilder',
           'pipeurlbuilder',
           'pipeitembuilder',
           'piperssitembuilder',
           'pipetextinput',
           'pipeurlinput',
           'pipestrconcat',
           'pipedateformat',
           'pipestrregex',
           'pipenumberinput',
           'pipeprivateinput',

           'pipefilter',
           'piperename',
           'piperegex',
           'pipeunion',
           'pipeloop',
           'pipesort',
           'pipecount',
           'pipetruncate',
           'pipereverse',
           'pipeuniq',
           'pipesubstr',
           'pipestrreplace',
           'pipesubelement',
           'pipetail',
           'pipecreaterss',
           'pipesplit',

           'pipesimplemath',

           'pipeoutput',

           'pipestringtokenizer',
          ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2013      Noe Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from decimal import Decimal
import re

from weboob.capabilities.bank import Account
from weboob.tools.browser import BasePage
from weboob.tools.capabilities.bank.transactions import FrenchTransaction

__all__ = ['LoginPage', 'DashboardPage', 'OperationsPage']

class LoginPage(BasePage):
    def login(self, username, password):
        self.browser.select_form(name="frmLogin")
        self.browser['username'] = username.encode('utf-8')
        self.browser['password'] = password.encode('utf-8')
        self.browser.find_control('lang').readonly = False
        self.browser['lang'] = 'fr'
        self.browser.submit(nologin=True)

class DashboardPage(BasePage):
    def iter_accounts(self):
        for line in self._accounts():
            yield self._get_account(line)

    def get_account(self, _id):
        xpath = './/a[@href="tbord.do?id=%s"]' % _id
        account = next(a for a in self._accounts() if a.xpath(xpath))
        return self._get_account(account)

    def _accounts(self):
        return self.document.getroot().cssselect('.tbord_account')

    _FIELD_XPATH = './/span[@class="%s"]//text()'
    _URL_XPATH = './/span[@class="accountLabel"]//a/@href'

    def _get_account(self, line):
        def get_field(field):
            return unicode(line.xpath(self._FIELD_XPATH % field)[0]).strip()

        account = Account()
        account._url = unicode(line.xpath(self._URL_XPATH)[0])
        account.id = account._url.replace("tbord.do?id=", "")
        account.balance = Decimal(FrenchTransaction.clean_amount(
            get_field('accountTotal')))
        account.label = get_field('accountLabel2')
        account.currency = Account.TXT2CUR.get(get_field('accountDev'),
                                               Account.CUR_UNKNOWN)

        return account

class OperationsPage(BasePage):
    _LINE_XPATH = '//tr[starts-with(@class,"PL_LIGLST_")]'

    def iter_history(self):
        i = 0
        for line in self.document.xpath(self._LINE_XPATH):
            i += 1
            operation = Transaction(i)

            date = line.xpath('.//td[@class="nlb d"]')[0].text_content().strip()
            raw = line.xpath('.//td[@class="t"]')[0].text_content().strip()

            amounts = line.xpath('.//td[@class="n"]')
            [debit, credit] = [amount.text_content().strip()
                               for amount in line.xpath('.//td[@class="n"]')]

            operation.parse(date=date, raw=raw)
            operation.set_amount(credit, debit)

            yield operation

class Transaction(FrenchTransaction):
    PATTERNS = [(re.compile('^(?:Vir(?:ement)?|VRT) (?P<text>.*)', re.I),
                 FrenchTransaction.TYPE_TRANSFER),
                (re.compile('^CARTE(?: ETR.)? ' +
                            '(?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{4}) ' +
                            '(?P<text>.*)'),
                 FrenchTransaction.TYPE_CARD),
                (re.compile('^CHQ (?P<text>.*)$'),
                 FrenchTransaction.TYPE_CHECK),
                (re.compile('^RCH (?P<text>.*)'),
                 FrenchTransaction.TYPE_DEPOSIT)]
<commit_msg>clean raw labels<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2013      Noe Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from decimal import Decimal
import re

from weboob.capabilities.bank import Account
from weboob.tools.browser import BasePage
from weboob.tools.capabilities.bank.transactions import FrenchTransaction

__all__ = ['LoginPage', 'DashboardPage', 'OperationsPage']

class LoginPage(BasePage):
    def login(self, username, password):
        self.browser.select_form(name="frmLogin")
        self.browser['username'] = username.encode('utf-8')
        self.browser['password'] = password.encode('utf-8')
        self.browser.find_control('lang').readonly = False
        self.browser['lang'] = 'fr'
        self.browser.submit(nologin=True)

class DashboardPage(BasePage):
    def iter_accounts(self):
        for line in self._accounts():
            yield self._get_account(line)

    def get_account(self, _id):
        xpath = './/a[@href="tbord.do?id=%s"]' % _id
        account = next(a for a in self._accounts() if a.xpath(xpath))
        return self._get_account(account)

    def _accounts(self):
        return self.document.getroot().cssselect('.tbord_account')

    _FIELD_XPATH = './/span[@class="%s"]//text()'
    _URL_XPATH = './/span[@class="accountLabel"]//a/@href'

    def _get_account(self, line):
        def get_field(field):
            return unicode(line.xpath(self._FIELD_XPATH % field)[0]).strip()

        account = Account()
        account._url = unicode(line.xpath(self._URL_XPATH)[0])
        account.id = account._url.replace("tbord.do?id=", "")
        account.balance = Decimal(FrenchTransaction.clean_amount(
            get_field('accountTotal')))
        account.label = get_field('accountLabel2')
        account.currency = Account.TXT2CUR.get(get_field('accountDev'),
                                               Account.CUR_UNKNOWN)

        return account

class OperationsPage(BasePage):
    _LINE_XPATH = '//tr[starts-with(@class,"PL_LIGLST_")]'

    def iter_history(self):
        i = 0
        for line in self.document.xpath(self._LINE_XPATH):
            i += 1
            operation = Transaction(i)

            date = line.xpath('.//td[@class="nlb d"]')[0].text_content().strip()
            raw = self.parser.tocleanstring(line.xpath('.//td[@class="t"]')[0])

            amounts = line.xpath('.//td[@class="n"]')
            [debit, credit] = [amount.text_content().strip()
                               for amount in amounts]

            operation.parse(date=date, raw=raw)
            operation.set_amount(credit, debit)

            yield operation

class Transaction(FrenchTransaction):
    PATTERNS = [(re.compile('^(?:Vir(?:ement)?|VRT) (?P<text>.*)', re.I),
                 FrenchTransaction.TYPE_TRANSFER),
                (re.compile('^CARTE(?: ETR.)? ' +
                            '(?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{4}) ' +
                            '(?P<text>.*)'),
                 FrenchTransaction.TYPE_CARD),
                (re.compile('^CHQ (?P<text>.*)$'),
                 FrenchTransaction.TYPE_CHECK),
                (re.compile('^RCH (?P<text>.*)'),
                 FrenchTransaction.TYPE_DEPOSIT)]
<|endoftext|>


<commit_before>from __future__ import absolute_import
from collections import defaultdict

import click
import glob
import logging
import os
import sys
import re

# required to load all vcs modules
from pip.vcs import git, mercurial, subversion, bazaar  # noqa
from piptools.datastructures import Spec, SpecSet, ConflictError
from piptools.logging import logger
from piptools.package_manager import PackageManager
from piptools.resolver import Resolver

from six import text_type

DEFAULT_REQUIREMENTS_FILE = 'requirements.in'
GLOB_PATTERN = '*requirements.in'

# Track external PyPi repos referenced
extra_index_urls = []
extra_find_links = []

def setup_logging(verbose):
    if verbose:
        level = logging.DEBUG
    else:
        level = logging.INFO

    handler = logging.StreamHandler()
    formatter = logging.Formatter('%(message)s', None)
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(level)

    from pip.log import logger as pip_logger
    pip_logger.consumers.append((pip_logger.VERBOSE_DEBUG, lambda msg: logger.debug('PIP said: ' + msg)))

def walk_specfile(filename):
    """Walks over the given file, and returns (req, filename, lineno)
    tuples for each entry.
    """
    with open(filename, 'rb') as f:
        reqs = f.read()

    for lineno, line in enumerate(reqs.splitlines(), 1):
        line = line.strip().decode('utf-8')
        if not line or line.startswith('#'):
            continue

        if line.startswith('-r'):
            requirement = line.split(None, 1)[1]
            # requirement file is relative to processed one
            requirement = os.path.join(os.path.dirname(filename), requirement)

            for spec in walk_specfile(requirement):
                yield spec.add_source('{0}:{1} -> {2}'.format(filename, lineno, spec.source))
        elif line.startswith('-f'):
            extra_find_links.append(line.split(None, 1)[1])
        elif line.startswith('--extra-index-url'):
            repo = re.split('=| ', line)
            if len(repo) > 1:
                repo = repo[1]
                logger.debug('Found a link to additional PyPi repo -> {0}'.format(repo))
                if repo not in extra_index_urls:
                    extra_index_urls.append(repo)
        else:
            spec = Spec.from_line(line, source='{0}:{1}'.format(filename, lineno))
            yield spec

def collect_source_specs(filenames):
    """This function collects all of the (primary) source specs into
    a flattened list of specs.
    """
    for filename in filenames:
        for spec in walk_specfile(filename):
            yield spec

def compile_specs(source_files, include_sources=False, dry_run=False):
    logger.debug('===> Collecting source requirements')
    top_level_specs = list(collect_source_specs(source_files))

    spec_set = SpecSet()
    spec_set.add_specs(top_level_specs)
    logger.debug('%s' % (spec_set,))

    logger.debug('')
    logger.debug('===> Normalizing source requirements')
    spec_set = spec_set.normalize()
    logger.debug('%s' % (spec_set,))

    package_manager = PackageManager(extra_index_urls, extra_find_links)

    logger.debug('')
    logger.debug('===> Resolving full tree')

    resolver = Resolver(spec_set, package_manager=package_manager)
    try:
        pinned_spec_set = resolver.resolve()
    except ConflictError as e:
        logger.error('error: {0}'.format(e))
        sys.exit(1)

    logger.debug('')
    logger.debug('===> Pinned spec set resolved')
    for spec in pinned_spec_set:
        logger.debug('- %s' % (spec,))

    if dry_run:
        return

    logger.debug('')
    logger.debug('===> Writing compiled files')

    # The spec set is global for all files passed to pip-compile. Here we go
    # through the resolver again (which will use its cache from the initial
    # run) to determine where to write each dependency.
    split = defaultdict(SpecSet)
    for spec in top_level_specs:
        split[spec.source.split(':')[0]].add_spec(spec)

    for source_file, spec_set in split.items():
        resolver = Resolver(spec_set, package_manager=package_manager)
        with logger.silent():
            local_pinned = resolver.resolve()
        name, ext = os.path.splitext(source_file)
        compiled_file = '{0}.txt'.format(name)
        assert source_file != compiled_file, "Can't overwrite %s" % source_file
        logger.debug('{0} -> {1}'.format(source_file, compiled_file))
        with open(compiled_file, 'wb') as f:
            for spec in sorted(local_pinned, key=text_type):
                f.write(text_type(spec).encode('utf-8'))
                if include_sources:
                    f.write(b'  # {}'.format(spec.source))
                f.write(b'\n')

            # Include external PyPi sources
            if len(extra_index_urls):
                for extra_index_url in extra_index_urls:
                    f.write(b'--extra-index-url {0}\n'.format(extra_index_url))

@click.command()
@click.option('--verbose', '-v', is_flag=True, help="Show more output")
@click.option('--dry-run', is_flag=True, help="Only show what would happen, don't change anything")
@click.option('--include-sources', '-i', is_flag=True,
              help="Write comments to the output file, indicating how the compiled dependencies where calculated")
@click.option('--find-links', '-f', help="Look for archives in this directory or on this HTML page")
@click.option('--extra-index-url', default=None, help="Add additional PyPi repo to search")
@click.argument('files', nargs=-1, type=click.Path(exists=True))
def cli(verbose, dry_run, include_sources, find_links, extra_index_url, files):
    """Compiles requirements.txt from requirements.in specs."""
    setup_logging(verbose)

    if find_links:
        extra_find_links.extend(find_links)

    if extra_index_url:
        urls = extra_index_url.split(',')
        extra_index_urls.extend(urls)

    src_files = files or glob.glob(GLOB_PATTERN)
    if not src_files:
        click.echo('No input files to process.')
        sys.exit(2)

    compile_specs(src_files, include_sources=include_sources, dry_run=dry_run)

    if dry_run:
        logger.info('Dry-run, so nothing updated.')
    else:
        logger.info('Dependencies updated.')
<commit_msg>Remove unused imports.<commit_after>from __future__ import absolute_import
from collections import defaultdict

import click
import glob
import logging
import os
import sys
import re

from piptools.datastructures import Spec, SpecSet, ConflictError
from piptools.logging import logger
from piptools.package_manager import PackageManager
from piptools.resolver import Resolver

from six import text_type

DEFAULT_REQUIREMENTS_FILE = 'requirements.in'
GLOB_PATTERN = '*requirements.in'

# Track external PyPi repos referenced
extra_index_urls = []
extra_find_links = []

def setup_logging(verbose):
    if verbose:
        level = logging.DEBUG
    else:
        level = logging.INFO

    handler = logging.StreamHandler()
    formatter = logging.Formatter('%(message)s', None)
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(level)

    from pip.log import logger as pip_logger
    pip_logger.consumers.append((pip_logger.VERBOSE_DEBUG, lambda msg: logger.debug('PIP said: ' + msg)))

def walk_specfile(filename):
    """Walks over the given file, and returns (req, filename, lineno)
    tuples for each entry.
    """
    with open(filename, 'rb') as f:
        reqs = f.read()

    for lineno, line in enumerate(reqs.splitlines(), 1):
        line = line.strip().decode('utf-8')
        if not line or line.startswith('#'):
            continue

        if line.startswith('-r'):
            requirement = line.split(None, 1)[1]
            # requirement file is relative to processed one
            requirement = os.path.join(os.path.dirname(filename), requirement)

            for spec in walk_specfile(requirement):
                yield spec.add_source('{0}:{1} -> {2}'.format(filename, lineno, spec.source))
        elif line.startswith('-f'):
            extra_find_links.append(line.split(None, 1)[1])
        elif line.startswith('--extra-index-url'):
            repo = re.split('=| ', line)
            if len(repo) > 1:
                repo = repo[1]
                logger.debug('Found a link to additional PyPi repo -> {0}'.format(repo))
                if repo not in extra_index_urls:
                    extra_index_urls.append(repo)
        else:
            spec = Spec.from_line(line, source='{0}:{1}'.format(filename, lineno))
            yield spec

def collect_source_specs(filenames):
    """This function collects all of the (primary) source specs into
    a flattened list of specs.
    """
    for filename in filenames:
        for spec in walk_specfile(filename):
            yield spec

def compile_specs(source_files, include_sources=False, dry_run=False):
    logger.debug('===> Collecting source requirements')
    top_level_specs = list(collect_source_specs(source_files))

    spec_set = SpecSet()
    spec_set.add_specs(top_level_specs)
    logger.debug('%s' % (spec_set,))

    logger.debug('')
    logger.debug('===> Normalizing source requirements')
    spec_set = spec_set.normalize()
    logger.debug('%s' % (spec_set,))

    package_manager = PackageManager(extra_index_urls, extra_find_links)

    logger.debug('')
    logger.debug('===> Resolving full tree')

    resolver = Resolver(spec_set, package_manager=package_manager)
    try:
        pinned_spec_set = resolver.resolve()
    except ConflictError as e:
        logger.error('error: {0}'.format(e))
        sys.exit(1)

    logger.debug('')
    logger.debug('===> Pinned spec set resolved')
    for spec in pinned_spec_set:
        logger.debug('- %s' % (spec,))

    if dry_run:
        return

    logger.debug('')
    logger.debug('===> Writing compiled files')

    # The spec set is global for all files passed to pip-compile. Here we go
    # through the resolver again (which will use its cache from the initial
    # run) to determine where to write each dependency.
    split = defaultdict(SpecSet)
    for spec in top_level_specs:
        split[spec.source.split(':')[0]].add_spec(spec)

    for source_file, spec_set in split.items():
        resolver = Resolver(spec_set, package_manager=package_manager)
        with logger.silent():
            local_pinned = resolver.resolve()
        name, ext = os.path.splitext(source_file)
        compiled_file = '{0}.txt'.format(name)
        assert source_file != compiled_file, "Can't overwrite %s" % source_file
        logger.debug('{0} -> {1}'.format(source_file, compiled_file))
        with open(compiled_file, 'wb') as f:
            for spec in sorted(local_pinned, key=text_type):
                f.write(text_type(spec).encode('utf-8'))
                if include_sources:
                    f.write(b'  # {}'.format(spec.source))
                f.write(b'\n')

            # Include external PyPi sources
            if len(extra_index_urls):
                for extra_index_url in extra_index_urls:
                    f.write(b'--extra-index-url {0}\n'.format(extra_index_url))

@click.command()
@click.option('--verbose', '-v', is_flag=True, help="Show more output")
@click.option('--dry-run', is_flag=True, help="Only show what would happen, don't change anything")
@click.option('--include-sources', '-i', is_flag=True,
              help="Write comments to the output file, indicating how the compiled dependencies where calculated")
@click.option('--find-links', '-f', help="Look for archives in this directory or on this HTML page")
@click.option('--extra-index-url', default=None, help="Add additional PyPi repo to search")
@click.argument('files', nargs=-1, type=click.Path(exists=True))
def cli(verbose, dry_run, include_sources, find_links, extra_index_url, files):
    """Compiles requirements.txt from requirements.in specs."""
    setup_logging(verbose)

    if find_links:
        extra_find_links.extend(find_links)

    if extra_index_url:
        urls = extra_index_url.split(',')
        extra_index_urls.extend(urls)

    src_files = files or glob.glob(GLOB_PATTERN)
    if not src_files:
        click.echo('No input files to process.')
        sys.exit(2)

    compile_specs(src_files, include_sources=include_sources, dry_run=dry_run)

    if dry_run:
        logger.info('Dry-run, so nothing updated.')
    else:
        logger.info('Dependencies updated.')
<|endoftext|>


<commit_before>"""
Module for managing a light via KNX.

It provides functionality for

* switching light 'on' and 'off'.
* setting the brightness.
* reading the current state from KNX bus.
"""
import asyncio
from xknx.knx import Address, DPTArray
from xknx.exceptions import CouldNotParseTelegram
from .device import Device
from .remote_value import RemoteValueSwitch1001

class Light(Device):
    """Class for managing a light."""

    def __init__(self,
                 xknx,
                 name,
                 group_address_switch=None,
                 group_address_switch_state=None,
                 group_address_brightness=None,
                 group_address_brightness_state=None,
                 device_updated_cb=None):
        """Initialize Light class."""
        # pylint: disable=too-many-arguments
        Device.__init__(self, xknx, name, device_updated_cb)
        if isinstance(group_address_brightness, (str, int)):
            group_address_brightness = Address(group_address_brightness)
        if isinstance(group_address_brightness_state, (str, int)):
            group_address_brightness_state = Address(group_address_brightness_state)

        self.switch = RemoteValueSwitch1001(
            xknx,
            group_address_switch,
            group_address_switch_state,
            after_update_cb=self.after_update)

        self.group_address_brightness = group_address_brightness
        self.group_address_brightness_state = group_address_brightness_state

        self.brightness = 0
        self.supports_dimming = \
            group_address_brightness is not None

    @classmethod
    def from_config(cls, xknx, name, config):
        """Initialize object from configuration structure."""
        group_address_switch = \
            config.get('group_address_switch')
        group_address_switch_state = \
            config.get('group_address_switch_state')
        group_address_brightness = \
            config.get('group_address_brightness')
        group_address_brightness_state = \
            config.get('group_address_brightness_state')

        return cls(xknx,
                   name,
                   group_address_switch=group_address_switch,
                   group_address_switch_state=group_address_switch_state,
                   group_address_brightness=group_address_brightness,
                   group_address_brightness_state=group_address_brightness_state)

    def has_group_address(self, group_address):
        """Test if device has given group address."""
        return self.switch.has_group_address(group_address) or \
               (self.group_address_brightness == group_address) or \
               (self.group_address_brightness_state == group_address)

    def __str__(self):
        """Return object as readable string."""
        if not self.supports_dimming:
            return '<Light name="{0}" ' \
                    'switch="{1}" />' \
                    .format(
                        self.name,
                        self.switch.group_addr_str())

        return '<Light name="{0}" ' \
            'switch="{1}" ' \
            'group_address_brightness="{2}" ' \
            'group_address_brightness_state="{3}" ' \
            'brightness="{4}" />' \
            .format(
                self.name,
                self.switch.group_addr_str(),
                self.group_address_brightness,
                self.group_address_brightness_state,
                self.brightness)

    @property
    def state(self):
        """Return the current switch state of the device."""
        return self.switch.value == RemoteValueSwitch1001.Value.ON

    @asyncio.coroutine
    def _set_internal_brightness(self, brightness):
        """Set the internal brightness of the device. If state was changed after update hooks are executed."""
        if brightness != self.brightness:
            self.brightness = brightness
            yield from self.after_update()

    @asyncio.coroutine
    def set_on(self):
        """Switch light on."""
        yield from self.switch.on()

    @asyncio.coroutine
    def set_off(self):
        """Switch light off."""
        yield from self.switch.off()

    @asyncio.coroutine
    def set_brightness(self, brightness):
        """Set brightness of light."""
        if not self.supports_dimming:
            return
        yield from self.send(self.group_address_brightness, DPTArray(brightness))
        yield from self._set_internal_brightness(brightness)

    @asyncio.coroutine
    def do(self, action):
        """Execute 'do' commands."""
        if action == "on":
            yield from self.set_on()
        elif action == "off":
            yield from self.set_off()
        elif action.startswith("brightness:"):
            yield from self.set_brightness(int(action[11:]))
        else:
            self.xknx.logger.warning("Could not understand action %s for device %s", action, self.get_name())

    def state_addresses(self):
        """Return group addresses which should be requested to sync state."""
        state_addresses = []
        state_addresses.extend(self.switch.state_addresses())
        if self.supports_dimming:
            state_address_brightness = \
                self.group_address_brightness_state or \
                self.group_address_brightness
            state_addresses.append(state_address_brightness)
        return state_addresses

    @asyncio.coroutine
    def process(self, telegram):
        """Process incoming telegram."""
        yield from self.switch.process(telegram)

        if (self.supports_dimming and
                (telegram.group_address == self.group_address_brightness or
                 telegram.group_address == self.group_address_brightness_state)):
            yield from self._process_brightness(telegram)

    @asyncio.coroutine
    def _process_brightness(self, telegram):
        """Process incoming telegram for brightness state."""
        if not isinstance(telegram.payload, DPTArray) or \
                len(telegram.payload.value) != 1:
            raise CouldNotParseTelegram()

        yield from self._set_internal_brightness(telegram.payload.value[0])

    def __eq__(self, other):
        """Equal operator."""
        return self.__dict__ == other.__dict__
<commit_msg>flake8 suggestion<commit_after>"""
Module for managing a light via KNX.

It provides functionality for

* switching light 'on' and 'off'.
* setting the brightness.
* reading the current state from KNX bus.
"""
import asyncio
from xknx.knx import Address, DPTArray
from xknx.exceptions import CouldNotParseTelegram
from .device import Device
from .remote_value import RemoteValueSwitch1001

class Light(Device):
    """Class for managing a light."""

    def __init__(self,
                 xknx,
                 name,
                 group_address_switch=None,
                 group_address_switch_state=None,
                 group_address_brightness=None,
                 group_address_brightness_state=None,
                 device_updated_cb=None):
        """Initialize Light class."""
        # pylint: disable=too-many-arguments
        Device.__init__(self, xknx, name, device_updated_cb)
        if isinstance(group_address_brightness, (str, int)):
            group_address_brightness = Address(group_address_brightness)
        if isinstance(group_address_brightness_state, (str, int)):
            group_address_brightness_state = Address(group_address_brightness_state)

        self.switch = RemoteValueSwitch1001(
            xknx,
            group_address_switch,
            group_address_switch_state,
            after_update_cb=self.after_update)

        self.group_address_brightness = group_address_brightness
        self.group_address_brightness_state = group_address_brightness_state

        self.brightness = 0
        self.supports_dimming = \
            group_address_brightness is not None

    @classmethod
    def from_config(cls, xknx, name, config):
        """Initialize object from configuration structure."""
        group_address_switch = \
            config.get('group_address_switch')
        group_address_switch_state = \
            config.get('group_address_switch_state')
        group_address_brightness = \
            config.get('group_address_brightness')
        group_address_brightness_state = \
            config.get('group_address_brightness_state')

        return cls(xknx,
                   name,
                   group_address_switch=group_address_switch,
                   group_address_switch_state=group_address_switch_state,
                   group_address_brightness=group_address_brightness,
                   group_address_brightness_state=group_address_brightness_state)

    def has_group_address(self, group_address):
        """Test if device has given group address."""
        return (self.switch.has_group_address(group_address) or
                (self.group_address_brightness == group_address) or
                (self.group_address_brightness_state == group_address))

    def __str__(self):
        """Return object as readable string."""
        if not self.supports_dimming:
            return '<Light name="{0}" ' \
                    'switch="{1}" />' \
                    .format(
                        self.name,
                        self.switch.group_addr_str())

        return '<Light name="{0}" ' \
            'switch="{1}" ' \
            'group_address_brightness="{2}" ' \
            'group_address_brightness_state="{3}" ' \
            'brightness="{4}" />' \
            .format(
                self.name,
                self.switch.group_addr_str(),
                self.group_address_brightness,
                self.group_address_brightness_state,
                self.brightness)

    @property
    def state(self):
        """Return the current switch state of the device."""
        return self.switch.value == RemoteValueSwitch1001.Value.ON

    @asyncio.coroutine
    def _set_internal_brightness(self, brightness):
        """Set the internal brightness of the device. If state was changed after update hooks are executed."""
        if brightness != self.brightness:
            self.brightness = brightness
            yield from self.after_update()

    @asyncio.coroutine
    def set_on(self):
        """Switch light on."""
        yield from self.switch.on()

    @asyncio.coroutine
    def set_off(self):
        """Switch light off."""
        yield from self.switch.off()

    @asyncio.coroutine
    def set_brightness(self, brightness):
        """Set brightness of light."""
        if not self.supports_dimming:
            return
        yield from self.send(self.group_address_brightness, DPTArray(brightness))
        yield from self._set_internal_brightness(brightness)

    @asyncio.coroutine
    def do(self, action):
        """Execute 'do' commands."""
        if action == "on":
            yield from self.set_on()
        elif action == "off":
            yield from self.set_off()
        elif action.startswith("brightness:"):
            yield from self.set_brightness(int(action[11:]))
        else:
            self.xknx.logger.warning("Could not understand action %s for device %s", action, self.get_name())

    def state_addresses(self):
        """Return group addresses which should be requested to sync state."""
        state_addresses = []
        state_addresses.extend(self.switch.state_addresses())
        if self.supports_dimming:
            state_address_brightness = \
                self.group_address_brightness_state or \
                self.group_address_brightness
            state_addresses.append(state_address_brightness)
        return state_addresses

    @asyncio.coroutine
    def process(self, telegram):
        """Process incoming telegram."""
        yield from self.switch.process(telegram)

        if (self.supports_dimming and
                (telegram.group_address == self.group_address_brightness or
                 telegram.group_address == self.group_address_brightness_state)):
            yield from self._process_brightness(telegram)

    @asyncio.coroutine
    def _process_brightness(self, telegram):
        """Process incoming telegram for brightness state."""
        if not isinstance(telegram.payload, DPTArray) or \
                len(telegram.payload.value) != 1:
            raise CouldNotParseTelegram()

        yield from self._set_internal_brightness(telegram.payload.value[0])

    def __eq__(self, other):
        """Equal operator."""
        return self.__dict__ == other.__dict__
<|endoftext|>


<commit_before>from copy import deepcopy
import simplejson as json
import parsable
from distributions.io.stream import open_compressed
import loom.schema_pb2
parsable = parsable.Parsable()

DEFAULTS = {
    'seed': 0,
    'schedule': {
        'cat_passes': 20.0,
        'kind_passes': 200.0,
        'max_reject_iters': 100,
    },
    'kernels': {
        'cat': {
            'empty_group_count': 1,
        },
        'hyper': {
            'run': True,
            'parallel': True,
        },
        'kind': {
            'iterations': 32,
            'empty_kind_count': 32,
            'row_queue_size': 0,
            'score_parallel': False,
        },
    },
    'posterior_enum': {
        'sample_count': 100,
        'sample_skip': 10,
    },
}

def fill_in_defaults(config, defaults=DEFAULTS):
    assert isinstance(config, dict), config
    assert isinstance(defaults, dict), defaults
    for key, default in defaults.iteritems():
        if key not in config:
            config[key] = deepcopy(default)
        elif isinstance(default, dict):
            fill_in_defaults(config[key], default)

def protobuf_dump(config, message, warn='WARN ignoring config'):
    for key, value in config.iteritems():
        warn_key = '{}.{}'.format(warn, key) if warn else None
        if hasattr(message, key):
            if isinstance(value, dict):
                protobuf_dump(value, getattr(message, key), warn_key)
            else:
                setattr(message, key, value)
        elif warn:
            print warn_key

def config_dump(config, filename):
    config = deepcopy(config)
    fill_in_defaults(config)
    message = loom.schema_pb2.Config()
    protobuf_dump(config, message)
    with open_compressed(filename, 'wb') as f:
        f.write(message.SerializeToString())

if __name__ == '__main__':
    json.dumps(DEFAULTS, indent=4, sort_keys=True)
<commit_msg>Fix printing of default config<commit_after>from copy import deepcopy
import simplejson as json
import parsable
from distributions.io.stream import open_compressed
import loom.schema_pb2
parsable = parsable.Parsable()

DEFAULTS = {
    'seed': 0,
    'schedule': {
        'cat_passes': 20.0,
        'kind_passes': 200.0,
        'max_reject_iters': 100,
    },
    'kernels': {
        'cat': {
            'empty_group_count': 1,
        },
        'hyper': {
            'run': True,
            'parallel': True,
        },
        'kind': {
            'iterations': 32,
            'empty_kind_count': 32,
            'row_queue_size': 0,
            'score_parallel': False,
        },
    },
    'posterior_enum': {
        'sample_count': 100,
        'sample_skip': 10,
    },
}

def fill_in_defaults(config, defaults=DEFAULTS):
    assert isinstance(config, dict), config
    assert isinstance(defaults, dict), defaults
    for key, default in defaults.iteritems():
        if key not in config:
            config[key] = deepcopy(default)
        elif isinstance(default, dict):
            fill_in_defaults(config[key], default)

def protobuf_dump(config, message, warn='WARN ignoring config'):
    for key, value in config.iteritems():
        warn_key = '{}.{}'.format(warn, key) if warn else None
        if hasattr(message, key):
            if isinstance(value, dict):
                protobuf_dump(value, getattr(message, key), warn_key)
            else:
                setattr(message, key, value)
        elif warn:
            print warn_key

def config_dump(config, filename):
    config = deepcopy(config)
    fill_in_defaults(config)
    message = loom.schema_pb2.Config()
    protobuf_dump(config, message)
    with open_compressed(filename, 'wb') as f:
        f.write(message.SerializeToString())

if __name__ == '__main__':
    print json.dumps(DEFAULTS, indent=4, sort_keys=True)
<|endoftext|>