<commit_before>import zc.sourcefactory.contextual
import zope.component
import zope.component.interfaces
import zope.interface
import zope.schema

import zeit.cms.content.contentsource
import zeit.cms.content.interfaces
import zeit.cms.relation.interfaces
from zeit.cms.i18n import MessageFactory as _

class ISyndicationManager(zope.interface.Interface):
    """Manages syndication of a content object."""

    targets = zope.interface.Attribute("List possible syndication targets.")

    canSyndicate = zope.interface.Attribute(
        "True if the object can be syndicated; False otherwise.")

    def syndicate(targets):
        """Syndicate the managed object to the given targets.

        raises SyndicationError if the object could not be syndicated.

        """

class IEntry(zope.interface.Interface):
    """An entry in a feed."""

    pinned = zope.schema.Bool(
        title=_('Pinned'))

    hidden = zope.schema.Bool(
        title=_('Hidden on HP'))

    big_layout = zope.schema.Bool(
        title=_('Big layout'))

    hidden_relateds = zope.schema.Bool(
        title=_('Hidden relateds'))

class IReadFeed(zope.interface.Interface):
    """Feed read interface."""

    title = zope.schema.TextLine(title=_("Title"))
    object_limit = zope.schema.Int(
        title=_("Limit amount"),
        description=_("limit-amount-description"),
        default=50,
        min=1,
        required=False)

    def __len__():
        """Return amount of objects syndicated."""

    def __iter__():
        """Iterate over published content.

        When content is pinned __iter__ is supposed to return pinned content
        at the right position.

        """

    def __contains__(content):
        """Return if content is syndicated in this channel."""

    def getPosition(content):
        """Returns the position of `content` in the feed.

        Returns 1-based position of the content object in the feed or None if
        the content is not syndicated in this feed.

        """

    def getMetadata(content):
        """Returns IEntry instance corresponding to content."""

class IWriteFeed(zope.interface.Interface):
    """Feed write interface."""

    def insert(position, content):
        """Add `content` to self at position `position`."""

    def remove(content):
        """Remove `content` from feed.

        raises ValueError if `content` not in feed.

        """

    def updateOrder(order):
        """Revise the order of keys, replacing the current ordering.

        order is a list or a tuple containing the set of existing keys in
        the new order. `order` must contain ``len(keys())`` items and cannot
        contain duplicate keys.

        Raises ``ValueError`` if order contains an invalid set of keys.
        """

    def updateMetadata(content):
        """Update the metadata of a contained object.

        Raises KeyError if the content is not syndicated in this feed.

        """

class IFeed(IReadFeed, IWriteFeed):
    """Documents can be published into a feed."""

class IContentSyndicatedEvent(zope.component.interfaces.IObjectEvent):
    """Issued when an object is syndicated."""

    targets = zope.schema.Set(
        title=u"The syndication target.",
        value_type=zope.schema.Object(IFeed))

class ContentSyndicatedEvent(zope.component.interfaces.ObjectEvent):
    """Issued when an object is syndicated."""

    zope.interface.implements(IContentSyndicatedEvent)

    def __init__(self, object, targets):
        super(ContentSyndicatedEvent, self).__init__(object)
        self.targets = set(targets)

class SyndicationError(Exception):
    """Raised when there is an error during syndication."""

class IMySyndicationTargets(zope.interface.Interface):

    def add(feed):
        """Add feed to my targets."""

    def remove(feed):
        """Remove feed from my targets."""

    def __contains__(feed):
        """Return (bool) whether feed is a target or not."""

    def __iter__():
        """Iterate over targets."""

class IFeedSource(zeit.cms.content.interfaces.ICMSContentSource):
    """A source for feeds."""

class FeedSource(zeit.cms.content.contentsource.CMSContentSource):

    zope.interface.implements(IFeedSource)
    name = 'zeit.cms.syndication.feed'
    check_interfaces = (IFeed,)

feedSource = FeedSource()
<commit_msg>Add helper method to interface<commit_after>import zc.sourcefactory.contextual
import zope.component
import zope.component.interfaces
import zope.interface
import zope.schema

import zeit.cms.content.contentsource
import zeit.cms.content.interfaces
import zeit.cms.relation.interfaces
from zeit.cms.i18n import MessageFactory as _

class ISyndicationManager(zope.interface.Interface):
    """Manages syndication of a content object."""

    targets = zope.interface.Attribute("List possible syndication targets.")

    canSyndicate = zope.interface.Attribute(
        "True if the object can be syndicated; False otherwise.")

    def syndicate(targets):
        """Syndicate the managed object to the given targets.

        raises SyndicationError if the object could not be syndicated.

        """

class IEntry(zope.interface.Interface):
    """An entry in a feed."""

    pinned = zope.schema.Bool(
        title=_('Pinned'))

    hidden = zope.schema.Bool(
        title=_('Hidden on HP'))

    big_layout = zope.schema.Bool(
        title=_('Big layout'))

    hidden_relateds = zope.schema.Bool(
        title=_('Hidden relateds'))

class IReadFeed(zope.interface.Interface):
    """Feed read interface."""

    title = zope.schema.TextLine(title=_("Title"))
    object_limit = zope.schema.Int(
        title=_("Limit amount"),
        description=_("limit-amount-description"),
        default=50,
        min=1,
        required=False)

    def __len__():
        """Return amount of objects syndicated."""

    def __iter__():
        """Iterate over published content.

        When content is pinned __iter__ is supposed to return pinned content
        at the right position.

        """

    def __contains__(content):
        """Return if content is syndicated in this channel."""

    def getPosition(content):
        """Returns the position of `content` in the feed.

        Returns 1-based position of the content object in the feed or None if
        the content is not syndicated in this feed.

        """

    def getMetadata(content):
        """Returns IEntry instance corresponding to content."""

class IWriteFeed(zope.interface.Interface):
    """Feed write interface."""

    def insert(position, content):
        """Add `content` to self at position `position`."""

    def append(content):
        """Add `content` to self at end."""

    def remove(content):
        """Remove `content` from feed.

        raises ValueError if `content` not in feed.

        """

    def updateOrder(order):
        """Revise the order of keys, replacing the current ordering.

        order is a list or a tuple containing the set of existing keys in
        the new order. `order` must contain ``len(keys())`` items and cannot
        contain duplicate keys.

        Raises ``ValueError`` if order contains an invalid set of keys.
        """

    def updateMetadata(content):
        """Update the metadata of a contained object.

        Raises KeyError if the content is not syndicated in this feed.

        """

class IFeed(IReadFeed, IWriteFeed):
    """Documents can be published into a feed."""

class IContentSyndicatedEvent(zope.component.interfaces.IObjectEvent):
    """Issued when an object is syndicated."""

    targets = zope.schema.Set(
        title=u"The syndication target.",
        value_type=zope.schema.Object(IFeed))

class ContentSyndicatedEvent(zope.component.interfaces.ObjectEvent):
    """Issued when an object is syndicated."""

    zope.interface.implements(IContentSyndicatedEvent)

    def __init__(self, object, targets):
        super(ContentSyndicatedEvent, self).__init__(object)
        self.targets = set(targets)

class SyndicationError(Exception):
    """Raised when there is an error during syndication."""

class IMySyndicationTargets(zope.interface.Interface):

    def add(feed):
        """Add feed to my targets."""

    def remove(feed):
        """Remove feed from my targets."""

    def __contains__(feed):
        """Return (bool) whether feed is a target or not."""

    def __iter__():
        """Iterate over targets."""

class IFeedSource(zeit.cms.content.interfaces.ICMSContentSource):
    """A source for feeds."""

class FeedSource(zeit.cms.content.contentsource.CMSContentSource):

    zope.interface.implements(IFeedSource)
    name = 'zeit.cms.syndication.feed'
    check_interfaces = (IFeed,)

feedSource = FeedSource()
<|endoftext|>


<commit_before>import time
import os

class MotorPositions(object):
    def __init__(self, path):
        self._path = path
        self._history = []
        #self._modification_time = os.path.getmtime(self._filename)
        self._filename = None
        self._modification_time = None
        self._date = None

    def get(self, timestamp):
        #timestamp += 18590000
        filename = self._time_to_filename(timestamp)
        if filename != self._filename:
            tmp_time = time.localtime(timestamp)
            self._date = time.mktime((tmp_time.tm_year, tmp_time.tm_mon, tmp_time.tm_mday, 0, 0, 0, 0, 0, 0))
            self._history = []
            self._update_file(filename)
        if os.path.getmtime(self._filename) > self._modification_time:
            self._update_file(self._filename)

        for index in xrange(len(self._history)):
            if self._history[-index][0] > timestamp:
                continue
            else:
                return self._history[-index][1]

    def _time_to_filename(self, timestamp):
        time_info = time.localtime(timestamp)
        return "{0}/stage-server_positions_{1:04}-{2:02}-{3:02}.log".format(self._path, time_info.tm_year, time_info.tm_mon, time_info.tm_mday)

    def _update_file(self, filename):
        print "load file", filename
        with open(filename, "r") as file_handle:
            total_history = file_handle.readlines()
        self._modification_time = os.path.getmtime(filename)
        if len(total_history) == len(self._history):
            return
        diff_history = total_history[len(self._history):]
        parsed_diff_history = [self._parse_line(l) for l in diff_history if len(l) > 150]
        self._history += parsed_diff_history
        self._filename = filename

    def _parse_time(self, time):
        return self._date + int(time[:2])*3600 + int(time[3:5])*60 + int(time[7:9])

    def _parse_line(self, line):
        line_data = line.split()
        data_dict = {"catcher_x": float(line_data[2]),
                     "catcher_y": float(line_data[3]),
                     "catcher_z": float(line_data[4]),
                     "aperture_x": float(line_data[6]),
                     "aperture_y": float(line_data[7]),
                     "aperture_z": float(line_data[8]),
                     "objective_x": float(line_data[10]),
                     "objective_y": float(line_data[11]),
                     "objective_z": float(line_data[12]),
                     "inline_mic": float(line_data[14]),
                     "inline_mirror_x": float(line_data[16]),
                     "inline_mirror_y": float(line_data[17]),
                     "fibre_x": float(line_data[19]),
                     "fibre_y": float(line_data[20]),
                     "fibre_z": float(line_data[21]),
                     "rod_x": float(line_data[23]),
                     "rod_y": float(line_data[24]),
                     "rod_z": float(line_data[25]),
                     "nozzle_x": float(line_data[27]),
                     "nozzle_y": float(line_data[28]),
                     "nozzle_z": float(line_data[29]),
                     "nozzle_rot": float(line_data[31]),
                     "dummy": float(line_data[33])}
        return (self._parse_time(line_data[0]), data_dict)

<commit_msg>holger_motors: Updated script to deal with live updating file (maybe)<commit_after>import time
import os

class MotorPositions(object):
    def __init__(self, path):
        self._path = path
        self._history = []
        #self._modification_time = os.path.getmtime(self._filename)
        self._filename = None
        self._filesize = None
        self._date = None
        self._num_lines = None

    def get(self, timestamp):
        #timestamp += 18590000
        self.timestamp = timestamp
        filename = self._time_to_filename(timestamp)
        if filename != self._filename:
            tmp_time = time.localtime(timestamp)
            self._date = time.mktime((tmp_time.tm_year, tmp_time.tm_mon, tmp_time.tm_mday, 0, 0, 0, 0, 0, 0))
            self._history = []
            self._update_file(filename)
        if os.path.getsize(self._filename) > self._filesize:
            print 'File size change'
            self._update_file(self._filename)

        for index in xrange(len(self._history)):
            if self._history[-index][0] > timestamp:
                continue
            else:
                return self._history[-index][1]
        return self._history[0][1]

    def _time_to_filename(self, timestamp):
        time_info = time.localtime(timestamp)
        return "{0}/stage-server_positions_{1:04}-{2:02}-{3:02}.log".format(self._path, time_info.tm_year, time_info.tm_mon, time_info.tm_mday)

    def _update_file(self, filename):
        print "load file", filename
        while not os.path.isfile(filename):
            self.timestamp -= self.timestamp%(24.*3600.) + 5.
            filename = self._time_to_filename(self.timestamp)
            print filename
        with open(filename, "r") as file_handle:
            total_history = file_handle.readlines()
        self._filesize = os.path.getsize(filename)
        if len(total_history) == self._num_lines:
            return
        else:
            self._num_lines = len(total_history)
        diff_history = total_history[len(self._history):]
        parsed_diff_history = [self._parse_line(l) for l in diff_history if len(l) > 150]
        self._history += parsed_diff_history
        self._filename = filename

    def _parse_time(self, time):
        return self._date + int(time[:2])*3600 + int(time[3:5])*60 + float(time[7:9])

    def _parse_line(self, line):
        line_data = line.split()
        data_dict = {"catcher_x": float(line_data[2]),
                     "catcher_y": float(line_data[3]),
                     "catcher_z": float(line_data[4]),
                     "aperture_x": float(line_data[6]),
                     "aperture_y": float(line_data[7]),
                     "aperture_z": float(line_data[8]),
                     "objective_x": float(line_data[10]),
                     "objective_y": float(line_data[11]),
                     "objective_z": float(line_data[12]),
                     "inline_mic": float(line_data[14]),
                     "inline_mirror_x": float(line_data[16]),
                     "inline_mirror_y": float(line_data[17]),
                     "fibre_x": float(line_data[19]),
                     "fibre_y": float(line_data[20]),
                     "fibre_z": float(line_data[21]),
                     "rod_x": float(line_data[23]),
                     "rod_y": float(line_data[24]),
                     "rod_z": float(line_data[25]),
                     "nozzle_x": float(line_data[27]),
                     "nozzle_y": float(line_data[28]),
                     "nozzle_z": float(line_data[29]),
                     "nozzle_rot": float(line_data[31]),
                     "dummy": float(line_data[33])}
        return (self._parse_time(line_data[0]), data_dict)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import StringIO
import zeit.cms.browser.view
import csv
import tinydav
import logging
import re
import sys
import lxml.etree
import posixpath
from zeit.cms.i18n import MessageFactory as _
from ordereddict import OrderedDict

log = logging.getLogger(__name__)

# TODO Product-ID's via ./work/source/zeit.cms/src/zeit/cms/content/products.xml ?
# TODO Check Unicode stuff
# TODO Naming stuff

class Toc(zeit.cms.browser.view.Base):
    """
    View for creating a Table of Content as a csv file.
    """
    # Get this form a config File
    DAV_SERVER_ROOT = "cms-backend.zeit.de"
    DAV_PORT = 9000
    DAV_ARCHIVE_ROOT = "/cms/archiv-wf/archiv"
    # The Volume Content Object will get extended so
    # this hardcoded stuff shouldn't be necassary
    # TODO Product-ID's via ./work/source/zeit.cms/src/zeit/cms/content/products.xml same
    # 'CW' in Ticket Description, changed it to 'ZWCW'
    PRODUCT_IDS = ['ZEI', 'ZESA', 'ZEIH', 'ZEOE', 'ZECH', 'ZECW']
    CSV_DELIMITER = '\t'

    def __init__(self, *args, **kwargs):
        super(Toc, self).__init__(*args, **kwargs)
        self.client = self._create_dav_client()

    def __call__(self):
        self.volume = self.context
        filename = self._generate_file_name()
        self.request.response.setHeader('Content-Type', 'text/csv')
        self.request.response.setHeader('Content-Disposition', 'attachment; filename="%s"' % filename)
        return self._create_toc_content()

    def _create_toc_content(self):
        """
        Create Table of Contents for the given Volume as a csv.
        :param volume: ..volume.Volume Content Instance
        :return: Specify
        """
        toc_data = self._get_via_dav()
        sorted_toc_data = self._sort_toc_data(toc_data)
        return self._create_csv(sorted_toc_data)

    def _generate_file_name(self):
        toc_file_string = _("Table of Content").lower().replace(" ", "_")
        return "{}_{}_{}.csv".format(toc_file_string, self.volume.year, self.volume.volume)

    def _get_via_dav(self):
        """
        Get and parse xml form webdav und create toc entries.
        :param volume: ..volume.Volume Content Instance
        :return: Sorted List of Toc entries
                 [{'ressort': str, 'page': str, 'title': str, "teaser": str}]
        """
        """
            {
                "Die Zeit: {
                        {"Politk: [sorted_toc_entrires]"}
                    }
            }
            sorted_toc_entries: [{'page': str, 'title': str, "teaser": str}, ...]
        """
        results = OrderedDict()
        product_ids = self._get_all_product_ids_for_volume()
        product_id_paths = self._get_all_paths_for_prodct_ids(product_ids)
        for product_path in product_id_paths:
            result_for_product = {}
            for ressort_path in self.list_relevant_dirs_with_dav(product_path):
                result_for_ressort = []
                for article_path in self._get_all_articles_in_path(ressort_path):
                    toc_entry = self._create_toc_element(article_path)
                    if toc_entry:
                        result_for_ressort.append(toc_entry)
                result_for_product[self._dir_name(ressort_path)] = result_for_ressort
            results[self._full_name_for_product_id(product_path)] = result_for_product
        return results

    def _get_all_articles_in_path(self, path):
        """ Expects like cms/archiv-wf/archiv/ZESA/2015/02/ """
        all_articles = []
        for article_path in self._get_all_files_in_folder(path):
            all_articles.append(article_path)
        return all_articles

    def _parse_article(self, doc_path):
        """
        Parse the artice XML.
        :param doc_path: path to artice xml on DAV-Server.
        """
        try:
            response = self.client.get(doc_path)
            return lxml.etree.fromstring(response.content)
        except:
            raise

    def _get_all_product_ids_for_volume(self):
        """ Returns sorted List [First Product in TOC, Second ...] """
        # Change this if the volume content object "knows" which Products it has...
        return self.PRODUCT_IDS

    def _get_all_paths_for_prodct_ids(self, product_ids):
        """
        Creates a list of unix-paths to  all given products
        :param product_ids: [str]
        :return: [str]
        """
        return [posixpath.join(*[str(e) for e in [self.DAV_ARCHIVE_ROOT, product_id, self.volume.year, self.volume.volume, '']])
                for product_id in product_ids]

    def _is_relevant_article(self, tree):
        """
        Predicate to decide if a doc is relevant for the toc.
        :param tree: lxml.etree  of the artice
        :return: bool
        """
        # Right now every article is relevant
        return True

    def _create_dav_client(self):
        # TODO Get uri from some config (zope.conf?)
        return tinydav.WebDAVClient(self.DAV_SERVER_ROOT, self.DAV_PORT)

    def _get_metadata_from_article_xml(self, tree):
        xpaths = {
            'title': "body/title/text()",
            'page': "//attribute[@name='page']/text()",
            'teaser': "body/subtitle/text()",
            'author': "//attribute[@name='author']/text()"
        }
        res = {}
        for key, xpath in xpaths.iteritems():
            res[key] = tree.xpath(xpath)
        return self._normalize_toc_element(res)

    def list_relevant_dirs_with_dav(self, path):
        """ Returns all paths to directories for a given path """
        response = self.client.propfind(path, depth=1)
        assert response.is_multistatus
        return [element.href for element in response if self._is_relevant_path_to_directory(path, element)]

    def _is_relevant_path_to_directory(self, root_path_of_element, element):
        try:
            folders_to_exclude = {'images', 'leserbriefe'}
            root_paths = {root_path_of_element, '/' + root_path_of_element}
            return self._is_dav_dir(element) \
                   and not any(folder in element.href for folder in folders_to_exclude) \
                   and element.href not in root_paths
        except:
            raise

    def _get_all_files_in_folder(self, ressort_path):
        try:
            response = self.client.propfind(ressort_path, depth=1)
            assert response.is_multistatus
            return [status_element.href for status_element in response if not self._is_dav_dir(status_element)]
        except:
            raise

    def _is_dav_dir(self, status_element):
        try:
            return 'directory' in status_element.get('getcontenttype').text
        except AttributeError:
            return False

    def _create_csv(self, toc_data):
        """
        Creates CSV File from TOC Data.
        SEITENZAHL [tab] TITEL + TEASER
        :param toc_data:
        :return: The CSV-File.
        """

        file_content = u''
        out = StringIO.StringIO()
        try:
            writer = csv.writer(out, delimiter=self.CSV_DELIMITER)
            for toc_element in self._generate_csv_rows(toc_data):
                try:
                    writer.writerow(
                        [val.encode('utf-8') for val in toc_element]
                    )
                except:
                    pass
            file_content = out.getvalue()
        finally:
            out.close()
            return file_content

    def _generate_csv_rows(self, toc_entries):
        """
        Generator to create the csv-rows.
        :param toc_entries: TODO
        :return: [CSV Row]
        """
        for product_name, ressort_dict in toc_entries.iteritems():
            yield [product_name]
            for ressort_name, toc_entries in ressort_dict.iteritems():
                yield [ressort_name]
                for toc_entry in toc_entries:
                    yield self._format_toc_element(toc_entry)
        return

    def _format_toc_element(self, toc_entry):
        title_and_tease = toc_entry.get("title") + u" " + toc_entry.get("teaser")
        return [toc_entry.get("page"), toc_entry.get("author"), title_and_tease]

    def _normalize_toc_element(self, toc_entry):
        for key, value in toc_entry.iteritems():
            toc_entry[key] = value[0] if len(value) > 0 else u""
        self._normalize_teaser(toc_entry)
        self._normalize_page(toc_entry)
        return toc_entry

    def _normalize_page(self, toc_dict):
        # CH1-1, 78, 021-
        page_string = toc_dict.get('page', u'')
        res = re.findall('\d+', page_string)
        toc_dict['page'] = res[0].lstrip("0") if res else u"-1"

    def _normalize_teaser(self, toc_entry):
        teaser = toc_entry.get('teaser', u'')
        # Delete Linebreaks and a whitespace
        toc_entry['teaser'] = teaser.replace('\n', u' ')
        toc_entry['teaser'] = re.sub(r'\s\s+', u' ', teaser)

    def _create_toc_element(self, doc_path):
        article_element = self._parse_article(doc_path)
        return self._get_metadata_from_article_xml(article_element) \
            if self._is_relevant_article(article_element) else None

    def _dir_name(self, path):
        return path.split('/')[-2].title()

    def _full_name_for_product_id(self, path):
        id_name_mapping = {
            'ZEI': u'Die Zeit',
            'ZESA': u'Zeit im Osten',
            'ZEIH': u'Zeit Hamburg',
            'ZEOE': u'Zeit Österreich',
            'ZECH': u'Zeit Schweiz',
            'ZECW': u'Christ und Welt'
        }

        for key, val in id_name_mapping.iteritems():
            folder_string = '/' + key + '/'
            if folder_string in path:
                return val

    def _sort_toc_data(self, toc_data):
        """
        :param toc_data:
        :return:
        """
        # TODO Think about a better way to sort the toc!
        for product_name, ressort_dict in toc_data.iteritems():
            for ressort_name, articles in ressort_dict.iteritems():
                toc_data[product_name][ressort_name] = self._sorted_articles(articles)
        for product_name, ressort_dict in toc_data.iteritems():
            toc_data[product_name] = self._sorted_ressorts(ressort_dict)
        return toc_data

    def _get_page_from_article(self, article):
        """
        :param article: {'page':str}
        :return: int
        """
        try:
            return int(article.get('page'))
        except ValueError:
            # The empty string will cause an Exception!
            return sys.maxint

    def _sorted_articles(self, articles):
        return sorted(articles, key=self._get_page_from_article)

    def _sorted_ressorts(self, ressorts):
        # Expects articles to be sorted by page
        ressort_min_page_number = [(resort_name, self._get_page_from_article(articles[0])) for resort_name, articles in ressorts.iteritems()]
        d = OrderedDict()
        for ressort in sorted(ressort_min_page_number, key=lambda resort_page_tup: resort_page_tup[1]):
            d[ressort[0]] = ressorts.get(ressort[0])
        return d
<commit_msg>Fix bug with volumes < 10 and change some docs<commit_after># -*- coding: utf-8 -*-
import StringIO
import zeit.cms.browser.view
import csv
import tinydav
import logging
import re
import sys
import lxml.etree
import posixpath
from zeit.cms.i18n import MessageFactory as _
from ordereddict import OrderedDict

log = logging.getLogger(__name__)

# TODO Product-ID's via ProductSource
# TODO Check Unicode
# TODO Naming

class Toc(zeit.cms.browser.view.Base):
    """
    View for creating a Table of Content as a csv file.
    """
    # Get this form a config File?
    DAV_SERVER_ROOT = "cms-backend.zeit.de"
    DAV_PORT = 9000
    DAV_ARCHIVE_ROOT = "/cms/archiv-wf/archiv"
    # The Volume Content Object will get extended soon
    # this hardcoded stuff shouldn't be necassary
    # 'CW' in Ticket Description, changed it to 'ZWCW'
    # The Order of the Product ID's matters, First in this list -> first in TOC
    PRODUCT_IDS = ['ZEI', 'ZESA', 'ZEIH', 'ZEOE', 'ZECH', 'ZECW']
    CSV_DELIMITER = '\t'

    def __init__(self, *args, **kwargs):
        super(Toc, self).__init__(*args, **kwargs)
        self.client = self._create_dav_client()

    def __call__(self):
        self.volume = self.context
        filename = self._generate_file_name()
        self.request.response.setHeader('Content-Type', 'text/csv')
        self.request.response.setHeader('Content-Disposition', 'attachment; filename="%s"' % filename)
        return self._create_toc_content()

    def _create_toc_content(self):
        """
        Create Table of Contents for the given Volume as a csv.
        :param volume: ..volume.Volume Content Instance
        :return: Specify
        """
        toc_data = self._get_via_dav()
        sorted_toc_data = self._sort_toc_data(toc_data)
        return self._create_csv(sorted_toc_data)

    def _generate_file_name(self):
        toc_file_string = _("Table of Content").lower().replace(" ", "_")
        return "{}_{}_{}.csv".format(toc_file_string, self.volume.year, self.volume.volume)

    def _get_via_dav(self):
        """
        Get and parse xml form webdav und create toc entries.
        :param volume: ..volume.Volume Content Instance
        :return: Sorted List of Toc entries
                 [{'ressort': str, 'page': str, 'title': str, "teaser": str}]
        """
        """
            {
                "Die Zeit: {
                        {"Politk: [sorted_toc_entrires]"}
                    }
            }
            sorted_toc_entries: [{'page': str, 'title': str, "teaser": str}, ...]
        """
        results = OrderedDict()
        product_ids = self._get_all_product_ids_for_volume()
        for product_path in self._get_all_paths_for_prodct_ids(product_ids):
            result_for_product = {}
            for ressort_path in self.list_relevant_dirs_with_dav(product_path):
                result_for_ressort = []
                for article_path in self._get_all_articles_in_path(ressort_path):
                    toc_entry = self._create_toc_element(article_path)
                    if toc_entry:
                        result_for_ressort.append(toc_entry)
                result_for_product[self._dir_name(ressort_path)] = result_for_ressort
            results[self._full_name_for_product_id(product_path)] = result_for_product
        return results

    def _get_all_articles_in_path(self, path):
        """ Expects like cms/archiv-wf/archiv/ZESA/2015/02/ """
        all_articles = []
        for article_path in self._get_all_files_in_folder(path):
            all_articles.append(article_path)
        return all_articles

    def _parse_article(self, doc_path):
        """
        Parse the artice XML.
        :param doc_path: path to artice xml on DAV-Server.
        """
        try:
            response = self.client.get(doc_path)
            return lxml.etree.fromstring(response.content)
        except:
            raise

    def _get_all_product_ids_for_volume(self):
        """ Returns sorted List [First Product in TOC, Second ...] """
        # Change this if the volume content object "knows" which Products it has...
        return self.PRODUCT_IDS

    def _get_all_paths_for_prodct_ids(self, product_ids):
        """
        Creates a list of unix-paths to all given products
        :param product_ids: [str]
        :return: [str]
        """
        volume_string = str(self.volume.volume)
        # Volumes 1-10 would lead to wrong paths like YEAR/1 instead of YEAR/01
        if self.volume.volume < 10:
            volume_string = '0' + volume_string
        return [posixpath.join(*[str(e) for e in [self.DAV_ARCHIVE_ROOT, product_id, self.volume.year, volume_string, '']])
                for product_id in product_ids]

    def _is_relevant_article(self, tree):
        """
        Predicate to decide if a doc is relevant for the toc.
        :param tree: lxml.etree  of the article
        :return: bool
        """
        # Right now every article is relevant
        return True

    def _create_dav_client(self):
        # TODO Get uri from some config (zope.conf)
        return tinydav.WebDAVClient(self.DAV_SERVER_ROOT, self.DAV_PORT)

    def _get_metadata_from_article_xml(self, tree):
        xpaths = {
            'title': "body/title/text()",
            'page': "//attribute[@name='page']/text()",
            'teaser': "body/subtitle/text()",
            'author': "//attribute[@name='author']/text()"
        }
        res = {}
        for key, xpath in xpaths.iteritems():
            res[key] = tree.xpath(xpath)
        return self._normalize_toc_element(res)

    def list_relevant_dirs_with_dav(self, path):
        """ Returns all paths to directories for a given path """
        response = self.client.propfind(path, depth=1)
        assert response.is_multistatus
        return [element.href for element in response if self._is_relevant_path_to_directory(path, element)]

    def _is_relevant_path_to_directory(self, root_path_of_element, element):
        try:
            folders_to_exclude = {'images', 'leserbriefe'}
            root_paths = {root_path_of_element, '/' + root_path_of_element}
            return self._is_dav_dir(element) \
                   and not any(folder in element.href for folder in folders_to_exclude) \
                   and element.href not in root_paths
        except:
            raise

    def _get_all_files_in_folder(self, ressort_path):
        response = self.client.propfind(ressort_path, depth=1)
        assert response.is_multistatus
        return [status_element.href for status_element in response if not self._is_dav_dir(status_element)]

    def _is_dav_dir(self, status_element):
        try:
            return 'directory' in status_element.get('getcontenttype').text
        except AttributeError:
            return False

    def _create_csv(self, toc_data):
        """
        Creates CSV File from TOC Data.
        SEITENZAHL [tab] TITEL + TEASER
        :param toc_data:
        :return: The CSV-File.
        """

        file_content = u''
        out = StringIO.StringIO()
        try:
            writer = csv.writer(out, delimiter=self.CSV_DELIMITER)
            for toc_element in self._generate_csv_rows(toc_data):
                try:
                    writer.writerow(
                        [val.encode('utf-8') for val in toc_element]
                    )
                except:
                    pass
            file_content = out.getvalue()
        finally:
            out.close()
            return file_content

    def _generate_csv_rows(self, toc_entries):
        """
        Generator to create the csv-rows.
        :param toc_entries: TODO
        :return: [CSV Row]
        """
        for product_name, ressort_dict in toc_entries.iteritems():
            yield [product_name]
            for ressort_name, toc_entries in ressort_dict.iteritems():
                yield [ressort_name]
                for toc_entry in toc_entries:
                    yield self._format_toc_element(toc_entry)
        return

    def _format_toc_element(self, toc_entry):
        title_and_tease = toc_entry.get("title") + u" " + toc_entry.get("teaser")
        return [toc_entry.get("page"), toc_entry.get("author"), title_and_tease]

    def _normalize_toc_element(self, toc_entry):
        for key, value in toc_entry.iteritems():
            toc_entry[key] = value[0] if len(value) > 0 else u""
        self._normalize_teaser(toc_entry)
        self._normalize_page(toc_entry)
        return toc_entry

    def _normalize_page(self, toc_dict):
        page_string = toc_dict.get('page', u'')
        res = re.findall('\d+', page_string)
        toc_dict['page'] = res[0].lstrip("0") if res else u"-1"

    def _normalize_teaser(self, toc_entry):
        """Delete linebreaks and a too much whitespace"""
        teaser = toc_entry.get('teaser', u'')
        toc_entry['teaser'] = teaser.replace('\n', u' ')
        toc_entry['teaser'] = re.sub(r'\s\s+', u' ', teaser)

    def _create_toc_element(self, doc_path):
        article_element = self._parse_article(doc_path)
        return self._get_metadata_from_article_xml(article_element) \
            if self._is_relevant_article(article_element) else None

    def _dir_name(self, path):
        return path.split('/')[-2].title()

    def _full_name_for_product_id(self, path):
        id_name_mapping = {
            'ZEI': u'Die Zeit',
            'ZESA': u'Zeit im Osten',
            'ZEIH': u'Zeit Hamburg',
            'ZEOE': u'Zeit Österreich',
            'ZECH': u'Zeit Schweiz',
            'ZECW': u'Christ und Welt'
        }

        for key, val in id_name_mapping.iteritems():
            folder_string = '/' + key + '/'
            if folder_string in path:
                return val

    def _sort_toc_data(self, toc_data):
        """
        :param toc_data:
        :return:
        """
        # TODO Think about a better way to sort the toc!
        for product_name, ressort_dict in toc_data.iteritems():
            for ressort_name, articles in ressort_dict.iteritems():
                toc_data[product_name][ressort_name] = self._sorted_articles(articles)
        for product_name, ressort_dict in toc_data.iteritems():
            toc_data[product_name] = self._sorted_ressorts(ressort_dict)
        return toc_data

    def _get_page_from_article(self, article):
        """
        :param article: {'page':str}
        :return: int
        """
        try:
            return int(article.get('page'))
        except ValueError:
            # The empty string will cause an Exception!
            return sys.maxint

    def _sorted_articles(self, articles):
        return sorted(articles, key=self._get_page_from_article)

    def _sorted_ressorts(self, ressorts):
        # Expects articles in ressorts dict to be sorted by page
        ressort_min_page_number = [(resort_name, self._get_page_from_article(articles[0])) for resort_name, articles in ressorts.iteritems()]
        d = OrderedDict()
        for ressort in sorted(ressort_min_page_number, key=lambda resort_page_tup: resort_page_tup[1]):
            d[ressort[0]] = ressorts.get(ressort[0])
        return d
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os, sys, argparse, logging, shutil, asyncio, time, signal

import appdirs
import hangups
from threading import Thread

from hangups.ui.utils import get_conv_name

import config
import handlers

# rpc sink
from sink2 import start_rpc_listener
from utils import simple_parse_to_segments

__version__ = '1.1'
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

class ConversationEvent(object):
    """Conversation event"""
    def __init__(self, bot, conv_event):
        self.conv_event = conv_event
        self.conv_id = conv_event.conversation_id
        self.conv = bot._conv_list.get(self.conv_id)
        self.user_id = conv_event.user_id
        self.user = self.conv.get_user(self.user_id)
        self.timestamp = conv_event.timestamp
        self.text = conv_event.text.strip() if isinstance(conv_event, hangups.ChatMessageEvent) else ''

    def print_debug(self):
        """Print informations about conversation event"""
        print('Conversation ID: {}'.format(self.conv_id))
        print('Conversation name: {}'.format(get_conv_name(self.conv, truncate=True)))
        print('User ID: {}'.format(self.user_id))
        print('User name: {}'.format(self.user.full_name))
        print('Timestamp: {}'.format(self.timestamp.astimezone(tz=None).strftime('%Y-%m-%d %H:%M:%S')))
        print('Text: {}'.format(self.text))
        print()

class HangupsBot(object):
    """Hangouts bot listening on all conversations"""
    def __init__(self, cookies_path, config_path, max_retries=5):
        self._client = None
        self._cookies_path = cookies_path
        self._max_retries = max_retries

        # These are populated by on_connect when it's called.
        self._conv_list = None        # hangups.ConversationList
        self._user_list = None        # hangups.UserList
        self._message_handler = None  # MessageHandler

        # Load config file
        self.config = config.Config(config_path)

        # Handle signals on Unix
        # (add_signal_handler is not implemented on Windows)
        try:
            loop = asyncio.get_event_loop()
            for signum in (signal.SIGINT, signal.SIGTERM):
                loop.add_signal_handler(signum, lambda: self.stop())
        except NotImplementedError:
            pass

    def login(self, cookies_path):
        """Login to Google account"""
        # Authenticate Google user and save auth cookies
        # (or load already saved cookies)
        try:
            cookies = hangups.auth.get_auth_stdin(cookies_path)
            return cookies
        except hangups.GoogleAuthError as e:
            print('Login failed ({})'.format(e))
            return False

    def run(self):
        """Connect to Hangouts and run bot"""
        cookies = self.login(self._cookies_path)
        if cookies:
            # Create Hangups client
            self._client = hangups.Client(cookies)
            self._client.on_connect.add_observer(self._on_connect)
            self._client.on_disconnect.add_observer(self._on_disconnect)

            # Start asyncio event loop and connect to Hangouts
            # If we are forcefully disconnected, try connecting again
            loop = asyncio.get_event_loop()

            if "jsonrpc" in self.config.keys():
                if self.config["jsonrpc"]:
                    print("starting json rpc sink")
                    # start up rpc listener in a separate thread
                    t = Thread(target=start_rpc_listener, args=(self, loop))
                    t.daemon = True
                    t.start()

            for retry in range(self._max_retries):
                try:
                    loop.run_until_complete(self._client.connect())
                    sys.exit(0)
                except Exception as e:
                    print('Client unexpectedly disconnected:\n{}'.format(e))
                    print('Waiting {} seconds...'.format(5 + retry * 5))
                    time.sleep(5 + retry * 5)
                    print('Trying to connect again (try {} of {})...'.format(retry + 1, self._max_retries))
            print('Maximum number of retries reached! Exiting...')
        sys.exit(1)

    def stop(self):
        """Disconnect from Hangouts"""
        asyncio.async(
            self._client.disconnect()
        ).add_done_callback(lambda future: future.result())

    def handle_chat_message(self, conv_event):
        """Handle chat messages"""
        event = ConversationEvent(self, conv_event)
        asyncio.async(self._message_handler.handle(event))

    def handle_membership_change(self, conv_event):
        """Handle conversation membership change"""
        event = ConversationEvent(self, conv_event)

        # Don't handle events caused by the bot himself
        if event.user.is_self:
            return

        # Test if watching for membership changes is enabled
        if not self.get_config_suboption(event.conv_id, 'membership_watching_enabled'):
            return

        # Generate list of added or removed users
        event_users = [event.conv.get_user(user_id) for user_id
                       in event.conv_event.participant_ids]
        names = ', '.join([user.full_name for user in event_users])

        # JOIN
        if event.conv_event.type_ == hangups.MembershipChangeType.JOIN:
            # Test if user who added new participants is admin
            admins_list = self.get_config_suboption(event.conv_id, 'admins')
            if event.user_id.chat_id in admins_list:
                self.send_message(event.conv, '{}: welcome'.format(names))
            else:
                segments = [hangups.ChatMessageSegment('!!! ATTENTION !!!', is_bold=True),
                            hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK),
                            hangups.ChatMessageSegment('{} added these users {}!'.format(
                                                       event.user.full_name, names)),
                            hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK),
                            hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK),
                            hangups.ChatMessageSegment('{}: please leave this hangout'.format(names))]
                self.send_message_segments(event.conv, segments)
        # LEAVE
        else:
            self.send_message(event.conv, '{}: left'.format(names))

    def handle_rename(self, conv_event):
        """Handle conversation rename"""
        event = ConversationEvent(self, conv_event)

        # Don't handle events caused by the bot himself
        if event.user.is_self:
            return

        # Test if watching for conversation rename is enabled
        if not self.get_config_suboption(event.conv_id, 'rename_watching_enabled'):
            return

        # Only print renames for now...
        if event.conv_event.new_name == '':
            print('{} cleared the conversation name'.format(event.user.first_name))
        else:
            print('{} renamed the conversation to {}'.format(event.user.first_name, event.conv_event.new_name))

    def send_message(self, conversation, text):
        """"Send simple chat message"""
        self.send_message_segments(conversation, [hangups.ChatMessageSegment(text)])

    def send_message_parsed(self, conversation, html):
        segments = simple_parse_to_segments(html)
        self.send_message_segments(conversation, segments)

    def send_message_segments(self, conversation, segments):
        """Send chat message segments"""
        # Ignore if the user hasn't typed a message.
        if len(segments) == 0:
            return
        # XXX: Exception handling here is still a bit broken. Uncaught
        # exceptions in _on_message_sent will only be logged.
        asyncio.async(
            conversation.send_message(segments)
        ).add_done_callback(self._on_message_sent)

    def list_conversations(self):
        """List all active conversations"""
        try:
            _all_conversations = self._conv_list.get_all()
            convs = sorted(_all_conversations, reverse=True, key=lambda c: c.last_modified)
            print('list_conversations() returned {} conversations'.format(len(convs)))
        except Exception as e:
            logging.warning("list_conversations()", e)
            raise

        return convs

    def get_config_suboption(self, conv_id, option):
        """Get config suboption for conversation (or global option if not defined)"""
        try:
            suboption = self.config['conversations'][conv_id][option]
        except KeyError:
            try:
                suboption = self.config[option]
            except KeyError:
                suboption = None
        return suboption

    def print_conversations(self):
        print('Conversations:')
        for c in self.list_conversations():
            print('  {} ({}) u:{}'.format(get_conv_name(c, truncate=True), c.id_, len(c.users)))
            for u in c.users:
                print('    {} ({}) {}'.format(u.first_name, u.full_name, u.id_.chat_id))
        print()

    def get_1on1_conversation(self, chat_id):
        conversation = None
        for c in self.list_conversations():
            if len(c.users) == 2:
                for u in c.users:
                    if u.id_.chat_id == chat_id:
                        conversation = c
                        break
        return conversation

    def _on_message_sent(self, future):
        """Handle showing an error if a message fails to send"""
        try:
            future.result()
        except hangups.NetworkError:
            print('Failed to send message!')

    def _on_connect(self, initial_data):
        """Handle connecting for the first time"""
        print('Connected!')
        self._message_handler = handlers.MessageHandler(self)

        self._user_list = hangups.UserList(self._client,
                                           initial_data.self_entity,
                                           initial_data.entities,
                                           initial_data.conversation_participants)
        self._conv_list = hangups.ConversationList(self._client,
                                                   initial_data.conversation_states,
                                                   self._user_list,
                                                   initial_data.sync_timestamp)
        self._conv_list.on_event.add_observer(self._on_event)

        self.print_conversations()

    def _on_event(self, conv_event):
        """Handle conversation events"""
        if isinstance(conv_event, hangups.ChatMessageEvent):
            self.handle_chat_message(conv_event)
        elif isinstance(conv_event, hangups.MembershipChangeEvent):
            self.handle_membership_change(conv_event)
        elif isinstance(conv_event, hangups.RenameEvent):
            self.handle_rename(conv_event)

    def _on_disconnect(self):
        """Handle disconnecting"""
        print('Connection lost!')

    def external_send_message(self, conversation_id, text):
        conversation = self._conv_list.get(conversation_id)
        print('sending message, conversation name:', get_conv_name(conversation))
        self.send_message(conversation, text)

    def external_send_message_parsed(self, conversation_id, html):
        conversation = self._conv_list.get(conversation_id)
        print('sending parsed message, conversation name:', get_conv_name(conversation))
        self.send_message_parsed(conversation, html)

def main():
    """Main entry point"""
    # Build default paths for files.
    dirs = appdirs.AppDirs('hangupsbot', 'hangupsbot')
    default_log_path = os.path.join(dirs.user_data_dir, 'hangupsbot.log')
    default_cookies_path = os.path.join(dirs.user_data_dir, 'cookies.json')
    default_config_path = os.path.join(dirs.user_data_dir, 'config.json')

    # Configure argument parser
    parser = argparse.ArgumentParser(prog='hangupsbot',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-d', '--debug', action='store_true',
                        help='log detailed debugging messages')
    parser.add_argument('--log', default=default_log_path,
                        help='log file path')
    parser.add_argument('--cookies', default=default_cookies_path,
                        help='cookie storage path')
    parser.add_argument('--config', default=default_config_path,
                        help='config storage path')
    args = parser.parse_args()

    # Create all necessary directories.
    for path in [args.log, args.cookies, args.config]:
        directory = os.path.dirname(path)
        if directory and not os.path.isdir(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                sys.exit('Failed to create directory: {}'.format(e))

    # If there is no config file in user data directory, copy default one there
    if not os.path.isfile(args.config):
        try:
            shutil.copy(os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'config.json')),
                        args.config)
        except (OSError, IOError) as e:
            sys.exit('Failed to copy default config file: {}'.format(e))

    # Configure logging
    log_level = logging.DEBUG if args.debug else logging.WARNING
    logging.basicConfig(filename=args.log, level=log_level, format=LOG_FORMAT)
    # asyncio's debugging logs are VERY noisy, so adjust the log level
    logging.getLogger('asyncio').setLevel(logging.WARNING)

    # initialise the bot
    bot = HangupsBot(args.cookies, args.config)
    bot.run()

if __name__ == '__main__':
    main()
<commit_msg>saner default log level: hangups.WARNING; hangupsbot.INFO<commit_after>#!/usr/bin/env python3
import os, sys, argparse, logging, shutil, asyncio, time, signal

import appdirs
import hangups
from threading import Thread

from hangups.ui.utils import get_conv_name

import config
import handlers

# rpc sink
from sink2 import start_rpc_listener
from utils import simple_parse_to_segments

__version__ = '1.1'
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

class ConversationEvent(object):
    """Conversation event"""
    def __init__(self, bot, conv_event):
        self.conv_event = conv_event
        self.conv_id = conv_event.conversation_id
        self.conv = bot._conv_list.get(self.conv_id)
        self.user_id = conv_event.user_id
        self.user = self.conv.get_user(self.user_id)
        self.timestamp = conv_event.timestamp
        self.text = conv_event.text.strip() if isinstance(conv_event, hangups.ChatMessageEvent) else ''

    def print_debug(self):
        """Print informations about conversation event"""
        print('Conversation ID: {}'.format(self.conv_id))
        print('Conversation name: {}'.format(get_conv_name(self.conv, truncate=True)))
        print('User ID: {}'.format(self.user_id))
        print('User name: {}'.format(self.user.full_name))
        print('Timestamp: {}'.format(self.timestamp.astimezone(tz=None).strftime('%Y-%m-%d %H:%M:%S')))
        print('Text: {}'.format(self.text))
        print()

class HangupsBot(object):
    """Hangouts bot listening on all conversations"""
    def __init__(self, cookies_path, config_path, max_retries=5):
        self._client = None
        self._cookies_path = cookies_path
        self._max_retries = max_retries

        # These are populated by on_connect when it's called.
        self._conv_list = None        # hangups.ConversationList
        self._user_list = None        # hangups.UserList
        self._message_handler = None  # MessageHandler

        # Load config file
        self.config = config.Config(config_path)

        # Handle signals on Unix
        # (add_signal_handler is not implemented on Windows)
        try:
            loop = asyncio.get_event_loop()
            for signum in (signal.SIGINT, signal.SIGTERM):
                loop.add_signal_handler(signum, lambda: self.stop())
        except NotImplementedError:
            pass

    def login(self, cookies_path):
        """Login to Google account"""
        # Authenticate Google user and save auth cookies
        # (or load already saved cookies)
        try:
            cookies = hangups.auth.get_auth_stdin(cookies_path)
            return cookies
        except hangups.GoogleAuthError as e:
            print('Login failed ({})'.format(e))
            return False

    def run(self):
        """Connect to Hangouts and run bot"""
        cookies = self.login(self._cookies_path)
        if cookies:
            # Create Hangups client
            self._client = hangups.Client(cookies)
            self._client.on_connect.add_observer(self._on_connect)
            self._client.on_disconnect.add_observer(self._on_disconnect)

            # Start asyncio event loop and connect to Hangouts
            # If we are forcefully disconnected, try connecting again
            loop = asyncio.get_event_loop()

            if "jsonrpc" in self.config.keys():
                if self.config["jsonrpc"]:
                    print("starting json rpc sink")
                    # start up rpc listener in a separate thread
                    t = Thread(target=start_rpc_listener, args=(self, loop))
                    t.daemon = True
                    t.start()

            for retry in range(self._max_retries):
                try:
                    loop.run_until_complete(self._client.connect())
                    sys.exit(0)
                except Exception as e:
                    print('Client unexpectedly disconnected:\n{}'.format(e))
                    print('Waiting {} seconds...'.format(5 + retry * 5))
                    time.sleep(5 + retry * 5)
                    print('Trying to connect again (try {} of {})...'.format(retry + 1, self._max_retries))
            print('Maximum number of retries reached! Exiting...')
        sys.exit(1)

    def stop(self):
        """Disconnect from Hangouts"""
        asyncio.async(
            self._client.disconnect()
        ).add_done_callback(lambda future: future.result())

    def handle_chat_message(self, conv_event):
        """Handle chat messages"""
        event = ConversationEvent(self, conv_event)
        asyncio.async(self._message_handler.handle(event))

    def handle_membership_change(self, conv_event):
        """Handle conversation membership change"""
        event = ConversationEvent(self, conv_event)

        # Don't handle events caused by the bot himself
        if event.user.is_self:
            return

        # Test if watching for membership changes is enabled
        if not self.get_config_suboption(event.conv_id, 'membership_watching_enabled'):
            return

        # Generate list of added or removed users
        event_users = [event.conv.get_user(user_id) for user_id
                       in event.conv_event.participant_ids]
        names = ', '.join([user.full_name for user in event_users])

        # JOIN
        if event.conv_event.type_ == hangups.MembershipChangeType.JOIN:
            # Test if user who added new participants is admin
            admins_list = self.get_config_suboption(event.conv_id, 'admins')
            if event.user_id.chat_id in admins_list:
                self.send_message(event.conv, '{}: welcome'.format(names))
            else:
                segments = [hangups.ChatMessageSegment('!!! ATTENTION !!!', is_bold=True),
                            hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK),
                            hangups.ChatMessageSegment('{} added these users {}!'.format(
                                                       event.user.full_name, names)),
                            hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK),
                            hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK),
                            hangups.ChatMessageSegment('{}: please leave this hangout'.format(names))]
                self.send_message_segments(event.conv, segments)
        # LEAVE
        else:
            self.send_message(event.conv, '{}: left'.format(names))

    def handle_rename(self, conv_event):
        """Handle conversation rename"""
        event = ConversationEvent(self, conv_event)

        # Don't handle events caused by the bot himself
        if event.user.is_self:
            return

        # Test if watching for conversation rename is enabled
        if not self.get_config_suboption(event.conv_id, 'rename_watching_enabled'):
            return

        # Only print renames for now...
        if event.conv_event.new_name == '':
            print('{} cleared the conversation name'.format(event.user.first_name))
        else:
            print('{} renamed the conversation to {}'.format(event.user.first_name, event.conv_event.new_name))

    def send_message(self, conversation, text):
        """"Send simple chat message"""
        self.send_message_segments(conversation, [hangups.ChatMessageSegment(text)])

    def send_message_parsed(self, conversation, html):
        segments = simple_parse_to_segments(html)
        self.send_message_segments(conversation, segments)

    def send_message_segments(self, conversation, segments):
        """Send chat message segments"""
        # Ignore if the user hasn't typed a message.
        if len(segments) == 0:
            return
        # XXX: Exception handling here is still a bit broken. Uncaught
        # exceptions in _on_message_sent will only be logged.
        asyncio.async(
            conversation.send_message(segments)
        ).add_done_callback(self._on_message_sent)

    def list_conversations(self):
        """List all active conversations"""
        try:
            _all_conversations = self._conv_list.get_all()
            convs = sorted(_all_conversations, reverse=True, key=lambda c: c.last_modified)
            logging.info("list_conversations() returned {} conversation(s)".format(len(convs)))
        except Exception as e:
            logging.warning("list_conversations()", e)
            raise

        return convs

    def get_config_suboption(self, conv_id, option):
        """Get config suboption for conversation (or global option if not defined)"""
        try:
            suboption = self.config['conversations'][conv_id][option]
        except KeyError:
            try:
                suboption = self.config[option]
            except KeyError:
                suboption = None
        return suboption

    def print_conversations(self):
        print('Conversations:')
        for c in self.list_conversations():
            print('  {} ({}) u:{}'.format(get_conv_name(c, truncate=True), c.id_, len(c.users)))
            for u in c.users:
                print('    {} ({}) {}'.format(u.first_name, u.full_name, u.id_.chat_id))
        print()

    def get_1on1_conversation(self, chat_id):
        conversation = None
        for c in self.list_conversations():
            if len(c.users) == 2:
                for u in c.users:
                    if u.id_.chat_id == chat_id:
                        conversation = c
                        break
        return conversation

    def _on_message_sent(self, future):
        """Handle showing an error if a message fails to send"""
        try:
            future.result()
        except hangups.NetworkError:
            print('Failed to send message!')

    def _on_connect(self, initial_data):
        """Handle connecting for the first time"""
        print('Connected!')
        self._message_handler = handlers.MessageHandler(self)

        self._user_list = hangups.UserList(self._client,
                                           initial_data.self_entity,
                                           initial_data.entities,
                                           initial_data.conversation_participants)
        self._conv_list = hangups.ConversationList(self._client,
                                                   initial_data.conversation_states,
                                                   self._user_list,
                                                   initial_data.sync_timestamp)
        self._conv_list.on_event.add_observer(self._on_event)

        self.print_conversations()

    def _on_event(self, conv_event):
        """Handle conversation events"""
        if isinstance(conv_event, hangups.ChatMessageEvent):
            self.handle_chat_message(conv_event)
        elif isinstance(conv_event, hangups.MembershipChangeEvent):
            self.handle_membership_change(conv_event)
        elif isinstance(conv_event, hangups.RenameEvent):
            self.handle_rename(conv_event)

    def _on_disconnect(self):
        """Handle disconnecting"""
        print('Connection lost!')

    def external_send_message(self, conversation_id, text):
        conversation = self._conv_list.get(conversation_id)
        print('sending message, conversation name:', get_conv_name(conversation))
        self.send_message(conversation, text)

    def external_send_message_parsed(self, conversation_id, html):
        conversation = self._conv_list.get(conversation_id)
        print('sending parsed message, conversation name:', get_conv_name(conversation))
        self.send_message_parsed(conversation, html)

def main():
    """Main entry point"""
    # Build default paths for files.
    dirs = appdirs.AppDirs('hangupsbot', 'hangupsbot')
    default_log_path = os.path.join(dirs.user_data_dir, 'hangupsbot.log')
    default_cookies_path = os.path.join(dirs.user_data_dir, 'cookies.json')
    default_config_path = os.path.join(dirs.user_data_dir, 'config.json')

    # Configure argument parser
    parser = argparse.ArgumentParser(prog='hangupsbot',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-d', '--debug', action='store_true',
                        help='log detailed debugging messages')
    parser.add_argument('--log', default=default_log_path,
                        help='log file path')
    parser.add_argument('--cookies', default=default_cookies_path,
                        help='cookie storage path')
    parser.add_argument('--config', default=default_config_path,
                        help='config storage path')
    args = parser.parse_args()

    # Create all necessary directories.
    for path in [args.log, args.cookies, args.config]:
        directory = os.path.dirname(path)
        if directory and not os.path.isdir(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                sys.exit('Failed to create directory: {}'.format(e))

    # If there is no config file in user data directory, copy default one there
    if not os.path.isfile(args.config):
        try:
            shutil.copy(os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'config.json')),
                        args.config)
        except (OSError, IOError) as e:
            sys.exit('Failed to copy default config file: {}'.format(e))

    # Configure logging
    log_level = logging.DEBUG if args.debug else logging.INFO
    logging.basicConfig(filename=args.log, level=log_level, format=LOG_FORMAT)
    # asyncio's debugging logs are VERY noisy, so adjust the log level
    logging.getLogger('asyncio').setLevel(logging.WARNING)
    # hangups log is quite verbose too, suppress so we can debug the bot
    logging.getLogger('hangups').setLevel(logging.WARNING)

    # initialise the bot
    bot = HangupsBot(args.cookies, args.config)
    bot.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import pytest
from nbformat.v4.nbbase import new_notebook
from nbformat.validator import NotebookValidationError
from tornado.web import HTTPError
import nbrmd
from nbrmd.contentsmanager import RmdFileContentsManager
from .utils import list_all_notebooks, remove_outputs, \
    remove_outputs_and_header

nbrmd.file_format_version.FILE_FORMAT_VERSION = {}

@pytest.mark.parametrize('nb_file', list_all_notebooks('.ipynb'))
def test_rmd_is_ok(nb_file, tmpdir):
    nb = nbrmd.readf(nb_file)
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    nb.metadata['nbrmd_formats'] = 'ipynb,Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)

    cm.save(
        model=dict(type='notebook', content=nb),
        path=tmp_ipynb)

    nb2 = nbrmd.readf(str(tmpdir.join(tmp_rmd)))

    assert remove_outputs(nb) == remove_outputs(nb2)

@pytest.mark.parametrize('nb_file', list_all_notebooks('.Rmd'))
def test_ipynb_is_ok(nb_file, tmpdir):
    nb = nbrmd.readf(nb_file)
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)

    cm.save(
        model=dict(type='notebook', content=nb),
        path=tmp_rmd)

    nb2 = nbrmd.readf(str(tmpdir.join(tmp_ipynb)))

    assert remove_outputs(nb) == remove_outputs(nb2)

@pytest.mark.parametrize('nb_file', list_all_notebooks('.ipynb'))
def test_all_files_created(nb_file, tmpdir):
    nb = nbrmd.readf(nb_file)
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'
    tmp_py = 'notebook.py'
    nb.metadata['nbrmd_formats'] = 'ipynb,Rmd,py'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)

    cm.save(
        model=dict(type='notebook', content=nb),
        path=tmp_ipynb)

    nb2 = nbrmd.readf(str(tmpdir.join(tmp_py)))
    assert remove_outputs_and_header(nb) == remove_outputs_and_header(nb2)

    nb3 = nbrmd.readf(str(tmpdir.join(tmp_rmd)))
    assert remove_outputs(nb) == remove_outputs(nb3)

def test_no_files_created_on_no_format(tmpdir):
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'
    tmp_py = 'notebook.py'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)
    cm.default_nbrmd_formats = ''

    cm.save(
        model=dict(type='notebook',
                   content=new_notebook(nbformat=4,
                                        metadata=dict())),
        path=tmp_ipynb)

    assert not os.path.isfile(str(tmpdir.join(tmp_py)))
    assert not os.path.isfile(str(tmpdir.join(tmp_rmd)))

def test_raise_on_wrong_format(tmpdir):
    tmp_ipynb = str(tmpdir.join('notebook.ipynb'))

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)

    with pytest.raises(HTTPError):
        cm.save(
            model=dict(type='notebook',
                       content=new_notebook(nbformat=4,
                                            metadata=dict(
                                                nbrmd_formats=['.doc']))),
            path=tmp_ipynb)

def test_no_rmd_on_not_notebook(tmpdir):
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)
    cm.default_nbrmd_formats = 'ipynb,Rmd'

    with pytest.raises(HTTPError):
        cm.save(model=dict(type='not notebook',
                           content=new_notebook()),
                path=tmp_ipynb)
    assert not os.path.isfile(str(tmpdir.join(tmp_rmd)))

def test_no_rmd_on_not_v4(tmpdir):
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)
    cm.default_nbrmd_formats = 'ipynb,Rmd'

    with pytest.raises(NotebookValidationError):
        cm.save(model=dict(type='notebook',
                           content=new_notebook(nbformat=3)),
                path=tmp_rmd)

    assert not os.path.isfile(str(tmpdir.join(tmp_ipynb)))
<commit_msg>Tests adapted to new default (empty) 'nbrmd_formats' mwouts/nbsrc#5<commit_after>import os
import pytest
from nbformat.v4.nbbase import new_notebook
from nbformat.validator import NotebookValidationError
from tornado.web import HTTPError
import nbrmd
from nbrmd.contentsmanager import RmdFileContentsManager
from .utils import list_all_notebooks, remove_outputs, \
    remove_outputs_and_header

nbrmd.file_format_version.FILE_FORMAT_VERSION = {}

@pytest.mark.parametrize('nb_file', list_all_notebooks('.ipynb'))
def test_rmd_is_ok(nb_file, tmpdir):
    nb = nbrmd.readf(nb_file)
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    nb.metadata['nbrmd_formats'] = 'ipynb,Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)

    cm.save(
        model=dict(type='notebook', content=nb),
        path=tmp_ipynb)

    nb2 = nbrmd.readf(str(tmpdir.join(tmp_rmd)))

    assert remove_outputs(nb) == remove_outputs(nb2)

@pytest.mark.parametrize('nb_file', list_all_notebooks('.Rmd'))
def test_ipynb_is_ok(nb_file, tmpdir):
    nb = nbrmd.readf(nb_file)
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)
    cm.default_nbrmd_formats = 'ipynb,Rmd'

    cm.save(
        model=dict(type='notebook', content=nb),
        path=tmp_rmd)

    nb2 = nbrmd.readf(str(tmpdir.join(tmp_ipynb)))

    assert remove_outputs(nb) == remove_outputs(nb2)

@pytest.mark.parametrize('nb_file', list_all_notebooks('.ipynb'))
def test_all_files_created(nb_file, tmpdir):
    nb = nbrmd.readf(nb_file)
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'
    tmp_py = 'notebook.py'
    nb.metadata['nbrmd_formats'] = 'ipynb,Rmd,py'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)

    cm.save(
        model=dict(type='notebook', content=nb),
        path=tmp_ipynb)

    nb2 = nbrmd.readf(str(tmpdir.join(tmp_py)))
    assert remove_outputs_and_header(nb) == remove_outputs_and_header(nb2)

    nb3 = nbrmd.readf(str(tmpdir.join(tmp_rmd)))
    assert remove_outputs(nb) == remove_outputs(nb3)

def test_no_files_created_on_no_format(tmpdir):
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'
    tmp_py = 'notebook.py'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)
    cm.default_nbrmd_formats = ''

    cm.save(
        model=dict(type='notebook',
                   content=new_notebook(nbformat=4,
                                        metadata=dict())),
        path=tmp_ipynb)

    assert not os.path.isfile(str(tmpdir.join(tmp_py)))
    assert not os.path.isfile(str(tmpdir.join(tmp_rmd)))

def test_raise_on_wrong_format(tmpdir):
    tmp_ipynb = str(tmpdir.join('notebook.ipynb'))

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)

    with pytest.raises(HTTPError):
        cm.save(
            model=dict(type='notebook',
                       content=new_notebook(nbformat=4,
                                            metadata=dict(
                                                nbrmd_formats=['.doc']))),
            path=tmp_ipynb)

def test_no_rmd_on_not_notebook(tmpdir):
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)
    cm.default_nbrmd_formats = 'ipynb,Rmd'

    with pytest.raises(HTTPError):
        cm.save(model=dict(type='not notebook',
                           content=new_notebook()),
                path=tmp_ipynb)
    assert not os.path.isfile(str(tmpdir.join(tmp_rmd)))

def test_no_rmd_on_not_v4(tmpdir):
    tmp_ipynb = 'notebook.ipynb'
    tmp_rmd = 'notebook.Rmd'

    cm = RmdFileContentsManager()
    cm.root_dir = str(tmpdir)
    cm.default_nbrmd_formats = 'ipynb,Rmd'

    with pytest.raises(NotebookValidationError):
        cm.save(model=dict(type='notebook',
                           content=new_notebook(nbformat=3)),
                path=tmp_rmd)

    assert not os.path.isfile(str(tmpdir.join(tmp_ipynb)))
<|endoftext|>


<commit_before># test_serialization.py

from __future__ import unicode_literals

import io
import random

import pytest

from concepts import Context, Definition
from concepts.lattices import Lattice

SERIALIZED = {
    'objects': (
        '1sg', '1pl', '2sg', '2pl', '3sg', '3pl',
    ),
    'properties': (
        '+1', '-1', '+2', '-2', '+3', '-3', '+sg', '+pl', '-sg', '-pl',
    ),
    'context': [
        (0, 3, 5, 6, 9),
        (0, 3, 5, 7, 8),
        (1, 2, 5, 6, 9),
        (1, 2, 5, 7, 8),
        (1, 3, 4, 6, 9),
        (1, 3, 4, 7, 8),
    ],
    'lattice': [
        ((), (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 5, 6), ()),
        ((0,), (0, 3, 5, 6, 9), (7, 8, 9), (0,)),
        ((1,), (0, 3, 5, 7, 8), (7, 10, 11), (0,)),
        ((2,), (1, 2, 5, 6, 9), (8, 12, 13), (0,)),
        ((3,), (1, 2, 5, 7, 8), (10, 12, 14), (0,)),
        ((4,), (1, 3, 4, 6, 9), (9, 13, 15), (0,)),
        ((5,), (1, 3, 4, 7, 8), (11, 14, 15), (0,)),
        ((0, 1), (0, 3, 5), (18, 19), (1, 2)),
        ((0, 2), (5, 6, 9), (16, 18), (1, 3)),
        ((0, 4), (3, 6, 9), (16, 19), (1, 5)),
        ((1, 3), (5, 7, 8), (17, 18), (2, 4)),
        ((1, 5), (3, 7, 8), (17, 19), (2, 6)),
        ((2, 3), (1, 2, 5), (18, 20), (3, 4)),
        ((2, 4), (1, 6, 9), (16, 20), (3, 5)),
        ((3, 5), (1, 7, 8), (17, 20), (4, 6)),
        ((4, 5), (1, 3, 4), (19, 20), (5, 6)),
        ((0, 2, 4), (6, 9), (21,), (8, 9, 13)),
        ((1, 3, 5), (7, 8), (21,), (10, 11, 14)),
        ((0, 1, 2, 3), (5,), (21,), (7, 8, 10, 12)),
        ((0, 1, 4, 5), (3,), (21,), (7, 9, 11, 15)),
        ((2, 3, 4, 5), (1,), (21,), (12, 13, 14, 15)),
        ((0, 1, 2, 3, 4, 5), (), (), (18, 19, 20, 16, 17)),
    ],
}

SERIALIZED_NOLATTICE = {'objects': SERIALIZED['objects'],
                        'properties': SERIALIZED['properties'],
                        'context': list(SERIALIZED['context'])}

@pytest.fixture(params=[SERIALIZED, SERIALIZED_NOLATTICE])
def d(request):
    return request.param

def test_todict(context, d):
    assert 'lattice' not in context.__dict__
    if 'lattice' not in d:
        assert context.todict() == context.todict(include_lattice=False) == d
        assert 'lattice' not in context.__dict__
    else:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__
        assert context.todict(include_lattice=True) == context.todict() == d
        assert 'lattice' in context.__dict__

@pytest.fixture
def d_invalid():
    return {'objects': SERIALIZED['objects'],
            'properties': SERIALIZED['properties'],
            'context': list(SERIALIZED['context']),
            'lattice': list(SERIALIZED['lattice'])}

@pytest.mark.parametrize('missing',
                         ['objects', 'properties', 'context', 'lattice'])
def test_fromdict_missing(d_invalid, missing):
    del d_invalid[missing]
    with pytest.raises(ValueError, match=r'missing .*%s' % missing):
        Context.fromdict(d_invalid, require_lattice=(missing == 'lattice'))

@pytest.mark.parametrize('nonstring', ['objects', 'properties'])
def test_fromdict_nonstring(d_invalid, nonstring):
    d_invalid[nonstring] = (42,) + d_invalid[nonstring][1:]
    with pytest.raises(ValueError, match=r'non-string %s' % nonstring):
        Context.fromdict(d_invalid)

@pytest.mark.parametrize('short', ['objects', 'context'])
def test_fromdict_mismatch(d_invalid, short):
    d_invalid[short] = d_invalid[short][1:]
    lens = (5, 6) if short == 'objects' else (6, 5)
    match = r'mismatch: %d objects with %d context' % lens
    with pytest.raises(ValueError, match=match):
        Context.fromdict(d_invalid)

def test_fromdict_context_duplicates(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (first[0], first[0]) + first[2:]
    with pytest.raises(ValueError, match='duplicate'):
        Context.fromdict(d_invalid)

def test_fromdict_context_invalid_index(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (42,) + first[1:]
    with pytest.raises(ValueError, match='invalid index'):
        Context.fromdict(d_invalid)

def test_fromdict_empty_lattice(d_invalid):
    d_invalid['lattice'] = []
    with pytest.raises(ValueError, match='empty lattice'):
        Context.fromdict(d_invalid)

@pytest.fixture(params=[False, True])
def require_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def ignore_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def raw(request):
    return request.param

def test_fromdict(context, lattice, d, require_lattice, ignore_lattice, raw):
    if require_lattice and 'lattice' not in d:
        return

    result = Context.fromdict(d,
                              require_lattice=require_lattice,
                              ignore_lattice=ignore_lattice,
                              raw=raw)

    assert result == context

    if ignore_lattice or 'lattice' not in d:
        assert 'lattice' not in result.__dict__
    else:
        assert 'lattice' in result.__dict__
        assert result.lattice == lattice

def test_raw(context, lattice, d, raw):
    def shuffled(items):
        result = list(items)
        random.shuffle(result)
        return result

    _lattice = d.get('lattice')
    d = {'objects': d['objects'], 'properties': d['properties'],
         'context': [shuffled(intent) for intent in d['context']]}

    if _lattice is not None:
        pairs = shuffled(enumerate(_lattice))
        index_map = {old: new for new, (old, _) in enumerate(pairs)}
        d['lattice'] = [(shuffled(ex), shuffled(in_),
                         shuffled(index_map[i] for i in up),
                         shuffled(index_map[i] for i in lo))
                        for _, (ex, in_, up, lo) in pairs]

    result = Context.fromdict(d, raw=raw)

    assert isinstance(result, Context)
    assert result == context
    if _lattice is not None:
        if raw:
            assert result.lattice == lattice
        else:
            # instance broken by shuffled(d['lattice'])
            assert result.lattice != lattice

@pytest.mark.parametrize('include_lattice', [False, None, True])
def test_roundtrip(context, include_lattice):
    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    d = context.todict(include_lattice=include_lattice)

    assert isinstance(d, dict) and d
    assert all(d[k] for k in ('objects', 'properties', 'context'))
    if include_lattice:
        assert 'lattice' in context.__dict__
        assert d['lattice']
    else:
        assert 'lattice' not in context.__dict__
        assert 'lattice' not in d

    result = Context.fromdict(d)

    assert isinstance(result, Context)
    assert result == context
    if include_lattice:
        assert 'lattice' in result.__dict__
        assert result.lattice == context.lattice

def test_json_invalid_path(context):
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        context.tojson(object())
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        context.fromjson(object())

def test_json_roundtrip(tmp_path, py2, context, encoding='utf-8'):
    path_obj = tmp_path / 'context.json'
    args = [str(path_obj), path_obj]
    args.append(io.BytesIO() if py2 else io.StringIO())
    for path in args:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__
        context.tojson(path)
        if isinstance(path, (io.BytesIO, io.StringIO)):
            path.seek(0)
        assert 'lattice' not in context.__dict__
        deserialized = context.fromjson(path, encoding=encoding)
        if isinstance(path, (io.BytesIO, io.StringIO)):
            path.seek(0)
        assert 'lattice' not in deserialized.__dict__
        assert deserialized == context

        assert isinstance(context.lattice, Lattice)
        assert 'lattice' in context.__dict__
        context.tojson(path, encoding=encoding)
        if isinstance(path, (io.BytesIO, io.StringIO)):
            path.seek(0)
        deserialized = context.fromjson(path, encoding=encoding)
        assert 'lattice' in deserialized.__dict__
        assert deserialized == context
        assert deserialized.lattice == context.lattice

def test_json_indent(py2, context):
    assert 'lattice' not in context.__dict__
    with (io.BytesIO() if py2 else io.StringIO()) as f:
        context.tojson(f, indent=4, sort_keys=True)
        assert 'lattice' not in context.__dict__
        serialized = f.getvalue()

    assert serialized.startswith('{\n    "context": [')

def test_json_newlinedelmited(py2, context):
    assert 'lattice' not in context.__dict__
    with (io.BytesIO() if py2 else io.StringIO()) as f:
        context.tojson(f, sort_keys=True)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        serialized = f.getvalue()
        assert serialized.startswith('{"context": [')
        context.tojson(f, sort_keys=True)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        second = f.getvalue()
        assert second == serialized * 2

def test_nonascii():
    d = Definition()
    abba = (u'Agneta F\xe4ltskog', u'Anni-Frid Lyngstat',
            u'Benny Andersson', u'Bj\xf6rn Ulvaeus')
    for o  in abba:
       d.add_object(o, ['human', 'singer'])
    d.add_property(u'female', abba[:2])
    d.add_property(u'male', abba[2:])
    d.add_property(u'keyboarder', abba[2])
    d.add_property(u'guitarrist', abba[3])
    d.add_property(u'sch\xf6n', abba[::2])
<commit_msg>finish nonascii-coverage<commit_after># test_serialization.py

from __future__ import unicode_literals

import io
import random

import pytest

from concepts import Context, Definition
from concepts.lattices import Lattice

SERIALIZED = {
    'objects': (
        '1sg', '1pl', '2sg', '2pl', '3sg', '3pl',
    ),
    'properties': (
        '+1', '-1', '+2', '-2', '+3', '-3', '+sg', '+pl', '-sg', '-pl',
    ),
    'context': [
        (0, 3, 5, 6, 9),
        (0, 3, 5, 7, 8),
        (1, 2, 5, 6, 9),
        (1, 2, 5, 7, 8),
        (1, 3, 4, 6, 9),
        (1, 3, 4, 7, 8),
    ],
    'lattice': [
        ((), (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 5, 6), ()),
        ((0,), (0, 3, 5, 6, 9), (7, 8, 9), (0,)),
        ((1,), (0, 3, 5, 7, 8), (7, 10, 11), (0,)),
        ((2,), (1, 2, 5, 6, 9), (8, 12, 13), (0,)),
        ((3,), (1, 2, 5, 7, 8), (10, 12, 14), (0,)),
        ((4,), (1, 3, 4, 6, 9), (9, 13, 15), (0,)),
        ((5,), (1, 3, 4, 7, 8), (11, 14, 15), (0,)),
        ((0, 1), (0, 3, 5), (18, 19), (1, 2)),
        ((0, 2), (5, 6, 9), (16, 18), (1, 3)),
        ((0, 4), (3, 6, 9), (16, 19), (1, 5)),
        ((1, 3), (5, 7, 8), (17, 18), (2, 4)),
        ((1, 5), (3, 7, 8), (17, 19), (2, 6)),
        ((2, 3), (1, 2, 5), (18, 20), (3, 4)),
        ((2, 4), (1, 6, 9), (16, 20), (3, 5)),
        ((3, 5), (1, 7, 8), (17, 20), (4, 6)),
        ((4, 5), (1, 3, 4), (19, 20), (5, 6)),
        ((0, 2, 4), (6, 9), (21,), (8, 9, 13)),
        ((1, 3, 5), (7, 8), (21,), (10, 11, 14)),
        ((0, 1, 2, 3), (5,), (21,), (7, 8, 10, 12)),
        ((0, 1, 4, 5), (3,), (21,), (7, 9, 11, 15)),
        ((2, 3, 4, 5), (1,), (21,), (12, 13, 14, 15)),
        ((0, 1, 2, 3, 4, 5), (), (), (18, 19, 20, 16, 17)),
    ],
}

SERIALIZED_NOLATTICE = {'objects': SERIALIZED['objects'],
                        'properties': SERIALIZED['properties'],
                        'context': list(SERIALIZED['context'])}

@pytest.fixture(params=[SERIALIZED, SERIALIZED_NOLATTICE])
def d(request):
    return request.param

def test_todict(context, d):
    assert 'lattice' not in context.__dict__
    if 'lattice' not in d:
        assert context.todict() == context.todict(include_lattice=False) == d
        assert 'lattice' not in context.__dict__
    else:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__
        assert context.todict(include_lattice=True) == context.todict() == d
        assert 'lattice' in context.__dict__

@pytest.fixture
def d_invalid():
    return {'objects': SERIALIZED['objects'],
            'properties': SERIALIZED['properties'],
            'context': list(SERIALIZED['context']),
            'lattice': list(SERIALIZED['lattice'])}

@pytest.mark.parametrize('missing',
                         ['objects', 'properties', 'context', 'lattice'])
def test_fromdict_missing(d_invalid, missing):
    del d_invalid[missing]
    with pytest.raises(ValueError, match=r'missing .*%s' % missing):
        Context.fromdict(d_invalid, require_lattice=(missing == 'lattice'))

@pytest.mark.parametrize('nonstring', ['objects', 'properties'])
def test_fromdict_nonstring(d_invalid, nonstring):
    d_invalid[nonstring] = (42,) + d_invalid[nonstring][1:]
    with pytest.raises(ValueError, match=r'non-string %s' % nonstring):
        Context.fromdict(d_invalid)

@pytest.mark.parametrize('short', ['objects', 'context'])
def test_fromdict_mismatch(d_invalid, short):
    d_invalid[short] = d_invalid[short][1:]
    lens = (5, 6) if short == 'objects' else (6, 5)
    match = r'mismatch: %d objects with %d context' % lens
    with pytest.raises(ValueError, match=match):
        Context.fromdict(d_invalid)

def test_fromdict_context_duplicates(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (first[0], first[0]) + first[2:]
    with pytest.raises(ValueError, match='duplicate'):
        Context.fromdict(d_invalid)

def test_fromdict_context_invalid_index(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (42,) + first[1:]
    with pytest.raises(ValueError, match='invalid index'):
        Context.fromdict(d_invalid)

def test_fromdict_empty_lattice(d_invalid):
    d_invalid['lattice'] = []
    with pytest.raises(ValueError, match='empty lattice'):
        Context.fromdict(d_invalid)

@pytest.fixture(params=[False, True])
def require_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def ignore_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def raw(request):
    return request.param

def test_fromdict(context, lattice, d, require_lattice, ignore_lattice, raw):
    if require_lattice and 'lattice' not in d:
        return

    result = Context.fromdict(d,
                              require_lattice=require_lattice,
                              ignore_lattice=ignore_lattice,
                              raw=raw)

    assert result == context

    if ignore_lattice or 'lattice' not in d:
        assert 'lattice' not in result.__dict__
    else:
        assert 'lattice' in result.__dict__
        assert result.lattice == lattice

def test_raw(context, lattice, d, raw):
    def shuffled(items):
        result = list(items)
        random.shuffle(result)
        return result

    _lattice = d.get('lattice')
    d = {'objects': d['objects'], 'properties': d['properties'],
         'context': [shuffled(intent) for intent in d['context']]}

    if _lattice is not None:
        pairs = shuffled(enumerate(_lattice))
        index_map = {old: new for new, (old, _) in enumerate(pairs)}
        d['lattice'] = [(shuffled(ex), shuffled(in_),
                         shuffled(index_map[i] for i in up),
                         shuffled(index_map[i] for i in lo))
                        for _, (ex, in_, up, lo) in pairs]

    result = Context.fromdict(d, raw=raw)

    assert isinstance(result, Context)
    assert result == context
    if _lattice is not None:
        if raw:
            assert result.lattice == lattice
        else:
            # instance broken by shuffled(d['lattice'])
            assert result.lattice != lattice

@pytest.mark.parametrize('include_lattice', [False, None, True])
def test_roundtrip(context, include_lattice):
    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    d = context.todict(include_lattice=include_lattice)

    assert isinstance(d, dict) and d
    assert all(d[k] for k in ('objects', 'properties', 'context'))
    if include_lattice:
        assert 'lattice' in context.__dict__
        assert d['lattice']
    else:
        assert 'lattice' not in context.__dict__
        assert 'lattice' not in d

    result = Context.fromdict(d)

    assert isinstance(result, Context)
    assert result == context
    if include_lattice:
        assert 'lattice' in result.__dict__
        assert result.lattice == context.lattice

def test_json_invalid_path(context):
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        context.tojson(object())
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        context.fromjson(object())

def test_json_roundtrip(tmp_path, py2, context, encoding='utf-8'):
    path_obj = tmp_path / 'context.json'
    args = [str(path_obj), path_obj]
    args.append(io.BytesIO() if py2 else io.StringIO())
    for path in args:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__
        context.tojson(path)
        if isinstance(path, (io.BytesIO, io.StringIO)):
            path.seek(0)
        assert 'lattice' not in context.__dict__
        deserialized = context.fromjson(path, encoding=encoding)
        if isinstance(path, (io.BytesIO, io.StringIO)):
            path.seek(0)
        assert 'lattice' not in deserialized.__dict__
        assert deserialized == context

        assert isinstance(context.lattice, Lattice)
        assert 'lattice' in context.__dict__
        context.tojson(path, encoding=encoding)
        if isinstance(path, (io.BytesIO, io.StringIO)):
            path.seek(0)
        deserialized = context.fromjson(path, encoding=encoding)
        assert 'lattice' in deserialized.__dict__
        assert deserialized == context
        assert deserialized.lattice == context.lattice

def test_json_indent(py2, context):
    assert 'lattice' not in context.__dict__
    with (io.BytesIO() if py2 else io.StringIO()) as f:
        context.tojson(f, indent=4, sort_keys=True)
        assert 'lattice' not in context.__dict__
        serialized = f.getvalue()

    assert serialized.startswith('{\n    "context": [')

def test_json_newlinedelmited(py2, context):
    assert 'lattice' not in context.__dict__
    with (io.BytesIO() if py2 else io.StringIO()) as f:
        context.tojson(f, sort_keys=True)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        serialized = f.getvalue()
        assert serialized.startswith('{"context": [')
        context.tojson(f, sort_keys=True)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        second = f.getvalue()
        assert second == serialized * 2

def _nonascii_context():
    d = Definition()
    abba = (u'Agneta F\xe4ltskog', u'Anni-Frid Lyngstat',
            u'Benny Andersson', u'Bj\xf6rn Ulvaeus')
    for o in abba:
       d.add_object(o, [u'human', u'singer'])
    d.add_property(u'female', abba[:2])
    d.add_property(u'male', abba[2:])
    d.add_property(u'keyboarder', [abba[2]])
    d.add_property(u'guitarrist', [abba[3]])
    d.add_property(u'sch\xf6n', abba[::2])
    return Context(*d)

def test_json_nonascii_context(py2, encoding='utf-8'):
    context = _nonascii_context()
    with (io.BytesIO() if py2 else io.StringIO()) as f:
        context.tojson(f, encoding=encoding)
        f.seek(0)
        serialized = f.getvalue()
        deserialized = context.fromjson(f, encoding=encoding)
        assert deserialized  == context
        assert u'"Agneta F\\u00e4ltskog"' in serialized
        assert u'"Bj\\u00f6rn Ulvaeus"' in serialized
<|endoftext|>