<commit_before># encoding: utf-8
#
# Author: Teemu Henrikki Pitkänen <teemu.pitkanen@helsinki.fi>
# University of Helsinki / Helsinki Institute for Information Technology 2016
#

import numpy as np
from scipy.spatial.distance import cdist
from Queue import PriorityQueue, Empty
from collections import deque
from math import log, ceil, floor

# The following imports used only for saving/loading trees
import os
import cPickle
from hashlib import sha1

class MRPTIndex(object):
    """
    The MRPT index is a data structure that allows us to answer approximate nearest neighbor queries quickly. The index
    is basically just a collection of random projection trees. This class implements the basic MRPT index as well as a
    number of small tricks to improve performance.
    """
    def __init__(self, data, n0, n_trees, degree=2, use_saved=False):
        """
        The initializer builds the MRPT index. It has to be called before ANN-queries can be made.
        :param data: The data fro which the index is built. At the moment this has to be done at once, no objects can be
        added later.
        :param n0: The maximum leaf size used in all of the trees of the index.
        :param n_trees: The number of trees used in the index.
        :param degree: Defines the degree of inner random projection tree nodes (the number of children). Use the
        default value unless you really know what you are doing.
        :param use_saved: Saves and loads trees for reuse instead of building new ones every time an index is created.
        :return: The index object.
        """
        self.data = data
        if use_saved:
            self.trees = []
            # The directory name is a hash of the data set contents to avoid using trees built for wrong data
            save_path = 'saved_trees/'+sha1(data.view(np.uint8)).hexdigest()[:8]+'/'+str(n0)+'/'+str(degree)
            self.trees = MRPTIndex.load_trees(save_path, n_trees)
            for t in range(len(self.trees), n_trees):
                t = RPTree(data, n0, degree=degree)
                self.trees.append(t)
                MRPTIndex.save_trees([t], save_path)
        else:
            self.trees = [RPTree(data, n0) for t in range(n_trees)]

    def ann(self, obj, k, extra_branches=0, n_elected=0):
        """
        The mrpt approximate nearest neighbor query. By default the function implements a basic query - the query object
        is routed to one leaf in each tree, and the nearest neighbors are searched in the union of these leaves. The
        extra parameters allow some tricks to improve performance. The parameter extra_brances allows the query to
        explore extra branches where the projection value was close to the split. The n_elected parameter causes the
        query to use the voting trick - not all objects in the union of leaves are searched, but the appearance counts
        as a vote and only the most voted objects get 'elected' to be searched. Thus an object must be returned by
        several trees to be included in the linear search.
        :param obj: The object whose neighbors are being searched
        :param k: The number of neighbors being searched
        :param extra_branches: The number of extra branches allowed in the index
        :param n_elected: The number of elected objects in the voting trick
        :return: The approximate neighbors. In extreme situations not strictly k, but slightly less (eg. 1 tree case)
        """
        priority_queue = PriorityQueue()
        all_projections = []
        votes = np.zeros(len(self.data))

        # First traverse each tree from root to leaf
        for tree_id in range(len(self.trees)):
            indexes, gaps, projections = self.trees[tree_id].full_tree_traversal(obj)
            votes[indexes] += 1
            all_projections.append(projections)
            for (gap_width, node, level) in gaps:
                priority_queue.put((gap_width, node, level, tree_id))

        # Optional branching trick: traverse down from #extra_branches nodes with the smallest d(projection, split)
        for i in range(extra_branches):
            try:
                gap_width, node, level, tree = priority_queue.get(block=False)
                indexes, gaps = RPTree.partial_tree_traversal(node, all_projections[tree][level:], level)
                votes[indexes] += 1
                for gap_width in gaps:
                    priority_queue.put((gap_width[0], gap_width[1], gap_width[2], tree))
            except Empty:
                print 'More branches than leaves. Will skip the extras.'

        # Decide which nodes to include in the brute force search
        if n_elected > 0:   # Optional voting trick
            elected = np.argsort(votes)[::-1][:n_elected]
        else:  # Basic mrpt
            elected = np.nonzero(votes)[0]

        # Find the nearest neighbors in the subset of objects
        return elected[np.argsort(cdist([obj], self.data[elected])[0])[:k]]

    @staticmethod
    def save_trees(trees, path):
        """
        A function for saving the trees to disk.
        :param trees: The trees to be saved
        :param path: The path where the trees are saved
        """
        if not os.path.exists(path):
            os.makedirs(path)
        ordinal = 0
        for tree in trees:
            while os.path.isfile(path + '/t' + str(ordinal) + '.idx'):
                ordinal += 1
            filename = path + '/t' + str(ordinal) + '.idx'
            with open(filename, 'w') as f:
                cPickle.dump(tree, f)

    @staticmethod
    def load_trees(path, n_trees):
        """
        A function for reading saved trees from the disk.
        :param path: The path from where the trees are loaded
        :param n_trees: The number of trees the user wants loaded
        :return: A list containing the trees. Empty if no such directory exists. If not enough trees exist the function
        will return as many as it finds.
        """
        trees = []
        if os.path.exists(path):
            files = os.listdir(path)
            for i in range(min(n_trees, len(files))):
                with open(path+'/'+files[i], 'r') as f:
                    trees.append(cPickle.load(f))
        return trees

class RPTree(object):
    """
    A Random Projection Tree is a spatial index data structure used in specific data mining and indexing tasks. Our
    implementation has been built with approximate nearest neighbor search (ANN) problems in mind. A tree is built by
    dividing the data space by random hyperplanes into small cells. In ANN-problems we usually achieve substantial
    improvements by quickly choosing just a subset of the data located in a single cell where the actual brute-force NN
    search is then performed, instead of using the whole data set.
    """

    def __init__(self, data, n0, degree=2):
        """
        Sets the main attributes and calls the build_tree routine which actually creates the nodes and builds the tree.
        :param data: The data for which the index is built
        :param n0: The maximum leaf size of the tree
        :param degree: The maximum number of children that each internal node of the tree has
        """
        self.seed = np.random.randint(0, int(1e9))
        self.degree = degree
        self.tree_height = ceil(log(len(data)/float(n0), degree))
        self.root = _Node()
        self._build_tree(data, n0)

    def _build_tree(self, data, n0):
        """
        An iterative method for building the random projection tree structure. The tree is built level-by-level by
        using a queue to handle the order in which the nodes are processed. The method is significantly faster to
        recursive building. This implementation uses same random vectors in each tree branch for running
        time efficiency. Note that although all projections are computed in a single matrix multiplication, the
        projection vector is different on each level.
        """
        # Restore rng settings for reproducibility and compute projections to random basis
        np.random.seed(self.seed)
        all_projections = np.dot(data, np.random.normal(size=(data.shape[1], self.tree_height)))

        # Main while loop that builds the tree one level at a time
        queue = deque([(self.root, np.arange(len(data)))])
        tracker = _FullTreeLevelTracker(self.degree)
        while len(queue) > 0:
            # Pop next node to be processed
            node, indexes = queue.popleft()

            # Divide the indexes into equal sized chunks (one for each child) and compute the split boundaries
            indexes_divided, node.splits = self._chunkify(indexes, all_projections[indexes, tracker.level], n0)

            # Set references to children, add child nodes to queue if further splits are required (node size > n0)
            for node_indexes in indexes_divided:
                if len(node_indexes) > n0:
                    child = _Node()
                    node.children.append(child)
                    queue.append((child, node_indexes))
                else:
                    node.children.append(node_indexes)

            tracker.object_added()  # Corresponds to adding _node_, not its children, thus called only once

    def _chunkify(self, indexes, projections, n0):
        """
        Divides the list 'indexes' into 'self.degree' equal-sized chunks. If the split is not even, the extra elements
        are added to the list entries on the left. The function aims at building as full leaf nodes as possible, so in
        case each child is not of size at least n0, the method may result in less than 'self.degree' chunks. Think of
        the case n0=10, self.degree=5, len(indexes)=12. Instead of splitting into 5 chunks of sizes 2-3, the function
        returns two chunks with sizes 6 and 5. Thus the leaf sizes are generally closer to n0 and the running time and
        performance stay more predictable.
        :param indexes: The list of indexes to be chunkified
        :param projections: The projections corresponding to the indexes
        :param n0: The maximum leaf size
        :return: The chunks as a list of lists, the projection values that separate these chunks
        """
        indexes = indexes[np.argsort(projections)]
        projections = np.sort(projections)
        n = len(indexes)

        n_chunks = min(int(ceil(n/float(n0))), self.degree)
        min_chunk_size = int(floor(n/n_chunks))
        chunk_sizes = np.repeat([min_chunk_size], n_chunks)
        chunk_sizes[range(n - min_chunk_size*n_chunks)] += 1
        chunk_bounds = np.cumsum(np.concatenate(([0], chunk_sizes)))

        return ([indexes[chunk_bounds[i]:chunk_bounds[i+1]] for i in range(n_chunks)],
                [(projections[i-1] + projections[i])/2 for i in chunk_bounds[1:-1]])

    def full_tree_traversal(self, obj):
        """
        The function places the query object 'obj' to a leaf. The function re-creates the same random vectors that were
        used in tree construction, computes the projections of the query vector and using the split information stored
        in the nodes places the query vector into a single leaf.
        :param obj: The query object, has to be given as a row vector
        :return: The indexes of the leaf, gap information and projection values from the path to leaf (for the extra
        branches trick)
        """
        # Restore rng settings, compute projections to random basis
        np.random.seed(self.seed)
        projections = np.dot(obj, np.random.normal(size=(len(obj), self.tree_height)))

        # Move down the tree according to the projections and split values stored in the tree
        indexes, gaps = self.partial_tree_traversal(self.root, projections, 0)
        return indexes, gaps, projections

    @staticmethod
    def partial_tree_traversal(node, projections, tree_level):
        """
        Moves down to a leaf starting from the specified node.
        """
        gaps = []
        for projection in projections:
            # Leaves as
            if not hasattr(node, 'splits'):
                break

            # Find the child where the query object belongs.
            child_index = len(node.splits)
            for i in range(len(node.splits)):
                if projection < node.splits[i]:
                    child_index = i
                    break

            # Store the distances to splits for the priority queue trick.
            for i in range(len(node.splits)):
                gap = abs(projection - node.splits[i])
                if i < child_index:
                    gaps.append((gap, node.children[i], tree_level + 1))
                elif i >= child_index:
                    gaps.append((gap, node.children[i+1], tree_level + 1))

            # Move down the tree for next iteration round
            node = node.children[child_index]
            tree_level += 1

        return node, gaps

class _FullTreeLevelTracker(object):
    """
    A _FullTreeLevelTracker object keeps track on the height of a tree with a fixed degree in internal nodes. Every time
    an object is added to the tree the object_added -method need to be called. The height can be read from the 'level'
    attribute.
    """
    def __init__(self, degree=2):
        self.level = 0
        self.level_capacity = 1
        self.level_occupancy = 0
        self.degree = degree

    def object_added(self):
        """
        Must be called every time a node is added to the tree, updates the attributes accordingly.
        """
        self.level_occupancy += 1
        if self.level_occupancy == self.level_capacity:
            self.level_occupancy = 0
            self.level_capacity *= self.degree
            self.level += 1

class _Node(object):
    """
    The class describes the structure of a single internal node. Only the split values used at this node and the links
    to the child nodes need to be stored. (The random vector can be generated without explicitly storing it.). Notice
    also that leaf nodes are not _Node-objects but just simple lists of data object indices.
    """
    def __init__(self):
        self.children = []
        self.splits = []
<commit_msg>Fixed possible future issue, now casting ceil to int<commit_after># encoding: utf-8
#
# Author: Teemu Henrikki Pitkänen <teemu.pitkanen@helsinki.fi>
# University of Helsinki / Helsinki Institute for Information Technology 2016
#

import numpy as np
from scipy.spatial.distance import cdist
from Queue import PriorityQueue, Empty
from collections import deque
from math import log, ceil, floor

# The following imports used only for saving/loading trees
import os
import cPickle
from hashlib import sha1

class MRPTIndex(object):
    """
    The MRPT index is a data structure that allows us to answer approximate nearest neighbor queries quickly. The index
    is basically just a collection of random projection trees. This class implements the basic MRPT index as well as a
    number of small tricks to improve performance.
    """
    def __init__(self, data, n0, n_trees, degree=2, use_saved=False):
        """
        The initializer builds the MRPT index. It has to be called before ANN-queries can be made.
        :param data: The data fro which the index is built. At the moment this has to be done at once, no objects can be
        added later.
        :param n0: The maximum leaf size used in all of the trees of the index.
        :param n_trees: The number of trees used in the index.
        :param degree: Defines the degree of inner random projection tree nodes (the number of children). Use the
        default value unless you really know what you are doing.
        :param use_saved: Saves and loads trees for reuse instead of building new ones every time an index is created.
        :return: The index object.
        """
        self.data = data
        if use_saved:
            self.trees = []
            # The directory name is a hash of the data set contents to avoid using trees built for wrong data
            save_path = 'saved_trees/'+sha1(data.view(np.uint8)).hexdigest()[:8]+'/'+str(n0)+'/'+str(degree)
            self.trees = MRPTIndex.load_trees(save_path, n_trees)
            for t in range(len(self.trees), n_trees):
                t = RPTree(data, n0, degree=degree)
                self.trees.append(t)
                MRPTIndex.save_trees([t], save_path)
        else:
            self.trees = [RPTree(data, n0) for t in range(n_trees)]

    def ann(self, obj, k, extra_branches=0, n_elected=0):
        """
        The mrpt approximate nearest neighbor query. By default the function implements a basic query - the query object
        is routed to one leaf in each tree, and the nearest neighbors are searched in the union of these leaves. The
        extra parameters allow some tricks to improve performance. The parameter extra_brances allows the query to
        explore extra branches where the projection value was close to the split. The n_elected parameter causes the
        query to use the voting trick - not all objects in the union of leaves are searched, but the appearance counts
        as a vote and only the most voted objects get 'elected' to be searched. Thus an object must be returned by
        several trees to be included in the linear search.
        :param obj: The object whose neighbors are being searched
        :param k: The number of neighbors being searched
        :param extra_branches: The number of extra branches allowed in the index
        :param n_elected: The number of elected objects in the voting trick
        :return: The approximate neighbors. In extreme situations not strictly k, but slightly less (eg. 1 tree case)
        """
        priority_queue = PriorityQueue()
        all_projections = []
        votes = np.zeros(len(self.data))

        # First traverse each tree from root to leaf
        for tree_id in range(len(self.trees)):
            indexes, gaps, projections = self.trees[tree_id].full_tree_traversal(obj)
            votes[indexes] += 1
            all_projections.append(projections)
            for (gap_width, node, level) in gaps:
                priority_queue.put((gap_width, node, level, tree_id))

        # Optional branching trick: traverse down from #extra_branches nodes with the smallest d(projection, split)
        for i in range(extra_branches):
            try:
                gap_width, node, level, tree = priority_queue.get(block=False)
                indexes, gaps = RPTree.partial_tree_traversal(node, all_projections[tree][level:], level)
                votes[indexes] += 1
                for gap_width in gaps:
                    priority_queue.put((gap_width[0], gap_width[1], gap_width[2], tree))
            except Empty:
                print 'More branches than leaves. Will skip the extras.'

        # Decide which nodes to include in the brute force search
        if n_elected > 0:   # Optional voting trick
            elected = np.argsort(votes)[::-1][:n_elected]
        else:  # Basic mrpt
            elected = np.nonzero(votes)[0]

        # Find the nearest neighbors in the subset of objects
        return elected[np.argsort(cdist([obj], self.data[elected])[0])[:k]]

    @staticmethod
    def save_trees(trees, path):
        """
        A function for saving the trees to disk.
        :param trees: The trees to be saved
        :param path: The path where the trees are saved
        """
        if not os.path.exists(path):
            os.makedirs(path)
        ordinal = 0
        for tree in trees:
            while os.path.isfile(path + '/t' + str(ordinal) + '.idx'):
                ordinal += 1
            filename = path + '/t' + str(ordinal) + '.idx'
            with open(filename, 'w') as f:
                cPickle.dump(tree, f)

    @staticmethod
    def load_trees(path, n_trees):
        """
        A function for reading saved trees from the disk.
        :param path: The path from where the trees are loaded
        :param n_trees: The number of trees the user wants loaded
        :return: A list containing the trees. Empty if no such directory exists. If not enough trees exist the function
        will return as many as it finds.
        """
        trees = []
        if os.path.exists(path):
            files = os.listdir(path)
            for i in range(min(n_trees, len(files))):
                with open(path+'/'+files[i], 'r') as f:
                    trees.append(cPickle.load(f))
        return trees

class RPTree(object):
    """
    A Random Projection Tree is a spatial index data structure used in specific data mining and indexing tasks. Our
    implementation has been built with approximate nearest neighbor search (ANN) problems in mind. A tree is built by
    dividing the data space by random hyperplanes into small cells. In ANN-problems we usually achieve substantial
    improvements by quickly choosing just a subset of the data located in a single cell where the actual brute-force NN
    search is then performed, instead of using the whole data set.
    """

    def __init__(self, data, n0, degree=2):
        """
        Sets the main attributes and calls the build_tree routine which actually creates the nodes and builds the tree.
        :param data: The data for which the index is built
        :param n0: The maximum leaf size of the tree
        :param degree: The maximum number of children that each internal node of the tree has
        """
        self.seed = np.random.randint(0, int(1e9))
        self.degree = degree
        self.tree_height = int(ceil(log(len(data)/float(n0), degree)))
        self.root = _Node()
        self._build_tree(data, n0)

    def _build_tree(self, data, n0):
        """
        An iterative method for building the random projection tree structure. The tree is built level-by-level by
        using a queue to handle the order in which the nodes are processed. The method is significantly faster to
        recursive building. This implementation uses same random vectors in each tree branch for running
        time efficiency. Note that although all projections are computed in a single matrix multiplication, the
        projection vector is different on each level.
        """
        # Restore rng settings for reproducibility and compute projections to random basis
        np.random.seed(self.seed)
        all_projections = np.dot(data, np.random.normal(size=(data.shape[1], self.tree_height)))

        # Main while loop that builds the tree one level at a time
        queue = deque([(self.root, np.arange(len(data)))])
        tracker = _FullTreeLevelTracker(self.degree)
        while len(queue) > 0:
            # Pop next node to be processed
            node, indexes = queue.popleft()

            # Divide the indexes into equal sized chunks (one for each child) and compute the split boundaries
            indexes_divided, node.splits = self._chunkify(indexes, all_projections[indexes, tracker.level], n0)

            # Set references to children, add child nodes to queue if further splits are required (node size > n0)
            for node_indexes in indexes_divided:
                if len(node_indexes) > n0:
                    child = _Node()
                    node.children.append(child)
                    queue.append((child, node_indexes))
                else:
                    node.children.append(node_indexes)

            tracker.object_added()  # Corresponds to adding _node_, not its children, thus called only once

    def _chunkify(self, indexes, projections, n0):
        """
        Divides the list 'indexes' into 'self.degree' equal-sized chunks. If the split is not even, the extra elements
        are added to the list entries on the left. The function aims at building as full leaf nodes as possible, so in
        case each child is not of size at least n0, the method may result in less than 'self.degree' chunks. Think of
        the case n0=10, self.degree=5, len(indexes)=12. Instead of splitting into 5 chunks of sizes 2-3, the function
        returns two chunks with sizes 6 and 5. Thus the leaf sizes are generally closer to n0 and the running time and
        performance stay more predictable.
        :param indexes: The list of indexes to be chunkified
        :param projections: The projections corresponding to the indexes
        :param n0: The maximum leaf size
        :return: The chunks as a list of lists, the projection values that separate these chunks
        """
        indexes = indexes[np.argsort(projections)]
        projections = np.sort(projections)
        n = len(indexes)

        n_chunks = min(int(ceil(n/float(n0))), self.degree)
        min_chunk_size = int(floor(n/n_chunks))
        chunk_sizes = np.repeat([min_chunk_size], n_chunks)
        chunk_sizes[range(n - min_chunk_size*n_chunks)] += 1
        chunk_bounds = np.cumsum(np.concatenate(([0], chunk_sizes)))

        return ([indexes[chunk_bounds[i]:chunk_bounds[i+1]] for i in range(n_chunks)],
                [(projections[i-1] + projections[i])/2 for i in chunk_bounds[1:-1]])

    def full_tree_traversal(self, obj):
        """
        The function places the query object 'obj' to a leaf. The function re-creates the same random vectors that were
        used in tree construction, computes the projections of the query vector and using the split information stored
        in the nodes places the query vector into a single leaf.
        :param obj: The query object, has to be given as a row vector
        :return: The indexes of the leaf, gap information and projection values from the path to leaf (for the extra
        branches trick)
        """
        # Restore rng settings, compute projections to random basis
        np.random.seed(self.seed)
        projections = np.dot(obj, np.random.normal(size=(len(obj), self.tree_height)))

        # Move down the tree according to the projections and split values stored in the tree
        indexes, gaps = self.partial_tree_traversal(self.root, projections, 0)
        return indexes, gaps, projections

    @staticmethod
    def partial_tree_traversal(node, projections, tree_level):
        """
        Moves down to a leaf starting from the specified node.
        """
        gaps = []
        for projection in projections:
            # Leaves as
            if not hasattr(node, 'splits'):
                break

            # Find the child where the query object belongs.
            child_index = len(node.splits)
            for i in range(len(node.splits)):
                if projection < node.splits[i]:
                    child_index = i
                    break

            # Store the distances to splits for the priority queue trick.
            for i in range(len(node.splits)):
                gap = abs(projection - node.splits[i])
                if i < child_index:
                    gaps.append((gap, node.children[i], tree_level + 1))
                elif i >= child_index:
                    gaps.append((gap, node.children[i+1], tree_level + 1))

            # Move down the tree for next iteration round
            node = node.children[child_index]
            tree_level += 1

        return node, gaps

class _FullTreeLevelTracker(object):
    """
    A _FullTreeLevelTracker object keeps track on the height of a tree with a fixed degree in internal nodes. Every time
    an object is added to the tree the object_added -method need to be called. The height can be read from the 'level'
    attribute.
    """
    def __init__(self, degree=2):
        self.level = 0
        self.level_capacity = 1
        self.level_occupancy = 0
        self.degree = degree

    def object_added(self):
        """
        Must be called every time a node is added to the tree, updates the attributes accordingly.
        """
        self.level_occupancy += 1
        if self.level_occupancy == self.level_capacity:
            self.level_occupancy = 0
            self.level_capacity *= self.degree
            self.level += 1

class _Node(object):
    """
    The class describes the structure of a single internal node. Only the split values used at this node and the links
    to the child nodes need to be stored. (The random vector can be generated without explicitly storing it.). Notice
    also that leaf nodes are not _Node-objects but just simple lists of data object indices.
    """
    def __init__(self):
        self.children = []
        self.splits = []
<|endoftext|>


<commit_before>import os.path
from operator import attrgetter
from itertools import groupby

from lxml import etree
from lxml.etree import SubElement

def html2xml(x, encoding='ascii'):
    if hasattr(x, 'wirite_c14n'): return x
    if not isinstance(x, basestring):
        if hasattr(x, '__unicode__'): x = unicode(x)
        else: x = str(x)
    if isinstance(x, str): x = unicode(x, encoding)
    return etree.HTML(x)

def normalize(html):
    # print etree.tostring(html, pretty_print=True), '\n'
    head = html.find('head')
    body = html.find('body')
    if body is None: body = SubElement(html, 'body')
    if head is None:
        head = html.makeelement('head')
        html.insert(-1, head)
    head_list = layout_tags_xpath(head)
    body[0:0] = head_list
    if head_list: pass
    elif layout_tags_xpath(body): pass
    else:
        for p in body.findall('p'):
            if layout_tags_xpath(p): break
        else: return
    i = 0
    while i < len(body):
        x = body[i]
        tag = x.tag
        if tag == 'p': unnest_p(x)
        elif tag in layout_tags:
            if tag == 'layout': unnest_layout(x)
            elif tag == 'content': unnest_content(x)
            else: unnest_element(x)
        i += 1
    # print etree.tostring(html, pretty_print=True), '\n'
    elements = []
    content = None
    text = body.text
    if text and not text.isspace():
        content = body.makeelement('content')
        content.text = text
        body.text = None
        elements.append(content)
    for x in body:
        tag = x.tag
        if tag in layout_tags and tag != 'column':
            if tag == 'content':
                if content is not None: normalize_content(content)
                content = None
            elements.append(x)
            tail = x.tail
            layout_tags[tag](x)
            if tail and not tail.isspace():
                content = body.makeelement('content')
                content.text = tail
                x.tail = None
                elements.append(content)
        else:
            if content is None:
                content = body.makeelement('content')
                elements.append(content)
            content.append(x)
    if content is not None: normalize_content(content)
    body[:] = elements
    # print etree.tostring(html, pretty_print=True), '\n'

def normalize_layout(layout):
    normalize_width(layout)

def normalize_header(header):
    pass

def normalize_footer(footer):
    pass

def normalize_sidebar(sidebar):
    normalize_width(sidebar)

def normalize_content(content):
    elements = []
    column = None
    text = content.text
    if text and not text.isspace():
        column = content.makeelement('column')
        column.text = text
        content.text = None
        elements.append(column)
    for x in content:
        if x.tag == 'column':
            if column is not None: normalize_column(column)
            column = None
            elements.append(x)
            tail = x.tail
            normalize_column(x)
            if tail and not tail.isspace():
                column = content.makeelement('column')
                column.text = tail
                x.tail = None
                elements.append(column)
        else:
            if column is None:
                column = content.makeelement('column')
                elements.append(column)
            column.append(x)
    if column is not None: normalize_column(column)
    content[:] = elements

def normalize_column(column):
    elements = []
    p = None
    text = column.text
    if text and not text.isspace():
        p = column.makeelement('p')
        p.text = text
        column.text = None
        elements.append(p)
    for x in column:
        if x.tag in block_level_tags:
            p = None
            elements.append(x)
            tail = x.tail
            if tail and not tail.isspace():
                p = column.makeelement('p')
                p.text = tail
                x.tail = None
                elements.append(p)
        else:
            if p is None:
                p = column.makeelement('p')
                elements.append(p)
            p.append(x)
    column[:] = elements

def normalize_width(element):
    width = element.get('width')
    if width is None or width[-2:] != 'px': return
    try: number = int(width[:-2])
    except ValueError: return
    element.set('width', width[:-2])

layout_tags = dict(
    layout=normalize_layout,
    header=normalize_header,
    footer=normalize_footer,
    sidebar=normalize_sidebar,
    content=normalize_content,
    column=normalize_column,
    )
layout_tags_xpath = etree.XPath('|'.join(layout_tags))

block_level_tags = set('''
    address blockquote center dir div dl fieldset form h1 h2 h3 h4 h5 h6 hr
    isindex menu noframes noscript ol p pre table ul'''.split())

def _unnest(x):
    parent = x.getparent()
    tail = parent.tail
    if tail and not tail.isspace():
        last = parent[-1]
        last.tail = (last.tail or '') + tail
        parent.tail = None
    parent2 = parent.getparent()
    i = parent.index(x)
    j = parent2.index(parent)
    parent2[j+1:j+1] = parent[i:]

def unnest_p(p):
    i = 0
    while i < len(p):
        x = p[i]
        if x.tag in layout_tags: _unnest(x)
        i += 1

def unnest_layout(layout):
    if len(layout): _unnest(layout[0])
    text = layout.text
    if text and not text.isspace():
        layout.text = None
        layout.tail = text

def unnest_content(content):
    i = 0
    while i < len(content):
        x = content[i]
        tag = x.tag
        if tag == 'p': unnest_p(x)
        elif tag == 'column': unnest_element(x)
        elif tag in layout_tags: _unnest(x)
        i += 1

def unnest_element(element):
    i = 0
    while i < len(element):
        x = element[i]
        tag = x.tag
        if tag == 'p': unnest_p(x)
        elif tag in layout_tags: _unnest(x)
        i += 1

xslt_filename = os.path.join(os.path.dirname(__file__), 'transform.xslt')
xslt = etree.XSLT(etree.parse(xslt_filename))

def transform(xml):
##    f = file('test.xml', 'w')
##    xml.getroottree().write(f)
##    f.close()
    return xslt(xml)

xml2html_template = """
<xsl:stylesheet version = '1.0'
                xmlns:xsl='http://www.w3.org/1999/XSL/Transform'>
  <xsl:output method="html" encoding="%s" indent="%s" %s/>
  <xsl:template match="/ | @* | node()">
    <xsl:copy>
      <xsl:apply-templates select="@* | node()" />
    </xsl:copy>
  </xsl:template>
</xsl:stylesheet>
"""

doctypes = dict(
    strict=['-//W3C//DTD HTML 4.01//EN',
            'http://www.w3.org/TR/html4/strict.dtd'],
    transitional=['-//W3C//DTD HTML 4.01 Transitional//EN',
                  'http://www.w3.org/TR/1999/REC-html401-19991224/loose.dtd'],
    frameset=['-//W3C//DTD HTML 4.01 Frameset//EN',
              'http://www.w3.org/TR/1999/REC-html401-19991224/frameset.dtd'],
    )
doctypes['loose'] = doctypes['transitional']

xml2html_cache = {}

def xml2html(xml, encoding='UTF-8', doctype='transitional', indent=True):
    encoding = encoding.upper().replace('_', '-')
    xslt = xml2html_cache.get((encoding, doctype, indent))
    if xslt is None:
        if len(doctype) == 2: pyblic, system = doctype
        else: public, system = doctypes.get(doctype, ['', ''])
        if public: public = 'doctype-public="%s"' % public
        if system: system = 'doctype-system="%s"' % system
        text = xml2html_template % (
            encoding, indent and 'yes' or 'no', ' '.join((public, system)))
        xslt = etree.XSLT(etree.XML(text))
        xml2html_cache[(encoding, doctype, indent)] = xslt
    result = xslt(xml)
    return (str(result).replace('<!--start of IE hack-->', '<!--[if IE]>')
                       .replace('\n<!--end of IE hack-->', '<![endif]-->\n')
                       .replace('<!--end of IE hack-->', '<![endif]-->\n'))
<commit_msg>Fixed bug in HTML normalization<commit_after>import os.path
from operator import attrgetter
from itertools import groupby

from lxml import etree
from lxml.etree import SubElement

def html2xml(x, encoding='ascii'):
    if hasattr(x, 'wirite_c14n'): return x
    if not isinstance(x, basestring):
        if hasattr(x, '__unicode__'): x = unicode(x)
        else: x = str(x)
    if isinstance(x, str): x = unicode(x, encoding)
    return etree.HTML(x)

def normalize(html):
    # print etree.tostring(html, pretty_print=True), '\n'
    head = html.find('head')
    body = html.find('body')
    if body is None: body = SubElement(html, 'body')
    if head is None:
        head = html.makeelement('head')
        html.insert(-1, head)
    head_list = layout_tags_xpath(head)
    body[0:0] = head_list
    if head_list: pass
    elif layout_tags_xpath(body): pass
    else:
        for p in body.findall('p'):
            if layout_tags_xpath(p): break
        else: return
    i = 0
    while i < len(body):
        x = body[i]
        tag = x.tag
        if tag == 'p': unnest_p(x)
        elif tag in layout_tags:
            if tag == 'layout': unnest_layout(x)
            elif tag == 'content': unnest_content(x)
            else: unnest_element(x)
        i += 1
    # print etree.tostring(html, pretty_print=True), '\n'
    elements = []
    content = None
    text = body.text
    if text and not text.isspace():
        content = body.makeelement('content')
        content.text = text
        body.text = None
        elements.append(content)
    for x in body:
        tag = x.tag
        if tag in layout_tags and tag != 'column':
            if content is not None: normalize_content(content)
            content = None
            elements.append(x)
            tail = x.tail
            layout_tags[tag](x)
            if tail and not tail.isspace():
                content = body.makeelement('content')
                content.text = tail
                x.tail = None
                elements.append(content)
        else:
            if content is None:
                content = body.makeelement('content')
                elements.append(content)
            content.append(x)
    if content is not None: normalize_content(content)
    body[:] = elements
    # print etree.tostring(html, pretty_print=True), '\n'

def normalize_layout(layout):
    normalize_width(layout)

def normalize_header(header):
    pass

def normalize_footer(footer):
    pass

def normalize_sidebar(sidebar):
    normalize_width(sidebar)

def normalize_content(content):
    elements = []
    column = None
    text = content.text
    if text and not text.isspace():
        column = content.makeelement('column')
        column.text = text
        content.text = None
        elements.append(column)
    for x in content:
        if x.tag == 'column':
            if column is not None: normalize_column(column)
            column = None
            elements.append(x)
            tail = x.tail
            normalize_column(x)
            if tail and not tail.isspace():
                column = content.makeelement('column')
                column.text = tail
                x.tail = None
                elements.append(column)
        else:
            if column is None:
                column = content.makeelement('column')
                elements.append(column)
            column.append(x)
    if column is not None: normalize_column(column)
    content[:] = elements

def normalize_column(column):
    elements = []
    p = None
    text = column.text
    if text and not text.isspace():
        p = column.makeelement('p')
        p.text = text
        column.text = None
        elements.append(p)
    for x in column:
        if x.tag in block_level_tags:
            p = None
            elements.append(x)
            tail = x.tail
            if tail and not tail.isspace():
                p = column.makeelement('p')
                p.text = tail
                x.tail = None
                elements.append(p)
        else:
            if p is None:
                p = column.makeelement('p')
                elements.append(p)
            p.append(x)
    column[:] = elements

def normalize_width(element):
    width = element.get('width')
    if width is None or width[-2:] != 'px': return
    try: number = int(width[:-2])
    except ValueError: return
    element.set('width', width[:-2])

layout_tags = dict(
    layout=normalize_layout,
    header=normalize_header,
    footer=normalize_footer,
    sidebar=normalize_sidebar,
    content=normalize_content,
    column=normalize_column,
    )
layout_tags_xpath = etree.XPath('|'.join(layout_tags))

block_level_tags = set('''
    address blockquote center dir div dl fieldset form h1 h2 h3 h4 h5 h6 hr
    isindex menu noframes noscript ol p pre table ul'''.split())

def _unnest(x):
    parent = x.getparent()
    tail = parent.tail
    if tail and not tail.isspace():
        last = parent[-1]
        last.tail = (last.tail or '') + tail
        parent.tail = None
    parent2 = parent.getparent()
    i = parent.index(x)
    j = parent2.index(parent)
    parent2[j+1:j+1] = parent[i:]

def unnest_p(p):
    i = 0
    while i < len(p):
        x = p[i]
        if x.tag in layout_tags: _unnest(x)
        i += 1

def unnest_layout(layout):
    if len(layout): _unnest(layout[0])
    text = layout.text
    if text and not text.isspace():
        layout.text = None
        layout.tail = text

def unnest_content(content):
    i = 0
    while i < len(content):
        x = content[i]
        tag = x.tag
        if tag == 'p': unnest_p(x)
        elif tag == 'column': unnest_element(x)
        elif tag in layout_tags: _unnest(x)
        i += 1

def unnest_element(element):
    i = 0
    while i < len(element):
        x = element[i]
        tag = x.tag
        if tag == 'p': unnest_p(x)
        elif tag in layout_tags: _unnest(x)
        i += 1

xslt_filename = os.path.join(os.path.dirname(__file__), 'transform.xslt')
xslt = etree.XSLT(etree.parse(xslt_filename))

def transform(xml):
##    f = file('test.xml', 'w')
##    xml.getroottree().write(f)
##    f.close()
    return xslt(xml)

xml2html_template = """
<xsl:stylesheet version = '1.0'
                xmlns:xsl='http://www.w3.org/1999/XSL/Transform'>
  <xsl:output method="html" encoding="%s" indent="%s" %s/>
  <xsl:template match="/ | @* | node()">
    <xsl:copy>
      <xsl:apply-templates select="@* | node()" />
    </xsl:copy>
  </xsl:template>
</xsl:stylesheet>
"""

doctypes = dict(
    strict=['-//W3C//DTD HTML 4.01//EN',
            'http://www.w3.org/TR/html4/strict.dtd'],
    transitional=['-//W3C//DTD HTML 4.01 Transitional//EN',
                  'http://www.w3.org/TR/1999/REC-html401-19991224/loose.dtd'],
    frameset=['-//W3C//DTD HTML 4.01 Frameset//EN',
              'http://www.w3.org/TR/1999/REC-html401-19991224/frameset.dtd'],
    )
doctypes['loose'] = doctypes['transitional']

xml2html_cache = {}

def xml2html(xml, encoding='UTF-8', doctype='transitional', indent=True):
    encoding = encoding.upper().replace('_', '-')
    xslt = xml2html_cache.get((encoding, doctype, indent))
    if xslt is None:
        if len(doctype) == 2: pyblic, system = doctype
        else: public, system = doctypes.get(doctype, ['', ''])
        if public: public = 'doctype-public="%s"' % public
        if system: system = 'doctype-system="%s"' % system
        text = xml2html_template % (
            encoding, indent and 'yes' or 'no', ' '.join((public, system)))
        xslt = etree.XSLT(etree.XML(text))
        xml2html_cache[(encoding, doctype, indent)] = xslt
    result = xslt(xml)
    return (str(result).replace('<!--start of IE hack-->', '<!--[if IE]>')
                       .replace('\n<!--end of IE hack-->', '<![endif]-->\n')
                       .replace('<!--end of IE hack-->', '<![endif]-->\n'))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# coding: utf8
# encoding=utf8
from vapory import *
from moviepy.editor import VideoClip
import json
import tweepy
import numpy as np
import math, os, time, sys
from math import pi
from credentials import *
from convertBytes import convert_bytes

#sys.setdefaultencoding('utf8')

def backend(len, dep):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    #os.system('stack exec create ' + str(len) + ' ' + str(dep) + ' > data')
    # data = os.popen('stack exec create ' + str(len) + ' ' + str(dep)).read()
    os.chdir('../python')
    return json.loads(data)

def nameToJSON(name):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    HSdata = os.popen("stack exec create n '" + str(name) + "'").read()
    print("HS Data: " + str(HSdata))
    if(HSdata == ""):
        print("Fehlerhafter Name")
        return False
    os.chdir('../python')
    try:
        return json.loads(HSdata)
    except TypeError:
        print("Fehlerhafte JSON")
        return False

#data = backend(5, 2)

lange = 3 #sek. Länge der gif !!! must be int !!!
obj = []
x_cam = []
y_cam = []
z_cam = []
x_max = [0]
x_min = [0]
y_max = [0]
y_min = [0]

def make_scene(t, center):
    #camera = Camera( 'location', [t*50,20,-((x_max[0]+y_max[0])/1.25)], 'look_at', [x_max[0]/2,0,0])
    t = int(t*25)
    camera = Camera( 'location', [x_cam[t],y_cam[t],z_cam[t]], 'look_at', center)
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)
    return make_scene(t, center).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def generate_circle(center=(0,0,0), r=50, n=100):
    return [
        (
            center[0] + (math.cos(2 * pi / n * x) * r),  # x
            center[1] + (math.sin(2 * pi / n * x) * r),  # y
            center[2] + (math.cos(2 * pi / n * x) * r)   # z

        ) for x in range(0, n + 1)]

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            if(x_new > x_max[0]):
                x_max[0] = x_new
            if(x_new < x_min[0]):
                x_min[0] = x_new
            if(y_new > y_max[0]):
                y_max[0] = y_new
            if(y_new < y_min[0]):
                y_min[0] = y_new
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

def plot(data, animation):
    print("Data: " + str(data))
    render(data, 15, 10, -1, 0, 0, 0)
    print("X max: " + str(x_max[0]) + ", Y max: " + str(y_max[0]))
    print("Kreispunkte: " + str(lange*25+1))
    center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)

    print("Kreismitte: " + str(center))

    obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
    obj.append(Background("color", [1,1,1]))

    if(animation):
        points = generate_circle(center=center, r=(x_max[0]+y_max[0]), n=lange*25+1)
        for x,y,z in points:
            x_cam.append(x)
            y_cam.append(z)
            z_cam.append(y)

        VideoClip(make_frame, duration=lange).write_gif("animation.gif",fps=25)
        size = convert_bytes(os.stat("animation.gif").st_size)

    else:
        camera = Camera( 'location', [0,0,-50], 'look_at', center)
        scene = Scene( camera, objects= obj, included=["glass.inc"])
        scene.render("image.png", width=1024, height=512)
        size = convert_bytes(os.stat("image.png").st_size)

    print("Filesize: " + size)

running = True

if __name__ == "__main__":

    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth)

    while running:
        print("Listening Tweets...")
        tweets = api.search(q="@AlkanPlotter")
        for tweet in tweets:
            if "@AlkanPlotter" in tweet.text:
                user = tweet.user.screen_name
                #Entferne Username und Leerzeichen
                name = tweet.text.replace("@AlkanPlotter", "").replace(" ", "")
                print("Name von Twitter: " + str(name))
                data = nameToJSON(name)
                if(data):
                    plot(data)
                    tweet = api.update_with_media("animation.gif", "@" + user + " " + name, in_reply_to_status_id = tweet.id)
                    running = False
                else:
                    print("Tweet ungultig!")
                    running = False
<commit_msg>Added static images<commit_after>#!/usr/bin/env python3
# coding: utf8
# encoding=utf8
from vapory import *
import json
import numpy as np
import math, os, time, sys
from math import pi

from convertBytes import convert_bytes

#sys.setdefaultencoding('utf8')

def backend(len, dep):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    #os.system('stack exec create ' + str(len) + ' ' + str(dep) + ' > data')
    # data = os.popen('stack exec create ' + str(len) + ' ' + str(dep)).read()
    os.chdir('../python')
    return json.loads(data)

def nameToJSON(name):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    HSdata = os.popen("stack exec create n '" + str(name) + "'").read()
    print("HS Data: " + str(HSdata))
    if(HSdata == ""):
        print("Fehlerhafter Name")
        return False
    os.chdir('../python')
    try:
        return json.loads(HSdata)
    except TypeError:
        print("Fehlerhafte JSON")
        return False

#data = backend(5, 2)

lange = 3 #sek. Länge der gif !!! must be int !!!
obj = []
x_cam = []
y_cam = []
z_cam = []
x_max = [0]
x_min = [0]
y_max = [0]
y_min = [0]

def make_scene(t, center):
    #camera = Camera( 'location', [t*50,20,-((x_max[0]+y_max[0])/1.25)], 'look_at', [x_max[0]/2,0,0])
    t = int(t*25)
    camera = Camera( 'location', [x_cam[t],y_cam[t],z_cam[t]], 'look_at', center)
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)
    return make_scene(t, center).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def generate_circle(center=(0,0,0), r=50, n=100):
    return [
        (
            center[0] + (math.cos(2 * pi / n * x) * r),  # x
            center[1] + (math.sin(2 * pi / n * x) * r),  # y
            center[2] + (math.cos(2 * pi / n * x) * r)   # z

        ) for x in range(0, n + 1)]

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            if(x_new > x_max[0]):
                x_max[0] = x_new
            if(x_new < x_min[0]):
                x_min[0] = x_new
            if(y_new > y_max[0]):
                y_max[0] = y_new
            if(y_new < y_min[0]):
                y_min[0] = y_new
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

def plot(data, animation):
    print("Data: " + str(data))
    render(data, 15, 10, -1, 0, 0, 0)
    print("X max: " + str(x_max[0]) + ", Y max: " + str(y_max[0]))
    print("Kreispunkte: " + str(lange*25+1))
    center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)

    print("Kreismitte: " + str(center))

    obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
    obj.append(Background("color", [1,1,1]))

    if(animation):
        from moviepy.editor import VideoClip
        points = generate_circle(center=center, r=(x_max[0]+y_max[0]), n=lange*25+1)
        for x,y,z in points:
            x_cam.append(x)
            y_cam.append(z)
            z_cam.append(y)

        VideoClip(make_frame, duration=lange).write_gif("animation.gif",fps=25)
        size = convert_bytes(os.stat("animation.gif").st_size)

    else:
        camera = Camera( 'location', [0,0,-50], 'look_at', center)
        scene = Scene( camera, objects= obj, included=["glass.inc"])
        scene.render("image.png", width=1024, height=512)
        size = convert_bytes(os.stat("image.png").st_size)

    print("Filesize: " + size)

running = True

if __name__ == "__main__":

    from credentials import *
    import tweepy

    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth)

    while running:
        print("Listening Tweets...")
        tweets = api.search(q="@AlkanPlotter")
        for tweet in tweets:
            if "@AlkanPlotter" in tweet.text:
                user = tweet.user.screen_name
                #Entferne Username und Leerzeichen
                name = tweet.text.replace("@AlkanPlotter", "").replace(" ", "")
                print("Name von Twitter: " + str(name))
                data = nameToJSON(name)
                if(data):
                    plot(data)
                    tweet = api.update_with_media("animation.gif", "@" + user + " " + name, in_reply_to_status_id = tweet.id)
                    running = False
                else:
                    print("Tweet ungultig!")
                    running = False
<|endoftext|>


<commit_before>#!/usr/bin/env python
# mammon - a useless ircd
#
# Copyright (c) 2015, William Pitcock <nenolod@dereferenced.org>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

from mammon.events import eventmgr_core, eventmgr_rfc1459
from mammon.capability import Capability, caplist

import base64
import binascii

valid_mechanisms = ['PLAIN']

cap_sasl = Capability('sasl', value=','.join(valid_mechanisms))

@eventmgr_core.handler('server start')
def m_sasl_start(info):
    ctx = info['server']
    if not ctx.hashing.enabled:
        ctx.logger.info('SASL PLAIN disabled because hashing is not available')
        valid_mechanisms.remove('PLAIN')
    if len(valid_mechanisms) == 0:
        ctx.logger.info('SASL disabled because no mechanisms are available')
        del caplist['sasl']

@eventmgr_rfc1459.message('AUTHENTICATE', min_params=1, allow_unregistered=True)
def m_AUTHENTICATE(cli, ev_msg):
    if len(ev_msg['params']) == 1 and ev_msg['params'][0] == '*':
        if getattr(cli, 'sasl', None):
            cli.dump_numeric('906', params=['SASL authentication aborted'])
            cli.sasl = None
        else:
            cli.dump_numeric('904', params=['SASL authentication failed'])
        return

    if getattr(cli, 'sasl', None):
        if len(ev_msg['params'][0]) > 400:
            cli.dump_numeric('905', params=['SASL message too long'])
            cli.sasl = None
            return

        try:
            data = base64.b64decode(ev_msg['params'][0])
        except binascii.Error:
            cli.dump_numeric('904', params=['SASL authentication failed'])
            return

        eventmgr_core.dispatch('sasl authenticate {}'.format(cli.sasl.casefold()), {
            'source': cli,
            'mechanism': cli.sasl,
            'data': data,
        })

    else:
        mechanism = ev_msg['params'][0].upper()
        if mechanism in valid_mechanisms:
            cli.sasl = mechanism
            cli.dump_verb('AUTHENTICATE', '+')
        else:
            cli.dump_numeric('904', params=['SASL authentication failed'])
            return

@eventmgr_core.handler('client registered')
def m_sasl_unreglocked(info):
    cli = info['client']
    if getattr(cli, 'sasl', None):
        cli.sasl = None
        cli.dump_numeric('906', params=['SASL authentication aborted'])

@eventmgr_core.handler('sasl authenticate plain')
def m_sasl_plain(info):
    cli = info['source']
    data = info['data']

    account, authorization_id, passphrase = data.split(b'\x00')
    account = str(account, 'utf8')
    passphrase = str(passphrase, 'utf8')

    account_info = cli.ctx.data.get('account.{}'.format(account), None)
    if (account_info and 'passphrase' in account_info['credentials'] and
            account_info['verified']):
        passphrase_hash = account_info['credentials']['passphrase']
        if cli.ctx.hashing.verify(passphrase, passphrase_hash):
            cli.account = account
            hostmask = cli.hostmask
            if hostmask is None:
                hostmask = '*'
            cli.dump_numeric('900', params=[hostmask, account, 'You are now logged in as {}'.format(account)])
            cli.dump_numeric('903', params=['SASL authentication successful'])
            return
    cli.dump_numeric('904', params=['SASL authentication failed'])
<commit_msg>mammon.ext.ircv3.sasl: Fix SASL auth and user registration<commit_after>#!/usr/bin/env python
# mammon - a useless ircd
#
# Copyright (c) 2015, William Pitcock <nenolod@dereferenced.org>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

from mammon.events import eventmgr_core, eventmgr_rfc1459
from mammon.capability import Capability, caplist

import base64
import binascii

valid_mechanisms = ['PLAIN']

cap_sasl = Capability('sasl', value=','.join(valid_mechanisms))

@eventmgr_core.handler('server start')
def m_sasl_start(info):
    ctx = info['server']
    if not ctx.hashing.enabled:
        ctx.logger.info('SASL PLAIN disabled because hashing is not available')
        valid_mechanisms.remove('PLAIN')
    if len(valid_mechanisms) == 0:
        ctx.logger.info('SASL disabled because no mechanisms are available')
        del caplist['sasl']

@eventmgr_rfc1459.message('AUTHENTICATE', min_params=1, allow_unregistered=True)
def m_AUTHENTICATE(cli, ev_msg):
    if len(ev_msg['params']) == 1 and ev_msg['params'][0] == '*':
        if getattr(cli, 'sasl', None):
            cli.dump_numeric('906', params=['SASL authentication aborted'])
            cli.sasl = None
        else:
            cli.dump_numeric('904', params=['SASL authentication failed'])
        return

    if getattr(cli, 'sasl', None):
        if len(ev_msg['params'][0]) > 400:
            cli.dump_numeric('905', params=['SASL message too long'])
            cli.sasl = None
            return

        try:
            data = base64.b64decode(ev_msg['params'][0])
        except binascii.Error:
            cli.dump_numeric('904', params=['SASL authentication failed'])
            return

        eventmgr_core.dispatch('sasl authenticate {}'.format(cli.sasl.casefold()), {
            'source': cli,
            'mechanism': cli.sasl,
            'data': data,
        })

    else:
        mechanism = ev_msg['params'][0].upper()
        if mechanism in valid_mechanisms:
            cli.sasl = mechanism
            cli.dump_verb('AUTHENTICATE', '+')
        else:
            cli.dump_numeric('904', params=['SASL authentication failed'])
            return

@eventmgr_core.handler('client registered')
def m_sasl_unreglocked(info):
    cli = info['client']
    if getattr(cli, 'sasl', None):
        cli.sasl = None
        cli.dump_numeric('906', params=['SASL authentication aborted'])

@eventmgr_core.handler('sasl authenticate plain')
def m_sasl_plain(info):
    cli = info['source']
    data = info['data']

    account, authorization_id, passphrase = data.split(b'\x00')
    account = str(account, 'utf8')
    passphrase = str(passphrase, 'utf8')

    account_info = cli.ctx.data.get('account.{}'.format(account), None)
    if (account_info and 'passphrase' in account_info['credentials'] and
            account_info['verified']):
        passphrase_hash = account_info['credentials']['passphrase']
        if cli.ctx.hashing.verify(passphrase, passphrase_hash):
            cli.account = account
            cli.sasl = None
            hostmask = cli.hostmask
            if hostmask is None:
                hostmask = '*'
            cli.dump_numeric('900', params=[hostmask, account, 'You are now logged in as {}'.format(account)])
            cli.dump_numeric('903', params=['SASL authentication successful'])
            return
    cli.dump_numeric('904', params=['SASL authentication failed'])
<|endoftext|>


<commit_before># -*- coding: UTF-8
#
#   wbtip
#   *****
#
#   Contains all the logic for handling tip related operations, managed by
#   the whistleblower, handled and executed within /wbtip/* URI PATH interaction.

from twisted.internet.defer import inlineCallbacks

from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.authentication import transport_security_check, authenticated
from globaleaks.rest import requests
from globaleaks.utils.utility import log, pretty_date_time
from globaleaks.utils.structures import Rosetta
from globaleaks.settings import transact, transact_ro, GLSetting
from globaleaks.models import WhistleblowerTip, Comment, Message, ReceiverTip
from globaleaks.rest import errors

def wb_serialize_tip(internaltip, language=GLSetting.memory_copy.default_language):
    itip_dict = {
        'context_gus': unicode(internaltip.context.id),
        'creation_date' : unicode(pretty_date_time(internaltip.creation_date)),
        'last_activity' : unicode(pretty_date_time(internaltip.creation_date)),
        'expiration_date' : unicode(pretty_date_time(internaltip.expiration_date)),
        'download_limit' : int(internaltip.download_limit),
        'access_limit' : int(internaltip.access_limit),
        'mark' : unicode(internaltip.mark),
        'pertinence' : unicode(internaltip.pertinence_counter),
        'escalation_threshold' : unicode(internaltip.escalation_threshold),
        'fields' : dict(internaltip.wb_fields),
    }

    # context_name and context_description are localized field
    mo = Rosetta()
    mo.acquire_storm_object(internaltip.context)
    for attr in ['name', 'description' ]:
        key = "context_%s" % attr
        itip_dict[key] = mo.dump_translated(attr, language)

    return itip_dict

def wb_serialize_file(internalfile):
    wb_file_desc = {
        'name' : unicode(internalfile.name),
        'sha2sum' : unicode(internalfile.sha2sum),
        'content_type' : unicode(internalfile.content_type),
        'creation_date' : unicode(pretty_date_time(internalfile.creation_date)),
        'size': int(internalfile.size),
    }
    return wb_file_desc

@transact_ro
def get_files_wb(store, wb_tip_id):
    wbtip = store.find(WhistleblowerTip, WhistleblowerTip.id == unicode(wb_tip_id)).one()

    file_list = []
    for internalfile in wbtip.internaltip.internalfiles:
        file_list.append(wb_serialize_file(internalfile))

    file_list.reverse()
    return file_list

@transact_ro
def get_internaltip_wb(store, tip_id, language=GLSetting.memory_copy.default_language):
    wbtip = store.find(WhistleblowerTip, WhistleblowerTip.id == unicode(tip_id)).one()

    if not wbtip:
        raise errors.TipReceiptNotFound

    tip_desc = wb_serialize_tip(wbtip.internaltip)

    # two elements from WhistleblowerTip
    tip_desc['access_counter'] = int(wbtip.access_counter)
    tip_desc['id'] = unicode(wbtip.id)

    return tip_desc

class WbTipInstance(BaseHandler):
    """
    WT1
    This interface expose the Whistleblower Tip.

    Tip is the safe area, created with an expiration time, where Receivers (and optionally)
    Whistleblower can discuss about the submission, comments, collaborative voting, forward,
    promote, and perform other operation in this closed environment.
    In the request is present an unique token, aim to authenticate the users accessing to the
    resource, and giving accountability in resource accesses.
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self):
        """
        Parameters: None
        Response: actorsTipDesc
        Errors: InvalidTipAuthToken

        Check the user id (in the whistleblower case, is authenticated and
        contain the internaltip)
        """

        answer = yield get_internaltip_wb(self.current_user['user_id'], self.request.language)
        answer['files'] = yield get_files_wb(self.current_user['user_id'])

        self.set_status(200)
        self.finish(answer)

def wb_serialize_comment(comment):
    comment_desc = {
        'comment_id' : unicode(comment.id),
        'type' : unicode(comment.type),
        'content' : unicode(comment.content),
        'system_content' : comment.system_content if comment.system_content else {},
        'author' : unicode(comment.author),
        'creation_date' : unicode(pretty_date_time(comment.creation_date))
    }

    return comment_desc

@transact_ro
def get_comment_list_wb(store, wb_tip_id):
    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        raise errors.TipReceiptNotFound

    comment_list = []
    for comment in wb_tip.internaltip.comments:
        comment_list.append(wb_serialize_comment(comment))

    return comment_list

@transact
def create_comment_wb(store, wb_tip_id, request):
    wbtip = store.find(WhistleblowerTip,
                       WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wbtip:
        raise errors.TipReceiptNotFound

    comment = Comment()
    comment.content = request['content']
    comment.internaltip_id = wbtip.internaltip.id
    comment.author = u'whistleblower' # The printed line
    comment.type = Comment._types[1] # WB
    comment.mark = Comment._marker[0] # Not notified

    wbtip.internaltip.comments.add(comment)

    return wb_serialize_comment(comment)

class WbTipCommentCollection(BaseHandler):
    """
    T2
    Interface use to read/write comments inside of a Tip, is not implemented as CRUD because we've not
    needs, at the moment, to delete/update comments once has been published. Comments is intended, now,
    as a stone written consideration about Tip reliability, therefore no editing and rethinking is
    permitted.
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self, *uriargs):
        """
        Parameters: None
        Response: actorsCommentList
        Errors: InvalidTipAuthToken
        """
        wb_comment_list = yield get_comment_list_wb(self.current_user['user_id'])

        self.set_status(200)
        self.finish(wb_comment_list)

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def post(self, *uriargs):
        """
        Request: actorsCommentDesc
        Response: actorsCommentDesc
        Errors: InvalidTipAuthToken, InvalidInputFormat, TipGusNotFound, TipReceiptNotFound
        """

        request = self.validate_message(self.request.body, requests.actorsCommentDesc)
        answer = yield create_comment_wb(self.current_user['user_id'], request)

        self.set_status(201) # Created
        self.finish(answer)

@transact_ro
def get_receiver_list_wb(store, wb_tip_id, language=GLSetting.memory_copy.default_language):
    """
    @return:
        This function contain the serialization of the receiver, this function is
        used only by /wbtip/receivers API

        The returned struct contain information on read/unread messages
    """

    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        raise errors.TipReceiptNotFound

    # this part has been refactored in an hackathon
    # the second part, was the only in place when we want to show only the
    # REAL receivers involved.
    # But, this give a bad feedback for the missing of rcvr list,
    # when WB access after few seconds. then has been handled the case:

    if not wb_tip.internaltip.receivertips.count():
        receiver_list = []

        log.debug("Early access from the WB to the Tip (creation_date: %s),"\
                  " Receiver not yet present: fallback on receiver list" %
                  pretty_date_time(wb_tip.creation_date))

        for receiver in wb_tip.internaltip.receivers:

            # This is the reduced version of Receiver serialization
            receiver_desc = {
                "name": unicode(receiver.name),
                "receiver_gus": unicode(receiver.id),
                "tags": receiver.tags,
                "access_counter" : 0,
                "unread_messages" : 0,
                "read_messages" : 0,
                "your_messages" : 0,
                # XXX ReceiverTip last activity ?
            }

            mo = Rosetta()
            mo.acquire_storm_object(receiver)
            receiver_desc["description"] = mo.dump_translated("description", language)
            receiver_list.append(receiver_desc)

        return receiver_list
    else:
        receiver_list = []
        for rtip in wb_tip.internaltip.receivertips:

            your_messages = store.find(Message,
                                       Message.receivertip_id == rtip.id,
                                       Message.type == u'whistleblower').count()

            unread_messages = store.find(Message,
                                         Message.receivertip_id == rtip.id,
                                         Message.type == u'receiver',
                                         Message.visualized == False).count()

            read_messages = store.find(Message,
                                       Message.receivertip_id == rtip.id,
                                       Message.type == u'receiver',
                                       Message.visualized == True).count()

            # if you change something here, check also 20 lines before!
            receiver_desc = {
                "name": unicode(rtip.receiver.name),
                "receiver_gus": unicode(rtip.receiver.id),
                "tags": rtip.receiver.tags,
                "access_counter" : rtip.access_counter,
                "unread_messages" : unread_messages,
                "read_messages" : read_messages,
                "your_messages" : your_messages
                # XXX ReceiverTip last activity ?
            }

            mo = Rosetta()
            mo.acquire_storm_object(rtip.receiver)
            receiver_desc["description"] = mo.dump_translated("description", language)

            receiver_list.append(receiver_desc)

        return receiver_list

class WbTipReceiversCollection(BaseHandler):
    """
    T3
    This interface return the list of the Receiver active in a Tip.
    GET /tip/<auth_tip_id>/receivers
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self):
        """
        Parameters: None
        Response: actorsReceiverList
        Errors: InvalidTipAuthToken
        """
        answer = yield get_receiver_list_wb(self.current_user['user_id'], self.request.language)

        self.set_status(200)
        self.finish(answer)

## Direct messages handling function

def wb_serialize_message(msg):
    return {
        'id' : unicode(msg.id),
        'creation_date' : unicode(pretty_date_time(msg.creation_date)),
        'content' : unicode(msg.content),
        'visualized' : bool(msg.visualized),
        'type' : unicode(msg.type),
        'author' : unicode(msg.author),
        'mark' : unicode(msg.mark)
    }

@transact
def get_messages_content(store, wb_tip_id, receiver_id):
    """
    Get the messages content and mark all the unread
    messages as "read"
    """
    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        raise errors.TipReceiptNotFound

    rtip = store.find(ReceiverTip, ReceiverTip.internaltip_id == wb_tip.internaltip.id,
                      ReceiverTip.receiver_id == unicode(receiver_id)).one()

    if not rtip:
        raise errors.TipGusNotFound

    messages = store.find(Message, Message.receivertip_id == rtip.id)

    messages_list = []
    for msg in messages:
        messages_list.append(wb_serialize_message(msg))

        if not msg.visualized and msg.type == u'receiver':
            log.debug("Marking as readed message [%s] from %s" % (msg.content, msg.author))
            msg.visualized = True

    store.commit()
    return messages_list

@transact
def create_message_wb(store, wb_tip_id, receiver_id, request):

    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        log.err("Invalid wb_tip supply: %s" % wb_tip)
        raise errors.TipReceiptNotFound

    rtip = store.find(ReceiverTip, ReceiverTip.internaltip_id == wb_tip.internaltip.id,
                                   ReceiverTip.receiver_id == unicode(receiver_id)).one()

    if not rtip:
        log.err("No ReceiverTip found: receiver_id %s itip %s" %
                (receiver_id, wb_tip.internaltip.id))
        raise errors.TipGusNotFound

    msg = Message()
    msg.content = request['content']
    msg.receivertip_id = rtip.id
    msg.author = u'Whistleblower'
    msg.visualized = False

    # remind: is safest use this convention, and probably we've to
    # change in the whole code the usage of Model._type[ndx]
    msg.type = u'whistleblower'
    msg.mark = u'not notified'

    try:
        store.add(msg)
    except Exception as dberror:
        log.err("Unable to add WB message from %s: %s" % (rtip.receiver.name, dberror))
        raise dberror

    return wb_serialize_message(msg)

class WbMessageCollection(BaseHandler):
    """
    W5
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self, receiver_id):

        messages = yield get_messages_content(self.current_user['user_id'], receiver_id)

        self.set_status(200)
        self.finish(messages)

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def post(self, receiver_id):

        request = self.validate_message(self.request.body, requests.actorsCommentDesc)

        message = yield create_message_wb(self.current_user['user_id'], receiver_id, request)

        self.set_status(201) # Created
        self.finish(message)
<commit_msg>fixed a non-localized string in whistleblower tip<commit_after># -*- coding: UTF-8
#
#   wbtip
#   *****
#
#   Contains all the logic for handling tip related operations, managed by
#   the whistleblower, handled and executed within /wbtip/* URI PATH interaction.

from twisted.internet.defer import inlineCallbacks

from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.authentication import transport_security_check, authenticated
from globaleaks.rest import requests
from globaleaks.utils.utility import log, pretty_date_time
from globaleaks.utils.structures import Rosetta
from globaleaks.settings import transact, transact_ro, GLSetting
from globaleaks.models import WhistleblowerTip, Comment, Message, ReceiverTip
from globaleaks.rest import errors

def wb_serialize_tip(internaltip, language=GLSetting.memory_copy.default_language):
    itip_dict = {
        'context_gus': unicode(internaltip.context.id),
        'creation_date' : unicode(pretty_date_time(internaltip.creation_date)),
        'last_activity' : unicode(pretty_date_time(internaltip.creation_date)),
        'expiration_date' : unicode(pretty_date_time(internaltip.expiration_date)),
        'download_limit' : int(internaltip.download_limit),
        'access_limit' : int(internaltip.access_limit),
        'mark' : unicode(internaltip.mark),
        'pertinence' : unicode(internaltip.pertinence_counter),
        'escalation_threshold' : unicode(internaltip.escalation_threshold),
        'fields' : dict(internaltip.wb_fields),
    }

    # context_name and context_description are localized field
    mo = Rosetta()
    mo.acquire_storm_object(internaltip.context)
    for attr in ['name', 'description' ]:
        key = "context_%s" % attr
        itip_dict[key] = mo.dump_translated(attr, language)

    return itip_dict

def wb_serialize_file(internalfile):
    wb_file_desc = {
        'name' : unicode(internalfile.name),
        'sha2sum' : unicode(internalfile.sha2sum),
        'content_type' : unicode(internalfile.content_type),
        'creation_date' : unicode(pretty_date_time(internalfile.creation_date)),
        'size': int(internalfile.size),
    }
    return wb_file_desc

@transact_ro
def get_files_wb(store, wb_tip_id):
    wbtip = store.find(WhistleblowerTip, WhistleblowerTip.id == unicode(wb_tip_id)).one()

    file_list = []
    for internalfile in wbtip.internaltip.internalfiles:
        file_list.append(wb_serialize_file(internalfile))

    file_list.reverse()
    return file_list

@transact_ro
def get_internaltip_wb(store, tip_id, language=GLSetting.memory_copy.default_language):
    wbtip = store.find(WhistleblowerTip, WhistleblowerTip.id == unicode(tip_id)).one()

    if not wbtip:
        raise errors.TipReceiptNotFound

    tip_desc = wb_serialize_tip(wbtip.internaltip, language)

    # two elements from WhistleblowerTip
    tip_desc['access_counter'] = int(wbtip.access_counter)
    tip_desc['id'] = unicode(wbtip.id)

    return tip_desc

class WbTipInstance(BaseHandler):
    """
    WT1
    This interface expose the Whistleblower Tip.

    Tip is the safe area, created with an expiration time, where Receivers (and optionally)
    Whistleblower can discuss about the submission, comments, collaborative voting, forward,
    promote, and perform other operation in this closed environment.
    In the request is present an unique token, aim to authenticate the users accessing to the
    resource, and giving accountability in resource accesses.
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self):
        """
        Parameters: None
        Response: actorsTipDesc
        Errors: InvalidTipAuthToken

        Check the user id (in the whistleblower case, is authenticated and
        contain the internaltip)
        """

        answer = yield get_internaltip_wb(self.current_user['user_id'], self.request.language)
        answer['files'] = yield get_files_wb(self.current_user['user_id'])

        self.set_status(200)
        self.finish(answer)

def wb_serialize_comment(comment):
    comment_desc = {
        'comment_id' : unicode(comment.id),
        'type' : unicode(comment.type),
        'content' : unicode(comment.content),
        'system_content' : comment.system_content if comment.system_content else {},
        'author' : unicode(comment.author),
        'creation_date' : unicode(pretty_date_time(comment.creation_date))
    }

    return comment_desc

@transact_ro
def get_comment_list_wb(store, wb_tip_id):
    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        raise errors.TipReceiptNotFound

    comment_list = []
    for comment in wb_tip.internaltip.comments:
        comment_list.append(wb_serialize_comment(comment))

    return comment_list

@transact
def create_comment_wb(store, wb_tip_id, request):
    wbtip = store.find(WhistleblowerTip,
                       WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wbtip:
        raise errors.TipReceiptNotFound

    comment = Comment()
    comment.content = request['content']
    comment.internaltip_id = wbtip.internaltip.id
    comment.author = u'whistleblower' # The printed line
    comment.type = Comment._types[1] # WB
    comment.mark = Comment._marker[0] # Not notified

    wbtip.internaltip.comments.add(comment)

    return wb_serialize_comment(comment)

class WbTipCommentCollection(BaseHandler):
    """
    T2
    Interface use to read/write comments inside of a Tip, is not implemented as CRUD because we've not
    needs, at the moment, to delete/update comments once has been published. Comments is intended, now,
    as a stone written consideration about Tip reliability, therefore no editing and rethinking is
    permitted.
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self, *uriargs):
        """
        Parameters: None
        Response: actorsCommentList
        Errors: InvalidTipAuthToken
        """
        wb_comment_list = yield get_comment_list_wb(self.current_user['user_id'])

        self.set_status(200)
        self.finish(wb_comment_list)

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def post(self, *uriargs):
        """
        Request: actorsCommentDesc
        Response: actorsCommentDesc
        Errors: InvalidTipAuthToken, InvalidInputFormat, TipGusNotFound, TipReceiptNotFound
        """

        request = self.validate_message(self.request.body, requests.actorsCommentDesc)
        answer = yield create_comment_wb(self.current_user['user_id'], request)

        self.set_status(201) # Created
        self.finish(answer)

@transact_ro
def get_receiver_list_wb(store, wb_tip_id, language=GLSetting.memory_copy.default_language):
    """
    @return:
        This function contain the serialization of the receiver, this function is
        used only by /wbtip/receivers API

        The returned struct contain information on read/unread messages
    """

    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        raise errors.TipReceiptNotFound

    # this part has been refactored in an hackathon
    # the second part, was the only in place when we want to show only the
    # REAL receivers involved.
    # But, this give a bad feedback for the missing of rcvr list,
    # when WB access after few seconds. then has been handled the case:

    if not wb_tip.internaltip.receivertips.count():
        receiver_list = []

        log.debug("Early access from the WB to the Tip (creation_date: %s),"\
                  " Receiver not yet present: fallback on receiver list" %
                  pretty_date_time(wb_tip.creation_date))

        for receiver in wb_tip.internaltip.receivers:

            # This is the reduced version of Receiver serialization
            receiver_desc = {
                "name": unicode(receiver.name),
                "receiver_gus": unicode(receiver.id),
                "tags": receiver.tags,
                "access_counter" : 0,
                "unread_messages" : 0,
                "read_messages" : 0,
                "your_messages" : 0,
                # XXX ReceiverTip last activity ?
            }

            mo = Rosetta()
            mo.acquire_storm_object(receiver)
            receiver_desc["description"] = mo.dump_translated("description", language)
            receiver_list.append(receiver_desc)

        return receiver_list
    else:
        receiver_list = []
        for rtip in wb_tip.internaltip.receivertips:

            your_messages = store.find(Message,
                                       Message.receivertip_id == rtip.id,
                                       Message.type == u'whistleblower').count()

            unread_messages = store.find(Message,
                                         Message.receivertip_id == rtip.id,
                                         Message.type == u'receiver',
                                         Message.visualized == False).count()

            read_messages = store.find(Message,
                                       Message.receivertip_id == rtip.id,
                                       Message.type == u'receiver',
                                       Message.visualized == True).count()

            # if you change something here, check also 20 lines before!
            receiver_desc = {
                "name": unicode(rtip.receiver.name),
                "receiver_gus": unicode(rtip.receiver.id),
                "tags": rtip.receiver.tags,
                "access_counter" : rtip.access_counter,
                "unread_messages" : unread_messages,
                "read_messages" : read_messages,
                "your_messages" : your_messages
                # XXX ReceiverTip last activity ?
            }

            mo = Rosetta()
            mo.acquire_storm_object(rtip.receiver)
            receiver_desc["description"] = mo.dump_translated("description", language)

            receiver_list.append(receiver_desc)

        return receiver_list

class WbTipReceiversCollection(BaseHandler):
    """
    T3
    This interface return the list of the Receiver active in a Tip.
    GET /tip/<auth_tip_id>/receivers
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self):
        """
        Parameters: None
        Response: actorsReceiverList
        Errors: InvalidTipAuthToken
        """
        answer = yield get_receiver_list_wb(self.current_user['user_id'], self.request.language)

        self.set_status(200)
        self.finish(answer)

## Direct messages handling function

def wb_serialize_message(msg):
    return {
        'id' : unicode(msg.id),
        'creation_date' : unicode(pretty_date_time(msg.creation_date)),
        'content' : unicode(msg.content),
        'visualized' : bool(msg.visualized),
        'type' : unicode(msg.type),
        'author' : unicode(msg.author),
        'mark' : unicode(msg.mark)
    }

@transact
def get_messages_content(store, wb_tip_id, receiver_id):
    """
    Get the messages content and mark all the unread
    messages as "read"
    """
    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        raise errors.TipReceiptNotFound

    rtip = store.find(ReceiverTip, ReceiverTip.internaltip_id == wb_tip.internaltip.id,
                      ReceiverTip.receiver_id == unicode(receiver_id)).one()

    if not rtip:
        raise errors.TipGusNotFound

    messages = store.find(Message, Message.receivertip_id == rtip.id)

    messages_list = []
    for msg in messages:
        messages_list.append(wb_serialize_message(msg))

        if not msg.visualized and msg.type == u'receiver':
            log.debug("Marking as readed message [%s] from %s" % (msg.content, msg.author))
            msg.visualized = True

    store.commit()
    return messages_list

@transact
def create_message_wb(store, wb_tip_id, receiver_id, request):

    wb_tip = store.find(WhistleblowerTip,
                        WhistleblowerTip.id == unicode(wb_tip_id)).one()

    if not wb_tip:
        log.err("Invalid wb_tip supply: %s" % wb_tip)
        raise errors.TipReceiptNotFound

    rtip = store.find(ReceiverTip, ReceiverTip.internaltip_id == wb_tip.internaltip.id,
                                   ReceiverTip.receiver_id == unicode(receiver_id)).one()

    if not rtip:
        log.err("No ReceiverTip found: receiver_id %s itip %s" %
                (receiver_id, wb_tip.internaltip.id))
        raise errors.TipGusNotFound

    msg = Message()
    msg.content = request['content']
    msg.receivertip_id = rtip.id
    msg.author = u'Whistleblower'
    msg.visualized = False

    # remind: is safest use this convention, and probably we've to
    # change in the whole code the usage of Model._type[ndx]
    msg.type = u'whistleblower'
    msg.mark = u'not notified'

    try:
        store.add(msg)
    except Exception as dberror:
        log.err("Unable to add WB message from %s: %s" % (rtip.receiver.name, dberror))
        raise dberror

    return wb_serialize_message(msg)

class WbMessageCollection(BaseHandler):
    """
    W5
    """

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def get(self, receiver_id):

        messages = yield get_messages_content(self.current_user['user_id'], receiver_id)

        self.set_status(200)
        self.finish(messages)

    @transport_security_check('wb')
    @authenticated('wb')
    @inlineCallbacks
    def post(self, receiver_id):

        request = self.validate_message(self.request.body, requests.actorsCommentDesc)

        message = yield create_message_wb(self.current_user['user_id'], receiver_id, request)

        self.set_status(201) # Created
        self.finish(message)
<|endoftext|>