<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Author:
Description:
'''
from core.broadcast import say, bang
#from core.config.settings import logger

class Reaction:
    """class Reaction"""
    response = ''
    request = ''

    def __str__(self):
        return 'My new reaction'

    @classmethod
    def __init__(self, *args, **kwargs):
        """ original request string """
        #logger.info(args)
        #logger.info(kwargs)
        #logger.info(kwargs.get('req_obj'))

        #get request object
        self.req_obj = kwargs.pop('req_obj')

        #request word sequence
        self.request = self.req_obj.get('request', '')

        #request received from (julius, jabber any other resources)
        self.req_from = self.req_obj.get('from', '')

        self.response = ''

    @classmethod
    def run(self):
        """default method"""

        response = "Just type any information that you want me to keep."
        response += "type: *find wifi* to browse wifi passwords"
        response += "\ntype: *register hours* if you want me to save your working hours automatically."
        response += "\nYou can ask me to remind about something with command: *remind me every 50 minutes with message hey buddy don't forget to fill the project timesheet!*"
        #response += "\nRemind command format is: *remind me after|each|every|in|before|during numeric(1,2,3 .. 100)|words(one,two.. ten etc.) minutes|mins|hours|days|years with|by \"hey buddy don't forget to fill the project timesheet!\"*"
        #response += "\nYou can also search for a command: *find command ....(any command)*"
        #response += "\nMore information can be found at http://www.smarty-bot.com/commands"

        if self.req_from == 'jabber':
            todo = {'text': response, 'jmsg': response, 'type': 'response'}
            self.response = todo

        if self.req_from == 'julius':
            bang()
            todo = {'say': response, 'text': response, 'type': 'response'}
            self.response = say(self.request.replace('say', '').upper())

        return self.response
<commit_msg>help text edited, help planned to be replaced with a better help system<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Author:
Description:
'''
from core.broadcast import say, bang
#from core.config.settings import logger

class Reaction:
    """class Reaction"""
    response = ''
    request = ''

    def __str__(self):
        return 'My new reaction'

    @classmethod
    def __init__(self, *args, **kwargs):
        """ original request string """
        #logger.info(args)
        #logger.info(kwargs)
        #logger.info(kwargs.get('req_obj'))

        #get request object
        self.req_obj = kwargs.pop('req_obj')

        #request word sequence
        self.request = self.req_obj.get('request', '')

        #request received from (julius, jabber any other resources)
        self.req_from = self.req_obj.get('from', '')

        self.response = ''

    @classmethod
    def run(self):
        """default method"""

        response = "Just type any information that you want me to keep."
        response += "type: *find ....* if you want to search for text that was added by yourself previously"
        response += "\ntype: *register hours* if you want me to save your working hours automatically."
        response += "\nYou can ask me to remind about something with command: *remind me every 50 minutes with message hey buddy don't forget to fill the project timesheet!*"
        #response += "\nRemind command format is: *remind me after|each|every|in|before|during numeric(1,2,3 .. 100)|words(one,two.. ten etc.) minutes|mins|hours|days|years with|by \"hey buddy don't forget to fill the project timesheet!\"*"
        #response += "\nYou can also search for a command: *find command ....(any command)*"  not ready
        #response += "\nMore information can be found at http://www.smarty-bot.com/commands" not ready

        if self.req_from == 'jabber':
            todo = {'text': response, 'jmsg': response, 'type': 'response'}
            self.response = todo

        if self.req_from == 'julius':
            bang()
            todo = {'say': response, 'text': response, 'type': 'response'}
            self.response = say(self.request.replace('say', '').upper())

        return self.response
<|endoftext|>


<commit_before>from .probability_space import ProbabilitySpace, BoxModel, DeckOfCards
from .random_variables import RV
from .random_processes import RandomProcess
from .time_index import TimeIndex
from .distributions import (
    Bernoulli,
    Binomial,
    Hypergeometric,
    Geometric,
    NegativeBinomial,
    Pascal,
    Poisson,
    Uniform,
    Normal,
    Exponential,
    Gamma,
    Beta,
    MultivariateNormal,
    BivariateNormal
)
from .markov_chains import MarkovChain, ContinuousTimeMarkovChain
from .poisson_process import PoissonProcess
from .plot import figure, xlabel, ylabel, xlim, ylim, plot
from .math import *
<commit_msg>add chisquare, t, f<commit_after>from .probability_space import ProbabilitySpace, BoxModel, DeckOfCards
from .random_variables import RV
from .random_processes import RandomProcess
from .time_index import TimeIndex
from .distributions import (
    Bernoulli,
    Binomial,
    Hypergeometric,
    Geometric,
    NegativeBinomial,
    Pascal,
    Poisson,
    Uniform,
    Normal,
    Exponential,
    Gamma,
    Beta,
    StudentT,
    ChiSquare,
    F,
    MultivariateNormal,
    BivariateNormal
)
from .markov_chains import MarkovChain, ContinuousTimeMarkovChain
from .poisson_process import PoissonProcess
from .plot import figure, xlabel, ylabel, xlim, ylim, plot
from .math import *
<|endoftext|>


<commit_before>import os
USE_SYMENGINE = os.getenv('USE_SYMENGINE')

if USE_SYMENGINE:
    from symengine.sympy_compat import (Symbol, Integer, sympify, S,
        SympifyError, exp, log, gamma, sqrt, I, E, pi, Matrix,
        sin, cos, tan, cot, csc, sec, asin, acos, atan, acot, acsc, asec,
        sinh, cosh, tanh, coth, asinh, acosh, atanh, acoth,
        lambdify, symarray, diff, zeros, eye, diag, ones, zeros,
        expand, Function, symbols, var, Add, Mul, Derivative)
    from symengine.sympy_compat import AppliedUndef
    #TODO: Fix this
    from symengine import Matrix as ImmutableMatrix
else:
    from sympy import (Symbol, Integer, sympify, S,
        SympifyError, exp, log, gamma, sqrt, I, E, pi, Matrix,
        sin, cos, tan, cot, csc, sec, asin, acos, atan, acot, acsc, asec,
        sinh, cosh, tanh, coth, asinh, acosh, atanh, acoth,
        lambdify, symarray, diff, zeros, eye, diag, ones, zeros,
        expand, Function, symbols, var, Add, Mul, Derivative)
    from sympy.core.function import AppliedUndef
    from sympy import ImmutableMatrix

<commit_msg>Fix whitespace<commit_after>import os
USE_SYMENGINE = os.getenv('USE_SYMENGINE')

if USE_SYMENGINE:
    from symengine.sympy_compat import (Symbol, Integer, sympify, S,
        SympifyError, exp, log, gamma, sqrt, I, E, pi, Matrix,
        sin, cos, tan, cot, csc, sec, asin, acos, atan, acot, acsc, asec,
        sinh, cosh, tanh, coth, asinh, acosh, atanh, acoth,
        lambdify, symarray, diff, zeros, eye, diag, ones, zeros,
        expand, Function, symbols, var, Add, Mul, Derivative)
    from symengine.sympy_compat import AppliedUndef
    #TODO: Fix this
    from symengine import Matrix as ImmutableMatrix
else:
    from sympy import (Symbol, Integer, sympify, S,
        SympifyError, exp, log, gamma, sqrt, I, E, pi, Matrix,
        sin, cos, tan, cot, csc, sec, asin, acos, atan, acot, acsc, asec,
        sinh, cosh, tanh, coth, asinh, acosh, atanh, acoth,
        lambdify, symarray, diff, zeros, eye, diag, ones, zeros,
        expand, Function, symbols, var, Add, Mul, Derivative)
    from sympy.core.function import AppliedUndef
    from sympy import ImmutableMatrix
<|endoftext|>


<commit_before>import time
import sublime
from sublime_plugin import TextCommand, EventListener

from ..git_command import GitCommand
from ...common.util import debug

class GsStatusBarEventListener(EventListener):

    def on_new(self, view):
        view.run_command("gs_update_status_bar")

    def on_load(self, view):
        view.run_command("gs_update_status_bar")

    def on_activated(self, view):
        view.run_command("gs_update_status_bar")

    def on_post_save(self, view):
        view.run_command("gs_update_status_bar")

last_execution = 0
update_status_bar_soon = False

class GsUpdateStatusBarCommand(TextCommand, GitCommand):

    """
    Update the short Git status in the Sublime status bar.
    """

    def run(self, edit):
        global last_execution, update_status_bar_soon
        if sublime.load_settings("GitSavvy.sublime-settings").get("git_status_in_status_bar"):

            millisec = int(round(time.time() * 1000))
            # If we updated to less then 100 ms we don't need to update now but
            # should update in 100 ms in case of current file change.
            #
            # So if this get called 4 timer with 20 ms in between each call
            # it will only update twice. Once at time 0 and one at time 100
            # even if it got called at 0, 20, 40, 60 and 80.

            if millisec - 100 > last_execution:
                sublime.set_timeout_async(self.run_async, 0)
            else:
                if not update_status_bar_soon:
                    update_status_bar_soon = True
                    sublime.set_timeout_async(self.run_async, 100)

            last_execution = int(round(time.time() * 1000))

    def run_async(self):
        # Short-circuit update attempts for files not part of Git repo.
        if not self._repo_path(throw_on_stderr=False):
            self.view.erase_status("gitsavvy-repo-status")
            return

        with debug.disable_logging():
            short_status = self.get_branch_status_short()
        self.view.set_status("gitsavvy-repo-status", short_status)

        global update_status_bar_soon
        update_status_bar_soon = False
<commit_msg>Enhancement: use async methods<commit_after>import time
import sublime
from sublime_plugin import TextCommand, EventListener

from ..git_command import GitCommand
from ...common.util import debug

class GsStatusBarEventListener(EventListener):

    def on_new_async(self, view):
        view.run_command("gs_update_status_bar")

    def on_load_async(self, view):
        view.run_command("gs_update_status_bar")

    def on_activated_async(self, view):
        view.run_command("gs_update_status_bar")

    def on_post_save_async(self, view):
        view.run_command("gs_update_status_bar")

last_execution = 0
update_status_bar_soon = False

class GsUpdateStatusBarCommand(TextCommand, GitCommand):

    """
    Update the short Git status in the Sublime status bar.
    """

    def run(self, edit):
        global last_execution, update_status_bar_soon
        if sublime.load_settings("GitSavvy.sublime-settings").get("git_status_in_status_bar"):

            millisec = int(round(time.time() * 1000))
            # If we updated to less then 100 ms we don't need to update now but
            # should update in 100 ms in case of current file change.
            #
            # So if this get called 4 timer with 20 ms in between each call
            # it will only update twice. Once at time 0 and one at time 100
            # even if it got called at 0, 20, 40, 60 and 80.

            if millisec - 100 > last_execution:
                sublime.set_timeout_async(self.run_async, 0)
            else:
                if not update_status_bar_soon:
                    update_status_bar_soon = True
                    sublime.set_timeout_async(self.run_async, 100)

            last_execution = int(round(time.time() * 1000))

    def run_async(self):
        # Short-circuit update attempts for files not part of Git repo.
        if not self._repo_path(throw_on_stderr=False):
            self.view.erase_status("gitsavvy-repo-status")
            return

        with debug.disable_logging():
            short_status = self.get_branch_status_short()
        self.view.set_status("gitsavvy-repo-status", short_status)

        global update_status_bar_soon
        update_status_bar_soon = False
<|endoftext|>


<commit_before>## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

# pylint: disable-msg=C0103
try:
    import editdist
except ImportError:
    # Okay, diffing will not be possible, but continue anyway,
    # since this package is only recommended, not mandatory.
    pass

def record_diff(rec1, rec2, compare_subfields, ind1='', ind2=''):
    """Compares two given records
    Considers the change of order of fields as a change
    @param rec1: First record
    @param rec2: Second record

    @return: dictionary of differences. Each difference is of a form:
     field_id: None - if field is the same for both records
     field_id: ('r',) - if field field_id exists in rec1 but not in
         rec2
     field_id: ('a',) - if field field_id exists in rec2 but not in rec1
     field_id: ('c', new_value) - if field field_id exists in both
         records, but it's value has changed
     new_value describes the new value of a given field (which
         allows to reconstruct new record from the old one)"""
    # Very simple test to save computing power.
    #if rec1 == rec2:
    #    return {}

    result = {}
    for tag in rec1:
        result[tag] = record_field_diff_generic(rec1, rec2, tag, compare_subfields, ind1, ind2)

    for tag in rec2:
        if tag not in rec1:
            result[tag] = record_field_diff_generic(rec1, rec2, tag, compare_subfields, ind1, ind2)
    return result

def record_field_diff_generic(rec1, rec2, tag, compare_subfields, ind1='', ind2=''):
    if tag not in rec2:
        return ('r',)
    if tag not in rec1:
        return ('a',)
    return record_field_diff(rec1[tag], rec2[tag], tag, compare_subfields, ind1, ind2)

def record_field_diff(fields1, fields2, tag, compare_subfields, ind1='', ind2=''):
    """Compares given field in two records.
    returns a list containing at most one element
    If the fields are identical (that means have the same order, the
     same subfields), empty list is returned.
    If the field is removed in second record, [(field, 'r')] is
     returned
    If the field is added in second record, [(field, 'a')]
     is returned
    If the field is changed [(field, 'c', comparison_table)] is returned.
    The comparison table is a table containing pairs of indexes showing
    the relations between fields."""
    # Extract the fields.
    fields1 = [field for field in enumerate(fields1)
               if _has_indicators(field[1], ind1, ind2)]
    fields2 = [field for field in enumerate(fields2)
               if _has_indicators(field[1], ind1, ind2)]
    if fields1 == fields2:
        return None

    fields_comparison = {}

    idx1, idx2 = 0, 0
    len1, len2 = len(fields1), len(fields2)

    while idx1 < len1 and idx2 < len2:
        list_index1, field1 = fields1[idx1]
        list_index2, field2 = fields2[idx2]

        subfields_are_similar, value = compare_subfields(field1[0], field2[0])
        if subfields_are_similar:
            ind_pair = (field1[1], field1[2])
            fields_comparison.setdefault(ind_pair, []).append((list_index1, list_index2, value))
            idx1 += 1
            idx2 += 1
        elif _field_in_fields(field1, fields2[idx2+1:], compare_subfields) \
            is None:
            ind_pair = (field1[1], field1[2])
            fields_comparison.setdefault(ind_pair, []).append((list_index1, None, None))
            idx1 += 1
        else:
            ind_pair = (field2[1], field2[2])
            fields_comparison.setdefault(ind_pair, []).append((None, list_index2, None))
            idx2 += 1

    # Add the remaining elements.
    for index in range(idx1, len(fields1)):
        field1 = fields1[index][1]
        list_index1 = fields1[index][0]
        ind_pair = (field1[1], field1[2])
        fields_comparison.setdefault(ind_pair, []).append((list_index1,None,None))
    for index in range(idx2, len(fields2)):
        field2 = fields2[index][1]
        list_index2 = fields2[index][0]
        ind_pair = (field2[1], field2[2])
        fields_comparison.setdefault(ind_pair, []).append((None,list_index2,None))

    comparisons_list = []
    indicators = fields_comparison.keys()
    indicators.sort()
    for indicator in indicators:
        comparisons_list.append((indicator, fields_comparison[indicator]))

    return ('c', comparisons_list)

def _has_indicators(field, ind1, ind2):
    """Checks if the field has the indicators. Consider an empty
    indicator as a wildcard."""
    if not ind1 and not ind2:
        return True
    else:
        return field[1:3] == (ind1, ind2)

def _same_indicators(field1, field2):
    """Checks if fields have the same indicators."""
    return field1[1:3] == field2[1:3]

def _field_in_fields(field, fields, compare_subfields):
    """Checks if a field 'field' has an equivalent in the list of fields
    'fields'. Uses the 'compare_subfields' method to achieve this."""
    if compare_subfields is None:
        compare_subfields = lambda a, b: a == b

    for index, field2 in fields:
        if compare_subfields(field[0], field2[0])[0]:
            return index

    return None

def compare_strings(str1, str2):
    """Compares 2 strings with the Levenshtein distance and returns a normalized
    value between 0.0 and 1.0 (meaning totally different and exactly the same
    respectively."""
    if str1 == str2:
        return 1.0
    max_len = max(len(str1), len(str2))
    if max_len == 0:
        return 0.0
    distance = editdist.distance(str1, str2)
    return (max_len - distance) / float(max_len)

def compare_subfields(subfield1, subfield2):
    """Compare two subfields taking into account the subfield code and the
    subfield string value."""
    #compare subfield code
    if subfield1[0] != subfield2[0]:
        return 0.0
    #compare subfield values
    return compare_strings(subfield1[1], subfield2[1])

def diff_subfields(subfields1, subfields2):
    """Return a list of diffs for the subfields. A diff is a tuple of:
    (index-of-subfield1, index-of-subfield2, distance-value)."""
    # The result of the comparisons.
    subfields_comparison = []

    # Two indexes used to retain the position of the subfields to compare.
    idx1, idx2 = 0, 0
    len1, len2 = len(subfields1), len(subfields2)

    while idx1 < len1 and idx2 < len2:
        subfield1 = subfields1[idx1]
        for i in range(idx2, len2):
            subfield2 = subfields2[i]
            # Compare the two current subfields.
            value = compare_subfields(subfield1, subfield2)
            if value >= 0.5: #we have a match
                for j in range(idx2, i): #no match for subfields2 in between, if any
                    subfields_comparison.append((None, j, 0.0))
                subfields_comparison.append((idx1, i, value))
                idx2 = i+1
                break
        else: #no match for subfield1
            subfields_comparison.append((idx1, None, 0.0))
        idx1 += 1

    # Add the remaining elements.
    for index in range(idx1, len1):
        subfields_comparison.append((index, None, 0.0))
    for index in range(idx2, len2):
        subfields_comparison.append((None, index, 0.0))

    return subfields_comparison

def match_subfields(subfields1, subfields2):
    """False if subfields dont match, True if they do and also their diff is
    returned."""
    subfield_diffs = diff_subfields(subfields1, subfields2)
    listofscores = [x[2] for x in subfield_diffs]
    if len(listofscores)==0: #in case of a controlfield like '001', '003',...
        return (False, None)
    #length of listofscores should be normally != 0
    average_score = sum(listofscores) / len(listofscores)
    if average_score >= 0.5:
        return (True, subfield_diffs)
    else:
        return (False, None)

def Levenshtein_diffs(str1, str2):
    """Actions (insert, delete, substitute, none) needed to perform on the two
    strings to make them identical."""
    actions = []
    matrix = _Levenshtein_matrix(str1, str2)
    i, j = len(str2), len(str1)
    counter = 0
    lastaction = None
    while i!=0 and j!=0:
        lastvalue = matrix[i][j]
        values = [matrix[i-1][j-1], matrix[i][j-1], matrix[i-1][j]]
        indexofmin = _min_index(values)
        if indexofmin == 0:
            if lastvalue == values[indexofmin]:
                action = 'n'
            else:
                action = 's'
            i, j = i-1, j-1
        elif indexofmin == 1:
            action = 'i'
            j = j-1
        else:  # if indexofmin == 2
            action = 'd'
            i = i-1
        if action != lastaction:
            actions.append( (lastaction, counter) )
            counter = 0
            lastaction = action
        counter = counter + 1
    actions.append( (lastaction, counter) )
    if i>0:
        actions.append( ('d', i) )
    if j>0:
        actions.append( ('i', j) )
    actions.pop(0)
    actions.reverse()
    return actions

def _Levenshtein_matrix(str1, str2):
    len1, len2 = len(str1), len(str2)
    # two-dimensional array of distances
    dist = []
    # initial values
    for i in range( len2 + 1 ):
        dist.append( [i] )
    for j in range(1,  len1 + 1 ):
        dist[0].append( j )
    # calculation of minimum distance
    for i in range(1, len2 + 1 ):
        for j in range(1, len1 + 1 ):
            if str1[j-1] == str2[i-1]:
                cost = 0
            else:
                cost = 1
            # choose between deletion, insertion, substitution
            dist[i].append( min( dist[i-1][j] + 1, \
                                 dist[i][j-1] + 1, \
                                 dist[i-1][j-1] + cost ) )
    return dist

def _min_index(alist):
    min_i = 0 #index of item with minimum value
    for i in range(1, len(alist)):
        if alist[i] < alist[min_i]:
            min_i = i
    return min_i

<commit_msg>BibMerge: safely operate when editdist is missing<commit_after>## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

# pylint: disable-msg=C0103
is_editdist_loaded = False
try:
    import editdist
    is_editdist_loaded = True
except ImportError:
    # Okay, diffing will not be possible, but continue anyway,
    # since this package is only recommended, not mandatory.
    pass

def record_diff(rec1, rec2, compare_subfields, ind1='', ind2=''):
    """Compares two given records
    Considers the change of order of fields as a change
    @param rec1: First record
    @param rec2: Second record

    @return: dictionary of differences. Each difference is of a form:
     field_id: None - if field is the same for both records
     field_id: ('r',) - if field field_id exists in rec1 but not in
         rec2
     field_id: ('a',) - if field field_id exists in rec2 but not in rec1
     field_id: ('c', new_value) - if field field_id exists in both
         records, but it's value has changed
     new_value describes the new value of a given field (which
         allows to reconstruct new record from the old one)"""
    # Very simple test to save computing power.
    #if rec1 == rec2:
    #    return {}

    result = {}
    for tag in rec1:
        result[tag] = record_field_diff_generic(rec1, rec2, tag, compare_subfields, ind1, ind2)

    for tag in rec2:
        if tag not in rec1:
            result[tag] = record_field_diff_generic(rec1, rec2, tag, compare_subfields, ind1, ind2)
    return result

def record_field_diff_generic(rec1, rec2, tag, compare_subfields, ind1='', ind2=''):
    if tag not in rec2:
        return ('r',)
    if tag not in rec1:
        return ('a',)
    return record_field_diff(rec1[tag], rec2[tag], tag, compare_subfields, ind1, ind2)

def record_field_diff(fields1, fields2, tag, compare_subfields, ind1='', ind2=''):
    """Compares given field in two records.
    returns a list containing at most one element
    If the fields are identical (that means have the same order, the
     same subfields), empty list is returned.
    If the field is removed in second record, [(field, 'r')] is
     returned
    If the field is added in second record, [(field, 'a')]
     is returned
    If the field is changed [(field, 'c', comparison_table)] is returned.
    The comparison table is a table containing pairs of indexes showing
    the relations between fields."""
    # Extract the fields.
    fields1 = [field for field in enumerate(fields1)
               if _has_indicators(field[1], ind1, ind2)]
    fields2 = [field for field in enumerate(fields2)
               if _has_indicators(field[1], ind1, ind2)]
    if fields1 == fields2:
        return None

    fields_comparison = {}

    idx1, idx2 = 0, 0
    len1, len2 = len(fields1), len(fields2)

    while idx1 < len1 and idx2 < len2:
        list_index1, field1 = fields1[idx1]
        list_index2, field2 = fields2[idx2]

        subfields_are_similar, value = compare_subfields(field1[0], field2[0])
        if subfields_are_similar:
            ind_pair = (field1[1], field1[2])
            fields_comparison.setdefault(ind_pair, []).append((list_index1, list_index2, value))
            idx1 += 1
            idx2 += 1
        elif _field_in_fields(field1, fields2[idx2+1:], compare_subfields) \
            is None:
            ind_pair = (field1[1], field1[2])
            fields_comparison.setdefault(ind_pair, []).append((list_index1, None, None))
            idx1 += 1
        else:
            ind_pair = (field2[1], field2[2])
            fields_comparison.setdefault(ind_pair, []).append((None, list_index2, None))
            idx2 += 1

    # Add the remaining elements.
    for index in range(idx1, len(fields1)):
        field1 = fields1[index][1]
        list_index1 = fields1[index][0]
        ind_pair = (field1[1], field1[2])
        fields_comparison.setdefault(ind_pair, []).append((list_index1,None,None))
    for index in range(idx2, len(fields2)):
        field2 = fields2[index][1]
        list_index2 = fields2[index][0]
        ind_pair = (field2[1], field2[2])
        fields_comparison.setdefault(ind_pair, []).append((None,list_index2,None))

    comparisons_list = []
    indicators = fields_comparison.keys()
    indicators.sort()
    for indicator in indicators:
        comparisons_list.append((indicator, fields_comparison[indicator]))

    return ('c', comparisons_list)

def _has_indicators(field, ind1, ind2):
    """Checks if the field has the indicators. Consider an empty
    indicator as a wildcard."""
    if not ind1 and not ind2:
        return True
    else:
        return field[1:3] == (ind1, ind2)

def _same_indicators(field1, field2):
    """Checks if fields have the same indicators."""
    return field1[1:3] == field2[1:3]

def _field_in_fields(field, fields, compare_subfields):
    """Checks if a field 'field' has an equivalent in the list of fields
    'fields'. Uses the 'compare_subfields' method to achieve this."""
    if compare_subfields is None:
        compare_subfields = lambda a, b: a == b

    for index, field2 in fields:
        if compare_subfields(field[0], field2[0])[0]:
            return index

    return None

def compare_strings(str1, str2):
    """Compares 2 strings with the Levenshtein distance and returns a normalized
    value between 0.0 and 1.0 (meaning totally different and exactly the same
    respectively."""
    if is_editdist_loaded:
        if str1 == str2:
            return 1.0
        max_len = max(len(str1), len(str2))
        if max_len == 0:
            return 0.0
        distance = editdist.distance(str1, str2)
        return (max_len - distance) / float(max_len)
    else:
        # the edit distance module is not loadable, we have to fail the comparison
        # all the strings will be treated as completely different
        return 0.0

def compare_subfields(subfield1, subfield2):
    """Compare two subfields taking into account the subfield code and the
    subfield string value."""
    #compare subfield code
    if subfield1[0] != subfield2[0]:
        return 0.0
    #compare subfield values
    return compare_strings(subfield1[1], subfield2[1])

def diff_subfields(subfields1, subfields2):
    """Return a list of diffs for the subfields. A diff is a tuple of:
    (index-of-subfield1, index-of-subfield2, distance-value)."""
    # The result of the comparisons.
    subfields_comparison = []

    # Two indexes used to retain the position of the subfields to compare.
    idx1, idx2 = 0, 0
    len1, len2 = len(subfields1), len(subfields2)

    while idx1 < len1 and idx2 < len2:
        subfield1 = subfields1[idx1]
        for i in range(idx2, len2):
            subfield2 = subfields2[i]
            # Compare the two current subfields.
            value = compare_subfields(subfield1, subfield2)
            if value >= 0.5: #we have a match
                for j in range(idx2, i): #no match for subfields2 in between, if any
                    subfields_comparison.append((None, j, 0.0))
                subfields_comparison.append((idx1, i, value))
                idx2 = i+1
                break
        else: #no match for subfield1
            subfields_comparison.append((idx1, None, 0.0))
        idx1 += 1

    # Add the remaining elements.
    for index in range(idx1, len1):
        subfields_comparison.append((index, None, 0.0))
    for index in range(idx2, len2):
        subfields_comparison.append((None, index, 0.0))

    return subfields_comparison

def match_subfields(subfields1, subfields2):
    """False if subfields dont match, True if they do and also their diff is
    returned."""
    subfield_diffs = diff_subfields(subfields1, subfields2)
    listofscores = [x[2] for x in subfield_diffs]
    if len(listofscores)==0: #in case of a controlfield like '001', '003',...
        return (False, None)
    #length of listofscores should be normally != 0
    average_score = sum(listofscores) / len(listofscores)
    if average_score >= 0.5:
        return (True, subfield_diffs)
    else:
        return (False, None)

def Levenshtein_diffs(str1, str2):
    """Actions (insert, delete, substitute, none) needed to perform on the two
    strings to make them identical."""
    actions = []
    matrix = _Levenshtein_matrix(str1, str2)
    i, j = len(str2), len(str1)
    counter = 0
    lastaction = None
    while i!=0 and j!=0:
        lastvalue = matrix[i][j]
        values = [matrix[i-1][j-1], matrix[i][j-1], matrix[i-1][j]]
        indexofmin = _min_index(values)
        if indexofmin == 0:
            if lastvalue == values[indexofmin]:
                action = 'n'
            else:
                action = 's'
            i, j = i-1, j-1
        elif indexofmin == 1:
            action = 'i'
            j = j-1
        else:  # if indexofmin == 2
            action = 'd'
            i = i-1
        if action != lastaction:
            actions.append( (lastaction, counter) )
            counter = 0
            lastaction = action
        counter = counter + 1
    actions.append( (lastaction, counter) )
    if i>0:
        actions.append( ('d', i) )
    if j>0:
        actions.append( ('i', j) )
    actions.pop(0)
    actions.reverse()
    return actions

def _Levenshtein_matrix(str1, str2):
    len1, len2 = len(str1), len(str2)
    # two-dimensional array of distances
    dist = []
    # initial values
    for i in range( len2 + 1 ):
        dist.append( [i] )
    for j in range(1,  len1 + 1 ):
        dist[0].append( j )
    # calculation of minimum distance
    for i in range(1, len2 + 1 ):
        for j in range(1, len1 + 1 ):
            if str1[j-1] == str2[i-1]:
                cost = 0
            else:
                cost = 1
            # choose between deletion, insertion, substitution
            dist[i].append( min( dist[i-1][j] + 1, \
                                 dist[i][j-1] + 1, \
                                 dist[i-1][j-1] + cost ) )
    return dist

def _min_index(alist):
    min_i = 0 #index of item with minimum value
    for i in range(1, len(alist)):
        if alist[i] < alist[min_i]:
            min_i = i
    return min_i

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 - 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import operator

from six import iteritems, itervalues
from six.moves import map

from twisted.internet import defer

from synapse.api.constants import EventTypes, Membership
from synapse.events.utils import prune_event
from synapse.storage import Storage
from synapse.storage.state import StateFilter
from synapse.types import get_domain_from_id

logger = logging.getLogger(__name__)

VISIBILITY_PRIORITY = ("world_readable", "shared", "invited", "joined")

MEMBERSHIP_PRIORITY = (
    Membership.JOIN,
    Membership.INVITE,
    Membership.KNOCK,
    Membership.LEAVE,
    Membership.BAN,
)

@defer.inlineCallbacks
def filter_events_for_client(
    storage: Storage,
    user_id,
    events,
    is_peeking=False,
    always_include_ids=frozenset(),
    filter_send_to_client=True,
):
    """
    Check which events a user is allowed to see. If the user can see the event but its
    sender asked for their data to be erased, prune the content of the event.

    Args:
        storage
        user_id(str): user id to be checked
        events(list[synapse.events.EventBase]): sequence of events to be checked
        is_peeking(bool): should be True if:
          * the user is not currently a member of the room, and:
          * the user has not been a member of the room since the given
            events
        always_include_ids (set(event_id)): set of event ids to specifically
            include (unless sender is ignored)
        filter_send_to_client (bool): Whether we're checking an event that's going to be
            sent to a client. This might not always be the case since this function can
            also be called to check whether a user can see the state at a given point.

    Returns:
        Deferred[list[synapse.events.EventBase]]
    """
    # Filter out events that have been soft failed so that we don't relay them
    # to clients.
    events = [e for e in events if not e.internal_metadata.is_soft_failed()]

    types = ((EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, user_id))
    event_id_to_state = yield storage.state.get_state_for_events(
        frozenset(e.event_id for e in events),
        state_filter=StateFilter.from_types(types),
    )

    ignore_dict_content = yield storage.main.get_global_account_data_by_type_for_user(
        "m.ignored_user_list", user_id
    )

    # FIXME: This will explode if people upload something incorrect.
    ignore_list = frozenset(
        ignore_dict_content.get("ignored_users", {}).keys()
        if ignore_dict_content
        else []
    )

    erased_senders = yield storage.main.are_users_erased((e.sender for e in events))

    if filter_send_to_client:
        room_ids = {e.room_id for e in events}
        retention_policies = {}

        for room_id in room_ids:
            retention_policies[
                room_id
            ] = yield storage.main.get_retention_policy_for_room(room_id)

    def allowed(event):
        """
        Args:
            event (synapse.events.EventBase): event to check

        Returns:
            None|EventBase:
               None if the user cannot see this event at all

               a redacted copy of the event if they can only see a redacted
               version

               the original event if they can see it as normal.
        """
        if event.type == "org.matrix.dummy_event" and filter_send_to_client:
            return None

        if (
            not event.is_state()
            and event.sender in ignore_list
            and filter_send_to_client
        ):
            return None

        # Until MSC2261 has landed we can't redact malicious alias events, so for
        # now we temporarily filter out m.room.aliases entirely to mitigate
        # abuse, while we spec a better solution to advertising aliases
        # on rooms.
        if event.type == EventTypes.Aliases:
            return None

        # Don't try to apply the room's retention policy if the event is a state event, as
        # MSC1763 states that retention is only considered for non-state events.
        if filter_send_to_client and not event.is_state():
            retention_policy = retention_policies[event.room_id]
            max_lifetime = retention_policy.get("max_lifetime")

            if max_lifetime is not None:
                oldest_allowed_ts = storage.main.clock.time_msec() - max_lifetime

                if event.origin_server_ts < oldest_allowed_ts:
                    return None

        if event.event_id in always_include_ids:
            return event

        state = event_id_to_state[event.event_id]

        # get the room_visibility at the time of the event.
        visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None)
        if visibility_event:
            visibility = visibility_event.content.get("history_visibility", "shared")
        else:
            visibility = "shared"

        if visibility not in VISIBILITY_PRIORITY:
            visibility = "shared"

        # Always allow history visibility events on boundaries. This is done
        # by setting the effective visibility to the least restrictive
        # of the old vs new.
        if event.type == EventTypes.RoomHistoryVisibility:
            prev_content = event.unsigned.get("prev_content", {})
            prev_visibility = prev_content.get("history_visibility", None)

            if prev_visibility not in VISIBILITY_PRIORITY:
                prev_visibility = "shared"

            new_priority = VISIBILITY_PRIORITY.index(visibility)
            old_priority = VISIBILITY_PRIORITY.index(prev_visibility)
            if old_priority < new_priority:
                visibility = prev_visibility

        # likewise, if the event is the user's own membership event, use
        # the 'most joined' membership
        membership = None
        if event.type == EventTypes.Member and event.state_key == user_id:
            membership = event.content.get("membership", None)
            if membership not in MEMBERSHIP_PRIORITY:
                membership = "leave"

            prev_content = event.unsigned.get("prev_content", {})
            prev_membership = prev_content.get("membership", None)
            if prev_membership not in MEMBERSHIP_PRIORITY:
                prev_membership = "leave"

            # Always allow the user to see their own leave events, otherwise
            # they won't see the room disappear if they reject the invite
            if membership == "leave" and (
                prev_membership == "join" or prev_membership == "invite"
            ):
                return event

            new_priority = MEMBERSHIP_PRIORITY.index(membership)
            old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)
            if old_priority < new_priority:
                membership = prev_membership

        # otherwise, get the user's membership at the time of the event.
        if membership is None:
            membership_event = state.get((EventTypes.Member, user_id), None)
            if membership_event:
                membership = membership_event.membership

        # if the user was a member of the room at the time of the event,
        # they can see it.
        if membership == Membership.JOIN:
            return event

        # otherwise, it depends on the room visibility.

        if visibility == "joined":
            # we weren't a member at the time of the event, so we can't
            # see this event.
            return None

        elif visibility == "invited":
            # user can also see the event if they were *invited* at the time
            # of the event.
            return event if membership == Membership.INVITE else None

        elif visibility == "shared" and is_peeking:
            # if the visibility is shared, users cannot see the event unless
            # they have *subequently* joined the room (or were members at the
            # time, of course)
            #
            # XXX: if the user has subsequently joined and then left again,
            # ideally we would share history up to the point they left. But
            # we don't know when they left. We just treat it as though they
            # never joined, and restrict access.
            return None

        # the visibility is either shared or world_readable, and the user was
        # not a member at the time. We allow it, provided the original sender
        # has not requested their data to be erased, in which case, we return
        # a redacted version.
        if erased_senders[event.sender]:
            return prune_event(event)

        return event

    # check each event: gives an iterable[None|EventBase]
    filtered_events = map(allowed, events)

    # remove the None entries
    filtered_events = filter(operator.truth, filtered_events)

    # we turn it into a list before returning it.
    return list(filtered_events)

@defer.inlineCallbacks
def filter_events_for_server(
    storage: Storage,
    server_name,
    events,
    redact=True,
    check_history_visibility_only=False,
):
    """Filter a list of events based on whether given server is allowed to
    see them.

    Args:
        storage
        server_name (str)
        events (iterable[FrozenEvent])
        redact (bool): Whether to return a redacted version of the event, or
            to filter them out entirely.
        check_history_visibility_only (bool): Whether to only check the
            history visibility, rather than things like if the sender has been
            erased. This is used e.g. during pagination to decide whether to
            backfill or not.

    Returns
        Deferred[list[FrozenEvent]]
    """

    def is_sender_erased(event, erased_senders):
        if erased_senders and erased_senders[event.sender]:
            logger.info("Sender of %s has been erased, redacting", event.event_id)
            return True
        return False

    def check_event_is_visible(event, state):
        history = state.get((EventTypes.RoomHistoryVisibility, ""), None)
        if history:
            visibility = history.content.get("history_visibility", "shared")
            if visibility in ["invited", "joined"]:
                # We now loop through all state events looking for
                # membership states for the requesting server to determine
                # if the server is either in the room or has been invited
                # into the room.
                for ev in itervalues(state):
                    if ev.type != EventTypes.Member:
                        continue
                    try:
                        domain = get_domain_from_id(ev.state_key)
                    except Exception:
                        continue

                    if domain != server_name:
                        continue

                    memtype = ev.membership
                    if memtype == Membership.JOIN:
                        return True
                    elif memtype == Membership.INVITE:
                        if visibility == "invited":
                            return True
                else:
                    # server has no users in the room: redact
                    return False

        return True

    # Lets check to see if all the events have a history visibility
    # of "shared" or "world_readable". If thats the case then we don't
    # need to check membership (as we know the server is in the room).
    event_to_state_ids = yield storage.state.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        state_filter=StateFilter.from_types(
            types=((EventTypes.RoomHistoryVisibility, ""),)
        ),
    )

    visibility_ids = set()
    for sids in itervalues(event_to_state_ids):
        hist = sids.get((EventTypes.RoomHistoryVisibility, ""))
        if hist:
            visibility_ids.add(hist)

    # If we failed to find any history visibility events then the default
    # is "shared" visiblity.
    if not visibility_ids:
        all_open = True
    else:
        event_map = yield storage.main.get_events(visibility_ids)
        all_open = all(
            e.content.get("history_visibility") in (None, "shared", "world_readable")
            for e in itervalues(event_map)
        )

    if not check_history_visibility_only:
        erased_senders = yield storage.main.are_users_erased((e.sender for e in events))
    else:
        # We don't want to check whether users are erased, which is equivalent
        # to no users having been erased.
        erased_senders = {}

    if all_open:
        # all the history_visibility state affecting these events is open, so
        # we don't need to filter by membership state. We *do* need to check
        # for user erasure, though.
        if erased_senders:
            to_return = []
            for e in events:
                if not is_sender_erased(e, erased_senders):
                    to_return.append(e)
                elif redact:
                    to_return.append(prune_event(e))

            return to_return

        # If there are no erased users then we can just return the given list
        # of events without having to copy it.
        return events

    # Ok, so we're dealing with events that have non-trivial visibility
    # rules, so we need to also get the memberships of the room.

    # first, for each event we're wanting to return, get the event_ids
    # of the history vis and membership state at those events.
    event_to_state_ids = yield storage.state.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        state_filter=StateFilter.from_types(
            types=((EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, None))
        ),
    )

    # We only want to pull out member events that correspond to the
    # server's domain.
    #
    # event_to_state_ids contains lots of duplicates, so it turns out to be
    # cheaper to build a complete event_id => (type, state_key) dict, and then
    # filter out the ones we don't want
    #
    event_id_to_state_key = {
        event_id: key
        for key_to_eid in itervalues(event_to_state_ids)
        for key, event_id in iteritems(key_to_eid)
    }

    def include(typ, state_key):
        if typ != EventTypes.Member:
            return True

        # we avoid using get_domain_from_id here for efficiency.
        idx = state_key.find(":")
        if idx == -1:
            return False
        return state_key[idx + 1 :] == server_name

    event_map = yield storage.main.get_events(
        [
            e_id
            for e_id, key in iteritems(event_id_to_state_key)
            if include(key[0], key[1])
        ]
    )

    event_to_state = {
        e_id: {
            key: event_map[inner_e_id]
            for key, inner_e_id in iteritems(key_to_eid)
            if inner_e_id in event_map
        }
        for e_id, key_to_eid in iteritems(event_to_state_ids)
    }

    to_return = []
    for e in events:
        erased = is_sender_erased(e, erased_senders)
        visible = check_event_is_visible(e, event_to_state[e.event_id])
        if visible and not erased:
            to_return.append(e)
        elif redact:
            to_return.append(prune_event(e))

    return to_return
<commit_msg>Also don't fail on aliases events in this case<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 - 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import operator

from six import iteritems, itervalues
from six.moves import map

from twisted.internet import defer

from synapse.api.constants import EventTypes, Membership
from synapse.events.utils import prune_event
from synapse.storage import Storage
from synapse.storage.state import StateFilter
from synapse.types import get_domain_from_id

logger = logging.getLogger(__name__)

VISIBILITY_PRIORITY = ("world_readable", "shared", "invited", "joined")

MEMBERSHIP_PRIORITY = (
    Membership.JOIN,
    Membership.INVITE,
    Membership.KNOCK,
    Membership.LEAVE,
    Membership.BAN,
)

@defer.inlineCallbacks
def filter_events_for_client(
    storage: Storage,
    user_id,
    events,
    is_peeking=False,
    always_include_ids=frozenset(),
    filter_send_to_client=True,
):
    """
    Check which events a user is allowed to see. If the user can see the event but its
    sender asked for their data to be erased, prune the content of the event.

    Args:
        storage
        user_id(str): user id to be checked
        events(list[synapse.events.EventBase]): sequence of events to be checked
        is_peeking(bool): should be True if:
          * the user is not currently a member of the room, and:
          * the user has not been a member of the room since the given
            events
        always_include_ids (set(event_id)): set of event ids to specifically
            include (unless sender is ignored)
        filter_send_to_client (bool): Whether we're checking an event that's going to be
            sent to a client. This might not always be the case since this function can
            also be called to check whether a user can see the state at a given point.

    Returns:
        Deferred[list[synapse.events.EventBase]]
    """
    # Filter out events that have been soft failed so that we don't relay them
    # to clients.
    events = [e for e in events if not e.internal_metadata.is_soft_failed()]

    types = ((EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, user_id))
    event_id_to_state = yield storage.state.get_state_for_events(
        frozenset(e.event_id for e in events),
        state_filter=StateFilter.from_types(types),
    )

    ignore_dict_content = yield storage.main.get_global_account_data_by_type_for_user(
        "m.ignored_user_list", user_id
    )

    # FIXME: This will explode if people upload something incorrect.
    ignore_list = frozenset(
        ignore_dict_content.get("ignored_users", {}).keys()
        if ignore_dict_content
        else []
    )

    erased_senders = yield storage.main.are_users_erased((e.sender for e in events))

    if filter_send_to_client:
        room_ids = {e.room_id for e in events}
        retention_policies = {}

        for room_id in room_ids:
            retention_policies[
                room_id
            ] = yield storage.main.get_retention_policy_for_room(room_id)

    def allowed(event):
        """
        Args:
            event (synapse.events.EventBase): event to check

        Returns:
            None|EventBase:
               None if the user cannot see this event at all

               a redacted copy of the event if they can only see a redacted
               version

               the original event if they can see it as normal.
        """
        if event.type == "org.matrix.dummy_event" and filter_send_to_client:
            return None

        if (
            not event.is_state()
            and event.sender in ignore_list
            and filter_send_to_client
        ):
            return None

        # Until MSC2261 has landed we can't redact malicious alias events, so for
        # now we temporarily filter out m.room.aliases entirely to mitigate
        # abuse, while we spec a better solution to advertising aliases
        # on rooms.
        if event.type == EventTypes.Aliases and filter_send_to_client:
            return None

        # Don't try to apply the room's retention policy if the event is a state event, as
        # MSC1763 states that retention is only considered for non-state events.
        if filter_send_to_client and not event.is_state():
            retention_policy = retention_policies[event.room_id]
            max_lifetime = retention_policy.get("max_lifetime")

            if max_lifetime is not None:
                oldest_allowed_ts = storage.main.clock.time_msec() - max_lifetime

                if event.origin_server_ts < oldest_allowed_ts:
                    return None

        if event.event_id in always_include_ids:
            return event

        state = event_id_to_state[event.event_id]

        # get the room_visibility at the time of the event.
        visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None)
        if visibility_event:
            visibility = visibility_event.content.get("history_visibility", "shared")
        else:
            visibility = "shared"

        if visibility not in VISIBILITY_PRIORITY:
            visibility = "shared"

        # Always allow history visibility events on boundaries. This is done
        # by setting the effective visibility to the least restrictive
        # of the old vs new.
        if event.type == EventTypes.RoomHistoryVisibility:
            prev_content = event.unsigned.get("prev_content", {})
            prev_visibility = prev_content.get("history_visibility", None)

            if prev_visibility not in VISIBILITY_PRIORITY:
                prev_visibility = "shared"

            new_priority = VISIBILITY_PRIORITY.index(visibility)
            old_priority = VISIBILITY_PRIORITY.index(prev_visibility)
            if old_priority < new_priority:
                visibility = prev_visibility

        # likewise, if the event is the user's own membership event, use
        # the 'most joined' membership
        membership = None
        if event.type == EventTypes.Member and event.state_key == user_id:
            membership = event.content.get("membership", None)
            if membership not in MEMBERSHIP_PRIORITY:
                membership = "leave"

            prev_content = event.unsigned.get("prev_content", {})
            prev_membership = prev_content.get("membership", None)
            if prev_membership not in MEMBERSHIP_PRIORITY:
                prev_membership = "leave"

            # Always allow the user to see their own leave events, otherwise
            # they won't see the room disappear if they reject the invite
            if membership == "leave" and (
                prev_membership == "join" or prev_membership == "invite"
            ):
                return event

            new_priority = MEMBERSHIP_PRIORITY.index(membership)
            old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)
            if old_priority < new_priority:
                membership = prev_membership

        # otherwise, get the user's membership at the time of the event.
        if membership is None:
            membership_event = state.get((EventTypes.Member, user_id), None)
            if membership_event:
                membership = membership_event.membership

        # if the user was a member of the room at the time of the event,
        # they can see it.
        if membership == Membership.JOIN:
            return event

        # otherwise, it depends on the room visibility.

        if visibility == "joined":
            # we weren't a member at the time of the event, so we can't
            # see this event.
            return None

        elif visibility == "invited":
            # user can also see the event if they were *invited* at the time
            # of the event.
            return event if membership == Membership.INVITE else None

        elif visibility == "shared" and is_peeking:
            # if the visibility is shared, users cannot see the event unless
            # they have *subequently* joined the room (or were members at the
            # time, of course)
            #
            # XXX: if the user has subsequently joined and then left again,
            # ideally we would share history up to the point they left. But
            # we don't know when they left. We just treat it as though they
            # never joined, and restrict access.
            return None

        # the visibility is either shared or world_readable, and the user was
        # not a member at the time. We allow it, provided the original sender
        # has not requested their data to be erased, in which case, we return
        # a redacted version.
        if erased_senders[event.sender]:
            return prune_event(event)

        return event

    # check each event: gives an iterable[None|EventBase]
    filtered_events = map(allowed, events)

    # remove the None entries
    filtered_events = filter(operator.truth, filtered_events)

    # we turn it into a list before returning it.
    return list(filtered_events)

@defer.inlineCallbacks
def filter_events_for_server(
    storage: Storage,
    server_name,
    events,
    redact=True,
    check_history_visibility_only=False,
):
    """Filter a list of events based on whether given server is allowed to
    see them.

    Args:
        storage
        server_name (str)
        events (iterable[FrozenEvent])
        redact (bool): Whether to return a redacted version of the event, or
            to filter them out entirely.
        check_history_visibility_only (bool): Whether to only check the
            history visibility, rather than things like if the sender has been
            erased. This is used e.g. during pagination to decide whether to
            backfill or not.

    Returns
        Deferred[list[FrozenEvent]]
    """

    def is_sender_erased(event, erased_senders):
        if erased_senders and erased_senders[event.sender]:
            logger.info("Sender of %s has been erased, redacting", event.event_id)
            return True
        return False

    def check_event_is_visible(event, state):
        history = state.get((EventTypes.RoomHistoryVisibility, ""), None)
        if history:
            visibility = history.content.get("history_visibility", "shared")
            if visibility in ["invited", "joined"]:
                # We now loop through all state events looking for
                # membership states for the requesting server to determine
                # if the server is either in the room or has been invited
                # into the room.
                for ev in itervalues(state):
                    if ev.type != EventTypes.Member:
                        continue
                    try:
                        domain = get_domain_from_id(ev.state_key)
                    except Exception:
                        continue

                    if domain != server_name:
                        continue

                    memtype = ev.membership
                    if memtype == Membership.JOIN:
                        return True
                    elif memtype == Membership.INVITE:
                        if visibility == "invited":
                            return True
                else:
                    # server has no users in the room: redact
                    return False

        return True

    # Lets check to see if all the events have a history visibility
    # of "shared" or "world_readable". If thats the case then we don't
    # need to check membership (as we know the server is in the room).
    event_to_state_ids = yield storage.state.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        state_filter=StateFilter.from_types(
            types=((EventTypes.RoomHistoryVisibility, ""),)
        ),
    )

    visibility_ids = set()
    for sids in itervalues(event_to_state_ids):
        hist = sids.get((EventTypes.RoomHistoryVisibility, ""))
        if hist:
            visibility_ids.add(hist)

    # If we failed to find any history visibility events then the default
    # is "shared" visiblity.
    if not visibility_ids:
        all_open = True
    else:
        event_map = yield storage.main.get_events(visibility_ids)
        all_open = all(
            e.content.get("history_visibility") in (None, "shared", "world_readable")
            for e in itervalues(event_map)
        )

    if not check_history_visibility_only:
        erased_senders = yield storage.main.are_users_erased((e.sender for e in events))
    else:
        # We don't want to check whether users are erased, which is equivalent
        # to no users having been erased.
        erased_senders = {}

    if all_open:
        # all the history_visibility state affecting these events is open, so
        # we don't need to filter by membership state. We *do* need to check
        # for user erasure, though.
        if erased_senders:
            to_return = []
            for e in events:
                if not is_sender_erased(e, erased_senders):
                    to_return.append(e)
                elif redact:
                    to_return.append(prune_event(e))

            return to_return

        # If there are no erased users then we can just return the given list
        # of events without having to copy it.
        return events

    # Ok, so we're dealing with events that have non-trivial visibility
    # rules, so we need to also get the memberships of the room.

    # first, for each event we're wanting to return, get the event_ids
    # of the history vis and membership state at those events.
    event_to_state_ids = yield storage.state.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        state_filter=StateFilter.from_types(
            types=((EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, None))
        ),
    )

    # We only want to pull out member events that correspond to the
    # server's domain.
    #
    # event_to_state_ids contains lots of duplicates, so it turns out to be
    # cheaper to build a complete event_id => (type, state_key) dict, and then
    # filter out the ones we don't want
    #
    event_id_to_state_key = {
        event_id: key
        for key_to_eid in itervalues(event_to_state_ids)
        for key, event_id in iteritems(key_to_eid)
    }

    def include(typ, state_key):
        if typ != EventTypes.Member:
            return True

        # we avoid using get_domain_from_id here for efficiency.
        idx = state_key.find(":")
        if idx == -1:
            return False
        return state_key[idx + 1 :] == server_name

    event_map = yield storage.main.get_events(
        [
            e_id
            for e_id, key in iteritems(event_id_to_state_key)
            if include(key[0], key[1])
        ]
    )

    event_to_state = {
        e_id: {
            key: event_map[inner_e_id]
            for key, inner_e_id in iteritems(key_to_eid)
            if inner_e_id in event_map
        }
        for e_id, key_to_eid in iteritems(event_to_state_ids)
    }

    to_return = []
    for e in events:
        erased = is_sender_erased(e, erased_senders)
        visible = check_event_is_visible(e, event_to_state[e.event_id])
        if visible and not erased:
            to_return.append(e)
        elif redact:
            to_return.append(prune_event(e))

    return to_return
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 - 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import logging
import operator

import six
from twisted.internet import defer

from synapse.api.constants import EventTypes, Membership
from synapse.events.utils import prune_event
from synapse.types import get_domain_from_id
from synapse.util.logcontext import make_deferred_yieldable, preserve_fn

logger = logging.getLogger(__name__)

VISIBILITY_PRIORITY = (
    "world_readable",
    "shared",
    "invited",
    "joined",
)

MEMBERSHIP_PRIORITY = (
    Membership.JOIN,
    Membership.INVITE,
    Membership.KNOCK,
    Membership.LEAVE,
    Membership.BAN,
)

@defer.inlineCallbacks
def filter_events_for_client(store, user_id, events, is_peeking=False,
                             always_include_ids=frozenset()):
    """
    Check which events a user is allowed to see

    Args:
        store (synapse.storage.DataStore): our datastore (can also be a worker
            store)
        user_id(str): user id to be checked
        events(list[synapse.events.EventBase]): sequence of events to be checked
        is_peeking(bool): should be True if:
          * the user is not currently a member of the room, and:
          * the user has not been a member of the room since the given
            events
        always_include_ids (set(event_id)): set of event ids to specifically
            include (unless sender is ignored)

    Returns:
        Deferred[list[synapse.events.EventBase]]
    """
    types = (
        (EventTypes.RoomHistoryVisibility, ""),
        (EventTypes.Member, user_id),
    )
    event_id_to_state = yield store.get_state_for_events(
        frozenset(e.event_id for e in events),
        types=types,
    )

    forgotten = yield make_deferred_yieldable(defer.gatherResults([
        defer.maybeDeferred(
            preserve_fn(store.who_forgot_in_room),
            room_id,
        )
        for room_id in frozenset(e.room_id for e in events)
    ], consumeErrors=True))

    # Set of membership event_ids that have been forgotten
    event_id_forgotten = frozenset(
        row["event_id"] for rows in forgotten for row in rows
    )

    ignore_dict_content = yield store.get_global_account_data_by_type_for_user(
        "m.ignored_user_list", user_id,
    )

    # FIXME: This will explode if people upload something incorrect.
    ignore_list = frozenset(
        ignore_dict_content.get("ignored_users", {}).keys()
        if ignore_dict_content else []
    )

    erased_senders = yield store.are_users_erased((e.sender for e in events))

    def allowed(event):
        """
        Args:
            event (synapse.events.EventBase): event to check

        Returns:
            None|EventBase:
               None if the user cannot see this event at all

               a redacted copy of the event if they can only see a redacted
               version

               the original event if they can see it as normal.
        """
        if not event.is_state() and event.sender in ignore_list:
            return None

        if event.event_id in always_include_ids:
            return event

        state = event_id_to_state[event.event_id]

        # get the room_visibility at the time of the event.
        visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None)
        if visibility_event:
            visibility = visibility_event.content.get("history_visibility", "shared")
        else:
            visibility = "shared"

        if visibility not in VISIBILITY_PRIORITY:
            visibility = "shared"

        # Always allow history visibility events on boundaries. This is done
        # by setting the effective visibility to the least restrictive
        # of the old vs new.
        if event.type == EventTypes.RoomHistoryVisibility:
            prev_content = event.unsigned.get("prev_content", {})
            prev_visibility = prev_content.get("history_visibility", None)

            if prev_visibility not in VISIBILITY_PRIORITY:
                prev_visibility = "shared"

            new_priority = VISIBILITY_PRIORITY.index(visibility)
            old_priority = VISIBILITY_PRIORITY.index(prev_visibility)
            if old_priority < new_priority:
                visibility = prev_visibility

        # likewise, if the event is the user's own membership event, use
        # the 'most joined' membership
        membership = None
        if event.type == EventTypes.Member and event.state_key == user_id:
            membership = event.content.get("membership", None)
            if membership not in MEMBERSHIP_PRIORITY:
                membership = "leave"

            prev_content = event.unsigned.get("prev_content", {})
            prev_membership = prev_content.get("membership", None)
            if prev_membership not in MEMBERSHIP_PRIORITY:
                prev_membership = "leave"

            # Always allow the user to see their own leave events, otherwise
            # they won't see the room disappear if they reject the invite
            if membership == "leave" and (
                prev_membership == "join" or prev_membership == "invite"
            ):
                return event

            new_priority = MEMBERSHIP_PRIORITY.index(membership)
            old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)
            if old_priority < new_priority:
                membership = prev_membership

        # otherwise, get the user's membership at the time of the event.
        if membership is None:
            membership_event = state.get((EventTypes.Member, user_id), None)
            if membership_event:
                # XXX why do we do this?
                # https://github.com/matrix-org/synapse/issues/3350
                if membership_event.event_id not in event_id_forgotten:
                    membership = membership_event.membership

        # if the user was a member of the room at the time of the event,
        # they can see it.
        if membership == Membership.JOIN:
            return event

        # otherwise, it depends on the room visibility.

        if visibility == "joined":
            # we weren't a member at the time of the event, so we can't
            # see this event.
            return None

        elif visibility == "invited":
            # user can also see the event if they were *invited* at the time
            # of the event.
            return (
                event if membership == Membership.INVITE else None
            )

        elif visibility == "shared" and is_peeking:
            # if the visibility is shared, users cannot see the event unless
            # they have *subequently* joined the room (or were members at the
            # time, of course)
            #
            # XXX: if the user has subsequently joined and then left again,
            # ideally we would share history up to the point they left. But
            # we don't know when they left. We just treat it as though they
            # never joined, and restrict access.
            return None

        # the visibility is either shared or world_readable, and the user was
        # not a member at the time. We allow it, provided the original sender
        # has not requested their data to be erased, in which case, we return
        # a redacted version.
        if erased_senders[event.sender]:
            return prune_event(event)

        return event

    # check each event: gives an iterable[None|EventBase]
    filtered_events = itertools.imap(allowed, events)

    # remove the None entries
    filtered_events = filter(operator.truth, filtered_events)

    # we turn it into a list before returning it.
    defer.returnValue(list(filtered_events))

@defer.inlineCallbacks
def filter_events_for_server(store, server_name, events):
    # First lets check to see if all the events have a history visibility
    # of "shared" or "world_readable". If thats the case then we don't
    # need to check membership (as we know the server is in the room).
    event_to_state_ids = yield store.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        types=(
            (EventTypes.RoomHistoryVisibility, ""),
        )
    )

    visibility_ids = set()
    for sids in event_to_state_ids.itervalues():
        hist = sids.get((EventTypes.RoomHistoryVisibility, ""))
        if hist:
            visibility_ids.add(hist)

    # If we failed to find any history visibility events then the default
    # is "shared" visiblity.
    if not visibility_ids:
        defer.returnValue(events)

    event_map = yield store.get_events(visibility_ids)
    all_open = all(
        e.content.get("history_visibility") in (None, "shared", "world_readable")
        for e in event_map.itervalues()
    )

    if all_open:
        defer.returnValue(events)

    # Ok, so we're dealing with events that have non-trivial visibility
    # rules, so we need to also get the memberships of the room.

    # first, for each event we're wanting to return, get the event_ids
    # of the history vis and membership state at those events.
    event_to_state_ids = yield store.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        types=(
            (EventTypes.RoomHistoryVisibility, ""),
            (EventTypes.Member, None),
        )
    )

    # We only want to pull out member events that correspond to the
    # server's domain.
    #
    # event_to_state_ids contains lots of duplicates, so it turns out to be
    # cheaper to build a complete set of unique
    # ((type, state_key), event_id) tuples, and then filter out the ones we
    # don't want.
    #
    state_key_to_event_id_set = {
        e
        for key_to_eid in six.itervalues(event_to_state_ids)
        for e in key_to_eid.items()
    }

    def include(typ, state_key):
        if typ != EventTypes.Member:
            return True
        idx = state_key.find(":")
        if idx == -1:
            return False
        return state_key[idx + 1:] == server_name

    event_map = yield store.get_events([
        e_id
        for key, e_id in state_key_to_event_id_set
        if include(key[0], key[1])
    ])

    event_to_state = {
        e_id: {
            key: event_map[inner_e_id]
            for key, inner_e_id in key_to_eid.iteritems()
            if inner_e_id in event_map
        }
        for e_id, key_to_eid in event_to_state_ids.iteritems()
    }

    erased_senders = yield store.are_users_erased(
        e.sender for e in events,
    )

    def redact_disallowed(event, state):
        # if the sender has been gdpr17ed, always return a redacted
        # copy of the event.
        if erased_senders[event.sender]:
            logger.info(
                "Sender of %s has been erased, redacting",
                event.event_id,
            )
            return prune_event(event)

        if not state:
            return event

        history = state.get((EventTypes.RoomHistoryVisibility, ''), None)
        if history:
            visibility = history.content.get("history_visibility", "shared")
            if visibility in ["invited", "joined"]:
                # We now loop through all state events looking for
                # membership states for the requesting server to determine
                # if the server is either in the room or has been invited
                # into the room.
                for ev in state.itervalues():
                    if ev.type != EventTypes.Member:
                        continue
                    try:
                        domain = get_domain_from_id(ev.state_key)
                    except Exception:
                        continue

                    if domain != server_name:
                        continue

                    memtype = ev.membership
                    if memtype == Membership.JOIN:
                        return event
                    elif memtype == Membership.INVITE:
                        if visibility == "invited":
                            return event
                else:
                    # server has no users in the room: redact
                    return prune_event(event)

        return event

    defer.returnValue([
        redact_disallowed(e, event_to_state[e.event_id])
        for e in events
    ])
<commit_msg>add a comment<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 - 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import logging
import operator

import six
from twisted.internet import defer

from synapse.api.constants import EventTypes, Membership
from synapse.events.utils import prune_event
from synapse.types import get_domain_from_id
from synapse.util.logcontext import make_deferred_yieldable, preserve_fn

logger = logging.getLogger(__name__)

VISIBILITY_PRIORITY = (
    "world_readable",
    "shared",
    "invited",
    "joined",
)

MEMBERSHIP_PRIORITY = (
    Membership.JOIN,
    Membership.INVITE,
    Membership.KNOCK,
    Membership.LEAVE,
    Membership.BAN,
)

@defer.inlineCallbacks
def filter_events_for_client(store, user_id, events, is_peeking=False,
                             always_include_ids=frozenset()):
    """
    Check which events a user is allowed to see

    Args:
        store (synapse.storage.DataStore): our datastore (can also be a worker
            store)
        user_id(str): user id to be checked
        events(list[synapse.events.EventBase]): sequence of events to be checked
        is_peeking(bool): should be True if:
          * the user is not currently a member of the room, and:
          * the user has not been a member of the room since the given
            events
        always_include_ids (set(event_id)): set of event ids to specifically
            include (unless sender is ignored)

    Returns:
        Deferred[list[synapse.events.EventBase]]
    """
    types = (
        (EventTypes.RoomHistoryVisibility, ""),
        (EventTypes.Member, user_id),
    )
    event_id_to_state = yield store.get_state_for_events(
        frozenset(e.event_id for e in events),
        types=types,
    )

    forgotten = yield make_deferred_yieldable(defer.gatherResults([
        defer.maybeDeferred(
            preserve_fn(store.who_forgot_in_room),
            room_id,
        )
        for room_id in frozenset(e.room_id for e in events)
    ], consumeErrors=True))

    # Set of membership event_ids that have been forgotten
    event_id_forgotten = frozenset(
        row["event_id"] for rows in forgotten for row in rows
    )

    ignore_dict_content = yield store.get_global_account_data_by_type_for_user(
        "m.ignored_user_list", user_id,
    )

    # FIXME: This will explode if people upload something incorrect.
    ignore_list = frozenset(
        ignore_dict_content.get("ignored_users", {}).keys()
        if ignore_dict_content else []
    )

    erased_senders = yield store.are_users_erased((e.sender for e in events))

    def allowed(event):
        """
        Args:
            event (synapse.events.EventBase): event to check

        Returns:
            None|EventBase:
               None if the user cannot see this event at all

               a redacted copy of the event if they can only see a redacted
               version

               the original event if they can see it as normal.
        """
        if not event.is_state() and event.sender in ignore_list:
            return None

        if event.event_id in always_include_ids:
            return event

        state = event_id_to_state[event.event_id]

        # get the room_visibility at the time of the event.
        visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None)
        if visibility_event:
            visibility = visibility_event.content.get("history_visibility", "shared")
        else:
            visibility = "shared"

        if visibility not in VISIBILITY_PRIORITY:
            visibility = "shared"

        # Always allow history visibility events on boundaries. This is done
        # by setting the effective visibility to the least restrictive
        # of the old vs new.
        if event.type == EventTypes.RoomHistoryVisibility:
            prev_content = event.unsigned.get("prev_content", {})
            prev_visibility = prev_content.get("history_visibility", None)

            if prev_visibility not in VISIBILITY_PRIORITY:
                prev_visibility = "shared"

            new_priority = VISIBILITY_PRIORITY.index(visibility)
            old_priority = VISIBILITY_PRIORITY.index(prev_visibility)
            if old_priority < new_priority:
                visibility = prev_visibility

        # likewise, if the event is the user's own membership event, use
        # the 'most joined' membership
        membership = None
        if event.type == EventTypes.Member and event.state_key == user_id:
            membership = event.content.get("membership", None)
            if membership not in MEMBERSHIP_PRIORITY:
                membership = "leave"

            prev_content = event.unsigned.get("prev_content", {})
            prev_membership = prev_content.get("membership", None)
            if prev_membership not in MEMBERSHIP_PRIORITY:
                prev_membership = "leave"

            # Always allow the user to see their own leave events, otherwise
            # they won't see the room disappear if they reject the invite
            if membership == "leave" and (
                prev_membership == "join" or prev_membership == "invite"
            ):
                return event

            new_priority = MEMBERSHIP_PRIORITY.index(membership)
            old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)
            if old_priority < new_priority:
                membership = prev_membership

        # otherwise, get the user's membership at the time of the event.
        if membership is None:
            membership_event = state.get((EventTypes.Member, user_id), None)
            if membership_event:
                # XXX why do we do this?
                # https://github.com/matrix-org/synapse/issues/3350
                if membership_event.event_id not in event_id_forgotten:
                    membership = membership_event.membership

        # if the user was a member of the room at the time of the event,
        # they can see it.
        if membership == Membership.JOIN:
            return event

        # otherwise, it depends on the room visibility.

        if visibility == "joined":
            # we weren't a member at the time of the event, so we can't
            # see this event.
            return None

        elif visibility == "invited":
            # user can also see the event if they were *invited* at the time
            # of the event.
            return (
                event if membership == Membership.INVITE else None
            )

        elif visibility == "shared" and is_peeking:
            # if the visibility is shared, users cannot see the event unless
            # they have *subequently* joined the room (or were members at the
            # time, of course)
            #
            # XXX: if the user has subsequently joined and then left again,
            # ideally we would share history up to the point they left. But
            # we don't know when they left. We just treat it as though they
            # never joined, and restrict access.
            return None

        # the visibility is either shared or world_readable, and the user was
        # not a member at the time. We allow it, provided the original sender
        # has not requested their data to be erased, in which case, we return
        # a redacted version.
        if erased_senders[event.sender]:
            return prune_event(event)

        return event

    # check each event: gives an iterable[None|EventBase]
    filtered_events = itertools.imap(allowed, events)

    # remove the None entries
    filtered_events = filter(operator.truth, filtered_events)

    # we turn it into a list before returning it.
    defer.returnValue(list(filtered_events))

@defer.inlineCallbacks
def filter_events_for_server(store, server_name, events):
    # First lets check to see if all the events have a history visibility
    # of "shared" or "world_readable". If thats the case then we don't
    # need to check membership (as we know the server is in the room).
    event_to_state_ids = yield store.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        types=(
            (EventTypes.RoomHistoryVisibility, ""),
        )
    )

    visibility_ids = set()
    for sids in event_to_state_ids.itervalues():
        hist = sids.get((EventTypes.RoomHistoryVisibility, ""))
        if hist:
            visibility_ids.add(hist)

    # If we failed to find any history visibility events then the default
    # is "shared" visiblity.
    if not visibility_ids:
        defer.returnValue(events)

    event_map = yield store.get_events(visibility_ids)
    all_open = all(
        e.content.get("history_visibility") in (None, "shared", "world_readable")
        for e in event_map.itervalues()
    )

    if all_open:
        defer.returnValue(events)

    # Ok, so we're dealing with events that have non-trivial visibility
    # rules, so we need to also get the memberships of the room.

    # first, for each event we're wanting to return, get the event_ids
    # of the history vis and membership state at those events.
    event_to_state_ids = yield store.get_state_ids_for_events(
        frozenset(e.event_id for e in events),
        types=(
            (EventTypes.RoomHistoryVisibility, ""),
            (EventTypes.Member, None),
        )
    )

    # We only want to pull out member events that correspond to the
    # server's domain.
    #
    # event_to_state_ids contains lots of duplicates, so it turns out to be
    # cheaper to build a complete set of unique
    # ((type, state_key), event_id) tuples, and then filter out the ones we
    # don't want.
    #
    state_key_to_event_id_set = {
        e
        for key_to_eid in six.itervalues(event_to_state_ids)
        for e in key_to_eid.items()
    }

    def include(typ, state_key):
        if typ != EventTypes.Member:
            return True

        # we avoid using get_domain_from_id here for efficiency.
        idx = state_key.find(":")
        if idx == -1:
            return False
        return state_key[idx + 1:] == server_name

    event_map = yield store.get_events([
        e_id
        for key, e_id in state_key_to_event_id_set
        if include(key[0], key[1])
    ])

    event_to_state = {
        e_id: {
            key: event_map[inner_e_id]
            for key, inner_e_id in key_to_eid.iteritems()
            if inner_e_id in event_map
        }
        for e_id, key_to_eid in event_to_state_ids.iteritems()
    }

    erased_senders = yield store.are_users_erased(
        e.sender for e in events,
    )

    def redact_disallowed(event, state):
        # if the sender has been gdpr17ed, always return a redacted
        # copy of the event.
        if erased_senders[event.sender]:
            logger.info(
                "Sender of %s has been erased, redacting",
                event.event_id,
            )
            return prune_event(event)

        if not state:
            return event

        history = state.get((EventTypes.RoomHistoryVisibility, ''), None)
        if history:
            visibility = history.content.get("history_visibility", "shared")
            if visibility in ["invited", "joined"]:
                # We now loop through all state events looking for
                # membership states for the requesting server to determine
                # if the server is either in the room or has been invited
                # into the room.
                for ev in state.itervalues():
                    if ev.type != EventTypes.Member:
                        continue
                    try:
                        domain = get_domain_from_id(ev.state_key)
                    except Exception:
                        continue

                    if domain != server_name:
                        continue

                    memtype = ev.membership
                    if memtype == Membership.JOIN:
                        return event
                    elif memtype == Membership.INVITE:
                        if visibility == "invited":
                            return event
                else:
                    # server has no users in the room: redact
                    return prune_event(event)

        return event

    defer.returnValue([
        redact_disallowed(e, event_to_state[e.event_id])
        for e in events
    ])
<|endoftext|>


<commit_before>import time
import os
from file_backup_config import FileBackupConfig
from s3_backup_config import S3BackupConfig
from inline_backup_config import InlineBackupConfig
import snapshot_manager
from boto import ec2
import kayvee
import logging

aws_region = os.environ['AWS_REGION']
config_path = os.environ['BACKUP_CONFIG']

def get_backup_conf(path):
    """ Gets backup config from file or S3 """
    if path.startswith("s3://"):
        return S3BackupConfig(path)
    elif ":" in path:
        # config is YAML or JSON
        return InlineBackupConfig(path)
    else:
        return FileBackupConfig(path)

def create_snapshots(backup_conf):
    ec2_connection = ec2.connect_to_region(aws_region)
    for volume, params in backup_conf.get().iteritems():
        logging.info(kayvee.formatLog("ebs-snapshots", "info", "about to take ebs snapshot {} - {}".format(volume, params)))
        interval = params.get('interval', 'daily')
        max_snapshots = params.get('max_snapshots', 0)
        name = params.get('name', '')
        snapshot_manager.run(
            ec2_connection, volume, interval, max_snapshots, name)

def snapshot_timer(interval=300):
    """ Gets backup conf, every x seconds checks for snapshots to create/delete,
        and performs the create/delete operations as needed """
    # Main loop gets the backup conf once.
    # Thereafter they are responsible for updating their own data
    backup_conf = get_backup_conf(config_path)
    while True:
        create_snapshots(backup_conf)
        time.sleep(interval)
<commit_msg>add backup region connection<commit_after>import time
import os
from file_backup_config import FileBackupConfig
from s3_backup_config import S3BackupConfig
from inline_backup_config import InlineBackupConfig
import snapshot_manager
from boto import ec2
import kayvee
import logging

aws_region = os.environ['AWS_REGION']
aws_backup_region = os.environ['AWS_BACKUP_REGION']
config_path = os.environ['BACKUP_CONFIG']

def get_backup_conf(path):
    """ Gets backup config from file or S3 """
    if path.startswith("s3://"):
        return S3BackupConfig(path)
    elif ":" in path:
        # config is YAML or JSON
        return InlineBackupConfig(path)
    else:
        return FileBackupConfig(path)

def create_snapshots(backup_conf):
    ec2_connection = ec2.connect_to_region(aws_region)
    ec2_backup_connection = ec2.connect_to_region(aws_backup_region)
    for volume, params in backup_conf.get().iteritems():
        logging.info(kayvee.formatLog("ebs-snapshots", "info", "about to take ebs snapshot {} - {}".format(volume, params)))
        interval = params.get('interval', 'daily')
        max_snapshots = params.get('max_snapshots', 0)
        name = params.get('name', '')
        snapshot_manager.run(
            ec2_connection, ec2_backup_connection, volume, interval, max_snapshots, name)

def snapshot_timer(interval=300):
    """ Gets backup conf, every x seconds checks for snapshots to create/delete,
        and performs the create/delete operations as needed """
    # Main loop gets the backup conf once.
    # Thereafter they are responsible for updating their own data
    backup_conf = get_backup_conf(config_path)
    while True:
        create_snapshots(backup_conf)
        time.sleep(interval)
<|endoftext|>