<commit_before>import sublime
import sublime_plugin
import re, sys

if not 'MODE' in globals(): MODE = {}
if not 'HIST' in globals(): HIST = []

def input_parser(user_input):
    m = re.match(r"(.+)/([lcr*()0-9]*)(f[0-9]*)?", user_input)

    if m and (m.group(2) or m.group(3)):
        regex = m.group(1)
        option = m.group(2)
        f = m.group(3)
    else:
        # print("No options!")
        return [user_input, [['l', 1]], 0]

    try:
        # for option
        rParan = re.compile(r"\(([^())]*)\)\*([0-9]+)")
        while True:
            if not rParan.search(option): break
            for r in rParan.finditer(option):
                option = option.replace(r.group(0), r.group(1)*int(r.group(2)),1)

        for r in re.finditer(r"([lcr][0-9]*)\*([0-9]+)", option):
            option = option.replace(r.group(0), r.group(1)*int(r.group(2)),1)

        option = re.findall(r"[lcr][0-9]*", option)
        option = list(map(lambda x: [x[0], 1] if len(x)==1 else [x[0], int(x[1:])], option))
        option = option if option else [['l', 1]]

        # for f
        f = 0 if not f else 1 if len(f)==1 else int(f[1:])
    except:
        [regex, option ,f]= [user_input, [['l', 1]], 0]

    return [regex, option ,f]

def update_colwidth(colwidth, content, option, strip_char):
    thiscolwidth = [len(c.strip(strip_char)) if i>0 else len(c.rstrip(strip_char).lstrip()) for i, c in enumerate(content)]
    for i,w in enumerate(thiscolwidth):
        if i<len(colwidth):
            colwidth[i] = max(colwidth[i], w)
        else:
            colwidth.append(w)

def get_named_pattern(user_input):
    patterns = sublime.load_settings('AlignTab.sublime-settings').get('named_patterns', {})
    user_input = patterns[user_input] if user_input in patterns else user_input
    user_input = HIST[-1] if HIST and user_input == 'last_rexp' else user_input
    return user_input

class AlignTabCommand(sublime_plugin.TextCommand):
    # Default the live_enabled setting so that if it doesn't get loaded nothing will break
    live_enabled = sublime.load_settings('AlignTab.sublime-settings').get('live_preview', True)
    live_change_made = False

    def run(self, edit, user_input=None, mode=False, event_type=None):
        view = self.view
        vid = view.id()
        if not user_input and not event_type:
            self.live_change_made = False
            v = self.view.window().show_input_panel('Align By RegEx:', '',
                    # On Done
                    lambda x: self.view.run_command("align_tab",{"user_input":x, "mode":mode, "event_type":"done"}),
                    # On Change
                    lambda x: self.on_change(x, mode),
                    # On Canel
                    lambda: self.on_change(None, mode) )
            v.set_syntax_file('Packages/AlignTab/AlignTab.hidden-tmLanguage')
            v.settings().set('is_widget', True)
            v.settings().set('gutter', False)
            v.settings().set('rulers', [])

        elif user_input:
            if not event_type: event_type = "done"

            # clear live status when ready to align
            if self.live_enabled: self.view.set_status("aligntab-live", "")

            # Don't update history if this is a live change
            if event_type == "done":
                # insert history and reset index
                if not HIST or (user_input!= HIST[-1] and user_input!= "last_rexp"): HIST.append(user_input)
                AlignTabHistory.index = None
                # If live is enabled, then don't re-change the text. We're done
                if self.live_enabled and self.live_change_made:
                    self.live_change_made = False
                    return
            success = self.align_tab(edit, user_input)

            global MODE
            if success:
                # If rows were found, then note that we have an undo due
                if event_type == "change" and self.live_enabled:
                    self.live_change_made = True
                    view.set_status("aligntab-live", "")
                if mode:
                    MODE[vid] = True
                    view.set_status("aligntab-table", "[Table Mode]")
            else:
                if event_type == "change" and self.live_enabled:
                    view.set_status("aligntab-live", "[Pattern not Found]")

                # this means no selection was detected to contain the regex,
                # therefore we check previous line and next line for each cursor
                if mode and not all(list(self.prev_next_match(user_input))):
                    MODE[vid] = False
                    view.set_status("aligntab-table", "")

    def on_change(self, user_input, mode):
        # Don't do anything if we're not live
        if not self.live_enabled:
            return
        # Undo the previous change if needed
        if self.live_change_made:
            self.view.run_command("undo")
            self.live_change_made = False
        # Run the align command
        if user_input:
            self.view.run_command("align_tab",{"user_input":user_input, "mode":mode, "event_type":"change"})

    def get_line_content(self, regex, f, row):
        view = self.view
        line = view.line(view.text_point(row,0))
        return [s for s in re.split(regex,view.substr(line),f)]

    def expand_sel(self, regex, option, f, rows, colwidth, strip_char):
        view = self.view
        lastrow = view.rowcol(view.size())[0]

        for sel in view.sel():
            for line in view.lines(sel):
                thisrow = view.rowcol(line.begin())[0]
                if (thisrow in rows): continue
                content = self.get_line_content(regex, f, thisrow)
                if len(content)<=1: continue
                update_colwidth(colwidth, content, option, strip_char)
                rows.append(thisrow)

            if sel.empty():
                thisrow = view.rowcol(sel.begin())[0]
                if not (thisrow in rows): continue
                beginrow = endrow = thisrow
                while endrow+1<=lastrow and not (endrow+1 in rows):
                    content = self.get_line_content(regex, f, endrow+1)
                    if len(content)<=1: break
                    update_colwidth(colwidth, content, option, strip_char)
                    endrow = endrow+1
                    rows.append(endrow)
                while beginrow-1>=0 and not (beginrow-1 in rows):
                    content = self.get_line_content(regex, f, beginrow-1)
                    if len(content)<=1: break
                    update_colwidth(colwidth, content, option, strip_char)
                    beginrow = beginrow-1
                    rows.append(beginrow)

    def prev_next_match(self, user_input):

        user_input = get_named_pattern(user_input)
        [regex, option, f] = input_parser(user_input)
        regex = '(' + regex + ')'
        # it is used to check whether table mode should be disabled
        view = self.view
        lastrow = view.rowcol(view.size())[0]
        rows = []
        for sel in view.sel():
            for line in view.lines(sel):
                rows.append(view.rowcol(line.begin())[0])
        rows = list(set(rows))
        for row in rows:
            if row-1>=0 and len(self.get_line_content(regex, f, row-1))>1:
                yield True
            elif row+1<=lastrow and len(self.get_line_content(regex, f, row+1))>1:
                yield True
            else:
                yield False

    def align_tab(self, edit, user_input):
        view = self.view

        user_input = get_named_pattern(user_input)
        [regex, option, f] = input_parser(user_input)
        regex = '(' + regex + ')'

        rows = []
        colwidth = []
        # do not strip \t if translate_tabs_to_spaces is false (which is the default)
        strip_char = ' ' if not view.settings().get("translate_tabs_to_spaces", False) else None
        self.expand_sel(regex, option, f , rows, colwidth, strip_char)
        rows = sorted(set(rows))
        if not rows: return False

        indentation = min([re.match("^(\s*)",
                view.substr(view.line(view.text_point(row,0)))).group(1) for row in rows])

        for row in reversed(rows):
            content = self.get_line_content(regex, f, row)
            # the last col
            begin = view.line(view.text_point(row, 0)).end()
            for i, c in reversed(list(enumerate(content))):
                # option cycles through the columns
                op = option[i % len(option)]
                # begin of current cell
                begin = begin-len(c)
                lenc = len(c.strip(strip_char)) if i>0 else len(c.rstrip(strip_char).lstrip())
                se = len(c) - len(c.rstrip(strip_char))
                sb = len(c)-lenc-se

                # oldpt is used to reset cursor position, since view.insert will change cursor's location

                if op[0] == "l":
                    fill = colwidth[i]-lenc+op[1] if i != len(content)-1 else 0
                    oldpt = [min(s.end()-sb,begin+lenc+fill) if lenc>0 else begin \
                                for s in view.sel() if s.empty() and begin+sb+lenc<=s.end()<=begin+sb+lenc+se]
                    view.erase(edit, sublime.Region(begin,begin+sb))
                    view.erase(edit, sublime.Region(begin+lenc,begin+lenc+se))
                    view.insert(edit, begin+lenc, " "*(fill))
                    if oldpt:
                        view.sel().subtract(sublime.Region(begin+lenc, begin+lenc+fill))
                        for s in [sublime.Region(b,b) for b in oldpt]: view.sel().add(s)

                if op[0] == "r":
                    fill = colwidth[i]-lenc
                    oldpt = [min(s.end()-sb+fill,begin+lenc+fill+op[1]) if lenc>0 else begin \
                                for s in view.sel() if s.empty() and begin+sb+lenc<=s.end()<=begin+sb+lenc+se]
                    view.erase(edit, sublime.Region(begin,begin+sb))
                    view.erase(edit, sublime.Region(begin+lenc,begin+lenc+se))
                    if i != len(content)-1: view.insert(edit, begin+lenc, " "*op[1])
                    view.insert(edit, begin, " "*fill)
                    if oldpt:
                        view.sel().subtract(sublime.Region(begin+fill+lenc, begin+fill+lenc+op[1]))
                        for s in [sublime.Region(b,b) for b in oldpt]: view.sel().add(s)

                if op[0] == "c":
                    lfill = int((colwidth[i]-lenc)/2)
                    rfill = colwidth[i]-lenc-lfill+op[1] if i != len(content)-1 else 0
                    oldpt = [min(s.end()-sb+lfill,begin+lenc+lfill+rfill) if lenc>0 else begin\
                                for s in view.sel() if s.empty() and begin+sb+lenc<=s.end()<=begin+sb+lenc+se]
                    view.erase(edit, sublime.Region(begin,begin+sb))
                    view.erase(edit, sublime.Region(begin+lenc,begin+lenc+se))
                    view.insert(edit, begin, " "*lfill)
                    view.insert(edit, begin+lfill+lenc, " "*rfill)
                    if oldpt:
                        view.sel().subtract(sublime.Region(begin+lfill+lenc, begin+lenc+lfill+rfill))
                        for s in [sublime.Region(b,b) for b in oldpt]: view.sel().add(s)

            view.insert(edit, view.text_point(row,0), indentation)

        return True

class AlignTabClearMode(sublime_plugin.TextCommand):
    def run(self, edit):
        view = self.view
        if view.is_scratch() or view.settings().get('is_widget'): return
        vid = view.id()
        global MODE
        print("Clear Table Mode!")
        if vid in MODE:
            MODE[vid] = False
            view.set_status("aligntab-table", "")

class AlignTabUndo(sublime_plugin.WindowCommand):
    def run(self):
        view = self.window.active_view()
        if view.is_scratch() or view.settings().get('is_widget'): return
        vid = view.id()
        global MODE
        if vid in MODE:
            MODE[vid] = False
            view.run_command("undo")
            view.run_command("undo")
            MODE[vid] = True

class AlignTabUpdater(sublime_plugin.EventListener):
    # Table mode
    def on_modified(self, view):
        if view.is_scratch() or view.settings().get('is_widget'): return
        cmdhist = view.command_history(0)
        if cmdhist[0] not in ["insert", "left_delete", "right_delete", "paste", "cut"]: return
        if cmdhist[0] == "insert" and cmdhist[1] == {'characters': ' '}: return
        vid = view.id()
        global MODE
        if vid in MODE:
            if MODE[vid]:
                view.run_command("align_tab", {"user_input": "last_rexp", "mode": True})

    def on_query_context(self, view, key, operator, operand, match_all):
        if view.is_scratch() or view.settings().get('is_widget'): return
        vid = view.id()
        if key == 'align_tab_mode':
            global MODE
            if vid in MODE:
                return MODE[vid]
            else:
                return False

    # restore History index
    def on_deactivated(self, view):
        if view.score_selector(0, 'text.aligntab') > 0:
            AlignTabHistory.index = None

    # remove MODE[vid] if file closes
    def on_close(self, view):
        vid = view.id()
        if vid in MODE: MODE.pop(vid)

# VintageEX teaches me the following
class AlignTabHistory(sublime_plugin.TextCommand):
    index = None
    def run(self, edit, backwards=False):
        if AlignTabHistory.index is None:
            AlignTabHistory.index = -1 if backwards else 0
        else:
            AlignTabHistory.index += -1 if backwards else 1

        if AlignTabHistory.index == len(HIST) or \
            AlignTabHistory.index < -len(HIST):
                AlignTabHistory.index = -1 if backwards else 0

        self.view.erase(edit, sublime.Region(0, self.view.size()))
        self.view.insert(edit, 0, HIST[AlignTabHistory.index])

<commit_msg>replace HIST and MODE by class variables<commit_after>import sublime
import sublime_plugin
import re, sys

def input_parser(user_input):
    m = re.match(r"(.+)/([lcr*()0-9]*)(f[0-9]*)?", user_input)

    if m and (m.group(2) or m.group(3)):
        regex = m.group(1)
        option = m.group(2)
        f = m.group(3)
    else:
        # print("No options!")
        return [user_input, [['l', 1]], 0]

    try:
        # for option
        rParan = re.compile(r"\(([^())]*)\)\*([0-9]+)")
        while True:
            if not rParan.search(option): break
            for r in rParan.finditer(option):
                option = option.replace(r.group(0), r.group(1)*int(r.group(2)),1)

        for r in re.finditer(r"([lcr][0-9]*)\*([0-9]+)", option):
            option = option.replace(r.group(0), r.group(1)*int(r.group(2)),1)

        option = re.findall(r"[lcr][0-9]*", option)
        option = list(map(lambda x: [x[0], 1] if len(x)==1 else [x[0], int(x[1:])], option))
        option = option if option else [['l', 1]]

        # for f
        f = 0 if not f else 1 if len(f)==1 else int(f[1:])
    except:
        [regex, option ,f]= [user_input, [['l', 1]], 0]

    return [regex, option ,f]

def update_colwidth(colwidth, content, option, strip_char):
    thiscolwidth = [len(c.strip(strip_char)) if i>0 else len(c.rstrip(strip_char).lstrip()) for i, c in enumerate(content)]
    for i,w in enumerate(thiscolwidth):
        if i<len(colwidth):
            colwidth[i] = max(colwidth[i], w)
        else:
            colwidth.append(w)

def get_named_pattern(user_input):
    patterns = sublime.load_settings('AlignTab.sublime-settings').get('named_patterns', {})
    user_input = patterns[user_input] if user_input in patterns else user_input
    user_input = AlignTabHistory.HIST[-1] if AlignTabHistory.HIST and user_input == 'last_rexp' else user_input
    return user_input

class AlignTabCommand(sublime_plugin.TextCommand):
    MODE = {}
    # Default the live_enabled setting so that if it doesn't get loaded nothing will break
    live_enabled = sublime.load_settings('AlignTab.sublime-settings').get('live_preview', True)
    live_change_made = False

    def run(self, edit, user_input=None, mode=False, event_type=None):
        view = self.view
        vid = view.id()
        if not user_input and not event_type:
            self.live_change_made = False
            v = self.view.window().show_input_panel('Align By RegEx:', '',
                    # On Done
                    lambda x: self.view.run_command("align_tab",{"user_input":x, "mode":mode, "event_type":"done"}),
                    # On Change
                    lambda x: self.on_change(x, mode),
                    # On Canel
                    lambda: self.on_change(None, mode) )
            v.set_syntax_file('Packages/AlignTab/AlignTab.hidden-tmLanguage')
            v.settings().set('is_widget', True)
            v.settings().set('gutter', False)
            v.settings().set('rulers', [])

        elif user_input:
            if not event_type: event_type = "done"

            # clear live status when ready to align
            if self.live_enabled: self.view.set_status("aligntab-live", "")

            # Don't update history if this is a live change
            if event_type == "done":
                # insert history and reset index
                if not AlignTabHistory.HIST or (user_input!= AlignTabHistory.HIST[-1] and user_input!= "last_rexp"): AlignTabHistory.HIST.append(user_input)
                AlignTabHistory.index = None
                # If live is enabled, then don't re-change the text. We're done
                if self.live_enabled and self.live_change_made:
                    self.live_change_made = False
                    return
            success = self.align_tab(edit, user_input)

            if success:
                # If rows were found, then note that we have an undo due
                if event_type == "change" and self.live_enabled:
                    self.live_change_made = True
                    view.set_status("aligntab-live", "")
                if mode:
                    AlignTabCommand.MODE[vid] = True
                    view.set_status("aligntab-table", "[Table Mode]")
            else:
                if event_type == "change" and self.live_enabled:
                    view.set_status("aligntab-live", "[Pattern not Found]")

                # this means no selection was detected to contain the regex,
                # therefore we check previous line and next line for each cursor
                if mode and not all(list(self.prev_next_match(user_input))):
                    AlignTabCommand.MODE[vid] = False
                    view.set_status("aligntab-table", "")

    def on_change(self, user_input, mode):
        # Don't do anything if we're not live
        if not self.live_enabled:
            return
        # Undo the previous change if needed
        if self.live_change_made:
            self.view.run_command("undo")
            self.live_change_made = False
        # Run the align command
        if user_input:
            self.view.run_command("align_tab",{"user_input":user_input, "mode":mode, "event_type":"change"})

    def get_line_content(self, regex, f, row):
        view = self.view
        line = view.line(view.text_point(row,0))
        return [s for s in re.split(regex,view.substr(line),f)]

    def expand_sel(self, regex, option, f, rows, colwidth, strip_char):
        view = self.view
        lastrow = view.rowcol(view.size())[0]

        for sel in view.sel():
            for line in view.lines(sel):
                thisrow = view.rowcol(line.begin())[0]
                if (thisrow in rows): continue
                content = self.get_line_content(regex, f, thisrow)
                if len(content)<=1: continue
                update_colwidth(colwidth, content, option, strip_char)
                rows.append(thisrow)

            if sel.empty():
                thisrow = view.rowcol(sel.begin())[0]
                if not (thisrow in rows): continue
                beginrow = endrow = thisrow
                while endrow+1<=lastrow and not (endrow+1 in rows):
                    content = self.get_line_content(regex, f, endrow+1)
                    if len(content)<=1: break
                    update_colwidth(colwidth, content, option, strip_char)
                    endrow = endrow+1
                    rows.append(endrow)
                while beginrow-1>=0 and not (beginrow-1 in rows):
                    content = self.get_line_content(regex, f, beginrow-1)
                    if len(content)<=1: break
                    update_colwidth(colwidth, content, option, strip_char)
                    beginrow = beginrow-1
                    rows.append(beginrow)

    def prev_next_match(self, user_input):

        user_input = get_named_pattern(user_input)
        [regex, option, f] = input_parser(user_input)
        regex = '(' + regex + ')'
        # it is used to check whether table mode should be disabled
        view = self.view
        lastrow = view.rowcol(view.size())[0]
        rows = []
        for sel in view.sel():
            for line in view.lines(sel):
                rows.append(view.rowcol(line.begin())[0])
        rows = list(set(rows))
        for row in rows:
            if row-1>=0 and len(self.get_line_content(regex, f, row-1))>1:
                yield True
            elif row+1<=lastrow and len(self.get_line_content(regex, f, row+1))>1:
                yield True
            else:
                yield False

    def align_tab(self, edit, user_input):
        view = self.view

        user_input = get_named_pattern(user_input)
        [regex, option, f] = input_parser(user_input)
        regex = '(' + regex + ')'

        rows = []
        colwidth = []
        # do not strip \t if translate_tabs_to_spaces is false (which is the default)
        strip_char = ' ' if not view.settings().get("translate_tabs_to_spaces", False) else None
        self.expand_sel(regex, option, f , rows, colwidth, strip_char)
        rows = sorted(set(rows))
        if not rows: return False

        indentation = min([re.match("^(\s*)",
                view.substr(view.line(view.text_point(row,0)))).group(1) for row in rows])

        for row in reversed(rows):
            content = self.get_line_content(regex, f, row)
            # the last col
            begin = view.line(view.text_point(row, 0)).end()
            for i, c in reversed(list(enumerate(content))):
                # option cycles through the columns
                op = option[i % len(option)]
                # begin of current cell
                begin = begin-len(c)
                lenc = len(c.strip(strip_char)) if i>0 else len(c.rstrip(strip_char).lstrip())
                se = len(c) - len(c.rstrip(strip_char))
                sb = len(c)-lenc-se

                # oldpt is used to reset cursor position, since view.insert will change cursor's location

                if op[0] == "l":
                    fill = colwidth[i]-lenc+op[1] if i != len(content)-1 else 0
                    oldpt = [min(s.end()-sb,begin+lenc+fill) if lenc>0 else begin \
                                for s in view.sel() if s.empty() and begin+sb+lenc<=s.end()<=begin+sb+lenc+se]
                    view.erase(edit, sublime.Region(begin,begin+sb))
                    view.erase(edit, sublime.Region(begin+lenc,begin+lenc+se))
                    view.insert(edit, begin+lenc, " "*(fill))
                    if oldpt:
                        view.sel().subtract(sublime.Region(begin+lenc, begin+lenc+fill))
                        for s in [sublime.Region(b,b) for b in oldpt]: view.sel().add(s)

                if op[0] == "r":
                    fill = colwidth[i]-lenc
                    oldpt = [min(s.end()-sb+fill,begin+lenc+fill+op[1]) if lenc>0 else begin \
                                for s in view.sel() if s.empty() and begin+sb+lenc<=s.end()<=begin+sb+lenc+se]
                    view.erase(edit, sublime.Region(begin,begin+sb))
                    view.erase(edit, sublime.Region(begin+lenc,begin+lenc+se))
                    if i != len(content)-1: view.insert(edit, begin+lenc, " "*op[1])
                    view.insert(edit, begin, " "*fill)
                    if oldpt:
                        view.sel().subtract(sublime.Region(begin+fill+lenc, begin+fill+lenc+op[1]))
                        for s in [sublime.Region(b,b) for b in oldpt]: view.sel().add(s)

                if op[0] == "c":
                    lfill = int((colwidth[i]-lenc)/2)
                    rfill = colwidth[i]-lenc-lfill+op[1] if i != len(content)-1 else 0
                    oldpt = [min(s.end()-sb+lfill,begin+lenc+lfill+rfill) if lenc>0 else begin\
                                for s in view.sel() if s.empty() and begin+sb+lenc<=s.end()<=begin+sb+lenc+se]
                    view.erase(edit, sublime.Region(begin,begin+sb))
                    view.erase(edit, sublime.Region(begin+lenc,begin+lenc+se))
                    view.insert(edit, begin, " "*lfill)
                    view.insert(edit, begin+lfill+lenc, " "*rfill)
                    if oldpt:
                        view.sel().subtract(sublime.Region(begin+lfill+lenc, begin+lenc+lfill+rfill))
                        for s in [sublime.Region(b,b) for b in oldpt]: view.sel().add(s)

            view.insert(edit, view.text_point(row,0), indentation)

        return True

class AlignTabClearMode(sublime_plugin.TextCommand):
    def run(self, edit):
        view = self.view
        if view.is_scratch() or view.settings().get('is_widget'): return
        vid = view.id()
        print("Clear Table Mode!")
        if vid in AlignTabCommand.MODE:
            AlignTabCommand.MODE[vid] = False
            view.set_status("aligntab-table", "")

class AlignTabUndo(sublime_plugin.WindowCommand):
    def run(self):
        view = self.window.active_view()
        if view.is_scratch() or view.settings().get('is_widget'): return
        vid = view.id()
        if vid in AlignTabCommand.MODE:
            AlignTabCommand.MODE[vid] = False
            view.run_command("undo")
            view.run_command("undo")
            AlignTabCommand.MODE[vid] = True

class AlignTabUpdater(sublime_plugin.EventListener):
    # Table mode
    def on_modified(self, view):
        if view.is_scratch() or view.settings().get('is_widget'): return
        cmdhist = view.command_history(0)
        if cmdhist[0] not in ["insert", "left_delete", "right_delete", "paste", "cut"]: return
        if cmdhist[0] == "insert" and cmdhist[1] == {'characters': ' '}: return
        vid = view.id()
        if vid in AlignTabCommand.MODE:
            if AlignTabCommand.MODE[vid]:
                view.run_command("align_tab", {"user_input": "last_rexp", "mode": True})

    def on_query_context(self, view, key, operator, operand, match_all):
        if view.is_scratch() or view.settings().get('is_widget'): return
        vid = view.id()
        if key == 'align_tab_mode':
            if vid in AlignTabCommand.MODE:
                return AlignTabCommand.MODE[vid]
            else:
                return False

    # restore History index
    def on_deactivated(self, view):
        if view.score_selector(0, 'text.aligntab') > 0:
            AlignTabHistory.index = None

    # remove AlignTabCommand.MODE[vid] if file closes
    def on_close(self, view):
        vid = view.id()
        if vid in AlignTabCommand.MODE: AlignTabCommand.MODE.pop(vid)

# VintageEX teaches me the following
class AlignTabHistory(sublime_plugin.TextCommand):
    HIST = []
    index = None
    def run(self, edit, backwards=False):
        if AlignTabHistory.index is None:
            AlignTabHistory.index = -1 if backwards else 0
        else:
            AlignTabHistory.index += -1 if backwards else 1

        if AlignTabHistory.index == len(AlignTabHistory.HIST) or \
            AlignTabHistory.index < -len(AlignTabHistory.HIST):
                AlignTabHistory.index = -1 if backwards else 0

        self.view.erase(edit, sublime.Region(0, self.view.size()))
        self.view.insert(edit, 0, AlignTabHistory.HIST[AlignTabHistory.index])

<|endoftext|>


<commit_before># Quantopian, Inc. licenses this file to you under the Apache License, Version
# 2.0 (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from base64 import b64encode
from collections import namedtuple, OrderedDict
import datetime
from distutils.version import LooseVersion
import glob
from hashlib import md5
from itertools import chain
import logbook
import os
import pickle
import re
import socket
import stat
from stopit import ThreadingTimeout
import subprocess
from tempfile import NamedTemporaryFile
import yaml

import penguindome.json as json

top_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
gpg_private_dir = os.path.join('server', 'keyring')
gpg_public_dir = os.path.join('client', 'keyring')
gpg_private_home = os.path.join(top_dir, gpg_private_dir)
gpg_public_home = os.path.join(top_dir, gpg_public_dir)
release_subdirs = ('client', 'penguindome')
settingses = {}
var_dir = os.path.join(top_dir, 'var')
releases_dir = os.path.join(var_dir, 'client_releases')
collected_dir = os.path.join(var_dir, 'collected')
release_file = os.path.join('client', 'release.txt')
plugins_dir = os.path.join('client', 'plugins')
commands_dir = os.path.join('client', 'commands')
signatures_dir = 'signatures'
gpg_mode = None
gpg_exe = None
got_logger = None
client_gpg_version = '2.1.11'
server_pipe_log_filter_re = re.compile(
    r'POST.*/server_pipe/.*/(?:send|receive).* 200 ')

SelectorVariants = namedtuple(
    'SelectorVariants', ['plain_mongo', 'plain_mem', 'enc_mongo', 'enc_mem'])

def release_files_iter(with_signatures=False, top_dir=top_dir):
    for dirpath, dirnames, filenames in os.walk(top_dir):
        if dirpath == top_dir:
            for i in range(len(dirnames) - 1, -1, -1):
                if dirnames[i] not in release_subdirs:
                    del dirnames[i]
            filenames = []
        for i in range(len(dirnames) - 1, -1, -1):
            if dirnames[i].startswith('.#'):
                del dirnames[i]
        for filename in filenames:
            if filename.startswith('.#'):
                continue
            if filename.endswith('~'):
                continue
            if filename.endswith('.pyc'):
                continue
            path = os.path.join(dirpath, filename)
            if not stat.S_ISREG(os.stat(path).st_mode):
                continue
            relative_path = path[len(top_dir) + 1:]
            if with_signatures:
                yield (relative_path,
                       os.path.join('signatures', relative_path + '.sig'))
            else:
                yield relative_path

def set_gpg(mode):
    global gpg_mode

    if mode == 'server':
        home = gpg_private_home
    elif mode == 'client':
        home = gpg_public_home
    else:
        raise Exception('Internal error: Unrecognized GPG mode {}'.format(
            mode))

    os.environ['GNUPGHOME'] = home
    os.chmod(home, 0o0700)
    # random seed gets corrupted sometimes because we're copying keyring from
    # server to client
    list(map(os.unlink, glob.glob(os.path.join(home, "random_seed*"))))
    gpg_mode = mode

def gpg_command(*cmd, with_trustdb=False, quiet=True,
                minimum_version='2.1.15'):
    global gpg_exe, gpg_exe

    if not gpg_mode:
        raise Exception('Attempt to use GPG before setting mode')
    if not gpg_exe:
        try:
            output = subprocess.check_output(
                ('gpg2', '--version'),
                stderr=subprocess.STDOUT).decode('utf8')
        except:
            output = subprocess.check_output(
                ('gpg', '--version'),
                stderr=subprocess.STDOUT).decode('utf8')
            gpg_exe = 'gpg'
        else:
            gpg_exe = 'gpg2'
        match = re.search(r'^gpg \(GnuPG\) (\d+(?:\.\d+(?:\.\d+)?)?)', output,
                          re.MULTILINE)
        if not match:
            raise Exception('Could not determine GnuPG version in output:\n{}'.
                            format(output))
        if LooseVersion(match.group(1)) < LooseVersion(minimum_version):
            raise Exception('GnuPG version {} or newer is required. '
                            'You have version {}.'.format(
                                minimum_version, match.group(1)))

    if with_trustdb:
        trustdb_args = ()
    else:
        trustdb_args = ('--trust-model', 'always')

    if quiet:
        quiet_args = ('--quiet',)
    else:
        quiet_args = ()

    cmd = tuple(chain((gpg_exe, '--batch', '--yes'), quiet_args, trustdb_args,
                      cmd))
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT).\
        decode('utf8')

def load_settings(which):
    settings_file = os.path.join(top_dir, which, 'settings.yml')
    try:
        mtime = os.stat(settings_file).st_mtime
    except:
        mtime = 0

    if which in settingses and mtime <= settingses[which]['mtime']:
        return settingses[which]['settings']

    if os.path.exists(settings_file):
        settings = yaml.load(open(settings_file))
        settings['loaded'] = True
    else:
        settings = {'loaded': False}

    defaults_file = os.path.join(top_dir, which, 'default-settings.yml')
    settings['defaults'] = yaml.load(open(defaults_file))

    if 'server_url' in settings:
        settings['server_url'] = re.sub(r'/+$', '', settings['server_url'])

    settingses[which] = {'settings': settings, 'mtime': mtime}

    return settings

def save_settings(which):
    settings = load_settings(which)
    bare = settings.copy()
    bare.pop('defaults', None)
    bare.pop('loaded', None)
    yaml.dump(bare, open(os.path.join(top_dir, which, 'settings.yml'), 'w'))
    settings['loaded'] = True

def get_setting(settings, setting, default=None, check_defaults=True):
    """Get a possibly recursive setting from a dictionary

    "settings" is a dictionary. "setting" is a colon-separated list of keys.
    Recurses through "settings" looking for the specified setting, and returns
    the specified default if the setting isn't present and there's no
    preconfigured default setting.
    """
    if check_defaults:
        defaults = settings.get('defaults', {})
    for key in setting.split(':'):
        try:
            settings = settings[key]
        except:
            if check_defaults:
                return get_setting(defaults, setting, default,
                                   check_defaults=False)
            return default
    return settings

def set_setting(settings, setting, value):
    keys = setting.split(':')
    subsettings = settings
    while len(keys) > 1:
        if keys[0] not in subsettings:
            subsettings[keys[0]] = {}
        subsettings = subsettings[keys[0]]
        keys.pop(0)
    if value is None:
        subsettings.pop(keys[0], None)
        if get_setting(settings, setting):
            # It has a default, so we need to store an override.
            subsettings[keys[0]] = None
    else:
        subsettings[keys[0]] = value

def verify_signature(file, top_dir=top_dir, raise_errors=False):
    signature_file = os.path.join(top_dir, signatures_dir, file + '.sig')
    file = os.path.join(top_dir, file)
    try:
        gpg_command('--verify', signature_file, file,
                    minimum_version=client_gpg_version)
    except subprocess.CalledProcessError:
        if raise_errors:
            raise
        return None
    return signature_file[len(top_dir) + 1:]

def get_logger(setting_getter, name, fail_to_local=False, filter=None):
    global got_logger
    if got_logger:
        # Yes, this means that if you try to change your logging within an
        # application, it won't work. This is intentional. You shouldn't do
        # that.
        return got_logger

    if filter:
        def log_filter(r, h):
            if server_pipe_log_filter_re.search(r.message):
                return False
            return filter(r, h)
    else:
        def log_filter(r, h):
            return not server_pipe_log_filter_re.search(r.message)

    logger_name = 'penguindome-' + name
    file_safe_logger_name = logger_name.replace(os.sep, '_')

    logger = logbook.Logger(logger_name)

    internal_log_dir = os.path.join(var_dir, 'log')
    internal_log_file = os.path.join(
        internal_log_dir, file_safe_logger_name + '.log')

    os.makedirs(internal_log_dir, 0o0700, exist_ok=True)

    # We always do local debug logging, regardless of whether we're also
    # logging elsewhere.
    logbook.RotatingFileHandler(
        internal_log_file, bubble=True, filter=log_filter).push_application()

    handler_name = setting_getter('logging:handler')
    if handler_name:
        handler_name = handler_name.lower()
        handler_name += 'handler'
        handler_name = next(d for d in dir(logbook)
                            if d.lower() == handler_name)
        handler = logbook.__dict__[handler_name]
        kwargs = {'bubble': True, 'filter': log_filter}
        level = setting_getter('logging:level')
        kwargs['level'] = logbook.__dict__[level.upper()]
        if handler_name == 'SyslogHandler':
            kwargs['facility'] = setting_getter('logging:syslog:facility')
            hostname = setting_getter('logging:syslog:host')
            if hostname:
                port = setting_getter('logging:syslog:port')
                try:
                    addrinfo = socket.getaddrinfo(
                        hostname, port, socket.AF_INET, socket.SOCK_STREAM)[0]
                except:
                    if not fail_to_local:
                        raise
                    logger.warn('Failed to resolve {}:{}, falling back to '
                                'local-only logging', hostname, port)
                    handler = None
                else:
                    kwargs['socktype'] = addrinfo[1]
                    kwargs['address'] = addrinfo[4]

        if handler:
            if fail_to_local:
                try:
                    with ThreadingTimeout(5, swallow_exc=False):
                        handler = handler(**kwargs)
                except:
                    logger.warn('Failed to create {}, falling back to '
                                'local-only logging', handler_name)
                else:
                    handler.push_application()
            else:
                handler(**kwargs).push_application()

    logbook.compat.redirect_logging()
    got_logger = logger
    return got_logger

def get_selectors(getter):
    return tuple(SelectorVariants(s, s.replace('.', ':'), s + '-encrypted',
                                  s.replace('.', ':') + '-encrypted')
                 for s in getter('secret_keeping:selectors', []))

def orderify(obj):
    """Convert dicts (recursively) to ordered dicts with sorted keys"""

    if isinstance(obj, dict):
        for key, value in obj.items():
            if isinstance(value, dict):
                obj[key] = orderify(value)
                continue
            if isinstance(value, (tuple, list)):
                obj[key] = orderify(value)
        obj = OrderedDict((key, obj[key]) for key in sorted(obj.keys()))
    elif isinstance(obj, (tuple, list)):
        obj = type(obj)(sorted((orderify(v) for v in obj),
                               key=str))

    return obj

def encrypt_document(getter, doc, log=None, selectors=None):
    if not getter('secret_keeping:enabled'):
        return doc, None
    key_id = getter('secret_keeping:key_id')
    if selectors is None:
        selectors = get_selectors(getter)
    update = {'$unset': {}, '$set': {}}
    for s in selectors:
        decrypted_data = get_setting(
            doc, s.plain_mem, check_defaults=False)
        if not decrypted_data:
            continue
        # This assures that the same decrypted data will always end up with the
        # same md5 hash by ensuring that the data are always encoded in the
        # same order.
        if isinstance(decrypted_data, dict):
            decrypted_data = orderify(decrypted_data)
        decrypted_data = json.dumps(decrypted_data).encode('utf-8')
        with NamedTemporaryFile('w+b') as unencrypted_file, \
                NamedTemporaryFile('w+b') as encrypted_file:
            unencrypted_file.write(decrypted_data)
            unencrypted_file.flush()
            try:
                gpg_command('--encrypt', '--recipient', key_id, '-o',
                            encrypted_file.name, unencrypted_file.name,
                            minimum_version=client_gpg_version)
            except subprocess.CalledProcessError as e:
                if log:
                    log.error('Gpg failed to encrypt. Output:\n{}',
                              e.output.decode('utf8'))
                raise
            encrypted_file.seek(0)
            encrypted_data = {
                'hash': md5(decrypted_data).hexdigest(),
                'data': b64encode(encrypted_file.read()).decode('utf8')}
        if s.plain_mongo != s.enc_mongo:
            update['$unset'][s.plain_mongo] = True
        update['$set'][s.enc_mongo] = encrypted_data
        set_setting(doc, s.plain_mem, None)
        set_setting(doc, s.enc_mem, encrypted_data)
    if update['$set']:
        if not update['$unset']:
            update.pop('$unset')
        return doc, update
    return doc, None

def cached_data(key, data=None, add_timestamp=False, check_logged_in=False,
                raise_exception=True):
    """Return or save cached data for the specified key

    If data is None and there is no cached data for the specified key, raises
    FileNotFoundError unless raise_exception is False in which case just
    returns None.

    If add_timestamp is true, then adds a cached_at timestamp to data (which
    must be a dict or dict-like object) before saving it.

    If check_logged_in is true, then checks if any users are currently logged
    in before saving or returning the data, and if not, then assumes that data
    is None even if it actually isn't.
    """

    if os.sep in key:
        raise Exception('Cache keys cannot have {} in them'.format(os.sep))

    cache_file = os.path.join(var_dir, 'data_cache', key)

    if check_logged_in and subprocess.check_output('who') == b'':
        # No one is logged in
        data = None

    if data is None:
        try:
            return pickle.load(open(cache_file, 'rb'))
        except FileNotFoundError:
            if raise_exception:
                raise
            return None

    os.makedirs(os.path.dirname(cache_file), exist_ok=True)

    if add_timestamp and isinstance(data, dict):
        save_data = data.copy()
        save_data['cached_at'] = datetime.datetime.utcnow()
    else:
        save_data = data

    pickle.dump(save_data, open(cache_file, 'wb'))
    return data
<commit_msg>Ignore GnuPG card reader status files when verifying signatures<commit_after># Quantopian, Inc. licenses this file to you under the Apache License, Version
# 2.0 (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from base64 import b64encode
from collections import namedtuple, OrderedDict
import datetime
from distutils.version import LooseVersion
import fnmatch
import glob
from hashlib import md5
from itertools import chain
import logbook
import os
import pickle
import re
import socket
import stat
from stopit import ThreadingTimeout
import subprocess
from tempfile import NamedTemporaryFile
import yaml

import penguindome.json as json

top_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
gpg_private_dir = os.path.join('server', 'keyring')
gpg_public_dir = os.path.join('client', 'keyring')
gpg_private_home = os.path.join(top_dir, gpg_private_dir)
gpg_public_home = os.path.join(top_dir, gpg_public_dir)
release_subdirs = ('client', 'penguindome')
settingses = {}
var_dir = os.path.join(top_dir, 'var')
releases_dir = os.path.join(var_dir, 'client_releases')
collected_dir = os.path.join(var_dir, 'collected')
release_file = os.path.join('client', 'release.txt')
plugins_dir = os.path.join('client', 'plugins')
commands_dir = os.path.join('client', 'commands')
signatures_dir = 'signatures'
gpg_mode = None
gpg_exe = None
got_logger = None
client_gpg_version = '2.1.11'
server_pipe_log_filter_re = re.compile(
    r'POST.*/server_pipe/.*/(?:send|receive).* 200 ')

SelectorVariants = namedtuple(
    'SelectorVariants', ['plain_mongo', 'plain_mem', 'enc_mongo', 'enc_mem'])

def release_files_iter(with_signatures=False, top_dir=top_dir):
    for dirpath, dirnames, filenames in os.walk(top_dir):
        if dirpath == top_dir:
            for i in range(len(dirnames) - 1, -1, -1):
                if dirnames[i] not in release_subdirs:
                    del dirnames[i]
            filenames = []
        for i in range(len(dirnames) - 1, -1, -1):
            if dirnames[i].startswith('.#'):
                del dirnames[i]
        for filename in filenames:
            if filename.startswith('.#'):
                continue
            if filename.endswith('~'):
                continue
            if filename.endswith('.pyc'):
                continue
            path = os.path.join(dirpath, filename)
            if not stat.S_ISREG(os.stat(path).st_mode):
                continue
            relative_path = path[len(top_dir) + 1:]
            if fnmatch.fnmatch(relative_path, '*/keyring/reader_*.status'):
                continue
            if with_signatures:
                yield (relative_path,
                       os.path.join('signatures', relative_path + '.sig'))
            else:
                yield relative_path

def set_gpg(mode):
    global gpg_mode

    if mode == 'server':
        home = gpg_private_home
    elif mode == 'client':
        home = gpg_public_home
    else:
        raise Exception('Internal error: Unrecognized GPG mode {}'.format(
            mode))

    os.environ['GNUPGHOME'] = home
    os.chmod(home, 0o0700)
    # random seed gets corrupted sometimes because we're copying keyring from
    # server to client
    list(map(os.unlink, glob.glob(os.path.join(home, "random_seed*"))))
    gpg_mode = mode

def gpg_command(*cmd, with_trustdb=False, quiet=True,
                minimum_version='2.1.15'):
    global gpg_exe, gpg_exe

    if not gpg_mode:
        raise Exception('Attempt to use GPG before setting mode')
    if not gpg_exe:
        try:
            output = subprocess.check_output(
                ('gpg2', '--version'),
                stderr=subprocess.STDOUT).decode('utf8')
        except:
            output = subprocess.check_output(
                ('gpg', '--version'),
                stderr=subprocess.STDOUT).decode('utf8')
            gpg_exe = 'gpg'
        else:
            gpg_exe = 'gpg2'
        match = re.search(r'^gpg \(GnuPG\) (\d+(?:\.\d+(?:\.\d+)?)?)', output,
                          re.MULTILINE)
        if not match:
            raise Exception('Could not determine GnuPG version in output:\n{}'.
                            format(output))
        if LooseVersion(match.group(1)) < LooseVersion(minimum_version):
            raise Exception('GnuPG version {} or newer is required. '
                            'You have version {}.'.format(
                                minimum_version, match.group(1)))

    if with_trustdb:
        trustdb_args = ()
    else:
        trustdb_args = ('--trust-model', 'always')

    if quiet:
        quiet_args = ('--quiet',)
    else:
        quiet_args = ()

    cmd = tuple(chain((gpg_exe, '--batch', '--yes'), quiet_args, trustdb_args,
                      cmd))
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT).\
        decode('utf8')

def load_settings(which):
    settings_file = os.path.join(top_dir, which, 'settings.yml')
    try:
        mtime = os.stat(settings_file).st_mtime
    except:
        mtime = 0

    if which in settingses and mtime <= settingses[which]['mtime']:
        return settingses[which]['settings']

    if os.path.exists(settings_file):
        settings = yaml.load(open(settings_file))
        settings['loaded'] = True
    else:
        settings = {'loaded': False}

    defaults_file = os.path.join(top_dir, which, 'default-settings.yml')
    settings['defaults'] = yaml.load(open(defaults_file))

    if 'server_url' in settings:
        settings['server_url'] = re.sub(r'/+$', '', settings['server_url'])

    settingses[which] = {'settings': settings, 'mtime': mtime}

    return settings

def save_settings(which):
    settings = load_settings(which)
    bare = settings.copy()
    bare.pop('defaults', None)
    bare.pop('loaded', None)
    yaml.dump(bare, open(os.path.join(top_dir, which, 'settings.yml'), 'w'))
    settings['loaded'] = True

def get_setting(settings, setting, default=None, check_defaults=True):
    """Get a possibly recursive setting from a dictionary

    "settings" is a dictionary. "setting" is a colon-separated list of keys.
    Recurses through "settings" looking for the specified setting, and returns
    the specified default if the setting isn't present and there's no
    preconfigured default setting.
    """
    if check_defaults:
        defaults = settings.get('defaults', {})
    for key in setting.split(':'):
        try:
            settings = settings[key]
        except:
            if check_defaults:
                return get_setting(defaults, setting, default,
                                   check_defaults=False)
            return default
    return settings

def set_setting(settings, setting, value):
    keys = setting.split(':')
    subsettings = settings
    while len(keys) > 1:
        if keys[0] not in subsettings:
            subsettings[keys[0]] = {}
        subsettings = subsettings[keys[0]]
        keys.pop(0)
    if value is None:
        subsettings.pop(keys[0], None)
        if get_setting(settings, setting):
            # It has a default, so we need to store an override.
            subsettings[keys[0]] = None
    else:
        subsettings[keys[0]] = value

def verify_signature(file, top_dir=top_dir, raise_errors=False):
    signature_file = os.path.join(top_dir, signatures_dir, file + '.sig')
    file = os.path.join(top_dir, file)
    try:
        gpg_command('--verify', signature_file, file,
                    minimum_version=client_gpg_version)
    except subprocess.CalledProcessError:
        if raise_errors:
            raise
        return None
    return signature_file[len(top_dir) + 1:]

def get_logger(setting_getter, name, fail_to_local=False, filter=None):
    global got_logger
    if got_logger:
        # Yes, this means that if you try to change your logging within an
        # application, it won't work. This is intentional. You shouldn't do
        # that.
        return got_logger

    if filter:
        def log_filter(r, h):
            if server_pipe_log_filter_re.search(r.message):
                return False
            return filter(r, h)
    else:
        def log_filter(r, h):
            return not server_pipe_log_filter_re.search(r.message)

    logger_name = 'penguindome-' + name
    file_safe_logger_name = logger_name.replace(os.sep, '_')

    logger = logbook.Logger(logger_name)

    internal_log_dir = os.path.join(var_dir, 'log')
    internal_log_file = os.path.join(
        internal_log_dir, file_safe_logger_name + '.log')

    os.makedirs(internal_log_dir, 0o0700, exist_ok=True)

    # We always do local debug logging, regardless of whether we're also
    # logging elsewhere.
    logbook.RotatingFileHandler(
        internal_log_file, bubble=True, filter=log_filter).push_application()

    handler_name = setting_getter('logging:handler')
    if handler_name:
        handler_name = handler_name.lower()
        handler_name += 'handler'
        handler_name = next(d for d in dir(logbook)
                            if d.lower() == handler_name)
        handler = logbook.__dict__[handler_name]
        kwargs = {'bubble': True, 'filter': log_filter}
        level = setting_getter('logging:level')
        kwargs['level'] = logbook.__dict__[level.upper()]
        if handler_name == 'SyslogHandler':
            kwargs['facility'] = setting_getter('logging:syslog:facility')
            hostname = setting_getter('logging:syslog:host')
            if hostname:
                port = setting_getter('logging:syslog:port')
                try:
                    addrinfo = socket.getaddrinfo(
                        hostname, port, socket.AF_INET, socket.SOCK_STREAM)[0]
                except:
                    if not fail_to_local:
                        raise
                    logger.warn('Failed to resolve {}:{}, falling back to '
                                'local-only logging', hostname, port)
                    handler = None
                else:
                    kwargs['socktype'] = addrinfo[1]
                    kwargs['address'] = addrinfo[4]

        if handler:
            if fail_to_local:
                try:
                    with ThreadingTimeout(5, swallow_exc=False):
                        handler = handler(**kwargs)
                except:
                    logger.warn('Failed to create {}, falling back to '
                                'local-only logging', handler_name)
                else:
                    handler.push_application()
            else:
                handler(**kwargs).push_application()

    logbook.compat.redirect_logging()
    got_logger = logger
    return got_logger

def get_selectors(getter):
    return tuple(SelectorVariants(s, s.replace('.', ':'), s + '-encrypted',
                                  s.replace('.', ':') + '-encrypted')
                 for s in getter('secret_keeping:selectors', []))

def orderify(obj):
    """Convert dicts (recursively) to ordered dicts with sorted keys"""

    if isinstance(obj, dict):
        for key, value in obj.items():
            if isinstance(value, dict):
                obj[key] = orderify(value)
                continue
            if isinstance(value, (tuple, list)):
                obj[key] = orderify(value)
        obj = OrderedDict((key, obj[key]) for key in sorted(obj.keys()))
    elif isinstance(obj, (tuple, list)):
        obj = type(obj)(sorted((orderify(v) for v in obj),
                               key=str))

    return obj

def encrypt_document(getter, doc, log=None, selectors=None):
    if not getter('secret_keeping:enabled'):
        return doc, None
    key_id = getter('secret_keeping:key_id')
    if selectors is None:
        selectors = get_selectors(getter)
    update = {'$unset': {}, '$set': {}}
    for s in selectors:
        decrypted_data = get_setting(
            doc, s.plain_mem, check_defaults=False)
        if not decrypted_data:
            continue
        # This assures that the same decrypted data will always end up with the
        # same md5 hash by ensuring that the data are always encoded in the
        # same order.
        if isinstance(decrypted_data, dict):
            decrypted_data = orderify(decrypted_data)
        decrypted_data = json.dumps(decrypted_data).encode('utf-8')
        with NamedTemporaryFile('w+b') as unencrypted_file, \
                NamedTemporaryFile('w+b') as encrypted_file:
            unencrypted_file.write(decrypted_data)
            unencrypted_file.flush()
            try:
                gpg_command('--encrypt', '--recipient', key_id, '-o',
                            encrypted_file.name, unencrypted_file.name,
                            minimum_version=client_gpg_version)
            except subprocess.CalledProcessError as e:
                if log:
                    log.error('Gpg failed to encrypt. Output:\n{}',
                              e.output.decode('utf8'))
                raise
            encrypted_file.seek(0)
            encrypted_data = {
                'hash': md5(decrypted_data).hexdigest(),
                'data': b64encode(encrypted_file.read()).decode('utf8')}
        if s.plain_mongo != s.enc_mongo:
            update['$unset'][s.plain_mongo] = True
        update['$set'][s.enc_mongo] = encrypted_data
        set_setting(doc, s.plain_mem, None)
        set_setting(doc, s.enc_mem, encrypted_data)
    if update['$set']:
        if not update['$unset']:
            update.pop('$unset')
        return doc, update
    return doc, None

def cached_data(key, data=None, add_timestamp=False, check_logged_in=False,
                raise_exception=True):
    """Return or save cached data for the specified key

    If data is None and there is no cached data for the specified key, raises
    FileNotFoundError unless raise_exception is False in which case just
    returns None.

    If add_timestamp is true, then adds a cached_at timestamp to data (which
    must be a dict or dict-like object) before saving it.

    If check_logged_in is true, then checks if any users are currently logged
    in before saving or returning the data, and if not, then assumes that data
    is None even if it actually isn't.
    """

    if os.sep in key:
        raise Exception('Cache keys cannot have {} in them'.format(os.sep))

    cache_file = os.path.join(var_dir, 'data_cache', key)

    if check_logged_in and subprocess.check_output('who') == b'':
        # No one is logged in
        data = None

    if data is None:
        try:
            return pickle.load(open(cache_file, 'rb'))
        except FileNotFoundError:
            if raise_exception:
                raise
            return None

    os.makedirs(os.path.dirname(cache_file), exist_ok=True)

    if add_timestamp and isinstance(data, dict):
        save_data = data.copy()
        save_data['cached_at'] = datetime.datetime.utcnow()
    else:
        save_data = data

    pickle.dump(save_data, open(cache_file, 'wb'))
    return data
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http.response import HttpResponseRedirect, StreamingHttpResponse
from django.db.models import get_model
from django.conf import settings
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.contrib import messages
from django.utils.translation import ugettext as _
from django.utils.encoding import smart_str
from yats.forms import TicketsForm, CommentForm, UploadFileForm, SearchForm, TicketCloseForm
from yats.models import tickets_files, tickets_comments, tickets_reports, ticket_resolution, tickets_participants, tickets_history
from yats.shortcuts import resize_image, touch_ticket, mail_ticket, mail_comment, mail_file, clean_search_values, check_references, remember_changes, add_history, prettyValues
import os
import io
try:
    import json
except ImportError:
    from django.utils import simplejson as json

def new(request):
    excludes = ['resolution']

    if request.method == 'POST':
        form = TicketsForm(request.POST, exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)
        if form.is_valid():
            tic = form.save()

            assigned = form.cleaned_data.get('assigned')
            if assigned:
                touch_ticket(assigned, tic.pk)

            touch_ticket(request.user, tic.pk)

            mail_ticket(request, tic.pk, form, rcpt=settings.TICKET_NEW_MAIL_RCPT)

            if form.cleaned_data.get('file_addition', False):
                return HttpResponseRedirect('/tickets/upload/%s/' % tic.pk)
            else:
                return HttpResponseRedirect('/tickets/view/%s/' % tic.pk)

    else:
        form = TicketsForm(exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)

    return render_to_response('tickets/new.html', {'layout': 'horizontal', 'form': form}, RequestContext(request))

def action(request, mode, ticket):
    mod_path, cls_name = settings.TICKET_CLASS.rsplit('.', 1)
    mod_path = mod_path.split('.').pop(0)
    tic = get_model(mod_path, cls_name).objects.get(pk=ticket)

    if mode == 'view':
        if request.method == 'POST':
            form = CommentForm(request.POST)
            if form.is_valid():
                com = tickets_comments()
                com.comment = form.cleaned_data['comment']
                com.ticket_id = ticket
                com.action = 6
                com.save(user=request.user)

                check_references(request, com)

                touch_ticket(request.user, ticket)

                add_history(request, tic, 6, com.comment)

                mail_comment(request, com.pk)

            else:
                if 'resolution' in request.POST:
                    if request.POST['resolution'] and int(request.POST['resolution']) > 0:
                        tic.resolution_id = request.POST['resolution']
                        tic.closed = True
                        tic.save(user=request.user)

                        com = tickets_comments()
                        com.comment = _('ticket closed - resolution: %(resolution)s\n\n%(comment)s') % {'resolution': ticket_resolution.objects.get(pk=request.POST['resolution']).name, 'comment': request.POST.get('close_comment', '')}
                        com.ticket_id = ticket
                        com.action = 1
                        com.save(user=request.user)

                        check_references(request, com)

                        touch_ticket(request.user, ticket)

                        add_history(request, tic, 1, request.POST.get('close_comment', ''))

                        mail_comment(request, com.pk)

                    else:
                        messages.add_message(request, messages.ERROR, _('no resolution selected'))

                else:
                    messages.add_message(request, messages.ERROR, _('comment invalid'))

        excludes = []
        form = TicketsForm(exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, instance=tic, customer=request.organisation.id, view_only=True)
        close = TicketCloseForm()

        participants = tickets_participants.objects.select_related('user').filter(ticket=ticket)

        files = tickets_files.objects.filter(ticket=ticket)
        paginator = Paginator(files, 10)
        page = request.GET.get('page')
        try:
            files_lines = paginator.page(page)
        except PageNotAnInteger:
            # If page is not an integer, deliver first page.
            files_lines = paginator.page(1)
        except EmptyPage:
            # If page is out of range (e.g. 9999), deliver last page of results.
            files_lines = paginator.page(paginator.num_pages)

        comments = tickets_comments.objects.select_related('c_user').filter(ticket=ticket).order_by('c_date')
        paginator = Paginator(comments, 10)
        page = request.GET.get('page')
        try:
            comments_lines = paginator.page(page)
        except PageNotAnInteger:
            # If page is not an integer, deliver first page.
            comments_lines = paginator.page(1)
        except EmptyPage:
            # If page is out of range (e.g. 9999), deliver last page of results.
            comments_lines = paginator.page(paginator.num_pages)

        breadcrumbs = request.session.get('breadcrumbs', [])
        # ignore dupplicates at the end of list
        if len(breadcrumbs) > 0:
            if breadcrumbs[-1] != long(ticket):
                breadcrumbs.append(long(ticket))
        else:
            breadcrumbs.append(long(ticket))
        while len(breadcrumbs) > 10:
            breadcrumbs.pop(0)
        request.session['breadcrumbs'] = breadcrumbs

        return render_to_response('tickets/view.html', {'layout': 'horizontal', 'ticket': tic, 'form': form, 'close': close, 'files': files_lines, 'comments': comments_lines, 'participants': participants}, RequestContext(request))

    elif mode == 'history':
        history = tickets_history.objects.filter(ticket=ticket)
        return render_to_response('tickets/history.html', {'layout': 'horizontal', 'ticket': tic, 'history': history}, RequestContext(request))

    elif mode == 'reopen':
        if tic.closed:
            tic.closed = False
            tic.resolution = None
            tic.save(user=request.user)

            com = tickets_comments()
            com.comment = _('ticket reopend - resolution deleted')
            com.ticket_id = ticket
            com.action = 2
            com.save(user=request.user)

            check_references(request, com)

            touch_ticket(request.user, ticket)

            add_history(request, tic, 2, None)

            mail_comment(request, com.pk)

        return HttpResponseRedirect('/tickets/view/%s/' % ticket)

    elif mode == 'edit':
        excludes = ['resolution']
        if request.method == 'POST':
            form = TicketsForm(request.POST, exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, instance=tic, customer=request.organisation.id)
            if form.is_valid():
                tic = form.save()

                assigned = form.cleaned_data.get('assigned')
                if assigned:
                    touch_ticket(assigned, tic.pk)

                remember_changes(request, form, tic)

                touch_ticket(request.user, tic.pk)

                mail_ticket(request, tic.pk, form)

                return HttpResponseRedirect('/tickets/view/%s/' % ticket)

        else:
            form = TicketsForm(exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, instance=tic, customer=request.organisation.id)

        return render_to_response('tickets/edit.html', {'ticket': tic, 'layout': 'horizontal', 'form': form}, RequestContext(request))

    elif mode == 'download':
        fileid = request.GET.get('file', -1)
        file_data = tickets_files.objects.get(id=fileid, ticket=ticket)
        src = '%s%s.dat' % (settings.FILE_UPLOAD_PATH, fileid)

        if request.GET.get('resize', 'no') == 'yes' and 'image' in file_data.content_type:
            img = resize_image('%s' % (src), (200, 150), 75)
            output = io.BytesIO()
            img.save(output, 'PNG')
            output.seek(0)
            response = StreamingHttpResponse(output, mimetype=smart_str(file_data.content_type))

        else:
            response = StreamingHttpResponse(open('%s' % (src),"rb"), mimetype=smart_str(file_data.content_type))
        response['Content-Disposition'] = 'attachment;filename=%s' % smart_str(file_data.name)
        return response

    elif mode == 'upload':
        if request.method == 'POST':
            form = UploadFileForm(request.POST, request.FILES)
            if form.is_valid():
                f = tickets_files()
                f.name = request.FILES['file'].name
                f.size = request.FILES['file'].size
                f.content_type = request.FILES['file'].content_type
                f.ticket_id = ticket
                f.public = True
                f.save(user=request.user)

                touch_ticket(request.user, ticket)

                add_history(request, tic, 5, request.FILES['file'].name)

                mail_file(request, f.pk)

                dest = settings.FILE_UPLOAD_PATH
                if not os.path.exists(dest):
                    os.makedirs(dest)

                with open('%s%s.dat' % (dest, f.id), 'wb+') as destination:
                    for chunk in request.FILES['file'].chunks():
                        destination.write(chunk)

                return HttpResponseRedirect('/tickets/view/%s/' % tic.pk)
        else:
            form = UploadFileForm()

        return render_to_response('tickets/file.html', {'ticketid': ticket, 'layout': 'horizontal', 'form': form}, RequestContext(request))

def table(request, **kwargs):
    search_params = {}
    mod_path, cls_name = settings.TICKET_CLASS.rsplit('.', 1)
    mod_path = mod_path.split('.').pop(0)
    tic = get_model(mod_path, cls_name).objects.select_related('type').all()

    if not request.user.is_staff:
        tic = tic.filter(customer=request.organisation)

    if 'search' in kwargs:
        is_search = True

        params = kwargs['search']
        for field in params:
            if params[field] != None and params[field] != '':
                search_params[field] = params[field]
        tic = tic.filter(**search_params)
    else:
        tic = tic.filter(closed=False)
        is_search = False

    pretty = prettyValues(search_params)

    paginator = Paginator(tic, 10)
    page = request.GET.get('page')
    try:
        tic_lines = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        tic_lines = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        tic_lines = paginator.page(paginator.num_pages)

    return render_to_response('tickets/list.html', {'lines': tic_lines, 'is_search': is_search, 'pretty': pretty}, RequestContext(request))

def search(request):
    searchable_fields = settings.TICKET_SEARCH_FIELDS

    if request.method == 'POST' and 'reportname' in request.POST and request.POST['reportname']:
        rep = tickets_reports()
        rep.name = request.POST['reportname']
        rep.search = json.dumps(request.session['last_search'])
        rep.save(user=request.user)

    if request.method == 'POST':
        form = SearchForm(request.POST, include_list=searchable_fields, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)
        form.is_valid()
        request.session['last_search'] = clean_search_values(form.cleaned_data)

        return table(request, search=request.session['last_search'])

    if 'last_search' in request.session and not 'new' in request.GET:
        return table(request, search=request.session['last_search'])

    form = SearchForm(include_list=searchable_fields, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)

    return render_to_response('tickets/search.html', {'layout': 'horizontal', 'form': form}, RequestContext(request))

def reports(request):
    if 'report' in request.GET:
        rep = tickets_reports.objects.get(pk=request.GET['report'])
        return table(request, search=json.loads(rep.search))

    reps = tickets_reports.objects.filter(c_user=request.user)

    paginator = Paginator(reps, 10)
    page = request.GET.get('page')
    try:
        rep_lines = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        rep_lines = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        rep_lines = paginator.page(paginator.num_pages)

    return render_to_response('tickets/reports.html', {'lines': rep_lines}, RequestContext(request))<commit_msg>= remember last search after select a report (from report)<commit_after># -*- coding: utf-8 -*-
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http.response import HttpResponseRedirect, StreamingHttpResponse
from django.db.models import get_model
from django.conf import settings
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.contrib import messages
from django.utils.translation import ugettext as _
from django.utils.encoding import smart_str
from yats.forms import TicketsForm, CommentForm, UploadFileForm, SearchForm, TicketCloseForm
from yats.models import tickets_files, tickets_comments, tickets_reports, ticket_resolution, tickets_participants, tickets_history
from yats.shortcuts import resize_image, touch_ticket, mail_ticket, mail_comment, mail_file, clean_search_values, check_references, remember_changes, add_history, prettyValues
import os
import io
try:
    import json
except ImportError:
    from django.utils import simplejson as json

def new(request):
    excludes = ['resolution']

    if request.method == 'POST':
        form = TicketsForm(request.POST, exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)
        if form.is_valid():
            tic = form.save()

            assigned = form.cleaned_data.get('assigned')
            if assigned:
                touch_ticket(assigned, tic.pk)

            touch_ticket(request.user, tic.pk)

            mail_ticket(request, tic.pk, form, rcpt=settings.TICKET_NEW_MAIL_RCPT)

            if form.cleaned_data.get('file_addition', False):
                return HttpResponseRedirect('/tickets/upload/%s/' % tic.pk)
            else:
                return HttpResponseRedirect('/tickets/view/%s/' % tic.pk)

    else:
        form = TicketsForm(exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)

    return render_to_response('tickets/new.html', {'layout': 'horizontal', 'form': form}, RequestContext(request))

def action(request, mode, ticket):
    mod_path, cls_name = settings.TICKET_CLASS.rsplit('.', 1)
    mod_path = mod_path.split('.').pop(0)
    tic = get_model(mod_path, cls_name).objects.get(pk=ticket)

    if mode == 'view':
        if request.method == 'POST':
            form = CommentForm(request.POST)
            if form.is_valid():
                com = tickets_comments()
                com.comment = form.cleaned_data['comment']
                com.ticket_id = ticket
                com.action = 6
                com.save(user=request.user)

                check_references(request, com)

                touch_ticket(request.user, ticket)

                add_history(request, tic, 6, com.comment)

                mail_comment(request, com.pk)

            else:
                if 'resolution' in request.POST:
                    if request.POST['resolution'] and int(request.POST['resolution']) > 0:
                        tic.resolution_id = request.POST['resolution']
                        tic.closed = True
                        tic.save(user=request.user)

                        com = tickets_comments()
                        com.comment = _('ticket closed - resolution: %(resolution)s\n\n%(comment)s') % {'resolution': ticket_resolution.objects.get(pk=request.POST['resolution']).name, 'comment': request.POST.get('close_comment', '')}
                        com.ticket_id = ticket
                        com.action = 1
                        com.save(user=request.user)

                        check_references(request, com)

                        touch_ticket(request.user, ticket)

                        add_history(request, tic, 1, request.POST.get('close_comment', ''))

                        mail_comment(request, com.pk)

                    else:
                        messages.add_message(request, messages.ERROR, _('no resolution selected'))

                else:
                    messages.add_message(request, messages.ERROR, _('comment invalid'))

        excludes = []
        form = TicketsForm(exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, instance=tic, customer=request.organisation.id, view_only=True)
        close = TicketCloseForm()

        participants = tickets_participants.objects.select_related('user').filter(ticket=ticket)

        files = tickets_files.objects.filter(ticket=ticket)
        paginator = Paginator(files, 10)
        page = request.GET.get('page')
        try:
            files_lines = paginator.page(page)
        except PageNotAnInteger:
            # If page is not an integer, deliver first page.
            files_lines = paginator.page(1)
        except EmptyPage:
            # If page is out of range (e.g. 9999), deliver last page of results.
            files_lines = paginator.page(paginator.num_pages)

        comments = tickets_comments.objects.select_related('c_user').filter(ticket=ticket).order_by('c_date')
        paginator = Paginator(comments, 10)
        page = request.GET.get('page')
        try:
            comments_lines = paginator.page(page)
        except PageNotAnInteger:
            # If page is not an integer, deliver first page.
            comments_lines = paginator.page(1)
        except EmptyPage:
            # If page is out of range (e.g. 9999), deliver last page of results.
            comments_lines = paginator.page(paginator.num_pages)

        breadcrumbs = request.session.get('breadcrumbs', [])
        # ignore dupplicates at the end of list
        if len(breadcrumbs) > 0:
            if breadcrumbs[-1] != long(ticket):
                breadcrumbs.append(long(ticket))
        else:
            breadcrumbs.append(long(ticket))
        while len(breadcrumbs) > 10:
            breadcrumbs.pop(0)
        request.session['breadcrumbs'] = breadcrumbs

        return render_to_response('tickets/view.html', {'layout': 'horizontal', 'ticket': tic, 'form': form, 'close': close, 'files': files_lines, 'comments': comments_lines, 'participants': participants}, RequestContext(request))

    elif mode == 'history':
        history = tickets_history.objects.filter(ticket=ticket)
        return render_to_response('tickets/history.html', {'layout': 'horizontal', 'ticket': tic, 'history': history}, RequestContext(request))

    elif mode == 'reopen':
        if tic.closed:
            tic.closed = False
            tic.resolution = None
            tic.save(user=request.user)

            com = tickets_comments()
            com.comment = _('ticket reopend - resolution deleted')
            com.ticket_id = ticket
            com.action = 2
            com.save(user=request.user)

            check_references(request, com)

            touch_ticket(request.user, ticket)

            add_history(request, tic, 2, None)

            mail_comment(request, com.pk)

        return HttpResponseRedirect('/tickets/view/%s/' % ticket)

    elif mode == 'edit':
        excludes = ['resolution']
        if request.method == 'POST':
            form = TicketsForm(request.POST, exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, instance=tic, customer=request.organisation.id)
            if form.is_valid():
                tic = form.save()

                assigned = form.cleaned_data.get('assigned')
                if assigned:
                    touch_ticket(assigned, tic.pk)

                remember_changes(request, form, tic)

                touch_ticket(request.user, tic.pk)

                mail_ticket(request, tic.pk, form)

                return HttpResponseRedirect('/tickets/view/%s/' % ticket)

        else:
            form = TicketsForm(exclude_list=excludes, is_stuff=request.user.is_staff, user=request.user, instance=tic, customer=request.organisation.id)

        return render_to_response('tickets/edit.html', {'ticket': tic, 'layout': 'horizontal', 'form': form}, RequestContext(request))

    elif mode == 'download':
        fileid = request.GET.get('file', -1)
        file_data = tickets_files.objects.get(id=fileid, ticket=ticket)
        src = '%s%s.dat' % (settings.FILE_UPLOAD_PATH, fileid)

        if request.GET.get('resize', 'no') == 'yes' and 'image' in file_data.content_type:
            img = resize_image('%s' % (src), (200, 150), 75)
            output = io.BytesIO()
            img.save(output, 'PNG')
            output.seek(0)
            response = StreamingHttpResponse(output, mimetype=smart_str(file_data.content_type))

        else:
            response = StreamingHttpResponse(open('%s' % (src),"rb"), mimetype=smart_str(file_data.content_type))
        response['Content-Disposition'] = 'attachment;filename=%s' % smart_str(file_data.name)
        return response

    elif mode == 'upload':
        if request.method == 'POST':
            form = UploadFileForm(request.POST, request.FILES)
            if form.is_valid():
                f = tickets_files()
                f.name = request.FILES['file'].name
                f.size = request.FILES['file'].size
                f.content_type = request.FILES['file'].content_type
                f.ticket_id = ticket
                f.public = True
                f.save(user=request.user)

                touch_ticket(request.user, ticket)

                add_history(request, tic, 5, request.FILES['file'].name)

                mail_file(request, f.pk)

                dest = settings.FILE_UPLOAD_PATH
                if not os.path.exists(dest):
                    os.makedirs(dest)

                with open('%s%s.dat' % (dest, f.id), 'wb+') as destination:
                    for chunk in request.FILES['file'].chunks():
                        destination.write(chunk)

                return HttpResponseRedirect('/tickets/view/%s/' % tic.pk)
        else:
            form = UploadFileForm()

        return render_to_response('tickets/file.html', {'ticketid': ticket, 'layout': 'horizontal', 'form': form}, RequestContext(request))

def table(request, **kwargs):
    search_params = {}
    mod_path, cls_name = settings.TICKET_CLASS.rsplit('.', 1)
    mod_path = mod_path.split('.').pop(0)
    tic = get_model(mod_path, cls_name).objects.select_related('type').all()

    if not request.user.is_staff:
        tic = tic.filter(customer=request.organisation)

    if 'search' in kwargs:
        is_search = True

        params = kwargs['search']
        for field in params:
            if params[field] != None and params[field] != '':
                search_params[field] = params[field]
        tic = tic.filter(**search_params)
    else:
        tic = tic.filter(closed=False)
        is_search = False

    pretty = prettyValues(search_params)

    paginator = Paginator(tic, 10)
    page = request.GET.get('page')
    try:
        tic_lines = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        tic_lines = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        tic_lines = paginator.page(paginator.num_pages)

    return render_to_response('tickets/list.html', {'lines': tic_lines, 'is_search': is_search, 'pretty': pretty}, RequestContext(request))

def search(request):
    searchable_fields = settings.TICKET_SEARCH_FIELDS

    if request.method == 'POST' and 'reportname' in request.POST and request.POST['reportname']:
        rep = tickets_reports()
        rep.name = request.POST['reportname']
        rep.search = json.dumps(request.session['last_search'])
        rep.save(user=request.user)

    if request.method == 'POST':
        form = SearchForm(request.POST, include_list=searchable_fields, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)
        form.is_valid()
        request.session['last_search'] = clean_search_values(form.cleaned_data)

        return table(request, search=request.session['last_search'])

    if 'last_search' in request.session and not 'new' in request.GET:
        return table(request, search=request.session['last_search'])

    form = SearchForm(include_list=searchable_fields, is_stuff=request.user.is_staff, user=request.user, customer=request.organisation.id)

    return render_to_response('tickets/search.html', {'layout': 'horizontal', 'form': form}, RequestContext(request))

def reports(request):
    if 'report' in request.GET:
        rep = tickets_reports.objects.get(pk=request.GET['report'])
        request.session['last_search'] = json.loads(rep.search)
        return table(request, search=request.session['last_search'])

    reps = tickets_reports.objects.filter(c_user=request.user)

    paginator = Paginator(reps, 10)
    page = request.GET.get('page')
    try:
        rep_lines = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        rep_lines = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        rep_lines = paginator.page(paginator.num_pages)

    return render_to_response('tickets/reports.html', {'lines': rep_lines}, RequestContext(request))<|endoftext|>


<commit_before>__author__ = 'kevin'

import sys
from subprocess import check_output
import os

class Call():

    input_functions = ['canonicalize_file_name', 'catgets', 'confstr', 'ctermid', 'ctermid', 'cuserid', 'dgettext',
                       'dngettext', 'fgetc', 'fgetc_unlocked', 'fgets', 'fgets_unlocked', 'fpathconf', 'fread',
                       'fread_unlocked', 'fscanf', 'getc', 'getchar', 'getchar_unlocked', 'getc_unlocked',
                       'get_current_dir_name', 'getcwd', 'getdelim', '__getdelim', 'getdelim', 'getdents', 'getenv',
                       'gethostbyaddr', 'gethostbyname', 'gethostbyname2', 'gethostent', 'gethostid', 'getline',
                       'getline', 'getlogin', 'getlogin_r', 'getmsg', 'getopt', '_getopt_internal', 'getopt_long',
                       'getopt_long_only', 'getpass', 'getpmsg', 'gets', 'gettext', 'getw', 'getwd', 'ngettext',
                       'pathconf', 'pread', 'pread64', 'ptsname', 'ptsname_r', 'read', 'readdir', 'readlink', 'readv',
                       'realpath', 'recv', 'recv_from', 'recvmesg', 'scanf', '__secure_getenv', 'signal', 'sysconf',
                       'ttyname', 'ttyname_r', 'vfscanf', 'vscanf']

    output_functions = ['dprintf', 'fprintf', 'fputc', 'fputchar_unlocked', 'fputc_unlocked', 'fputs', 'fputs_unlocked',
                        'fwrite', 'fwrite_unlocked', 'perror', 'printf', 'psignal', 'putc', 'putchar', 'putc_unlocked',
                        'putenv', 'putmsg', 'putpmsg', 'puts', 'putw', 'pwrite', 'pwrite64', 'send', 'sendmsg',
                        'sendto', 'setenv', 'sethostid', 'setlogin', 'ungetc', 'vdprintf', 'vfprintf', 'vsyslog',
                        'write', 'writev']

    def __init__(self, clfow_line):
        split_line = clfow_line.split(Cflow.indent)

        self.function_info = split_line[-1].strip()
        self.level = len(split_line) - 1

    def __str__(self):
        return "Level: " + str(self.level) + " - " + self.function_info

    def is_input_function(self):
        is_input = self._leaf_belongs_to(Call.input_functions)
        return is_input

    def is_output_function(self):
        is_output = self._leaf_belongs_to(Call.output_functions)
        return is_output

    def _leaf_belongs_to(self, function_set):
        belongs = False

        if self.is_leaf():
            belongs = self.leaf_function_name in function_set

        return belongs

    def is_leaf(self):
        """Checks if the function call represented by this object is a leaf in the call graph generated by cflow.
        Being a leaf means that the function is not defined in the files given to cflow as arguments to analyze.
        """
        return self.function_info.endswith("()")

    @property
    def leaf_function_name(self):
        return self.function_info[:-2]

class Cflow():
    """Wraps around some of cflow's command line interface"""

    indent = "    "

    _file_patterns = ["*.c", "*.h"]
    _cflow_exec = "cflow"

    def __init__(self, source_dir):
        self.source_dir = source_dir

    def _get_cflow_args(self):
        dirs_found = []

        for path, dirs, files in os.walk(os.path.abspath(self.source_dir)):
            for dir in dirs:
                for pattern in Cflow._file_patterns:
                    dirs_found.append(os.path.join(path, dir, pattern))

        return dirs_found

    def _build_cflow_command(self):
        cflow_args = self._get_cflow_args()
        command = "cd {0}; ".format(self.source_dir) + Cflow._cflow_exec + ' ' + ' '.join(cflow_args) + ";"

        return command

    def get_text_call_graph(self):
        cmd_cflow = self._build_cflow_command()
        cflow_output = check_output(cmd_cflow, shell=True).decode("utf-8")

        return cflow_output

class Stack():

    _collection = list()

    def __init__(self):
        pass

    def push(self, data):
        self._collection.append(data)

    def pop(self):
        return self._collection.pop()

    @property
    def top(self):
        return self._collection[-1]

def main(args):
    call_graph_file = args[1]

    entry_points = list()
    exit_points = list()

    with open(call_graph_file) as call_graph_file:
        for i, line in enumerate(call_graph_file):
            if i == 0:
                main_call = Call(line)

                parent = Stack()
                parent.push(main_call)
                previous = main_call
            else:
                current = Call(line)

                if current.level > previous.level:
                    parent.push(previous)
                elif current.level < previous.level:
                    for i in range(current.level - previous.level):
                        parent.pop()

                if current.is_input_function():
                    entry_points.append(parent.top)

                if current.is_output_function():
                    exit_points.append(parent.top)

                previous = current

    print("entry points: " + str(len(entry_points)))
    print("exit points: " + str(len(exit_points)))

if __name__ == '__main__':
    main(sys.argv)<commit_msg>Changed io_points.py to read cflow's output directly from the console. No file is generated from cflow's output. Also, minor refactoring done to the algorithm.<commit_after>__author__ = 'kevin'

import sys
import subprocess
import os

class Call():

    input_functions = ['canonicalize_file_name', 'catgets', 'confstr', 'ctermid', 'ctermid', 'cuserid', 'dgettext',
                       'dngettext', 'fgetc', 'fgetc_unlocked', 'fgets', 'fgets_unlocked', 'fpathconf', 'fread',
                       'fread_unlocked', 'fscanf', 'getc', 'getchar', 'getchar_unlocked', 'getc_unlocked',
                       'get_current_dir_name', 'getcwd', 'getdelim', '__getdelim', 'getdelim', 'getdents', 'getenv',
                       'gethostbyaddr', 'gethostbyname', 'gethostbyname2', 'gethostent', 'gethostid', 'getline',
                       'getline', 'getlogin', 'getlogin_r', 'getmsg', 'getopt', '_getopt_internal', 'getopt_long',
                       'getopt_long_only', 'getpass', 'getpmsg', 'gets', 'gettext', 'getw', 'getwd', 'ngettext',
                       'pathconf', 'pread', 'pread64', 'ptsname', 'ptsname_r', 'read', 'readdir', 'readlink', 'readv',
                       'realpath', 'recv', 'recv_from', 'recvmesg', 'scanf', '__secure_getenv', 'signal', 'sysconf',
                       'ttyname', 'ttyname_r', 'vfscanf', 'vscanf']

    output_functions = ['dprintf', 'fprintf', 'fputc', 'fputchar_unlocked', 'fputc_unlocked', 'fputs', 'fputs_unlocked',
                        'fwrite', 'fwrite_unlocked', 'perror', 'printf', 'psignal', 'putc', 'putchar', 'putc_unlocked',
                        'putenv', 'putmsg', 'putpmsg', 'puts', 'putw', 'pwrite', 'pwrite64', 'send', 'sendmsg',
                        'sendto', 'setenv', 'sethostid', 'setlogin', 'ungetc', 'vdprintf', 'vfprintf', 'vsyslog',
                        'write', 'writev']

    def __init__(self, clfow_line):
        split_line = clfow_line.split(Cflow.indent)

        self.function_info = split_line[-1].strip()
        self.level = len(split_line) - 1

    def __str__(self):
        return "Level: " + str(self.level) + " - " + self.function_info

    def is_input_function(self):
        is_input = self._leaf_belongs_to(Call.input_functions)
        return is_input

    def is_output_function(self):
        is_output = self._leaf_belongs_to(Call.output_functions)
        return is_output

    def _leaf_belongs_to(self, function_set):
        belongs = False

        if self.is_leaf():
            belongs = self.leaf_function_name in function_set

        return belongs

    def is_leaf(self):
        """Checks if the function call represented by this object is a leaf in the call graph generated by cflow.
        Being a leaf means that the function is not defined in the files given to cflow as arguments to analyze.
        """
        return self.function_info.endswith("()")

    @property
    def leaf_function_name(self):
        return self.function_info[:-2]

class Cflow():
    """Wraps around some of cflow's command line interface"""

    indent = "    "

    _file_patterns = ["*.c", "*.h"]
    _cflow_exec = "cflow"

    def __init__(self, source_dir):
        self.source_dir = source_dir

    def _get_cflow_args(self):
        dirs_found = []

        for path, dirs, files in os.walk(os.path.abspath(self.source_dir)):
            for dir in dirs:
                for pattern in Cflow._file_patterns:
                    dirs_found.append(os.path.join(path, dir, pattern))

        return dirs_found

    def _build_cflow_command(self):
        cflow_args = self._get_cflow_args()
        command = "cd {0}; ".format(self.source_dir) + Cflow._cflow_exec + ' ' + ' '.join(cflow_args) + ";"

        return command

    def get_text_call_graph(self):
        cmd_cflow = self._build_cflow_command()
        cflow_output = check_output(cmd_cflow, shell=True).decode("utf-8")

        return cflow_output

class Stack():

    _collection = list()

    def __init__(self):
        pass

    def push(self, data):
        self._collection.append(data)

    def pop(self):
        return self._collection.pop()

    @property
    def top(self):
        return self._collection[-1]

def main(args):
    source_dir = args[1]

    entry_points_count = 0
    exit_points_count = 0
    # entry_points = list()
    # exit_points = list()

    parent = Stack()
    line = "INITIAL"
    i = 0

    # os.chdir(source_dir)
    proc = subprocess.Popen(['sh', 'run_cflow.sh'], stdout=subprocess.PIPE)

    while line != '':
        line = proc.stdout.readline().decode(encoding='UTF-8')
        print("test:", line.rstrip())

        current = Call(line)

        if i != 0:
            if current.level > previous.level:
                parent.push(previous)
            elif current.level < previous.level:
                for i in range(current.level - previous.level):
                    parent.pop()

            if current.is_input_function():
                # entry_points.append(parent.top)
                entry_points_count += 1

            if current.is_output_function():
                # exit_points.append(parent.top)
                exit_points_count += 1

        previous = current
        i += 1

    # print("entry points: " + str(len(entry_points)))
    # print("exit points: " + str(len(exit_points)))
    print("entry points: " + str(entry_points_count))
    print("exit points: " + str(exit_points_count))

if __name__ == '__main__':
    main(sys.argv)<|endoftext|>