<commit_before>import sublime, sublime_plugin

import sys
import os

from .lib.util import communicate, popen, create_environment
from .edit import Edit

from collections import defaultdict
import json
import platform
import queue
import re
import shlex
import subprocess
import _thread
import time
import traceback

INDENTATION = '  '
backspace_re = re.compile('.\b')

xiki_settings = sublime.load_settings('SublimeXiki.sublime-settings')

class BoundaryError(Exception): pass

if not 'already' in globals():
    already = True
    commands = defaultdict(dict)

def spawn(view, indent, cmd, sel):
    local_commands = commands[view.id()]
    q = queue.Queue()
    def fold(region):
        regions = view.get_regions(region)
        for region in regions:
            lines = view.split_by_newlines(region)
            if len(lines) > 24:

                lines = lines[1:-24]
                try:
                    area = lines.pop()
                except IndexError:
                    return

                for sub in lines:
                    area = area.cover(sub)

                view.unfold(area)
                view.fold(area)

    def merge(region):
        if q.empty(): return
        regions = view.get_regions(region)
        if not regions: return

        pos = view.line(regions[0].end() - 1)

        restore_sel = []
        for sel in view.sel():
            if pos.end() in (sel.a, sel.b):
                restore_sel.append(sel)
                view.sel().subtract(sel)

        with Edit(view) as edit:
            try:
                start = time.time()
                lines = []
                while time.time() - start < 0.05 and len(lines) < 200:
                    try:
                        lines.append(q.get(False))
                        q.task_done()
                    except queue.Empty:
                        break

                if not lines: return
                lines = backspace_re.sub('', '\n'.join(lines))
                insert(view, edit, pos, lines, indent + INDENTATION)

                fold(region)
            except:
                print(traceback.format_exc())
            finally:
                def restore_selections():
                    for sel in restore_sel:
                        view.sel().add(sel)

                edit.callback(restore_selections)

    def poll(p, region, fd):
        line = True
        while p.poll() is None or line:
            line = fd.readline()
            if isinstance(line, bytes):
                line = line.decode('utf8')

            if line:
                q.put(line.rstrip('\r\n'))

        # if the process wasn't terminated
        out = fd.read()
        if isinstance(out, bytes):
            out = out.decode('utf8')

        if out:
            q.put(out.rstrip('\r\n'))
            merge(region)

    def out(p, region):
        last = 0
        while p.poll() is None:
            since = time.time() - last
            if since > 0.05 or since > 0.01 and q.qsize() < 10:
                last = time.time()
                merge(region)
            else:
                time.sleep(max(0.1 - since, 0.1))

        if p.returncode not in (-9, -15):
            del local_commands[region]
            while not q.empty():
                merge(region)
                time.sleep(0.05)

        view.erase_regions(region)

    p = popen(cmd, use_pty=False)
    if isinstance(p, subprocess.Popen):
        region = 'xiki sub %i' % p.pid
        line = view.full_line(sel.b)
        spread = sublime.Region(line.a, line.b)
        local_commands[region] = p
        view.add_regions(region, [spread], 'keyword', '', sublime.DRAW_OUTLINED)

        if p.pty:
            _thread.start_new_thread(poll, (p, region, p.stdout))
        else:
            _thread.start_new_thread(poll, (p, region, p.stderr))
            _thread.start_new_thread(poll, (p, region, p.stdout))
        _thread.start_new_thread(out, (p, region))
    else:
        with Edit(view) as edit:
            insert(view, edit, sel, 'Error: ' + (p or ''), indent + INDENTATION)

def xiki(view, cont=False):
    if is_xiki_buffer(view):
        for sel in view.sel():
            output = None
            cmd = None
            persist = False
            oldcwd = None
            op = None
            scroll = False

            view.sel().subtract(sel)

            if sel.end() == view.size():
                with Edit(view) as edit:
                    edit.insert(view.size(), '\n')

            row, _ = view.rowcol(sel.b)
            indent, sign, path, tag, tree = find_tree(view, row)

            pos = view.line(sel.b).b
            if get_line(view, row+1).startswith(indent + INDENTATION):
                if sign == '-':
                    with Edit(view) as edit:
                        replace_line(view, edit, pos, indent + '+ ' + tag)

                do_clean = True
                check = sublime.Region(sel.b, sel.b)
                for name, process in list(commands[view.id()].items()):
                    regions = view.get_regions(name)
                    for region in regions:
                        if region.contains(check):
                            try:
                                process.terminate()
                            except OSError:
                                pass

                            do_clean = False

                if do_clean and not cont:
                    op = 'cleanup'
                    with Edit(view) as edit:
                        cleanup(view, edit, pos, indent + INDENTATION)
                # select(view, pos)
            elif sign == '$' or sign == '$$':
                op = 'command'
                error = None

                p = os.path.expanduser('~')
                if path:
                    p = dirname(path, tree, tag)
                try:
                    oldcwd = os.getcwd()
                except FileNotFoundError:
                    pass
                try:
                    os.chdir(p)
                except Exception as err:
                    error = err

                env = create_environment()
                if sign == '$$' or xiki_settings.get('single_dollar_shell'):
                    shell = env.get('SHELL')
                    if shell:
                        cmd = [shell, '-c', tag]
                    elif os.name == 'nt':
                        cmd = ['cmd', '/c', tag]

                if not cmd:
                    try:
                        cmd = shlex.split(tag, True)
                    except ValueError as err:
                        error = err

                if error:
                    cmd = None
                    output = str(error) + '\n'

                persist = True
            elif path:
                # directory listing or file open
                target = os.path.join(path, tree)
                d, f = os.path.split(target)
                f = unslash(f)
                target = os.path.join(d, f)

                if os.path.isfile(target):
                    op = 'file'
                    if platform.system() == 'Windows':
                        target = os.path.abspath(target)

                    if not cont:
                        sublime.active_window().open_file(target)
                elif os.path.isdir(target):
                    op = 'dir'
                    dirs = ''
                    files = ''
                    listing = []
                    try:
                        listing = os.listdir(target)
                    except OSError as err:
                        dirs = '- ' + err.strerror + '\n'

                    for entry in listing:
                        absolute = os.path.join(target, entry)
                        if os.path.isdir(absolute):
                            dirs += '+ %s/\n' % entry
                        else:
                            entry = slash(entry, '\\+$-')
                            files += '%s\n' % entry

                    output = (dirs + files) or ' '
            elif sign == '-':
                # dunno here
                pass
            elif tree:
                op = 'xiki'
                cmd = ['xiki']
                cmd += tree.split(' ')

            if cmd:
                if persist:
                    end = view.line(sel.b).b
                    with Edit(view) as edit:
                        edit.insert(end, '\n' + indent + INDENTATION)
                    spawn(view, indent, cmd, sel)
                else:
                    output = communicate(cmd, None, 3)

                if oldcwd:
                    os.chdir(oldcwd)

            if output:
                with Edit(view) as edit:
                    if sign == '+':
                        replace_line(view, edit, pos, indent + '- ' + tag)

                    insert(view, edit, sel, output, indent + INDENTATION)

            if cont:
                region = find_region(view, pos, indent + INDENTATION)
                end = view.line(region.end()).begin()

                added = ''
                if op == 'file':
                    added += '$ '
                elif op == 'dir':
                    added += INDENTATION + '$ '
                elif sign in ('$$', '$'):
                    added += sign + ' '

                def set_selection():
                    cur = view.line(end).end()
                    sel = sublime.Region(cur, cur)
                    view.sel().add(sel)
                    view.show_at_center(sel)

                with Edit(view) as edit:
                    edit.insert(end, indent + added + '\n')
                    edit.callback(set_selection)
            else:
                view.sel().add(sel)

            if scroll:
                view.show_at_center(sel)

def find_tree(view, row):
    regex = re.compile(r'^(\s*)(\$\$|[-+$]\s*)?(.*)$')

    line = get_line(view, row)
    match = regex.match(line)

    line_indent = last_indent = match.group(1)
    sign = (match.group(2) or '').strip()
    tag = match.group(3)
    tree = [tag]
    if tag.startswith('/'):
        sign = '/'

    offset = -1
    while last_indent != '':
        try:
            line = get_line(view, row+offset)
        except BoundaryError:
            break

        offset -= 1

        match = regex.match(line)
        if match:
            indent = match.group(1)
            part = match.group(3)

            if len(indent) < len(last_indent) and part:
                last_indent = indent
                tree.insert(0, part)

    new_tree = []
    path = None
    for part in reversed(tree):
        if part.startswith('@'):
            new_tree.insert(0, part.strip('@'))
        elif part.startswith('/'):
            path = part
        elif part.startswith('~'):
            path = os.path.expanduser(part)
        else:
            new_tree.insert(0, part)
            continue

        break

    return line_indent, sign, path, tag, '/'.join(new_tree).replace('//', '/')

# helpers

def slash(s, chars):
    if re.match(r'^[%s]' % re.escape(chars), s):
        s = '\\' + s

    return s

def unslash(s):
    out = ''
    escaped = False
    for c in s:
        if escaped:
            escaped = False
            out += c
        elif c == '\\':
            escaped = True
        else:
            out += c

    return out

def replace_line(view, edit, point, text):
    text = text.rstrip()
    line = view.full_line(point)

    edit.insert(line.b, text + '\n')
    edit.erase(line)

def find_region(view, pos, indent):
    line, _ = view.rowcol(pos)

    point = view.text_point(line + 1, 0)
    text = view.substr(sublime.Region(point, view.size()))
    count = 0
    for l in text.split('\n'):
        if not l.startswith(indent) and l.strip():
            break
        else:
            count += 1

    start = view.text_point(line + 1, 0)
    end = view.text_point(line + count, 0)
    region = sublime.Region(
        view.full_line(start).begin(),
        view.full_line(end).end(),
    )
    return region

def cleanup(view, edit, pos, indent):
    region = find_region(view, pos, indent)
    edit.erase(region)

def insert(view, edit, sel, text, indent='', cleanup=True):
    pos = view.line(sel.b).b

    lines = reversed(text.split('\n'))
    for line in lines:
        line = '\n' + indent + line
        edit.insert(pos, line)

def get_line(view, row=0):
    point = view.text_point(row, 0)
    if row < 0:
        raise BoundaryError

    line = view.line(point)
    return view.substr(line).strip('\r\n')

def dirname(path, tree, tag):
    path_re = r'^(.+)/%s$' % re.escape(tag)
    match = re.match(path_re, tree)
    if match:
        return os.path.join(path, match.group(1))
    else:
        return path

def completions(base, partial, executable=False):
    if os.path.isdir(base):
        ret = []
        partial = partial.lower()

        for name in os.listdir(base):
            path = os.path.join(base, name)
            if name.lower().startswith(partial):
                if not executable or os.access(path, os.X_OK):
                    ret.append((name, name))

        return ret

def apply_xiki_settings(view):
    settings = view.settings()
    settings.set('tab_size', 2)
    settings.set('translate_tabs_to_spaces', True)
    settings.set('syntax', 'Packages/SublimeXiki/Xiki.tmLanguage')

def is_xiki_buffer(view):
    if view is None or not view.settings().has('syntax'):
        return False

    return view.settings().get('syntax').endswith('/Xiki.tmLanguage')

# sublime event classes

class XikiListener(sublime_plugin.EventListener):
    def on_query_completions(self, view, prefix, locations):
        if is_xiki_buffer(view):
            sel = view.sel()
            if len(sel) == 1:
                row, _ = view.rowcol(sel[0].b)
                indent, sign, path, tag, tree = find_tree(view, row)

                if sign == '$':
                    # command completion
                    pass
                elif path:
                    # directory/file completion
                    target, partial = os.path.split(dirname(path, tree, tag))
                    return completions(target, partial), sublime.INHIBIT_WORD_COMPLETIONS

    def on_query_context(self, view, key, operator, operand, match_all):
        if key == 'xiki' and is_xiki_buffer(view):
            return True

    def on_load(self, view):
        # handle new user preferences file
        if view.file_name() and os.path.split(view.file_name())[1] == 'SublimeXiki.sublime-settings':
            if view.size() == 0:
                with Edit(view) as edit:
                    template = {
                        "double_click": False,
                        "single_dollar_shell": False,
                    }
                    edit.insert(0, json.dumps(template, indent=4))
        elif is_xiki_buffer(view):
            apply_xiki_settings(view)

    def on_close(self, view):
        vid = view.id()
        for process in list(commands[vid].values()):
            try:
                process.terminate()
            except OSError:
                pass

        del commands[vid]

class Xiki(sublime_plugin.TextCommand):
    def run(self, edit):
        xiki(self.view)

    def is_enabled(self):
        if is_xiki_buffer(self.view):
            return True
        return False

class XikiContinue(Xiki):
    def run(self, edit):
        xiki(self.view, cont=True)

class NewXiki(sublime_plugin.WindowCommand):
    def run(self):
        view = self.window.new_file()
        apply_xiki_settings(view)

class XikiClick(sublime_plugin.WindowCommand):
    def run(self):
        view = self.window.active_view()
        if is_xiki_buffer(view) and xiki_settings.get('double_click'):
            sel = view.sel()
            s = sel[0]

            text = view.substr(s)
            is_word = r'^(\w+|[^\w]+)$'
            if not re.match(is_word, text.strip('\n')):
                return

            sel.clear()
            sel.add(sublime.Region(s.b, s.a))
            xiki(view)
<commit_msg>always use single_dollar_shell<commit_after>import sublime, sublime_plugin

import sys
import os

from .lib.util import communicate, popen, create_environment, which
from .edit import Edit

from collections import defaultdict
import json
import platform
import queue
import re
import shlex
import subprocess
import _thread
import time
import traceback

INDENTATION = '  '
backspace_re = re.compile('.\b')

xiki_settings = sublime.load_settings('SublimeXiki.sublime-settings')

class BoundaryError(Exception): pass

if not 'already' in globals():
    already = True
    commands = defaultdict(dict)

def spawn(view, indent, cmd, sel):
    local_commands = commands[view.id()]
    q = queue.Queue()
    def fold(region):
        regions = view.get_regions(region)
        for region in regions:
            lines = view.split_by_newlines(region)
            if len(lines) > 24:

                lines = lines[1:-24]
                try:
                    area = lines.pop()
                except IndexError:
                    return

                for sub in lines:
                    area = area.cover(sub)

                view.unfold(area)
                view.fold(area)

    def merge(region):
        if q.empty(): return
        regions = view.get_regions(region)
        if not regions: return

        pos = view.line(regions[0].end() - 1)

        restore_sel = []
        for sel in view.sel():
            if pos.end() in (sel.a, sel.b):
                restore_sel.append(sel)
                view.sel().subtract(sel)

        with Edit(view) as edit:
            try:
                start = time.time()
                lines = []
                while time.time() - start < 0.05 and len(lines) < 200:
                    try:
                        lines.append(q.get(False))
                        q.task_done()
                    except queue.Empty:
                        break

                if not lines: return
                lines = backspace_re.sub('', '\n'.join(lines))
                insert(view, edit, pos, lines, indent + INDENTATION)

                fold(region)
            except:
                print(traceback.format_exc())
            finally:
                def restore_selections():
                    for sel in restore_sel:
                        view.sel().add(sel)

                edit.callback(restore_selections)

    def poll(p, region, fd):
        line = True
        while p.poll() is None or line:
            line = fd.readline()
            if isinstance(line, bytes):
                line = line.decode('utf8')

            if line:
                q.put(line.rstrip('\r\n'))

        # if the process wasn't terminated
        out = fd.read()
        if isinstance(out, bytes):
            out = out.decode('utf8')

        if out:
            q.put(out.rstrip('\r\n'))
            merge(region)

    def out(p, region):
        last = 0
        while p.poll() is None:
            since = time.time() - last
            if since > 0.05 or since > 0.01 and q.qsize() < 10:
                last = time.time()
                merge(region)
            else:
                time.sleep(max(0.1 - since, 0.1))

        if p.returncode not in (-9, -15):
            del local_commands[region]
            while not q.empty():
                merge(region)
                time.sleep(0.05)

        view.erase_regions(region)

    p = popen(cmd, use_pty=False)
    if isinstance(p, subprocess.Popen):
        region = 'xiki sub %i' % p.pid
        line = view.full_line(sel.b)
        spread = sublime.Region(line.a, line.b)
        local_commands[region] = p
        view.add_regions(region, [spread], 'keyword', '', sublime.DRAW_OUTLINED)

        if p.pty:
            _thread.start_new_thread(poll, (p, region, p.stdout))
        else:
            _thread.start_new_thread(poll, (p, region, p.stderr))
            _thread.start_new_thread(poll, (p, region, p.stdout))
        _thread.start_new_thread(out, (p, region))
    else:
        with Edit(view) as edit:
            insert(view, edit, sel, 'Error: ' + (p or ''), indent + INDENTATION)

def xiki(view, cont=False):
    if is_xiki_buffer(view):
        for sel in view.sel():
            output = None
            cmd = None
            persist = False
            oldcwd = None
            op = None
            scroll = False

            view.sel().subtract(sel)

            if sel.end() == view.size():
                with Edit(view) as edit:
                    edit.insert(view.size(), '\n')

            row, _ = view.rowcol(sel.b)
            indent, sign, path, tag, tree = find_tree(view, row)

            pos = view.line(sel.b).b
            if get_line(view, row+1).startswith(indent + INDENTATION):
                if sign == '-':
                    with Edit(view) as edit:
                        replace_line(view, edit, pos, indent + '+ ' + tag)

                do_clean = True
                check = sublime.Region(sel.b, sel.b)
                for name, process in list(commands[view.id()].items()):
                    regions = view.get_regions(name)
                    for region in regions:
                        if region.contains(check):
                            try:
                                process.terminate()
                            except OSError:
                                pass

                            do_clean = False

                if do_clean and not cont:
                    op = 'cleanup'
                    with Edit(view) as edit:
                        cleanup(view, edit, pos, indent + INDENTATION)
                # select(view, pos)
            elif sign == '$' or sign == '$$':
                op = 'command'
                error = None

                p = os.path.expanduser('~')
                if path:
                    p = dirname(path, tree, tag)
                try:
                    oldcwd = os.getcwd()
                except FileNotFoundError:
                    pass
                try:
                    os.chdir(p)
                except Exception as err:
                    error = err

                env = create_environment()
                shell = env.get('SHELL')
                if shell and which(shell):
                    cmd = [shell, '-c', tag]
                elif os.name == 'nt':
                    cmd = ['cmd', '/c', tag]
                if not cmd:
                    try:
                        cmd = shlex.split(tag, True)
                    except ValueError as err:
                        error = err

                if error:
                    cmd = None
                    output = str(error) + '\n'

                persist = True
            elif path:
                # directory listing or file open
                target = os.path.join(path, tree)
                d, f = os.path.split(target)
                f = unslash(f)
                target = os.path.join(d, f)

                if os.path.isfile(target):
                    op = 'file'
                    if platform.system() == 'Windows':
                        target = os.path.abspath(target)

                    if not cont:
                        sublime.active_window().open_file(target)
                elif os.path.isdir(target):
                    op = 'dir'
                    dirs = ''
                    files = ''
                    listing = []
                    try:
                        listing = os.listdir(target)
                    except OSError as err:
                        dirs = '- ' + err.strerror + '\n'

                    for entry in listing:
                        absolute = os.path.join(target, entry)
                        if os.path.isdir(absolute):
                            dirs += '+ %s/\n' % entry
                        else:
                            entry = slash(entry, '\\+$-')
                            files += '%s\n' % entry

                    output = (dirs + files) or ' '
            elif sign == '-':
                # dunno here
                pass
            elif tree:
                op = 'xiki'
                cmd = ['xiki']
                cmd += tree.split(' ')

            if cmd:
                if persist:
                    end = view.line(sel.b).b
                    with Edit(view) as edit:
                        edit.insert(end, '\n' + indent + INDENTATION)
                    spawn(view, indent, cmd, sel)
                else:
                    output = communicate(cmd, None, 3)

                if oldcwd:
                    os.chdir(oldcwd)

            if output:
                with Edit(view) as edit:
                    if sign == '+':
                        replace_line(view, edit, pos, indent + '- ' + tag)

                    insert(view, edit, sel, output, indent + INDENTATION)

            if cont:
                region = find_region(view, pos, indent + INDENTATION)
                end = view.line(region.end()).begin()

                added = ''
                if op == 'file':
                    added += '$ '
                elif op == 'dir':
                    added += INDENTATION + '$ '
                elif sign == '$':
                    added += sign + ' '

                def set_selection():
                    cur = view.line(end).end()
                    sel = sublime.Region(cur, cur)
                    view.sel().add(sel)
                    view.show_at_center(sel)

                with Edit(view) as edit:
                    edit.insert(end, indent + added + '\n')
                    edit.callback(set_selection)
            else:
                view.sel().add(sel)

            if scroll:
                view.show_at_center(sel)

def find_tree(view, row):
    regex = re.compile(r'^(\s*)(\$\$|[-+$]\s*)?(.*)$')

    line = get_line(view, row)
    match = regex.match(line)

    line_indent = last_indent = match.group(1)
    sign = (match.group(2) or '').strip()
    tag = match.group(3)
    tree = [tag]
    if tag.startswith('/'):
        sign = '/'

    offset = -1
    while last_indent != '':
        try:
            line = get_line(view, row+offset)
        except BoundaryError:
            break

        offset -= 1

        match = regex.match(line)
        if match:
            indent = match.group(1)
            part = match.group(3)

            if len(indent) < len(last_indent) and part:
                last_indent = indent
                tree.insert(0, part)

    new_tree = []
    path = None
    for part in reversed(tree):
        if part.startswith('@'):
            new_tree.insert(0, part.strip('@'))
        elif part.startswith('/'):
            path = part
        elif part.startswith('~'):
            path = os.path.expanduser(part)
        else:
            new_tree.insert(0, part)
            continue

        break

    return line_indent, sign, path, tag, '/'.join(new_tree).replace('//', '/')

# helpers

def slash(s, chars):
    if re.match(r'^[%s]' % re.escape(chars), s):
        s = '\\' + s

    return s

def unslash(s):
    out = ''
    escaped = False
    for c in s:
        if escaped:
            escaped = False
            out += c
        elif c == '\\':
            escaped = True
        else:
            out += c

    return out

def replace_line(view, edit, point, text):
    text = text.rstrip()
    line = view.full_line(point)

    edit.insert(line.b, text + '\n')
    edit.erase(line)

def find_region(view, pos, indent):
    line, _ = view.rowcol(pos)

    point = view.text_point(line + 1, 0)
    text = view.substr(sublime.Region(point, view.size()))
    count = 0
    for l in text.split('\n'):
        if not l.startswith(indent) and l.strip():
            break
        else:
            count += 1

    start = view.text_point(line + 1, 0)
    end = view.text_point(line + count, 0)
    region = sublime.Region(
        view.full_line(start).begin(),
        view.full_line(end).end(),
    )
    return region

def cleanup(view, edit, pos, indent):
    region = find_region(view, pos, indent)
    edit.erase(region)

def insert(view, edit, sel, text, indent='', cleanup=True):
    pos = view.line(sel.b).b

    lines = reversed(text.split('\n'))
    for line in lines:
        line = '\n' + indent + line
        edit.insert(pos, line)

def get_line(view, row=0):
    point = view.text_point(row, 0)
    if row < 0:
        raise BoundaryError

    line = view.line(point)
    return view.substr(line).strip('\r\n')

def dirname(path, tree, tag):
    path_re = r'^(.+)/%s$' % re.escape(tag)
    match = re.match(path_re, tree)
    if match:
        return os.path.join(path, match.group(1))
    else:
        return path

def completions(base, partial, executable=False):
    if os.path.isdir(base):
        ret = []
        partial = partial.lower()

        for name in os.listdir(base):
            path = os.path.join(base, name)
            if name.lower().startswith(partial):
                if not executable or os.access(path, os.X_OK):
                    ret.append((name, name))

        return ret

def apply_xiki_settings(view):
    settings = view.settings()
    settings.set('tab_size', 2)
    settings.set('translate_tabs_to_spaces', True)
    settings.set('syntax', 'Packages/SublimeXiki/Xiki.tmLanguage')

def is_xiki_buffer(view):
    if view is None or not view.settings().has('syntax'):
        return False

    return view.settings().get('syntax').endswith('/Xiki.tmLanguage')

# sublime event classes

class XikiListener(sublime_plugin.EventListener):
    def on_query_completions(self, view, prefix, locations):
        if is_xiki_buffer(view):
            sel = view.sel()
            if len(sel) == 1:
                row, _ = view.rowcol(sel[0].b)
                indent, sign, path, tag, tree = find_tree(view, row)

                if sign == '$':
                    # command completion
                    pass
                elif path:
                    # directory/file completion
                    target, partial = os.path.split(dirname(path, tree, tag))
                    return completions(target, partial), sublime.INHIBIT_WORD_COMPLETIONS

    def on_query_context(self, view, key, operator, operand, match_all):
        if key == 'xiki' and is_xiki_buffer(view):
            return True

    def on_load(self, view):
        # handle new user preferences file
        if view.file_name() and os.path.split(view.file_name())[1] == 'SublimeXiki.sublime-settings':
            if view.size() == 0:
                with Edit(view) as edit:
                    template = {
                        "double_click": False,
                    }
                    edit.insert(0, json.dumps(template, indent=4))
        elif is_xiki_buffer(view):
            apply_xiki_settings(view)

    def on_close(self, view):
        vid = view.id()
        for process in list(commands[vid].values()):
            try:
                process.terminate()
            except OSError:
                pass

        del commands[vid]

class Xiki(sublime_plugin.TextCommand):
    def run(self, edit):
        xiki(self.view)

    def is_enabled(self):
        if is_xiki_buffer(self.view):
            return True
        return False

class XikiContinue(Xiki):
    def run(self, edit):
        xiki(self.view, cont=True)

class NewXiki(sublime_plugin.WindowCommand):
    def run(self):
        view = self.window.new_file()
        apply_xiki_settings(view)

class XikiClick(sublime_plugin.WindowCommand):
    def run(self):
        view = self.window.active_view()
        if is_xiki_buffer(view) and xiki_settings.get('double_click'):
            sel = view.sel()
            s = sel[0]

            text = view.substr(s)
            is_word = r'^(\w+|[^\w]+)$'
            if not re.match(is_word, text.strip('\n')):
                return

            sel.clear()
            sel.add(sublime.Region(s.b, s.a))
            xiki(view)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# encoding: utf-8
"""
ylmir.py

Created by Karl Dubost on 2011-12-03.
Copyright (c) 2011 Grange. All rights reserved.
"""

import sys, os
import argparse
from lxml.html import tostring, html5parser
from lxml import etree
from lxml.etree import Element, SubElement

# CONFIG SITE
DOMAIN = u"la-grange.net"
SITE = u"http://www.%s/" % (DOMAIN)
FAVICON = SITE + "favicon"

SITENAME = u"Les carnets Web de La Grange"
TAGLINE = u"Rêveries le long d'un brin de chèvrefeuille"

FEEDTAGID = u"tag:la-grange.net,2000-04-12:karl"
FEEDLANG = u"fr"
FEEDATOMNOM = u"feed.atom"
FEEDATOMURL = u"%s%s" % (SITE,FEEDATOMNOM)

STATUSLIST = [u'draft',u'pub',u'acl']
DATETYPELIST = [u'created',u'modified']
LICENSELIST = {u'ccby': u'http://creativecommons.org/licenses/by/2.0/fr/',
               u'copy': u'©'}

AUTHOR = u"Karl Dubost"
AUTHORURI = u"http://www.la-grange.net/karl/"

HTMLNS = u"http://www.w3.org/1999/xhtml"
ATOMNS = u"http://www.w3.org/2005/Atom"
HTML = "{%s}" % HTMLNS
ATOM = "{%s}" % ATOMNS
NSMAP = {None : HTMLNS}
NSMAP2 = {None : ATOMNS}
NSMAP3 = {'html' : HTMLNS}

# CONFIG with cli (TODO)
STYLESHEET = "/2011/12/01/proto/style/article.css"
STATUS = ""
MAXFEEDITEM = 20
LICENSE = "ccby"

# PATHS

help_message = '''
This script has been entirely created
for processing text files for the site
La Grange http://www.la-grange.net/.
'''

# General processing features

def parserawpost(rawpostpath):
    """Given a path, parse an html file
    TODO check if the file is correct.
    """
    doc = html5parser.parse(rawpostpath).getroot()
    #TODO find a way to send the element with only the namespace on the first element.
    print "INFO: Document parsed"
    return doc

# Extracting information from the blog posts

def getdocstatus(doc):
    """Check the publication status of the document
    returns a string
    if there are multiple meta, returns the first one and issues a warning"""
    findstatus = etree.ETXPath("//{%s}meta[@name='status']" % HTMLNS)
    docstatus = findstatus(doc)
    if len(docstatus) >= 1:
        status = docstatus[0].attrib['content']
        if status in STATUSLIST:
            if len(docstatus) > 1:
                print "WARNING: There are more than one status. Taking the first one : " + status
            else:
                print "INFO: The document status is " + status
            return status
        else:
            raise Exception, "ERROR: No valid status for your document: %s not in %s" % (status, STATUSLIST)
    if len(docstatus) == 0:
        print "WARNING: There is no status for this document."
        status = "undefined"
        return status

def getdocdate(doc, DATETYPE):
    """return the creation date of the document in ISO format YYYY-MM-DD
    Input the document, typeofdate in between created and modified"""
    # TODO: check if the format is correct aka YYYY-MM-DD
    if DATETYPE not in DATETYPELIST:
        sys.exit("ERROR: No valid type for the date: " + DATETYPE)
    finddate = etree.ETXPath("string(//{%s}time[@class=%r]/@datetime)" % (HTMLNS,DATETYPE))
    date = finddate(doc)
    return date

def getcontent(doc):
    """return the full content of an article"""
    findcontent = etree.ETXPath("//{%s}article" % HTMLNS)
    content = findcontent(doc)
    return content[0]

def gettitle(doc):
    """return a list of markup and text being the title of the document"""
    findtitle =  etree.ETXPath("//{%s}h1[text()]" % HTMLNS)
    if len(findtitle(doc)) == 0:
        sys.exit("ERROR: The document has no title")
    title = findtitle(doc)[0]
    titlemarkup = etree.tostring(title,encoding="utf-8")
    titletext = etree.tostring(title,encoding="utf-8",method="text")
    return titlemarkup, titletext

def makeblogpost(doc):
    """create a blog post ready to be publish from a raw or already published document"""
    pass

def makefeedskeleton(websitetitle, tagline, feedtagid, lang, feedatomurl, site, license, faviconlink, authorname, authoruri):
    """create the feed skeleton for a specific Web site"""
    feed = Element('feed')
    feed.attrib['lang'] = lang

    # Web site title
    title = SubElement(feed, 'title')
    title.text = websitetitle

    # Tagline
    subtitle = SubElement(feed, 'subtitle')
    subtitle.text = tagline

    # feedid
    feedid = SubElement(feed, 'id')
    feedid.text = feedtagid

    # updated
    updated = SubElement(feed, 'updated')

    # link self atom
    linkselfatom = SubElement(feed, 'link')
    linkselfatom.attrib["rel"] = "self"
    linkselfatom.attrib["type"] = "application/atom+xml"
    linkselfatom.attrib["href"] = FEEDATOMURL

    # link alternate blog
    linkselfatom = SubElement(feed, 'link')
    linkselfatom.attrib["rel"] = "alternate"
    linkselfatom.attrib["type"] = "application/xhtml+xml"
    linkselfatom.attrib["href"] = site

    # link license
    linkselfatom = SubElement(feed, 'link')
    linkselfatom.attrib["rel"] = "license"
    linkselfatom.attrib["href"] = license

    # icon
    icon = SubElement(feed, 'icon')
    icon.text = faviconlink

    # author
    author = SubElement(feed, 'author')
    name = SubElement(author, 'name')
    name.text = authorname
    # email = SubElement(author, 'email')
    # email.text = FEEDEMAIL
    uri = SubElement(author, 'uri')
    uri.text = authoruri

    return feed

def makefeedentry(url, tagid, posttitle, created, modified, postcontent):
    """create an individual Atom feed entry from a ready to be publish post"""
    entry = Element('entry',nsmap=NSMAP2)
    id = SubElement(entry, 'id')
    id.text = tagid
    linkfeedentry = SubElement(entry, 'link')
    linkfeedentry.attrib["rel"] = "alternate"
    # TODO: This should be probably on a case by case.
    linkfeedentry.attrib["type"] = "text/html"
    linkfeedentry.attrib["href"] = url
    title = SubElement(entry, 'title')
    title.text = posttitle
    published = SubElement(entry, 'published')
    published.text = created
    updated = SubElement(entry, 'updated')
    updated.text = modified
    content = SubElement(entry, 'content')
    content.attrib["type"] = "xhtml"
    # changing the namespace to HTML
    # so only the local root element (div) will get the namespace
    divcontent = SubElement(content, "{%s}div"%HTMLNS, nsmap=NSMAP)
    # Adding a full tree fragment.
    divcontent.append(postcontent)
    linkselfatom = SubElement(entry, 'link',nsmap=NSMAP2)
    linkselfatom.attrib["rel"] = "license"
    linkselfatom.attrib["href"] = LICENSELIST['ccby']
    return entry

def createtagid(urlpath,isodate):
    """Create a unide tagid for a given blog post
    tag:la-grange.net,2012-01-24:2012/01/24/silence"""
    tagid = "tag:%s,%s:%s" % (DOMAIN,isodate,urlpath)
    return tagid

def nowdate(format=""):
    """Compute date in different formats I need"""
    # date in French please
    my_locale = "fr_FR"
    locale.setlocale(locale.LC_ALL, my_locale)
    if format == "rfc3339":
        return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
    elif format == "iso":
        return strftime("%Y-%m-%d", gmtime())
    elif format == "path":
        return strftime("%Y/%m/%d", gmtime())
    elif format == "humain":
        # remove the leading 0 of the date
        dategeek = strftime("%d %B %Y", gmtime())
        if dategeek.startswith('0'):
            dategeek = dategeek.lstrip('0')
        return dategeek
    elif format == "humainlong":
        # Remove the leading 0
        # And add the day of the week
        # "Vendredi "+ "3 février 2012"
        dategeek = strftime("%d %B %Y", localtime())
        if dategeek.startswith('0'):
            dategeek = dategeek.lstrip('0')
        return strftime("%A ", localtime()) + dategeek
    else:
        print "wrong format"
        sys.exit(1)

def updatefeed(feedentry):
    """Update the feed with the last individual feed entry"""
    pass

def updateannualindex(feedentry):
    """update the HTML Annual index with the feedendry"""
    pass

def updatemonthlyindex(indexmarkup, monthindexpath):
    """update the HTML Annual index with the feedendry"""
    print etree.tostring(indexmarkup, encoding="utf-8")
    # is there a monthly index.
    if os.path.isfile(monthindexpath):
        monthlyindex = parserawpost(monthindexpath)
    else:
        # TODO
        print "need to create index file"
        createmonthlyindex(monthindexpath)
    # grab the path and the modified date for the blog post
    anchor = indexmarkup.xpath("/li/a")
    newmodified = indexmarkup.xpath("/html:li/time[@class='modified']/text()", namespaces={'html':'http://www.w3.org/1999/xhtml'})[0]
    link = anchor[0].get('href')
    # check if the element is already in the list
    findli = etree.ETXPath("//{%s}li/{%s}a" % (HTMLNS,HTMLNS))
    fulllist = findli(monthlyindex)
    ENDLIST = True
    for item in fulllist:
        # if yes replace it with the new one.
        if item.get('href') == link:
            ENDLIST = False
            for timeelt in item.itersiblings(preceding=True):
                if timeelt.get('class') == 'modified':
                    timeelt.set('datetime',newmodified)
                    timeelt.text = newmodified
            return etree.tostring(monthlyindex, encoding="utf-8")

    if ENDLIST:
        findul = etree.ETXPath("//{%s}ul" % HTMLNS)
        ul = findul(monthlyindex)[0]
        print ul
        ul.append(indexmarkup)
        print "Add markup at the end"
        return etree.tostring(monthlyindex, encoding="utf-8")

    # if NO add it to the end of the list?
    # hmmm what about if the date is not in order :)

def createindexmarkup(postpath, created, title):
    """Create the Markup necessary to update the indexes"""
    dcreated = {'class':'created', 'datetime':created}
    # Creating the Markup
    li = etree.Element("{%s}li" % HTMLNS, nsmap=NSMAP)
    ctime = etree.SubElement(li,'time', dcreated)
    ctime.text = created[:10]
    ctime.tail = u" : "
    anchor = etree.SubElement(li, 'a', {'href':postpath})
    anchor.text = title.strip()
    return li

def updatearchivemap():
    """update the archive map page for new months and/or new years.
    not sure it is necessary. Manually is kind of cool with less
    dependencies."""
    pass

def createmonthlyindex(monthindexpath):
    """create a monthly index when it doesn't exist"""
    pass

def createannualindex(year):
    """create an annual index when it doesn't exist"""
    pass

# MAIN

def main():

    # Parsing the cli
    parser = argparse.ArgumentParser(description="Managing Web site blog posts")

    parser.add_argument('rawpost', metavar='FILE', help='file to be processed', action='store', nargs=1, type=argparse.FileType('rt'))
    parser.add_argument('-o', '--output', help='the blog post ready to be sync', nargs=1, dest="output", type=argparse.FileType('wt'))
    atomgroup = parser.add_mutually_exclusive_group()
    atomgroup.add_argument('--atom', help='create an atom feed. DEFAULT', action='store_true', dest="createfeed", default=True)
    atomgroup.add_argument('--noatom', help='do not create the atom feed', action='store_false', dest='createfeed', default=False)

    args = parser.parse_args()
    rawpostpath = args.rawpost[0]

    # Parse the document
    rawpost = parserawpost(rawpostpath)
    abspathpost = os.path.abspath(rawpostpath.name)
    # A few tests when developing
    STATUS = getdocstatus(rawpost)
    titlemarkup, title = gettitle(rawpost)
    title = title.decode("utf-8")
    print "TITLE: ", title
    print "TITLEMARKUP: ", titlemarkup
    created = getdocdate(rawpost, 'created')
    modified = getdocdate(rawpost, 'modified')
    print "CREATED:  ", created
    print "MODIFIED: ", modified
    content = getcontent(rawpost)

    # What are the paths?
    monthabspath = os.path.dirname(os.path.dirname(abspathpost))
    yearabspath = os.path.dirname(monthabspath)
    rootabspath = os.path.dirname(yearabspath)
    postpath = abspathpost[len(rootabspath):]
    monthindexpath = monthabspath+"/index.html"
    tagid =  createtagid(abspathpost,created)
    feedentry = makefeedentry(abspathpost, tagid, title, created, modified, content)
    print etree.tostring(feedentry, pretty_print=True, encoding='utf-8')
    indexmarkup = createindexmarkup(postpath[:-5], created, title)
    print etree.tostring(indexmarkup, pretty_print=True, encoding='utf-8')
    # print updatemonthlyindex(indexmarkup, monthindexpath)

    # feedbase = makefeedskeleton(SITENAME, TAGLINE, FEEDTAGID, FEEDLANG, FEEDATOMURL, SITE, LICENSELIST['ccby'], FAVICON, AUTHOR, AUTHORURI)
    # print etree.tostring(monthlyindex, encoding="utf-8",pretty_print=True)
    # two cases
    # either it is already in the list: update
    # It is not in the list: new post
    # check the path

if __name__ == "__main__":
    sys.exit(main())

<commit_msg>adding a few modules for the time computation<commit_after>#!/usr/bin/env python2.7
# encoding: utf-8
"""
ylmir.py

Created by Karl Dubost on 2011-12-03.
Copyright (c) 2011 Grange. All rights reserved.
"""

import sys, os, locale
from time import gmtime, strftime, localtime
import argparse
from lxml.html import tostring, html5parser
from lxml import etree
from lxml.etree import Element, SubElement

# CONFIG SITE
DOMAIN = u"la-grange.net"
SITE = u"http://www.%s/" % (DOMAIN)
FAVICON = SITE + "favicon"

SITENAME = u"Les carnets Web de La Grange"
TAGLINE = u"Rêveries le long d'un brin de chèvrefeuille"

FEEDTAGID = u"tag:la-grange.net,2000-04-12:karl"
FEEDLANG = u"fr"
FEEDATOMNOM = u"feed.atom"
FEEDATOMURL = u"%s%s" % (SITE,FEEDATOMNOM)

STATUSLIST = [u'draft',u'pub',u'acl']
DATETYPELIST = [u'created',u'modified']
LICENSELIST = {u'ccby': u'http://creativecommons.org/licenses/by/2.0/fr/',
               u'copy': u'©'}

AUTHOR = u"Karl Dubost"
AUTHORURI = u"http://www.la-grange.net/karl/"

HTMLNS = u"http://www.w3.org/1999/xhtml"
ATOMNS = u"http://www.w3.org/2005/Atom"
HTML = "{%s}" % HTMLNS
ATOM = "{%s}" % ATOMNS
NSMAP = {None : HTMLNS}
NSMAP2 = {None : ATOMNS}
NSMAP3 = {'html' : HTMLNS}

# CONFIG with cli (TODO)
STYLESHEET = "/2011/12/01/proto/style/article.css"
STATUS = ""
MAXFEEDITEM = 20
LICENSE = "ccby"

# PATHS

help_message = '''
This script has been entirely created
for processing text files for the site
La Grange http://www.la-grange.net/.
'''

# General processing features

def parserawpost(rawpostpath):
    """Given a path, parse an html file
    TODO check if the file is correct.
    """
    doc = html5parser.parse(rawpostpath).getroot()
    #TODO find a way to send the element with only the namespace on the first element.
    print "INFO: Document parsed"
    return doc

# Extracting information from the blog posts

def getdocstatus(doc):
    """Check the publication status of the document
    returns a string
    if there are multiple meta, returns the first one and issues a warning"""
    findstatus = etree.ETXPath("//{%s}meta[@name='status']" % HTMLNS)
    docstatus = findstatus(doc)
    if len(docstatus) >= 1:
        status = docstatus[0].attrib['content']
        if status in STATUSLIST:
            if len(docstatus) > 1:
                print "WARNING: There are more than one status. Taking the first one : " + status
            else:
                print "INFO: The document status is " + status
            return status
        else:
            raise Exception, "ERROR: No valid status for your document: %s not in %s" % (status, STATUSLIST)
    if len(docstatus) == 0:
        print "WARNING: There is no status for this document."
        status = "undefined"
        return status

def getdocdate(doc, DATETYPE):
    """return the creation date of the document in ISO format YYYY-MM-DD
    Input the document, typeofdate in between created and modified"""
    # TODO: check if the format is correct aka YYYY-MM-DD
    if DATETYPE not in DATETYPELIST:
        sys.exit("ERROR: No valid type for the date: " + DATETYPE)
    finddate = etree.ETXPath("string(//{%s}time[@class=%r]/@datetime)" % (HTMLNS,DATETYPE))
    date = finddate(doc)
    return date

def getcontent(doc):
    """return the full content of an article"""
    findcontent = etree.ETXPath("//{%s}article" % HTMLNS)
    content = findcontent(doc)
    return content[0]

def gettitle(doc):
    """return a list of markup and text being the title of the document"""
    findtitle =  etree.ETXPath("//{%s}h1[text()]" % HTMLNS)
    if len(findtitle(doc)) == 0:
        sys.exit("ERROR: The document has no title")
    title = findtitle(doc)[0]
    titlemarkup = etree.tostring(title,encoding="utf-8")
    titletext = etree.tostring(title,encoding="utf-8",method="text")
    return titlemarkup, titletext

def makeblogpost(doc):
    """create a blog post ready to be publish from a raw or already published document"""
    pass

def makefeedskeleton(websitetitle, tagline, feedtagid, lang, feedatomurl, site, license, faviconlink, authorname, authoruri):
    """create the feed skeleton for a specific Web site"""
    feed = Element('feed')
    feed.attrib['lang'] = lang

    # Web site title
    title = SubElement(feed, 'title')
    title.text = websitetitle

    # Tagline
    subtitle = SubElement(feed, 'subtitle')
    subtitle.text = tagline

    # feedid
    feedid = SubElement(feed, 'id')
    feedid.text = feedtagid

    # updated
    updated = SubElement(feed, 'updated')

    # link self atom
    linkselfatom = SubElement(feed, 'link')
    linkselfatom.attrib["rel"] = "self"
    linkselfatom.attrib["type"] = "application/atom+xml"
    linkselfatom.attrib["href"] = FEEDATOMURL

    # link alternate blog
    linkselfatom = SubElement(feed, 'link')
    linkselfatom.attrib["rel"] = "alternate"
    linkselfatom.attrib["type"] = "application/xhtml+xml"
    linkselfatom.attrib["href"] = site

    # link license
    linkselfatom = SubElement(feed, 'link')
    linkselfatom.attrib["rel"] = "license"
    linkselfatom.attrib["href"] = license

    # icon
    icon = SubElement(feed, 'icon')
    icon.text = faviconlink

    # author
    author = SubElement(feed, 'author')
    name = SubElement(author, 'name')
    name.text = authorname
    # email = SubElement(author, 'email')
    # email.text = FEEDEMAIL
    uri = SubElement(author, 'uri')
    uri.text = authoruri

    return feed

def makefeedentry(url, tagid, posttitle, created, modified, postcontent):
    """create an individual Atom feed entry from a ready to be publish post"""
    entry = Element('entry',nsmap=NSMAP2)
    id = SubElement(entry, 'id')
    id.text = tagid
    linkfeedentry = SubElement(entry, 'link')
    linkfeedentry.attrib["rel"] = "alternate"
    # TODO: This should be probably on a case by case.
    linkfeedentry.attrib["type"] = "text/html"
    linkfeedentry.attrib["href"] = url
    title = SubElement(entry, 'title')
    title.text = posttitle
    published = SubElement(entry, 'published')
    published.text = created
    updated = SubElement(entry, 'updated')
    updated.text = modified
    content = SubElement(entry, 'content')
    content.attrib["type"] = "xhtml"
    # changing the namespace to HTML
    # so only the local root element (div) will get the namespace
    divcontent = SubElement(content, "{%s}div"%HTMLNS, nsmap=NSMAP)
    # Adding a full tree fragment.
    divcontent.append(postcontent)
    linkselfatom = SubElement(entry, 'link',nsmap=NSMAP2)
    linkselfatom.attrib["rel"] = "license"
    linkselfatom.attrib["href"] = LICENSELIST['ccby']
    return entry

def createtagid(urlpath,isodate):
    """Create a unide tagid for a given blog post
    tag:la-grange.net,2012-01-24:2012/01/24/silence"""
    tagid = "tag:%s,%s:%s" % (DOMAIN,isodate,urlpath)
    return tagid

def nowdate(format=""):
    """Compute date in different formats I need"""
    # date in French please
    my_locale = "fr_FR"
    locale.setlocale(locale.LC_ALL, my_locale)
    if format == "rfc3339":
        return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
    elif format == "iso":
        return strftime("%Y-%m-%d", gmtime())
    elif format == "path":
        return strftime("%Y/%m/%d", gmtime())
    elif format == "humain":
        # remove the leading 0 of the date
        dategeek = strftime("%d %B %Y", gmtime())
        if dategeek.startswith('0'):
            dategeek = dategeek.lstrip('0')
        return dategeek
    elif format == "humainlong":
        # Remove the leading 0
        # And add the day of the week
        # "Vendredi "+ "3 février 2012"
        dategeek = strftime("%d %B %Y", localtime())
        if dategeek.startswith('0'):
            dategeek = dategeek.lstrip('0')
        return strftime("%A ", localtime()) + dategeek
    else:
        print "wrong format"
        sys.exit(1)

def updatefeed(feedentry):
    """Update the feed with the last individual feed entry"""
    pass

def updateannualindex(feedentry):
    """update the HTML Annual index with the feedendry"""
    pass

def updatemonthlyindex(indexmarkup, monthindexpath):
    """update the HTML Annual index with the feedendry"""
    print etree.tostring(indexmarkup, encoding="utf-8")
    # is there a monthly index.
    if os.path.isfile(monthindexpath):
        monthlyindex = parserawpost(monthindexpath)
    else:
        # TODO
        print "need to create index file"
        createmonthlyindex(monthindexpath)
    # grab the path and the modified date for the blog post
    anchor = indexmarkup.xpath("/li/a")
    newmodified = indexmarkup.xpath("/html:li/time[@class='modified']/text()", namespaces={'html':'http://www.w3.org/1999/xhtml'})[0]
    link = anchor[0].get('href')
    # check if the element is already in the list
    findli = etree.ETXPath("//{%s}li/{%s}a" % (HTMLNS,HTMLNS))
    fulllist = findli(monthlyindex)
    ENDLIST = True
    for item in fulllist:
        # if yes replace it with the new one.
        if item.get('href') == link:
            ENDLIST = False
            for timeelt in item.itersiblings(preceding=True):
                if timeelt.get('class') == 'modified':
                    timeelt.set('datetime',newmodified)
                    timeelt.text = newmodified
            return etree.tostring(monthlyindex, encoding="utf-8")

    if ENDLIST:
        findul = etree.ETXPath("//{%s}ul" % HTMLNS)
        ul = findul(monthlyindex)[0]
        print ul
        ul.append(indexmarkup)
        print "Add markup at the end"
        return etree.tostring(monthlyindex, encoding="utf-8")

    # if NO add it to the end of the list?
    # hmmm what about if the date is not in order :)

def createindexmarkup(postpath, created, title):
    """Create the Markup necessary to update the indexes"""
    dcreated = {'class':'created', 'datetime':created}
    # Creating the Markup
    li = etree.Element("{%s}li" % HTMLNS, nsmap=NSMAP)
    ctime = etree.SubElement(li,'time', dcreated)
    ctime.text = created[:10]
    ctime.tail = u" : "
    anchor = etree.SubElement(li, 'a', {'href':postpath})
    anchor.text = title.strip()
    return li

def updatearchivemap():
    """update the archive map page for new months and/or new years.
    not sure it is necessary. Manually is kind of cool with less
    dependencies."""
    pass

def createmonthlyindex(monthindexpath):
    """create a monthly index when it doesn't exist"""
    pass

def createannualindex(year):
    """create an annual index when it doesn't exist"""
    pass

# MAIN

def main():

    # Parsing the cli
    parser = argparse.ArgumentParser(description="Managing Web site blog posts")

    parser.add_argument('rawpost', metavar='FILE', help='file to be processed', action='store', nargs=1, type=argparse.FileType('rt'))
    parser.add_argument('-o', '--output', help='the blog post ready to be sync', nargs=1, dest="output", type=argparse.FileType('wt'))
    atomgroup = parser.add_mutually_exclusive_group()
    atomgroup.add_argument('--atom', help='create an atom feed. DEFAULT', action='store_true', dest="createfeed", default=True)
    atomgroup.add_argument('--noatom', help='do not create the atom feed', action='store_false', dest='createfeed', default=False)

    args = parser.parse_args()
    rawpostpath = args.rawpost[0]

    # Parse the document
    rawpost = parserawpost(rawpostpath)
    abspathpost = os.path.abspath(rawpostpath.name)
    # A few tests when developing
    STATUS = getdocstatus(rawpost)
    titlemarkup, title = gettitle(rawpost)
    title = title.decode("utf-8")
    print "TITLE: ", title
    print "TITLEMARKUP: ", titlemarkup
    created = getdocdate(rawpost, 'created')
    modified = getdocdate(rawpost, 'modified')
    print "CREATED:  ", created
    print "MODIFIED: ", modified
    content = getcontent(rawpost)
    print nowdate('rfc3339')

    # What are the paths?
    monthabspath = os.path.dirname(os.path.dirname(abspathpost))
    yearabspath = os.path.dirname(monthabspath)
    rootabspath = os.path.dirname(yearabspath)
    postpath = abspathpost[len(rootabspath):]
    monthindexpath = monthabspath+"/index.html"
    tagid =  createtagid(abspathpost,created)
    feedentry = makefeedentry(abspathpost, tagid, title, created, modified, content)
    print etree.tostring(feedentry, pretty_print=True, encoding='utf-8')
    indexmarkup = createindexmarkup(postpath[:-5], created, title)
    print etree.tostring(indexmarkup, pretty_print=True, encoding='utf-8')
    # print updatemonthlyindex(indexmarkup, monthindexpath)

    # feedbase = makefeedskeleton(SITENAME, TAGLINE, FEEDTAGID, FEEDLANG, FEEDATOMURL, SITE, LICENSELIST['ccby'], FAVICON, AUTHOR, AUTHORURI)
    # print etree.tostring(monthlyindex, encoding="utf-8",pretty_print=True)
    # two cases
    # either it is already in the list: update
    # It is not in the list: new post
    # check the path

if __name__ == "__main__":
    sys.exit(main())

<|endoftext|>


<commit_before>import os
import sys

import click

from modules import *

sys.path.insert(1, os.getcwd())

@click.group(cls=alias.Alias)
@click.pass_context
def cli(ctx):
    """
    Yoda PA: A personal assistant based on the command line
    """

# The alias module
cli.add_command(alias.alias)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def chat(ctx, input):
    """
    A simple chatbot\n
    To use, type: yoda chat <message>
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += i + ' '
        data = sys.modules['modules.chat'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

# The devtools module
cli.add_command(dev.dev)
cli.add_command(dev.speedtest)
cli.add_command(dev.url)
cli.add_command(dev.hackernews)
cli.add_command(dev.coinflip)
cli.add_command(dev.iplookup)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def love(ctx, input):
    """
    maintain a profile of someone you love\n

    commands:\n
    setup: Setup love\n
    status: check love status\n
    note: Add a note\n
    notes: View notes\n
    like: Add things they like\n
    likes: View things they like\n
    addbirth: Add birthday\n
    showbirth: Show birthday\n
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.love'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def diary(ctx, input):
    """
    Maintain a personal diary\n
    roughly based on the concept of Bullet Journal (http://bulletjournal.com/) \n\n

    Commands:\n
    nn: New note\n
    nt: new Task\n
    dt: delete task\n
    notes: view all notes\n
    ut: update task\n
    un: update note\n
    dct: delete all completed tasks\n
    dn:delete particular note\n
    tasks: view all completed and incomplete tasks\n
    ct: complete task
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.diary'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def money(ctx, input):
    """
    For tracking money \n\n
    Commands:\n
    setup: set a profile with default currency and initial money\n
    status: check config\n
    exp: add an expense\n
    exps: view all expenses\n
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.money'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

# The food module
cli.add_command(food.food)
# The learn module
cli.add_command(learn.learn)
cli.add_command(learn.vocabulary)
cli.add_command(learn.flashcards)
cli.add_command(learn.define)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def setup(ctx, input):
    """
    create a setup configuration for you to save some information locally
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.setup'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

# feedback
@cli.command()
def feedback():
    """
    Provide feedback for this package by:\n
    - Reporting a bug
    - Suggesting a feature
    - General suggestion
    """
    click.echo('For:\n\
    1. reporting a bug\n\
    2. For suggesting a feature\n\
    3. Any general suggestion or question\n\
Please create an issue in the Github repository:\nhttps://github.com/yoda-pa/yoda/issues/new')

# the life module
cli.add_command(life.rlist)
cli.add_command(life.ideas)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def goals(ctx, input):
    """
    Set your goals \n\n

    Commands:\n
    new: New goal\n
    view: View all completed and incomplete goals\n
    tasks: View tasks related to the goal\n
    complete: Complete a goal\n
    analyze: Analyze goals\n
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.goals'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

@cli.command()
@click.pass_context
@click.argument('input', nargs=1,required=True, callback=alias.alias_checker)
def ascii_transform(ctx, input):
    """
    Transform an image into ascii \n\n
    Pass the path for the image as the argument\n
    """
    input = util.get_arguments(ctx,-1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i+'')
        data = sys.modules['modules.asciiator'].process(str(test_string))
    else:
        click.echo('No input specified. Run with --help for info')
    print(data)
<commit_msg>hopefully test pass<commit_after>import os
import sys

import click

from modules import *

sys.path.insert(1, os.getcwd())

@click.group(cls=alias.Alias)
@click.pass_context
def cli(ctx):
    """
    Yoda PA: A personal assistant based on the command line
    """

# The alias module
cli.add_command(alias.alias)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def chat(ctx, input):
    """
    A simple chatbot\n
    To use, type: yoda chat <message>
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += i + ' '
        data = sys.modules['modules.chat'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

# The devtools module
cli.add_command(dev.dev)
cli.add_command(dev.speedtest)
cli.add_command(dev.url)
cli.add_command(dev.hackernews)
cli.add_command(dev.coinflip)
cli.add_command(dev.iplookup)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def love(ctx, input):
    """
    maintain a profile of someone you love\n

    commands:\n
    setup: Setup love\n
    status: check love status\n
    note: Add a note\n
    notes: View notes\n
    like: Add things they like\n
    likes: View things they like\n
    addbirth: Add birthday\n
    showbirth: Show birthday\n
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.love'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def diary(ctx, input):
    """
    Maintain a personal diary\n
    roughly based on the concept of Bullet Journal (http://bulletjournal.com/) \n\n

    Commands:\n
    nn: New note\n
    nt: new Task\n
    dt: delete task\n
    notes: view all notes\n
    ut: update task\n
    un: update note\n
    dct: delete all completed tasks\n
    dn:delete particular note\n
    tasks: view all completed and incomplete tasks\n
    ct: complete task
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.diary'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def money(ctx, input):
    """
    For tracking money \n\n
    Commands:\n
    setup: set a profile with default currency and initial money\n
    status: check config\n
    exp: add an expense\n
    exps: view all expenses\n
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.money'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

# The food module
cli.add_command(food.food)
# The learn module
cli.add_command(learn.learn)
cli.add_command(learn.vocabulary)
cli.add_command(learn.flashcards)
cli.add_command(learn.define)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def setup(ctx, input):
    """
    create a setup configuration for you to save some information locally
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.setup'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

# feedback
@cli.command()
def feedback():
    """
    Provide feedback for this package by:\n
    - Reporting a bug
    - Suggesting a feature
    - General suggestion
    """
    click.echo('For:\n\
    1. reporting a bug\n\
    2. For suggesting a feature\n\
    3. Any general suggestion or question\n\
Please create an issue in the Github repository:\nhttps://github.com/yoda-pa/yoda/issues/new')

# the life module
cli.add_command(life.rlist)
cli.add_command(life.ideas)

@cli.command()
@click.pass_context
@click.argument('input', nargs=-1, required=False, callback=alias.alias_checker)
def goals(ctx, input):
    """
    Set your goals \n\n

    Commands:\n
    new: New goal\n
    view: View all completed and incomplete goals\n
    tasks: View tasks related to the goal\n
    complete: Complete a goal\n
    analyze: Analyze goals\n
    """
    input = util.get_arguments(ctx, -1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i + ' ')
        data = sys.modules['modules.goals'].process(test_string)
    else:
        click.echo('No input specified. Run with --help for info')

@cli.command()
@click.pass_context
@click.argument('input', nargs=1,required=True, callback=alias.alias_checker)
def ascii_transform(ctx, input):
    """
    Transform an image into ascii \n\n
    Pass the absolute path to  the image as the argument\n
    """
    input = util.get_arguments(ctx,-1)
    if input:
        test_string = ''
        for i in input:
            test_string += (i+'')
        data = sys.modules['modules.asciiator'].process(str(test_string))
    else:
        click.echo('No input specified. Run with --help for info')
    print(data)
<|endoftext|>


<commit_before>"""Check for consistency of users to projects

"""
import logging
from website.app import init_app
from website import models
from modularodm import Q

app = init_app()
logger = logging.getLogger(__name__)

def check_consistency_of_users_to_projects():

    nodes = models.Node.find(
        Q('is_deleted', 'eq', False)
    )
    count = 0
    for node in nodes:
        if node._id not in node.creator.created:
            logger.info('Creator {} is inconsistent in reference with node {}'.format(node.creator._id, node._id))
            count += 1
        for contributor in node.contributors:
            if node._id not in contributor.contributed:
                logger.info('Contributor {} is inconsistent in reference with node {}'.format(contributor._id, node._id))
                count += 1

    if count == 0:
        logger.info("Consistency of users to projects' check is done. There is no inconsistency found.")
    else:
        logger.info("Consistency of users to projects' check is done. There are {} inconsistency found.".format(count))

if __name__ == '__main__':
    check_consistency_of_users_to_projects()
<commit_msg>add file logger<commit_after>"""Check for consistency of users to projects

"""
import logging
from website.app import init_app
from website import models
from modularodm import Q
from scripts import utils as script_utils

app = init_app()
logger = logging.getLogger(__name__)

def check_consistency_of_users_to_projects():

    nodes = models.Node.find(
        Q('is_deleted', 'eq', False)
    )
    count = 0
    for node in nodes:
        if node._id not in node.creator.created:
            logger.info('Creator {} is inconsistent in reference with node {}'.format(node.creator._id, node._id))
            count += 1
        for contributor in node.contributors:
            if node._id not in contributor.contributed:
                logger.info('Contributor {} is inconsistent in reference with node {}'.format(contributor._id, node._id))
                count += 1

    if count == 0:
        logger.info("Consistency of users to projects' check is done. There is no inconsistency found.")
    else:
        logger.info("Consistency of users to projects' check is done. There are {} inconsistency found.".format(count))

if __name__ == '__main__':
    script_utils.add_file_logger(logger, __file__)
    check_consistency_of_users_to_projects()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Standard library imports
# Third party imports
from django.db.models import get_model
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django.views.generic import ListView

# Local application / specific library imports
from machina.conf import settings as machina_settings
from machina.core.loading import get_class
from machina.views.mixins import PermissionRequiredMixin

Forum = get_model('forum', 'Forum')

PermissionHandler = get_class('permission.handler', 'PermissionHandler')
perm_handler = PermissionHandler()

class IndexView(ListView):
    template_name = 'forum/index.html'
    context_object_name = 'forums'

    def get_queryset(self):
        return perm_handler.forum_list_filter(
            Forum.objects.filter(
                # Category, top-level forums and links
                Q(parent__isnull=True) |
                # Forums that have a top-level category has parent
                Q(parent__parent__isnull=True, parent__type=Forum.TYPE_CHOICES.forum_cat) |
                # Sub forums that can be displayed
                Q(parent__parent__isnull=True, display_sub_forum_list=True) |
                # Children of forums that have a category as parent
                Q(parent__parent__parent__isnull=True,
                    parent__parent__type=Forum.TYPE_CHOICES.forum_cat,
                    parent__type=Forum.TYPE_CHOICES.forum_post,
                    display_sub_forum_list=True)),
            self.request.user
        )

class ForumView(PermissionRequiredMixin, ListView):
    template_name = 'forum/forum_detail.html'
    context_object_name = 'topics'
    permission_required = ['can_read_forum', ]
    paginate_by = machina_settings.FORUM_TOPICS_NUMBER_PER_PAGE

    def get_forum(self):
        return get_object_or_404(Forum, pk=self.kwargs['pk'])

    def get_queryset(self):
        self.forum = self.get_forum()
        qs = self.forum.topics.all()
        return qs

    def get_object(self):
        """
        Return the considered forum in order to allow permission checks.
        """
        if not hasattr(self, 'forum'):
            self.forum = self.get_forum()
        return self.forum

    def get_context_data(self, **kwargs):
        context = super(ForumView, self).get_context_data(**kwargs)

        # Insert the considered forum into the context
        context['forum'] = self.forum

        # Get the list of forums that have the current forum as parent
        sub_forums = Forum.objects.filter(
            # Category, top-level forums and links
            Q(parent=self.forum) |
            # Forums that have a top-level category has parent
            Q(parent__parent=self.forum, parent__type=Forum.TYPE_CHOICES.forum_cat) |
            # Sub forums that can be displayed
            Q(parent__parent=self.forum, display_sub_forum_list=True) |
            # Children of forums that have a category as parent
            Q(parent__parent__parent=self.forum,
                parent__parent__type=Forum.TYPE_CHOICES.forum_cat,
                parent__type=Forum.TYPE_CHOICES.forum_post,
                display_sub_forum_list=True))
        context['sub_forums'] = perm_handler.forum_list_filter(sub_forums, self.request.user)

        return context
<commit_msg>Forum view updated<commit_after># -*- coding: utf-8 -*-

# Standard library imports
# Third party imports
from django.db.models import get_model
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.views.generic import ListView

# Local application / specific library imports
from machina.conf import settings as machina_settings
from machina.core.loading import get_class
from machina.views.mixins import PermissionRequiredMixin

Forum = get_model('forum', 'Forum')

PermissionHandler = get_class('permission.handler', 'PermissionHandler')
perm_handler = PermissionHandler()

class IndexView(ListView):
    template_name = 'forum/index.html'
    context_object_name = 'forums'

    def get_queryset(self):
        return perm_handler.forum_list_filter(
            Forum.objects.filter(
                # Category, top-level forums and links
                Q(parent__isnull=True) |
                # Forums that have a top-level category has parent
                Q(parent__parent__isnull=True, parent__type=Forum.TYPE_CHOICES.forum_cat) |
                # Sub forums that can be displayed
                Q(parent__parent__isnull=True, display_sub_forum_list=True) |
                # Children of forums that have a category as parent
                Q(parent__parent__parent__isnull=True,
                    parent__parent__type=Forum.TYPE_CHOICES.forum_cat,
                    parent__type=Forum.TYPE_CHOICES.forum_post,
                    display_sub_forum_list=True)),
            self.request.user
        )

class ForumView(PermissionRequiredMixin, ListView):
    template_name = 'forum/forum_detail.html'
    context_object_name = 'topics'
    permission_required = ['can_read_forum', ]
    paginate_by = machina_settings.FORUM_TOPICS_NUMBER_PER_PAGE

    def get_forum(self):
        return get_object_or_404(Forum, pk=self.kwargs['pk'])

    def get_queryset(self):
        self.forum = self.get_forum()
        qs = self.forum.topics.all()
        return qs

    def get_object(self):
        """
        Return the considered forum in order to allow permission checks.
        """
        if not hasattr(self, 'forum'):
            self.forum = self.get_forum()
        return self.forum

    def get_context_data(self, **kwargs):
        context = super(ForumView, self).get_context_data(**kwargs)

        # Insert the considered forum into the context
        context['forum'] = self.forum

        # Get the list of forums that have the current forum as parent
        sub_forums = Forum.objects.filter(
            # Category, top-level forums and links
            Q(parent=self.forum) |
            # Forums that have a top-level category has parent
            Q(parent__parent=self.forum, parent__type=Forum.TYPE_CHOICES.forum_cat) |
            # Sub forums that can be displayed
            Q(parent__parent=self.forum, display_sub_forum_list=True) |
            # Children of forums that have a category as parent
            Q(parent__parent__parent=self.forum,
                parent__parent__type=Forum.TYPE_CHOICES.forum_cat,
                parent__type=Forum.TYPE_CHOICES.forum_post,
                display_sub_forum_list=True))
        context['sub_forums'] = perm_handler.forum_list_filter(sub_forums, self.request.user)

        return context

    def dispatch(self, request, *args, **kwargs):
        forum = self.get_object()
        if forum.is_link:
            return HttpResponseRedirect(forum.link)
        return super(ForumView, self).dispatch(request, *args, **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
imguralbum.py - Download a whole imgur album in one go.

Provides both a class and a command line utility in a single script
to download Imgur albums.

MIT License
Copyright Alex Gisby <alex@solution10.com>
"""

import sys
import re
import urllib.request, urllib.parse, urllib.error
import os
import math

help_message = """
Quickly and easily download an album from Imgur.

Format:
    $ python imguralbum.py [album URL] [destination folder]

Example:
    $ python imguralbum.py http://imgur.com/a/uOOju#6 /Users/alex/images

If you omit the dest folder name, the utility will create one with the same name
as the album
(for example for http://imgur.com/a/uOOju it'll create uOOju/ in the cwd)
"""

class ImgurAlbumException(Exception):
    def __init__(self, msg=False):
        self.msg = msg

class ImgurAlbumDownloader:
    def __init__(self, album_url):
        """
        Constructor. Pass in the album_url that you want to download.
        """
        self.album_url = album_url

        # Callback members:
        self.image_callbacks = []
        self.complete_callbacks = []

        # Check the URL is actually imgur:
        match = re.match("(https?)\:\/\/(www\.)?(?:m\.)?imgur\.com/a/([a-zA-Z0-9]+)(#[0-9]+)?", album_url)
        if not match:
            raise ImgurAlbumException("URL must be a valid Imgur Album")

        self.protocol = match.group(1)
        self.album_key = match.group(3)

        # Read the no-script version of the page for all the images:
        noscriptURL = "http://imgur.com/a/" + self.album_key + "/noscript"
        self.response = urllib.request.urlopen(url=noscriptURL)
        if self.response.getcode() != 200:
            raise ImgurAlbumException("Error reading Imgur: Error Code %d" % self.response.getcode())

        # Read in the images now so we can get stats and stuff:
        html = self.response.read().decode('utf-8')
        self.images = re.findall('<img src="(\/\/i\.imgur\.com\/([a-zA-Z0-9]+\.(jpg|jpeg|png|gif)))"', html)#,html

    def num_images(self):
        """
        Returns the number of images that are present in this album.
        """
        return len(self.images)

    def album_key(self):
        """
        Returns the key of this album. Helpful if you plan on generating your own
        folder names.
        """
        return self.album_key

    def on_image_download(self, callback):
        """
        Allows you to bind a function that will be called just before an image is
        about to be downloaded. You'll be given the 1-indexed position of the image, it's URL
        and it's destination file in the callback like so:
            my_awesome_callback(1, "http://i.imgur.com/fGWX0.jpg", "~/Downloads/1-fGWX0.jpg")
        """
        self.image_callbacks.append(callback)

    def on_complete(self, callback):
        """
        Allows you to bind onto the end of the process, displaying any lovely messages
        to your users, or carrying on with the rest of the program. Whichever.
        """
        self.complete_callbacks.append(callback)

    def save_images(self, foldername=False):
        """
        Saves the images from the album into a folder given by foldername.
        If no foldername is given, it'll use the cwd and the album key.
        And if the folder doesn't exist, it'll try and create it.
        """
        # Try and create the album folder:
        if foldername:
            albumFolder = foldername
        else:
            albumFolder = self.album_key

        if not os.path.exists(albumFolder):
            os.makedirs(albumFolder)

        # And finally loop through and save the images:
        for (counter, image) in enumerate(self.images, start=1):
            image_url = "%s:%s" % (self.protocol, image[0])

            # Fetch hi-res images (Fixes https://github.com/alexgisby/imgur-album-downloader/issues/5)
            image_url = re.sub(r"([a-zA-Z0-9]+)(h)\.(jpg|jpeg|png|gif)$", r"\1.\3", image_url)

            prefix = "%0*d-" % (
                int(math.ceil(math.log(len(self.images) + 1, 10))),
                counter
            )
            path = os.path.join(albumFolder, prefix + image[1])

            # Run the callbacks:
            for fn in self.image_callbacks:
                fn(counter, image_url, path)

            # Actually download the thing
            urllib.request.urlretrieve(image_url, path)

        # Run the complete callbacks:
        for fn in self.complete_callbacks:
            fn()

if __name__ == '__main__':
    args = sys.argv

    if len(args) == 1:
        # Print out the help message and exit:
        print (help_message)
        exit()

    try:
        # Fire up the class:
        downloader = ImgurAlbumDownloader(args[1])
        print(("Found {0} images in album".format(downloader.num_images())))

        # Called when an image is about to download:
        def print_image_progress(index, url, dest):
            print(("Downloading Image %d" % index))
            print(("    %s >> %s" % (url, dest)))
        downloader.on_image_download(print_image_progress)

        # Called when the downloads are all done.
        def all_done():
            print ("")
            print ("Done!")
        downloader.on_complete(all_done)

        # Work out if we have a foldername or not:
        if len(args) == 3:
            albumFolder = args[2]
        else:
            albumFolder = False

        # Enough talk, let's save!
        downloader.save_images(albumFolder)
        exit()

    except ImgurAlbumException as e:
        print(("Error: " + e.msg))
        print ("")
        print ("How to use")
        print ("=============")
        print (help_message)
        exit(1)
<commit_msg>A large album that I tried to download (~3500 wallpapers) had a few broken links. Might want to skip files that are already downloaded, just a thought.<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
imguralbum.py - Download a whole imgur album in one go.

Provides both a class and a command line utility in a single script
to download Imgur albums.

MIT License
Copyright Alex Gisby <alex@solution10.com>
"""

import sys
import re
import urllib.request, urllib.parse, urllib.error
import os
import math

help_message = """
Quickly and easily download an album from Imgur.

Format:
    $ python imguralbum.py [album URL] [destination folder]

Example:
    $ python imguralbum.py http://imgur.com/a/uOOju#6 /Users/alex/images

If you omit the dest folder name, the utility will create one with the same name
as the album
(for example for http://imgur.com/a/uOOju it'll create uOOju/ in the cwd)
"""

class ImgurAlbumException(Exception):
    def __init__(self, msg=False):
        self.msg = msg

class ImgurAlbumDownloader:
    def __init__(self, album_url):
        """
        Constructor. Pass in the album_url that you want to download.
        """
        self.album_url = album_url

        # Callback members:
        self.image_callbacks = []
        self.complete_callbacks = []

        # Check the URL is actually imgur:
        match = re.match("(https?)\:\/\/(www\.)?(?:m\.)?imgur\.com/a/([a-zA-Z0-9]+)(#[0-9]+)?", album_url)
        if not match:
            raise ImgurAlbumException("URL must be a valid Imgur Album")

        self.protocol = match.group(1)
        self.album_key = match.group(3)

        # Read the no-script version of the page for all the images:
        noscriptURL = "http://imgur.com/a/" + self.album_key + "/noscript"
        self.response = urllib.request.urlopen(url=noscriptURL)
        if self.response.getcode() != 200:
            raise ImgurAlbumException("Error reading Imgur: Error Code %d" % self.response.getcode())

        # Read in the images now so we can get stats and stuff:
        html = self.response.read().decode('utf-8')
        self.images = re.findall('<img src="(\/\/i\.imgur\.com\/([a-zA-Z0-9]+\.(jpg|jpeg|png|gif)))"', html)#,html

    def num_images(self):
        """
        Returns the number of images that are present in this album.
        """
        return len(self.images)

    def album_key(self):
        """
        Returns the key of this album. Helpful if you plan on generating your own
        folder names.
        """
        return self.album_key

    def on_image_download(self, callback):
        """
        Allows you to bind a function that will be called just before an image is
        about to be downloaded. You'll be given the 1-indexed position of the image, it's URL
        and it's destination file in the callback like so:
            my_awesome_callback(1, "http://i.imgur.com/fGWX0.jpg", "~/Downloads/1-fGWX0.jpg")
        """
        self.image_callbacks.append(callback)

    def on_complete(self, callback):
        """
        Allows you to bind onto the end of the process, displaying any lovely messages
        to your users, or carrying on with the rest of the program. Whichever.
        """
        self.complete_callbacks.append(callback)

    def save_images(self, foldername=False):
        """
        Saves the images from the album into a folder given by foldername.
        If no foldername is given, it'll use the cwd and the album key.
        And if the folder doesn't exist, it'll try and create it.
        """
        # Try and create the album folder:
        if foldername:
            albumFolder = foldername
        else:
            albumFolder = self.album_key

        if not os.path.exists(albumFolder):
            os.makedirs(albumFolder)

        # And finally loop through and save the images:
        for (counter, image) in enumerate(self.images, start=1):
            image_url = "%s:%s" % (self.protocol, image[0])

            # Fetch hi-res images (Fixes https://github.com/alexgisby/imgur-album-downloader/issues/5)
            image_url = re.sub(r"([a-zA-Z0-9]+)(h)\.(jpg|jpeg|png|gif)$", r"\1.\3", image_url)

            prefix = "%0*d-" % (
                int(math.ceil(math.log(len(self.images) + 1, 10))),
                counter
            )
            path = os.path.join(albumFolder, prefix + image[1])

            # Run the callbacks:
            for fn in self.image_callbacks:
                fn(counter, image_url, path)

            # Actually download the thing
            if os.path.isfile(path):
                print ("Skipping, already exists.")
            else:
                try:
                    urllib.request.urlretrieve(image_url, path)
                except:
                    print ("Download failed.")

        # Run the complete callbacks:
        for fn in self.complete_callbacks:
            fn()

if __name__ == '__main__':
    args = sys.argv

    if len(args) == 1:
        # Print out the help message and exit:
        print (help_message)
        exit()

    try:
        # Fire up the class:
        downloader = ImgurAlbumDownloader(args[1])
        print(("Found {0} images in album".format(downloader.num_images())))

        # Called when an image is about to download:
        def print_image_progress(index, url, dest):
            print(("Downloading Image %d" % index))
            print(("    %s >> %s" % (url, dest)))
        downloader.on_image_download(print_image_progress)

        # Called when the downloads are all done.
        def all_done():
            print ("")
            print ("Done!")
        downloader.on_complete(all_done)

        # Work out if we have a foldername or not:
        if len(args) == 3:
            albumFolder = args[2]
        else:
            albumFolder = False

        # Enough talk, let's save!
        downloader.save_images(albumFolder)
        exit()

    except ImgurAlbumException as e:
        print(("Error: " + e.msg))
        print ("")
        print ("How to use")
        print ("=============")
        print (help_message)
        exit(1)
<|endoftext|>


<commit_before>"""
A simple demonstration of Impala UDF generation.
"""

from numba.ext.impala import udf, IntVal, FunctionContext

@udf(IntVal(FunctionContext, IntVal, IntVal))
def add_udf(context, arg1, arg2):
    if arg1.is_null or arg2.is_null:
        return IntVal.null
    return IntVal(arg1.val + arg2.val)

# Simply print the module IR
print(add_udf.llvm_module)

<commit_msg>Add simple test to exercise DoubleVal, TinyIntVal and BooleanVal<commit_after>"""
A simple demonstration of Impala UDF generation.
"""

from numba.ext.impala import (udf, IntVal, FunctionContext, BooleanVal,
                              DoubleVal, TinyIntVal)

@udf(IntVal(FunctionContext, IntVal, IntVal))
def add_udf(context, arg1, arg2):
    if arg1.is_null or arg2.is_null:
        return IntVal.null
    return IntVal(arg1.val + arg2.val)

# Simply print the module IR
print(add_udf.llvm_module)

@udf(BooleanVal(FunctionContext, DoubleVal, TinyIntVal))
def exercise_double_tinyint_bool(context, arg1, arg2):
    if arg1.is_null or arg2.is_null:
        return BooleanVal.null

    lt = arg1.val < arg2.val
    return BooleanVal(lt)

print(exercise_double_tinyint_bool.llvm_module)
<|endoftext|>