<commit_before>"""Usage:
    episode new <project_name>
    episode server <host> <port>
    episode build
    episode watch
    episode -h | --help | --version
"""

import os
import re
from http.server import HTTPServer, BaseHTTPRequestHandler
import shutil
from datetime import date
from jinja2 import Environment, FileSystemLoader
from markdown import Markdown
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from docopt import docopt
import yaml
import time

SEED_PATH = os.path.abspath("seed")

md_pattern = re.compile(r"(\n)*\-+(\n)*(?P<meta>(.*?\n)*?)\-+\n*?")
post_name_pattern = re.compile(r"(?P<year>(\d{4}))\-(?P<month>(\d{1,2}))\-(?P<day>(\d{1,2}))\-(?P<alias>(.+))")

md = Markdown()

def chunks(l, n):
    """ Yield successive n-sized chunks from l.
    """
    for i in range(0, len(l), n):
        yield l[i:i+n]

class Page:
    def __init__(self, file, config, path_templete="", site_path="site", date_template="%Y/%m/%d"):

        self._path_template = path_templete
        self._date_template = date_template
        self.site_path = site_path
        self.config = config
        self._filename, self._filename_extension = os.path.splitext(os.path.basename(file))

        self.formatted_date = None

        self._file = open(file, "r").read()
        self._parse_file_name()
        self._parse_file()

    @property
    def path(self):
        if self.type == "post":
            return os.path.join(self.site_path, self._path_template.format(year=self.date.year,
                                                           month=self.date.month,
                                                           day=self.date.day))
        else:
            return self.site_path

    @property
    def url_path(self):
        if self.type == "post":
            return os.path.join(self._path_template.format(year=self.date.year,
                                                           month=self.date.month,
                                                           day=self.date.day))
        else:
            return ""
    def _parse_file_name(self):
        matched = re.match(post_name_pattern, self._filename)
        if matched:
            year = int(matched.group("year"))
            month = int(matched.group("month"))
            day = int(matched.group("day"))
            self.date = date(year, month, day)
            self.alias = matched.group("alias")
            self.formatted_date = self.date.strftime(self._date_template)
            self.type = "post"
        else:
            self.alias = self._filename
            self.type = "page"

    def _parse_file(self):
        matched = md_pattern.match(self._file)
        meta = matched.group("meta")
        self.data = yaml.load(meta)
        self.data["date"] = self.formatted_date if self.formatted_date else None
        self.data["content"] = md.convert(self._file[matched.end():])
        self.data["path"] = self.path
        self.data["alias"] = self.alias
        self.data["template"] += ".html"
        self.data["url"] = os.path.join(self.config.get("url"), self.url_path, self.alias)

class Episode:
    def __init__(self):
        self.posts = []
        self.pages = []

        self.project_path = os.getcwd()
        self._get_config()
        self.env = Environment(loader=FileSystemLoader(self._get_path(self.config.get("template_folder"))))
        self.env.globals["site"] = self.config

    def _get_path(self, folder):
        return os.path.join(self.project_path, folder)

    def _get_config(self):
        config_path = os.path.join(self.project_path, "config.yaml")
        stream = open(config_path, "r")
        self.config = yaml.load(stream)
        # print("="*10)
        # print(self.config)

    def _get_template_by_name(self, template_name):
        return self.env.get_template("{}.html".format(template_name))

    def _copy_static_files(self):
        for d in self.config.get("static_folders"):
            from_dir = os.path.join(self.project_path, d)
            to_dir = os.path.join(self._get_path(self.config.get("destination")), d)
            if os.path.exists(from_dir):
                if os.path.exists(to_dir):
                    shutil.rmtree(to_dir)
                shutil.copytree(from_dir, to_dir)

    def _walk_files(self):
        for dirpath, dirnames, filenames in os.walk(self.project_path):
            for name in filenames:
                if os.path.splitext(name)[-1] in self.config.get("file_ext"):
                    file_obj = Page(os.path.join(dirpath, name), config=self.config, path_templete=self.config.get("path_template"))
                    if file_obj.type == "post":
                        self.posts.append(file_obj.data)
                    else:
                        self.pages.append(file_obj.data)
        self.posts.sort(key=lambda x: x["date"], reverse=True)
        self.env.globals.update({
            "posts": self.posts,
            "pages": self.pages
        })

    def _render_html_file(self, page):
        target_file = os.path.join(page.get("path"), page.get("alias")) + ".html"
        print(target_file)
        if not os.path.exists(page.get("path")):
            os.makedirs(page.get("path"))
        f = open(target_file, 'w')
        f.write(self.env.get_template(page.get("template")).render(page))
        f.close()

    def _render_pagination(self):
        pagination = self.config.get("paginate")
        pagination_folder = os.path.join(self._get_path(self.config.get("destination")), "page")
        print(pagination_folder)
        if len(self.posts) > pagination:
            os.makedirs(pagination_folder)
        for index, posts in enumerate(chunks(self.posts, pagination)):
            if index == 0:
                f = open(os.path.join(self._get_path(self.config.get("destination")), "index.html"), 'w')
            else:
                f = open(os.path.join(pagination_folder, "{}.html".format(str(index))), 'w')
            f.write(self.env.get_template("index.html").render({"pagination_posts": posts}))
            f.close()

    def _render(self):
        for page in self.pages:
            self._render_html_file(page)
        for post in self.posts:
            self._render_html_file(post)
        self._render_pagination()

    def generate_project(self, project_name):
        if os.path.exists(project_name):
            shutil.rmtree(project_name)
        shutil.copytree(SEED_PATH, project_name)

    def build(self):
        start = time.clock()
        # shutil.rmtree(self._get_path(self.config.get("destination")))
        self._copy_static_files()
        self._walk_files()
        self._render()

        end = time.clock()
        print("run time: {time}s".format(time=end-start))

    def server(self, address="0.0.0.0", port=8000, server_class=HTTPServer, handler_class=BaseHTTPRequestHandler):
        self.build()
        print("start server")
        server_address = (address, port)
        httpd = server_class(server_address, handler_class)
        print("Serving at http://{address}:{port}".format(address=address, port=port))
        httpd.serve_forever()

    def watch(self):
        self.build()
        event_handler = FileChangeEventHandler(self)
        observer = Observer()
        observer.schedule(event_handler, self.config.get("root"), recursive=False)
        # for item in os.listdir("."):
        #     if os.path.isdir(item) and item != "site":
        #         observer.schedule(event_handler, item, recursive=False)
        observer.start()
        try:
            while True:
                time.sleep(1)
        except KeyboardInterrupt:
            observer.stop()

class FileChangeEventHandler(FileSystemEventHandler):
    def __init__(self, episode):
        self.episode = episode

    # def on_moved(self, event):
    #     self.episode.build()
    #     print("==move====src"+event.src_path)

    def on_created(self, event):
        self.episode.build()
        print("==create====src"+event.src_path)

    def on_deleted(self, event):
        self.episode.build()
        print("==delete====src"+event.src_path)

    def on_modified(self, event):
        self.episode.build()
        print("===modified===src"+event.src_path)

def start_server():
    print("start server")
    Episode().server()

def start_watch():
    print("start watch")
    Episode().watch()

def start_build():
    print("start build")
    Episode().build()

def start_new(project_name):
    print("create a new project")
    Episode().generate_project(project_name)

def command_options(arguments):
    if arguments["new"]:
        start_new(arguments["<project_name>"])
    elif arguments["build"]:
        start_build()
    elif arguments["server"]:
        start_server()
    elif arguments["watch"]:
        start_watch()

if __name__ == '__main__':
    arguments = docopt(__doc__, version='0.0.1')
    command_options(arguments)<commit_msg>import git support, not impement yet<commit_after>"""Usage:
    episode new <project_name>
    episode server <host> <port>
    episode build
    episode watch
    episode -h | --help | --version
"""

import os
import re
import sh
from http.server import HTTPServer, BaseHTTPRequestHandler
import shutil
from datetime import date
from jinja2 import Environment, FileSystemLoader
from markdown import Markdown
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from docopt import docopt
import yaml
import time

SEED_PATH = os.path.abspath("seed")

md_pattern = re.compile(r"(\n)*\-+(\n)*(?P<meta>(.*?\n)*?)\-+\n*?")
post_name_pattern = re.compile(r"(?P<year>(\d{4}))\-(?P<month>(\d{1,2}))\-(?P<day>(\d{1,2}))\-(?P<alias>(.+))")

md = Markdown()

def chunks(l, n):
    """ Yield successive n-sized chunks from l.
    """
    for i in range(0, len(l), n):
        yield l[i:i+n]

class Page:
    def __init__(self, file, config, path_templete="", site_path="site", date_template="%Y/%m/%d"):

        self._path_template = path_templete
        self._date_template = date_template
        self.site_path = site_path
        self.config = config
        self._filename, self._filename_extension = os.path.splitext(os.path.basename(file))

        self.formatted_date = None

        self._file = open(file, "r").read()
        self._parse_file_name()
        self._parse_file()

    @property
    def path(self):
        if self.type == "post":
            return os.path.join(self.site_path, self._path_template.format(year=self.date.year,
                                                           month=self.date.month,
                                                           day=self.date.day))
        else:
            return self.site_path

    @property
    def url_path(self):
        if self.type == "post":
            return os.path.join(self._path_template.format(year=self.date.year,
                                                           month=self.date.month,
                                                           day=self.date.day))
        else:
            return ""
    def _parse_file_name(self):
        matched = re.match(post_name_pattern, self._filename)
        if matched:
            year = int(matched.group("year"))
            month = int(matched.group("month"))
            day = int(matched.group("day"))
            self.date = date(year, month, day)
            self.alias = matched.group("alias")
            self.formatted_date = self.date.strftime(self._date_template)
            self.type = "post"
        else:
            self.alias = self._filename
            self.type = "page"

    def _parse_file(self):
        matched = md_pattern.match(self._file)
        meta = matched.group("meta")
        self.data = yaml.load(meta)
        self.data["date"] = self.formatted_date if self.formatted_date else None
        self.data["content"] = md.convert(self._file[matched.end():])
        self.data["path"] = self.path
        self.data["alias"] = self.alias
        self.data["template"] += ".html"
        self.data["url"] = os.path.join(self.config.get("url"), self.url_path, self.alias)

class Episode:
    def __init__(self):
        self.posts = []
        self.pages = []

        self.project_path = os.getcwd()
        self._get_config()
        self.env = Environment(loader=FileSystemLoader(self._get_path(self.config.get("template_folder"))))
        self.env.globals["site"] = self.config

    def _get_path(self, folder):
        return os.path.join(self.project_path, folder)

    def _get_config(self):
        config_path = os.path.join(self.project_path, "config.yaml")
        stream = open(config_path, "r")
        self.config = yaml.load(stream)
        # print("="*10)
        # print(self.config)

    def _get_template_by_name(self, template_name):
        return self.env.get_template("{}.html".format(template_name))

    def _copy_static_files(self):
        for d in self.config.get("static_folders"):
            from_dir = os.path.join(self.project_path, d)
            to_dir = os.path.join(self._get_path(self.config.get("destination")), d)
            if os.path.exists(from_dir):
                if os.path.exists(to_dir):
                    shutil.rmtree(to_dir)
                shutil.copytree(from_dir, to_dir)

    def _walk_files(self):
        for dirpath, dirnames, filenames in os.walk(self.project_path):
            for name in filenames:
                if os.path.splitext(name)[-1] in self.config.get("file_ext"):
                    file_obj = Page(os.path.join(dirpath, name), config=self.config, path_templete=self.config.get("path_template"))
                    if file_obj.type == "post":
                        self.posts.append(file_obj.data)
                    else:
                        self.pages.append(file_obj.data)
        self.posts.sort(key=lambda x: x["date"], reverse=True)
        self.env.globals.update({
            "posts": self.posts,
            "pages": self.pages
        })

    def _render_html_file(self, page):
        target_file = os.path.join(page.get("path"), page.get("alias")) + ".html"
        print(target_file)
        if not os.path.exists(page.get("path")):
            os.makedirs(page.get("path"))
        f = open(target_file, 'w')
        f.write(self.env.get_template(page.get("template")).render(page))
        f.close()

    def _render_pagination(self):
        pagination = self.config.get("paginate")
        pagination_folder = os.path.join(self._get_path(self.config.get("destination")), "page")
        print(pagination_folder)
        if len(self.posts) > pagination:
            os.makedirs(pagination_folder)
        for index, posts in enumerate(chunks(self.posts, pagination)):
            if index == 0:
                f = open(os.path.join(self._get_path(self.config.get("destination")), "index.html"), 'w')
            else:
                f = open(os.path.join(pagination_folder, "{}.html".format(str(index))), 'w')
            f.write(self.env.get_template("index.html").render({"pagination_posts": posts}))
            f.close()

    def _render(self):
        for page in self.pages:
            self._render_html_file(page)
        for post in self.posts:
            self._render_html_file(post)
        self._render_pagination()

    def generate_project(self, project_name):
        if os.path.exists(project_name):
            shutil.rmtree(project_name)
        shutil.copytree(SEED_PATH, project_name)

    def build(self):
        start = time.clock()
        # shutil.rmtree(self._get_path(self.config.get("destination")))
        self._copy_static_files()
        self._walk_files()
        self._render()

        end = time.clock()
        print("run time: {time}s".format(time=end-start))

    def server(self, address="0.0.0.0", port=8000, server_class=HTTPServer, handler_class=BaseHTTPRequestHandler):
        self.build()
        print("start server")
        server_address = (address, port)
        httpd = server_class(server_address, handler_class)
        print("Serving at http://{address}:{port}".format(address=address, port=port))
        httpd.serve_forever()

    def watch(self):
        self.build()
        event_handler = FileChangeEventHandler(self)
        observer = Observer()
        observer.schedule(event_handler, self.config.get("root"), recursive=False)
        # for item in os.listdir("."):
        #     if os.path.isdir(item) and item != "site":
        #         observer.schedule(event_handler, item, recursive=False)
        observer.start()
        try:
            while True:
                time.sleep(1)
        except KeyboardInterrupt:
            observer.stop()

    def deploy(self):
        self.build()
        self._git_push()

    def _git_clone(url):
        sh.git.clone(url)
        sh.git.checkout("master")

    def _checkout_or_create(branch="master"):
        try:
            sh.git.checkout(branch)
        except sh.ErrorReturnCode_1 as e:
            sh.git.checkout("-b", branch)

    def _git_branch():
        os.chdir('windfarer.github.io')
        # print(sh.git.checkout("-b", "gh-pages"))
        try:
            sh.git.checkout("gh-pages")
        except sh.ErrorReturnCode_1 as e:
            sh.git.checkout("-b", "gh-pages")

    def _git_push():
        sh.git.push("origin", "gh-pages")

class FileChangeEventHandler(FileSystemEventHandler):
    def __init__(self, episode):
        self.episode = episode

    # def on_moved(self, event):
    #     self.episode.build()
    #     print("==move====src"+event.src_path)

    def on_created(self, event):
        self.episode.build()
        print("==create====src"+event.src_path)

    def on_deleted(self, event):
        self.episode.build()
        print("==delete====src"+event.src_path)

    def on_modified(self, event):
        self.episode.build()
        print("===modified===src"+event.src_path)

def start_server():
    print("start server")
    Episode().server()

def start_watch():
    print("start watch")
    Episode().watch()

def start_build():
    print("start build")
    Episode().build()

def start_new(project_name):
    print("create a new project")
    Episode().generate_project(project_name)

def start_deploy():
    print("deploy to github")
    Episode().deploy()

def command_options(arguments):
    if arguments["new"]:
        start_new(arguments["<project_name>"])
    elif arguments["build"]:
        start_build()
    elif arguments["server"]:
        start_server()
    elif arguments["watch"]:
        start_watch()
    elif arguments["deploy"]:
        start_deploy()

if __name__ == '__main__':
    arguments = docopt(__doc__, version='0.0.1')
    command_options(arguments)<|endoftext|>


<commit_before># Copyright (C) 2012
# See LICENSE.txt for details.

"""
Epytext (and general Python docstring) wrapper
==============================================

Utility for wrapping docstrings in Python; specifically, docstrings in U{Epytext
<http://epydoc.sourceforge.net/manual-epytext.html>} format, or those that are
close enough.

The wrapping herein generally aheres to all the conventions set forth by the
Twisted project U{http://twistedmatrix.com/}.

Currently (obviously) the only supported editor is U{Sublime Text 2
<http://www.sublimetext.com/>} but a sufficiently enterprising individual could
either use this file as a script (no dependencies!) by piping the contents of
the docstring to it, or call L{wrapPythonDocstring} and preserve point position.
"""

from __future__ import unicode_literals

import re
from uuid import uuid4

__all__ = [
    "wrapPythonDocstring"
]

def isUnderline(expr):
    return bool(re.match("[=]+$", expr) or re.match("[-]+$", expr))

class RegularParagraph(object):
    otherIndent = ""

    def __init__(self, pointTracker, fixedIndent="", hangIndent="",
                 followIndent=""):
        self.words = []
        self.fixedIndent = fixedIndent
        self.hangIndent = hangIndent
        self.followIndent = followIndent
        self.more = None
        self.pointTracker = pointTracker
        self._unwrappedLines = 0
        self._headingType = None
        self._headingPoints = []

    def matchesTag(self, other):
        return False

    def __nonzero__(self):
        return bool(self.words)

    def all(self):
        while self is not None:
            #print self.__class__.__name__
            if self:
                yield self
            self = self.more

    def setIsHeading(self, headingType):
        self._headingType = headingType

    def isHeading(self):
        return bool(self._headingType)

    def add(self, line):
        clean = self.pointTracker.peek(line)
        stripped = clean.strip()

        if stripped:
            self._unwrappedLines += 1
            active = self
            firstword = list(self.pointTracker.filterWords(line.split()))[0]
            if stripped.startswith("@"):
                fp = FieldParagraph(pointTracker=self.pointTracker)
                fp.words.extend(line.split())
                active = self.more = fp
            elif isUnderline(stripped) and self._unwrappedLines == 2:
                # This paragraph is actually a section heading.
                active.setIsHeading(stripped[0])
                self._headingPoints = self.pointTracker.extractPoints(line)
                # FIXME: should respect leading indentation.
                active = self.more = self.nextRegular()
            elif (firstword == '-' or
                  (firstword.endswith(".") and firstword[:-1].isdigit())):
                # Aesthetically I prefer a 2-space indent here, but the
                # convention in the codebase seems to be 4 spaces.
                hangIndent = self.pointTracker.lengthOf(firstword) + 1
                fp = RegularParagraph(
                    pointTracker=self.pointTracker, fixedIndent="    ",
                    hangIndent=" " * hangIndent, followIndent=self.followIndent
                )
                fp.words.extend(line.split())
                active = self.more = fp
            else:
                self.words.extend(line.split())
            if stripped.endswith("::"):
                active.more = PreFormattedParagraph(
                    pointTracker=self.pointTracker,
                    fixedIndent=(active.fixedIndent + active.hangIndent +
                                 active.otherIndent),
                    indentBegins=len(clean) - len(clean.lstrip())
                )
                active = active.more
            return active
        else:
            rawstrip = line.strip()
            if rawstrip:
                self.words.append(rawstrip)
            if len(list(self.pointTracker.filterWords(self.words))):
                return self.nextRegular()
        return self

    def wrap(self, output, indentation, width):
        if not self.words:
            return
        thisLine = self.firstIndent(indentation)
        first = True
        prevWord = ''
        for word in self.words:
            if not self.pointTracker.isWord(word):
                thisLine += word
                continue
            if ((prevWord.endswith(".") or prevWord.endswith("?") or
                 prevWord.endswith("!")) and not prevWord[:-1].isdigit()):
                words = prevWord.split(".")[:-1]
                if ( len(words) > 1 and
                     [self.pointTracker.lengthOf(x) for x in words] ==
                     [1] * len(words) ):
                    # acronym
                    spaces = 1
                else:
                    spaces = 2
            else:
                spaces = 1
            prevWord = word
            if ( self.pointTracker.lengthOf(thisLine) +
                 self.pointTracker.lengthOf(word) + spaces <= width ):
                if first:
                    first = not first
                else:
                    thisLine += (" " * spaces)
                thisLine += word
            else:
                output.write(self.pointTracker.scan(thisLine, output.tell()))
                output.write("\n")
                thisLine = self.restIndent(indentation) + word
        output.write(self.pointTracker.scan(thisLine, output.tell()))
        output.write("\n")
        if self.isHeading():
            indentText = self.firstIndent(indentation)
            lineSize = self.pointTracker.lengthOf(thisLine) - len(indentText)
            output.write(self.pointTracker.scan(
                indentText + ''.join(self._headingPoints) +
                (self._headingType * lineSize), output.tell()
            ))
            output.write("\n")

    def firstIndent(self, indentation):
        return indentation + self.fixedIndent

    def restIndent(self, indentation):
        return (indentation + self.fixedIndent + self.hangIndent +
                self.otherIndent)

    def nextRegular(self):
        self.more = RegularParagraph(pointTracker=self.pointTracker,
                                     fixedIndent=self.nextIndent(),
                                     followIndent=self.nextIndent())
        return self.more

    def nextIndent(self):
        return self.followIndent

class FieldParagraph(RegularParagraph):

    otherIndent = "    "

    def nextIndent(self):
        return "    "

    def matchesTag(self, other):
        if isinstance(other, FieldParagraph):
            myWords = list(self.pointTracker.filterWords(self.words))
            theirWords = list(self.pointTracker.filterWords(other.words))
            if ( set([myWords[0], theirWords[0]]) ==
                 set(["@return:", "@rtype:"]) ):
                 # matching @return and @rtype fields.
                return True
            elif len(myWords) > 1 and len(theirWords) > 1:
                # matching @param and @type fields.
                return myWords[1] == theirWords[1]
            return False
        else:
            return False

class PreFormattedParagraph(object):

    def __init__(self, pointTracker, fixedIndent, indentBegins):
        self.lines = []
        self.indentBegins = indentBegins
        self.fixedIndent = fixedIndent
        self.more = None
        self.pointTracker = pointTracker

    def matchesTag(self, other):
        return False

    def add(self, line):
        actualLine = self.pointTracker.peek(line)

        if actualLine.strip():
            if len(actualLine) - len(actualLine.lstrip()) <= self.indentBegins:
                next = self.more = RegularParagraph(
                    pointTracker=self.pointTracker,
                    fixedIndent=self.fixedIndent
                )
                return next.add(line)
            self.lines.append(line.rstrip())
        else:
            self.lines.append(line.strip())
        return self

    def fixIndentation(self):
        while self.lines and not self.lines[0].strip():
            self.lines.pop(0)
        while self.lines and not self.lines[-1].strip():
            self.lines.pop()
        if not self.lines:
            return
        cleanLines = map(self.pointTracker.peek, self.lines)
        commonLeadingIndent = min([len(x) - len(x.lstrip()) for x in cleanLines
                                   if x.strip()])
        newLines = []
        for actualLine, line in zip(cleanLines, self.lines):
            if actualLine != line and line[:commonLeadingIndent].strip():
                # There's a marker, and it's in the leading whitespace.
                # Explicitly reposition the marker at the beginning of the fixed
                # indentation.
                line = (self.pointTracker.marker +
                        actualLine[commonLeadingIndent:])
            else:
                line = line.rstrip()[commonLeadingIndent:]
            newLines.append(line)
        self.lines = newLines

    def wrap(self, output, indentation, width):
        # OK, now we know about all the lines we're going to know about.
        self.fixIndentation()
        for line in self.lines:
            if self.pointTracker.peek(line):
                output.write(indentation + "    " + self.fixedIndent)
            output.write(self.pointTracker.scan(line, output.tell()))
            output.write("\n")

class PointTracker(object):
    """
    Object for keeping track of where the insertion points are.
    """

    def __init__(self, point):
        self.point = point
        self.marker = "{" + unicode(uuid4()) + "}"
        self.outPoints = []

    def annotate(self, text):
        """
        Add point references to a block of text.
        """
        return text[:self.point] + self.marker + text[self.point:]

    def filterWords(self, words):
        for word in words:
            if self.isWord(word):
                yield self.peek(word)

    def isWord(self, text):
        """
        Is the given word actually a word, or just an artifact of the
        point-tracking process?  If it's just the point marker by itself, then
        no, it isn't, and don't insert additional whitespace after it.
        """
        return not (text == self.marker)

    def lengthOf(self, word):
        """
        How long would this word be if it didn't have any point-markers in it?
        """
        return len(self.peek(word))

    def peek(self, word):
        """
        What would this word look like if it didn't have any point-markers in
        it?
        """
        return word.replace(self.marker, "")

    def extractPoints(self, text):
        """
        Return a C{list} of all point markers contained in the text.
        """
        if self.marker in text:
            return [self.marker]
        return []

    def scan(self, text, offset):
        """
        Scan some text for point markers, remember them, and remove them.
        """
        idx = text.find(self.marker)
        if idx == -1:
            return text
        self.outPoints.append(idx + offset)
        return self.peek(text)

def wrapPythonDocstring(docstring, output, indentation="    ",
                        width=79, point=0):
    """
    Wrap a given Python docstring.

    @param docstring: the docstring itself (just the stuff between the quotes).
    @type docstring: unicode

    @param output: The unicode output file to write the wrapped docstring to.
    @type output: L{file}-like (C{write} takes unicode.)

    @param indentation: a string (consisting only of spaces) indicating the
        amount of space to shift by.  Don't adjust this.  It's always 4 spaces.
        PEP8 says so.
    @type indentation: L{unicode}

    @param width: The maximum number of characters allowed in a wrapped line.
    @type width: L{int}

    @param point: The location of the cursor in the text, as an offset from the
        beginning of the docstring.  If this function is being used from within
        a graphical editor, this parameter can be used (in addition to the
        return value of this function) to reposition the cursor at the relative
        position which the user will expect.

    @return: The new location of the cursor.
    """
    # TODO: multiple points; usable, for example, for start and end of a
    # currently active selection.
    pt = PointTracker(point)
    start = paragraph = RegularParagraph(pt)
    docstring = pt.annotate(docstring)
    for line in docstring.split("\n"):
        paragraph = paragraph.add(line)
    prevp = None
    for paragraph in start.all():
        if not paragraph.matchesTag(prevp):
            output.write("\n")
        prevp = paragraph
        paragraph.wrap(output, indentation, width)
    output.write(indentation)
    return pt.outPoints[0]

if __name__ == '__main__':
    import sys
    wrapPythonDocstring(sys.stdin.read(), sys.stdout)

<commit_msg>Fix wrapping the paragraph following the paragraph following a pre-formatted paragraph while inside an epytext field (@param, @var, etc).<commit_after># Copyright (C) 2012
# See LICENSE.txt for details.

"""
Epytext (and general Python docstring) wrapper
==============================================

Utility for wrapping docstrings in Python; specifically, docstrings in U{Epytext
<http://epydoc.sourceforge.net/manual-epytext.html>} format, or those that are
close enough.

The wrapping herein generally aheres to all the conventions set forth by the
Twisted project U{http://twistedmatrix.com/}.

Currently (obviously) the only supported editor is U{Sublime Text 2
<http://www.sublimetext.com/>} but a sufficiently enterprising individual could
either use this file as a script (no dependencies!) by piping the contents of
the docstring to it, or call L{wrapPythonDocstring} and preserve point position.
"""

from __future__ import unicode_literals

import re
from uuid import uuid4

__all__ = [
    "wrapPythonDocstring"
]

def isUnderline(expr):
    return bool(re.match("[=]+$", expr) or re.match("[-]+$", expr))

class RegularParagraph(object):
    otherIndent = ""

    def __init__(self, pointTracker, fixedIndent="", hangIndent="",
                 followIndent=""):
        self.words = []
        self.fixedIndent = fixedIndent
        self.hangIndent = hangIndent
        self.followIndent = followIndent
        self.more = None
        self.pointTracker = pointTracker
        self._unwrappedLines = 0
        self._headingType = None
        self._headingPoints = []

    def matchesTag(self, other):
        return False

    def __nonzero__(self):
        return bool(self.words)

    def all(self):
        while self is not None:
            #print self.__class__.__name__
            if self:
                yield self
            self = self.more

    def setIsHeading(self, headingType):
        self._headingType = headingType

    def isHeading(self):
        return bool(self._headingType)

    def add(self, line):
        clean = self.pointTracker.peek(line)
        stripped = clean.strip()

        if stripped:
            self._unwrappedLines += 1
            active = self
            firstword = list(self.pointTracker.filterWords(line.split()))[0]
            if stripped.startswith("@"):
                fp = FieldParagraph(pointTracker=self.pointTracker)
                fp.words.extend(line.split())
                active = self.more = fp
            elif isUnderline(stripped) and self._unwrappedLines == 2:
                # This paragraph is actually a section heading.
                active.setIsHeading(stripped[0])
                self._headingPoints = self.pointTracker.extractPoints(line)
                # FIXME: should respect leading indentation.
                active = self.nextRegular()
            elif (firstword == '-' or
                  (firstword.endswith(".") and firstword[:-1].isdigit())):
                # Aesthetically I prefer a 2-space indent here, but the
                # convention in the codebase seems to be 4 spaces.
                hangIndent = self.pointTracker.lengthOf(firstword) + 1
                fp = RegularParagraph(
                    pointTracker=self.pointTracker,
                    fixedIndent=active.fixedIndent,
                    hangIndent=" " * hangIndent, followIndent=self.followIndent
                )
                fp.words.extend(line.split())
                active = self.more = fp
            else:
                self.words.extend(line.split())
            if stripped.endswith("::"):
                active.more = PreFormattedParagraph(
                    active,
                    indentBegins=len(clean) - len(clean.lstrip())
                )
                active = active.more
            return active
        else:
            rawstrip = line.strip()
            if rawstrip:
                self.words.append(rawstrip)
            if len(list(self.pointTracker.filterWords(self.words))):
                return self.nextRegular()
        return self

    def wrap(self, output, indentation, width):
        if not self.words:
            return
        thisLine = self.firstIndent(indentation)
        first = True
        prevWord = ''
        for word in self.words:
            if not self.pointTracker.isWord(word):
                thisLine += word
                continue
            if ((prevWord.endswith(".") or prevWord.endswith("?") or
                 prevWord.endswith("!")) and not prevWord[:-1].isdigit()):
                words = prevWord.split(".")[:-1]
                if ( len(words) > 1 and
                     [self.pointTracker.lengthOf(x) for x in words] ==
                     [1] * len(words) ):
                    # acronym
                    spaces = 1
                else:
                    spaces = 2
            else:
                spaces = 1
            prevWord = word
            if ( self.pointTracker.lengthOf(thisLine) +
                 self.pointTracker.lengthOf(word) + spaces <= width ):
                if first:
                    first = not first
                else:
                    thisLine += (" " * spaces)
                thisLine += word
            else:
                output.write(self.pointTracker.scan(thisLine, output.tell()))
                output.write("\n")
                thisLine = self.restIndent(indentation) + word
        output.write(self.pointTracker.scan(thisLine, output.tell()))
        output.write("\n")
        if self.isHeading():
            indentText = self.firstIndent(indentation)
            lineSize = self.pointTracker.lengthOf(thisLine) - len(indentText)
            output.write(self.pointTracker.scan(
                indentText + ''.join(self._headingPoints) +
                (self._headingType * lineSize), output.tell()
            ))
            output.write("\n")

    def firstIndent(self, indentation):
        return indentation + self.fixedIndent

    def restIndent(self, indentation):
        return (indentation + self.fixedIndent + self.hangIndent +
                self.otherIndent)

    def genRegular(self):
        return RegularParagraph(pointTracker=self.pointTracker,
                                fixedIndent=self.nextIndent(),
                                followIndent=self.nextIndent())

    def nextRegular(self):
        self.more = self.genRegular()
        return self.more

    def nextIndent(self):
        return self.followIndent

class FieldParagraph(RegularParagraph):

    otherIndent = "    "

    def nextIndent(self):
        return "    "

    def matchesTag(self, other):
        if isinstance(other, FieldParagraph):
            myWords = list(self.pointTracker.filterWords(self.words))
            theirWords = list(self.pointTracker.filterWords(other.words))
            if ( set([myWords[0], theirWords[0]]) ==
                 set(["@return:", "@rtype:"]) ):
                 # matching @return and @rtype fields.
                return True
            elif len(myWords) > 1 and len(theirWords) > 1:
                # matching @param and @type fields.
                return myWords[1] == theirWords[1]
            return False
        else:
            return False

class PreFormattedParagraph(object):

    def __init__(self, before, indentBegins):
        self.lines = []
        self.before = before

        pointTracker = before.pointTracker

        fixedIndent = (before.fixedIndent + before.hangIndent +
                       before.otherIndent)

        self.indentBegins = indentBegins
        self.fixedIndent = fixedIndent
        self.more = None
        self.pointTracker = pointTracker

    def matchesTag(self, other):
        return False

    def add(self, line):
        actualLine = self.pointTracker.peek(line)

        if actualLine.strip():
            if len(actualLine) - len(actualLine.lstrip()) <= self.indentBegins:
                next = self.more = self.before.genRegular()
                return next.add(line)
            self.lines.append(line.rstrip())
        else:
            self.lines.append(line.strip())
        return self

    def fixIndentation(self):
        while self.lines and not self.lines[0].strip():
            self.lines.pop(0)
        while self.lines and not self.lines[-1].strip():
            self.lines.pop()
        if not self.lines:
            return
        cleanLines = map(self.pointTracker.peek, self.lines)
        commonLeadingIndent = min([len(x) - len(x.lstrip()) for x in cleanLines
                                   if x.strip()])
        newLines = []
        for actualLine, line in zip(cleanLines, self.lines):
            if actualLine != line and line[:commonLeadingIndent].strip():
                # There's a marker, and it's in the leading whitespace.
                # Explicitly reposition the marker at the beginning of the fixed
                # indentation.
                line = (self.pointTracker.marker +
                        actualLine[commonLeadingIndent:])
            else:
                line = line.rstrip()[commonLeadingIndent:]
            newLines.append(line)
        self.lines = newLines

    def wrap(self, output, indentation, width):
        # OK, now we know about all the lines we're going to know about.
        self.fixIndentation()
        for line in self.lines:
            if self.pointTracker.peek(line):
                output.write(indentation + "    " + self.fixedIndent)
            output.write(self.pointTracker.scan(line, output.tell()))
            output.write("\n")

class PointTracker(object):
    """
    Object for keeping track of where the insertion points are.
    """

    def __init__(self, point):
        self.point = point
        self.marker = "{" + unicode(uuid4()) + "}"
        self.outPoints = []

    def annotate(self, text):
        """
        Add point references to a block of text.
        """
        return text[:self.point] + self.marker + text[self.point:]

    def filterWords(self, words):
        for word in words:
            if self.isWord(word):
                yield self.peek(word)

    def isWord(self, text):
        """
        Is the given word actually a word, or just an artifact of the
        point-tracking process?  If it's just the point marker by itself, then
        no, it isn't, and don't insert additional whitespace after it.
        """
        return not (text == self.marker)

    def lengthOf(self, word):
        """
        How long would this word be if it didn't have any point-markers in it?
        """
        return len(self.peek(word))

    def peek(self, word):
        """
        What would this word look like if it didn't have any point-markers in
        it?
        """
        return word.replace(self.marker, "")

    def extractPoints(self, text):
        """
        Return a C{list} of all point markers contained in the text.
        """
        if self.marker in text:
            return [self.marker]
        return []

    def scan(self, text, offset):
        """
        Scan some text for point markers, remember them, and remove them.
        """
        idx = text.find(self.marker)
        if idx == -1:
            return text
        self.outPoints.append(idx + offset)
        return self.peek(text)

def wrapPythonDocstring(docstring, output, indentation="    ",
                        width=79, point=0):
    """
    Wrap a given Python docstring.

    @param docstring: the docstring itself (just the stuff between the quotes).
    @type docstring: unicode

    @param output: The unicode output file to write the wrapped docstring to.
    @type output: L{file}-like (C{write} takes unicode.)

    @param indentation: a string (consisting only of spaces) indicating the
        amount of space to shift by.  Don't adjust this.  It's always 4 spaces.
        PEP8 says so.
    @type indentation: L{unicode}

    @param width: The maximum number of characters allowed in a wrapped line.
    @type width: L{int}

    @param point: The location of the cursor in the text, as an offset from the
        beginning of the docstring.  If this function is being used from within
        a graphical editor, this parameter can be used (in addition to the
        return value of this function) to reposition the cursor at the relative
        position which the user will expect.

    @return: The new location of the cursor.
    """
    # TODO: multiple points; usable, for example, for start and end of a
    # currently active selection.
    pt = PointTracker(point)
    start = paragraph = RegularParagraph(pt)
    docstring = pt.annotate(docstring)
    for line in docstring.split("\n"):
        paragraph = paragraph.add(line)
    prevp = None
    for paragraph in start.all():
        if not paragraph.matchesTag(prevp):
            output.write("\n")
        prevp = paragraph
        paragraph.wrap(output, indentation, width)
    output.write(indentation)
    return pt.outPoints[0]

if __name__ == '__main__':
    import sys
    from cStringIO import StringIO
    io = StringIO()
    indata = sys.stdin.read()
    firstline = [line for line in indata.split("\n") if line][0]
    wrapPythonDocstring(indata, io,
                        indentation=" " * (len(firstline) - len(firstline.lstrip())))
    sys.stdout.write(io.getvalue())
    sys.stdout.flush()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Elasticsearch related funcionality."""
import hashlib
import logging
import os
import time

from urlparse import urlparse

import elasticsearch.helpers

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import RequestError

from sqlalchemy.types import (
    BIGINT,
    BOOLEAN,
    CHAR,
    CLOB,
    DATE,
    DATETIME,
    DECIMAL,
    FLOAT,
    INTEGER,
    NCHAR,
    NUMERIC,
    NVARCHAR,
    NullType,
    REAL,
    SMALLINT,
    TEXT,
    TIME,
    TIMESTAMP,
    VARCHAR,
)

from esis.fs import TreeExplorer
from esis.db import (
    DBReader,
    Database,
    TableReader,
)

logger = logging.getLogger(__name__)

class Client(object):

    """Elasticsearch client wrapper."""

    def __init__(self):
        """Create low level client."""
        self.es_client = Elasticsearch()

    def index(self, directory):
        """Index all the information available in a directory.

        In elasticsearch there will be an index for each database and a
        document type for each table in the database.

        :param directory: Directory that should be indexed
        :type directory: str

        """
        logger.debug('Indexing %r...', directory)
        start = time.time()
        documents_indexed = self._index_directory(directory)
        end = time.time()
        logger.info('%d documents indexed in %.2f seconds',
                    documents_indexed, end-start)

    def _index_directory(self, directory):
        """Index all databases under a given directory.

        :param directory: Path to the directory to explore
        :type directory: str
        :return: Documents indexed for this directory
        :rtype: int

        """
        documents_indexed = 0

        tree_explorer = TreeExplorer(directory)
        for db_path in tree_explorer.paths():
            index_name = get_index_name(db_path)
            self._recreate_index(index_name)
            with Database(db_path) as database:
                documents_indexed = self._index_database(index_name, database)

        return documents_indexed

    def _recreate_index(self, index_name):
        """Recreate elasticsearch index.

        It's checked that the index exists before trying to delete it to avoid
        failures.

        :param index_name: Elasticsearch index to delete
        :type index_name: str

        """
        logger.debug('Recreating index (%s)...', index_name)
        if self.es_client.indices.exists(index_name):
            self.es_client.indices.delete(index_name)
        self.es_client.indices.create(index_name)

    def _index_database(self, index_name, database):
        """Index all tables in a database file.

        :param index_name: Elasticsearch index name
        :type index_name: str
        :param database: Database to be indexed
        :type database: :class:`esis.db.Database`
        :return: Documents indexed for this database
        :rtype: int

        """
        # Recreate index for the given database
        documents_indexed = 0

        logger.debug('Populating index (%s)...', index_name)
        db_reader = DBReader(database)

        # Index the content of every database table
        for table_name in db_reader.tables():
            table_reader = TableReader(database, table_name)
            documents_indexed += self._index_table(
                index_name, table_name, table_reader)

        return documents_indexed

    def _index_table(self, index_name, table_name, table_reader):
        """Index all rows in a database table.

        :param index_name: Elasticsearch index to use when indexing documents
        :type index_name: str
        :table_name: Table name for which rows are indexed in elasticsearch
        :type table: str
        :param table_reader: Object to iterate through all rows in a table
        :type table_reader: :class:`esis.db.TableReader`
        :return: Documents indexed for this table
        :rtype: int

        """
        documents_indexed = 0

        # TODO: Transform table name to avoid names not allowed in elasticsearch
        document_type = table_name

        # Translate database schema into an elasticsearch mapping
        table_schema = table_reader.get_schema()
        table_mapping = Mapping(table_name, table_schema)
        try:
            self.es_client.indices.put_mapping(
                index=index_name,
                doc_type=document_type,
                body=table_mapping.mapping)
        except RequestError as exception:
            logger.error(exception)

        actions = (
            get_index_action(index_name, document_type, row)
            for row in table_reader.rows()
        )
        documents_indexed, errors = elasticsearch.helpers.bulk(
            self.es_client, actions)

        if errors:
            logger.warning('Indexing errors reported: %s', errors)

        return documents_indexed

    def search(self, query):
        """Yield all documents that match a given query.

        :param query: A simple query with data to search in elasticsearch
        :type query: str
        :return: Records that matched the query as returned by elasticsearch
        :rtype: list(dict)

        """
        body = {
            'query': {
                'match': {
                    '_all': query,
                },
            },
        }

        response = self.es_client.search(
            body=body,
            scroll='5m',
            size=100,
        )

        hits_info = response['hits']
        hits_total = hits_info['total']
        logger.info('%d documents matched', hits_total)
        hits = hits_info['hits']
        yield hits

        if '_scroll_id' in response:
            scroll_id = response['_scroll_id']

            while True:
                response = self.es_client.scroll(
                    scroll_id=scroll_id,
                    scroll='5m',
                )
                hits = response['hits']['hits']
                if not hits:
                    break

                yield hits

class Mapping(object):

    """ElasticSearch mapping.

    :param table_name: Database table name
    :type table_name: str
    :param table_schema: Database table schema from sqlalchemy
    :type table_schema: dict(str, sqlalchemy.types.*)

    """

    # Mapping from sqlalchemy types to elasticsearch ones
    # Note: The columns that have a type that maps to None, will be removed
    # from the final mapping to let elastic search figure out the type by
    # itself. This is because SQLite works with storage classes and type
    # affinities and it's not always clear what datatype data will really have.
    # In particular, values with NUMERIC affinity might be stored using any of
    # the five available storage classes, so it's not possible to predict for
    # all cases what type of data will be stored without looking at it as
    # elasticsearch does.
    SQL_TYPE_MAPPING = {
        BIGINT: 'long',
        BOOLEAN: 'boolean',
        CHAR: 'string',
        CLOB: 'string',
        DATE: None,
        DATETIME: 'date',
        DECIMAL: None,
        FLOAT: 'float',
        # TODO: Use 'integer' when data is in range
        INTEGER: 'long',
        NCHAR: 'string',
        NVARCHAR: 'string',
        NullType: None,
        NUMERIC: None,
        # TODO: Use 'float' when data is in range
        REAL: 'double',
        SMALLINT: 'integer',
        TEXT: 'string',
        TIME: 'date',  # TODO: Map to something time specific?
        TIMESTAMP: 'date',
        VARCHAR: 'string',
    }

    def __init__(self, table_name, table_schema):
        """Map every column type to an elasticsearch mapping."""
        columns_mapping = {}

        for column_name, column_sql_type in table_schema.iteritems():
            column_mapping = self._get_column_mapping(column_sql_type)

            # Skip columns that don't have an mapping defined and let
            # elasticsearch figure out the mapping itself
            if column_mapping is None:
                continue

            columns_mapping[column_name] = column_mapping

        self.mapping = {
            table_name: {
                'properties': columns_mapping,
            }
        }

    def _get_column_mapping(self, column_sql_type):
        """Return column mapping based on its name and type.

        :param column_sql_type: Database column type
        :type column_sql_type: sqlalchemy.types.*
        :return: Mapping for the given column name and type (if possible)
        :rtype: dict(str) | None

        """
        column_es_type = self.SQL_TYPE_MAPPING[type(column_sql_type)]
        if column_es_type is None:
            return None

        column_mapping = {'type': column_es_type}
        return column_mapping

def get_index_name(path):
    """Get index name for a database file.

    :param path: Path to file to be indexed
    :type path: str
    :return: Index name for the DB file
    :rtype: str

    """
    # There's a limit in elasticsearch index name length and there are some
    # characters that are not allowed. Using an md5 hash seems to be a
    # reasonable way to get to a unique index name for each database file.
    index_name = hashlib.md5(path).hexdigest()
    return index_name

def get_index_action(index_name, document_type, row):
    """Generate index action for a given database row.

    :param index_name: Elasticsearch index to use
    :type index_name: str
    :param document_type: Elasticsearch document type to use
    :type index_name: str
    :param row: The row to be indexed in elasticsearch
    :type row: sqlalchemy.engine.result.RowProxy
    :return: Action to be passed in bulk request
    :rtype: dict

    """
    source = dict(row)

    # Avoid indexing binary data
    for field_name, field_data in source.items():
        # Avoid indexing binary data
        if isinstance(field_data, buffer):
            logger.debug('%r field discarded before indexing', field_name)
            del source[field_name]

        # Avoid indexing local paths
        elif isinstance(field_data, basestring):
            url = urlparse(field_data)
            if (url.scheme == 'file'
                    and os.path.exists(url.path)):
                logger.debug(
                    '%r field discarded before indexing', field_name)
                del source[field_name]

    action = {
        '_index': index_name,
        '_type': document_type,
        '_source': source,
    }

    # Use the same _id field in elasticsearch as in the database table
    if '_id' in source:
        action['_id'] = source['_id']

    return action

<commit_msg>Add logging to search<commit_after># -*- coding: utf-8 -*-
"""Elasticsearch related funcionality."""
import hashlib
import logging
import os
import time

from urlparse import urlparse

import elasticsearch.helpers

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import RequestError

from sqlalchemy.types import (
    BIGINT,
    BOOLEAN,
    CHAR,
    CLOB,
    DATE,
    DATETIME,
    DECIMAL,
    FLOAT,
    INTEGER,
    NCHAR,
    NUMERIC,
    NVARCHAR,
    NullType,
    REAL,
    SMALLINT,
    TEXT,
    TIME,
    TIMESTAMP,
    VARCHAR,
)

from esis.fs import TreeExplorer
from esis.db import (
    DBReader,
    Database,
    TableReader,
)

logger = logging.getLogger(__name__)

class Client(object):

    """Elasticsearch client wrapper."""

    def __init__(self):
        """Create low level client."""
        self.es_client = Elasticsearch()

    def index(self, directory):
        """Index all the information available in a directory.

        In elasticsearch there will be an index for each database and a
        document type for each table in the database.

        :param directory: Directory that should be indexed
        :type directory: str

        """
        logger.debug('Indexing %r...', directory)
        start = time.time()
        documents_indexed = self._index_directory(directory)
        end = time.time()
        logger.info('%d documents indexed in %.2f seconds',
                    documents_indexed, end-start)

    def _index_directory(self, directory):
        """Index all databases under a given directory.

        :param directory: Path to the directory to explore
        :type directory: str
        :return: Documents indexed for this directory
        :rtype: int

        """
        documents_indexed = 0

        tree_explorer = TreeExplorer(directory)
        for db_path in tree_explorer.paths():
            index_name = get_index_name(db_path)
            self._recreate_index(index_name)
            with Database(db_path) as database:
                documents_indexed = self._index_database(index_name, database)

        return documents_indexed

    def _recreate_index(self, index_name):
        """Recreate elasticsearch index.

        It's checked that the index exists before trying to delete it to avoid
        failures.

        :param index_name: Elasticsearch index to delete
        :type index_name: str

        """
        logger.debug('Recreating index (%s)...', index_name)
        if self.es_client.indices.exists(index_name):
            self.es_client.indices.delete(index_name)
        self.es_client.indices.create(index_name)

    def _index_database(self, index_name, database):
        """Index all tables in a database file.

        :param index_name: Elasticsearch index name
        :type index_name: str
        :param database: Database to be indexed
        :type database: :class:`esis.db.Database`
        :return: Documents indexed for this database
        :rtype: int

        """
        # Recreate index for the given database
        documents_indexed = 0

        logger.debug('Populating index (%s)...', index_name)
        db_reader = DBReader(database)

        # Index the content of every database table
        for table_name in db_reader.tables():
            table_reader = TableReader(database, table_name)
            documents_indexed += self._index_table(
                index_name, table_name, table_reader)

        return documents_indexed

    def _index_table(self, index_name, table_name, table_reader):
        """Index all rows in a database table.

        :param index_name: Elasticsearch index to use when indexing documents
        :type index_name: str
        :table_name: Table name for which rows are indexed in elasticsearch
        :type table: str
        :param table_reader: Object to iterate through all rows in a table
        :type table_reader: :class:`esis.db.TableReader`
        :return: Documents indexed for this table
        :rtype: int

        """
        documents_indexed = 0

        # TODO: Transform table name to avoid names not allowed in elasticsearch
        document_type = table_name

        # Translate database schema into an elasticsearch mapping
        table_schema = table_reader.get_schema()
        table_mapping = Mapping(table_name, table_schema)
        try:
            self.es_client.indices.put_mapping(
                index=index_name,
                doc_type=document_type,
                body=table_mapping.mapping)
        except RequestError as exception:
            logger.error(exception)

        actions = (
            get_index_action(index_name, document_type, row)
            for row in table_reader.rows()
        )
        documents_indexed, errors = elasticsearch.helpers.bulk(
            self.es_client, actions)

        if errors:
            logger.warning('Indexing errors reported: %s', errors)

        return documents_indexed

    def search(self, query):
        """Yield all documents that match a given query.

        :param query: A simple query with data to search in elasticsearch
        :type query: str
        :return: Records that matched the query as returned by elasticsearch
        :rtype: list(dict)

        """
        logger.debug('Searching %r...', query)
        body = {
            'query': {
                'match': {
                    '_all': query,
                },
            },
        }

        response = self.es_client.search(
            body=body,
            scroll='5m',
            size=100,
        )

        hits_info = response['hits']
        hits_total = hits_info['total']
        logger.info('%d documents matched', hits_total)
        hits = hits_info['hits']
        yield hits

        if '_scroll_id' in response:
            scroll_id = response['_scroll_id']

            while True:
                response = self.es_client.scroll(
                    scroll_id=scroll_id,
                    scroll='5m',
                )
                hits = response['hits']['hits']
                if not hits:
                    break

                yield hits

class Mapping(object):

    """ElasticSearch mapping.

    :param table_name: Database table name
    :type table_name: str
    :param table_schema: Database table schema from sqlalchemy
    :type table_schema: dict(str, sqlalchemy.types.*)

    """

    # Mapping from sqlalchemy types to elasticsearch ones
    # Note: The columns that have a type that maps to None, will be removed
    # from the final mapping to let elastic search figure out the type by
    # itself. This is because SQLite works with storage classes and type
    # affinities and it's not always clear what datatype data will really have.
    # In particular, values with NUMERIC affinity might be stored using any of
    # the five available storage classes, so it's not possible to predict for
    # all cases what type of data will be stored without looking at it as
    # elasticsearch does.
    SQL_TYPE_MAPPING = {
        BIGINT: 'long',
        BOOLEAN: 'boolean',
        CHAR: 'string',
        CLOB: 'string',
        DATE: None,
        DATETIME: 'date',
        DECIMAL: None,
        FLOAT: 'float',
        # TODO: Use 'integer' when data is in range
        INTEGER: 'long',
        NCHAR: 'string',
        NVARCHAR: 'string',
        NullType: None,
        NUMERIC: None,
        # TODO: Use 'float' when data is in range
        REAL: 'double',
        SMALLINT: 'integer',
        TEXT: 'string',
        TIME: 'date',  # TODO: Map to something time specific?
        TIMESTAMP: 'date',
        VARCHAR: 'string',
    }

    def __init__(self, table_name, table_schema):
        """Map every column type to an elasticsearch mapping."""
        columns_mapping = {}

        for column_name, column_sql_type in table_schema.iteritems():
            column_mapping = self._get_column_mapping(column_sql_type)

            # Skip columns that don't have an mapping defined and let
            # elasticsearch figure out the mapping itself
            if column_mapping is None:
                continue

            columns_mapping[column_name] = column_mapping

        self.mapping = {
            table_name: {
                'properties': columns_mapping,
            }
        }

    def _get_column_mapping(self, column_sql_type):
        """Return column mapping based on its name and type.

        :param column_sql_type: Database column type
        :type column_sql_type: sqlalchemy.types.*
        :return: Mapping for the given column name and type (if possible)
        :rtype: dict(str) | None

        """
        column_es_type = self.SQL_TYPE_MAPPING[type(column_sql_type)]
        if column_es_type is None:
            return None

        column_mapping = {'type': column_es_type}
        return column_mapping

def get_index_name(path):
    """Get index name for a database file.

    :param path: Path to file to be indexed
    :type path: str
    :return: Index name for the DB file
    :rtype: str

    """
    # There's a limit in elasticsearch index name length and there are some
    # characters that are not allowed. Using an md5 hash seems to be a
    # reasonable way to get to a unique index name for each database file.
    index_name = hashlib.md5(path).hexdigest()
    return index_name

def get_index_action(index_name, document_type, row):
    """Generate index action for a given database row.

    :param index_name: Elasticsearch index to use
    :type index_name: str
    :param document_type: Elasticsearch document type to use
    :type index_name: str
    :param row: The row to be indexed in elasticsearch
    :type row: sqlalchemy.engine.result.RowProxy
    :return: Action to be passed in bulk request
    :rtype: dict

    """
    source = dict(row)

    # Avoid indexing binary data
    for field_name, field_data in source.items():
        # Avoid indexing binary data
        if isinstance(field_data, buffer):
            logger.debug('%r field discarded before indexing', field_name)
            del source[field_name]

        # Avoid indexing local paths
        elif isinstance(field_data, basestring):
            url = urlparse(field_data)
            if (url.scheme == 'file'
                    and os.path.exists(url.path)):
                logger.debug(
                    '%r field discarded before indexing', field_name)
                del source[field_name]

    action = {
        '_index': index_name,
        '_type': document_type,
        '_source': source,
    }

    # Use the same _id field in elasticsearch as in the database table
    if '_id' in source:
        action['_id'] = source['_id']

    return action

<|endoftext|>


<commit_before># encoding: utf-8
# Copyright (c) 2015 Grzegorz Chrupała
# A simple encoder-decoder example with funktional
from __future__ import division
import theano
import numpy
import random
import itertools
import cPickle as pickle
import argparse
import gzip
import sys
import util
import copy
from layer import *

class EncoderDecoder(Layer):
    """A simple encoder-decoder net with shared input and output vocabulary."""
    def __init__(self, size_vocab, size, depth):
        self.size_vocab  = size_vocab
        self.size     = size
        self.depth    = depth
        self.Embed    = Embedding(self.size_vocab, self.size)
        self.Unembed  = self.Embed.unembed
        encoder = lambda size_in, size: StackedGRUH0(size_in, size, self.depth)
        decoder = lambda size_in, size: StackedGRU(size_in, size, self.depth)
        self.Encdec   = EncoderDecoderGRU(self.size, self.size, self.size,
                                          encoder=encoder,
                                          decoder=decoder)
        self.Out      = Dense(size_in=self.size, size_out=self.size)
        self.params   = self.Embed.params + self.Encdec.params + self.Out.params

    def __call__(self, inp, out_prev):
        return softmax3d(self.Unembed(self.Out(self.Encdec(self.Embed(inp), self.Embed(out_prev)))))

class Model(object):
    """Trainable encoder-decoder model."""
    def __init__(self, size_vocab, size, depth):
        self.size = size
        self.size_vocab = size_vocab
        self.depth = depth
        self.network = EncoderDecoder(self.size_vocab, self.size, self.depth)
        self.input       = T.imatrix()
        self.output_prev = T.imatrix()
        self.output      = T.imatrix()
        OH = OneHot(size_in=self.size_vocab)
        self.output_oh   = OH(self.output)
        self.output_pred = self.network(self.input, self.output_prev)
        self.cost = CrossEntropy(self.output_oh, self.output_pred)
        self.updater = Adam()
        self.updates = self.updater.get_updates(self.network.params, self.cost)
        self.train = theano.function([self.input, self.output_prev, self.output ],
                                      self.cost, updates=self.updates)
        self.predict = theano.function([self.input, self.output_prev], self.output_pred)
        # Like train, but no updates
        self.loss = theano.function([self.input, self.output_prev, self.output ], self.cost)

def pad(xss, padding):
    max_len = max((len(xs) for xs in xss))
    def pad_one(xs):
        return xs + [ padding for _ in range(0,(max_len-len(xs))) ]
    return [ pad_one(xs) for xs in xss ]

def grouper(iterable, n):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3) --> ABC DEF G
    args = [iter(iterable)] * n
    return itertools.izip(*args)

def batch(item, BEG, END):
    """Prepare minibatch."""
    mb = numpy.array(pad([[BEG]+s+[END] for s in item], END), dtype='int32')
    inp = mb[:,1:]
    out = mb[:,1:]
    out_prev = mb[:,0:-1]
    return (inp, out_prev, out)

def valid_loss(model, valid, BEG, END):
    costs = 0.0; N = 0
    for _j, item in enumerate(grouper(valid, 256)):
        j = _j + 1
        inp, out_prev, out = batch(item, BEG, END)
        costs = costs + model.loss(inp, out_prev, out) ; N = N + 1
    return costs / N

def shuffled(x):
    y = copy.copy(x)
    random.shuffle(y)
    return y

def main():
    parser = argparse.ArgumentParser(description="Stacked recurrent autoencoder for sentences.")
    parser.add_argument('--size',   type=int, default=512,       help="Size of embeddings and hidden layers")
    parser.add_argument('--depth',  type=int, default=2,         help="Number of hidden layers")
    parser.add_argument('--epochs', type=int, default=1,         help="Number of training epochs")
    parser.add_argument('--seed',   type=int, default=None,      help="Random seed")
    parser.add_argument('--log',    type=str, default='log.txt', help="Path to log file")
    parser.add_argument('train',    type=str,                    help="Path to training data")
    parser.add_argument('valid',    type=str,                    help="Path to validation data")
    args = parser.parse_args()
    if args.seed is not None:
        random.seed(args.seed)
    mapper = util.IdMapper(min_df=10)
    sents = shuffled(list(mapper.fit_transform([ line.split() for line in open(args.train) ])))
    sents_valid = list(mapper.transform([line.split() for line in open(args.valid) ]))
    mb_size = 128
    model = Model(size_vocab=mapper.size(), size=args.size, depth=args.depth)
    with open(args.log,'w') as log:
        for epoch in range(1,args.epochs + 1):
            costs = 0 ; N = 0
            for _j, item in enumerate(grouper(sents, 128)):
                j = _j + 1
                inp, out_prev, out = batch(item, mapper.BEG_ID, mapper.END_ID)
                costs = costs + model.train(inp, out_prev, out) ; N = N + 1
                print epoch, j, "train", costs / N
                if j % 500 == 0:
                    cost_valid = valid_loss(model, sents_valid, mapper.BEG_ID, mapper.END_ID)
                    print epoch, j, "valid", cost_valid
                if j % 100 == 0:
                    pred = model.predict(inp, out_prev)
                    for i in range(len(pred)):
                        orig = [ w for w in list(mapper.inverse_transform([inp[i]]))[0]
                                 if w != mapper.END ]
                        res =  [ w for w in list(mapper.inverse_transform([numpy.argmax(pred, axis=2)[i]]))[0]
                                 if w != mapper.END ]
                        log.write("{}".format(' '.join(orig)))
                        log.write("\n")
                        log.write("{}".format(' '.join(res)))
                        log.write("\n")
                    log.flush()
        pickle.dump(model, gzip.open('model.{0}.pkl.gz'.format(epoch),'w'))

if __name__ == '__main__':
    main()
<commit_msg>dump model at each epoch<commit_after># encoding: utf-8
# Copyright (c) 2015 Grzegorz Chrupała
# A simple encoder-decoder example with funktional
from __future__ import division
import theano
import numpy
import random
import itertools
import cPickle as pickle
import argparse
import gzip
import sys
import util
import copy
from layer import *

class EncoderDecoder(Layer):
    """A simple encoder-decoder net with shared input and output vocabulary."""
    def __init__(self, size_vocab, size, depth):
        self.size_vocab  = size_vocab
        self.size     = size
        self.depth    = depth
        self.Embed    = Embedding(self.size_vocab, self.size)
        self.Unembed  = self.Embed.unembed
        encoder = lambda size_in, size: StackedGRUH0(size_in, size, self.depth)
        decoder = lambda size_in, size: StackedGRU(size_in, size, self.depth)
        self.Encdec   = EncoderDecoderGRU(self.size, self.size, self.size,
                                          encoder=encoder,
                                          decoder=decoder)
        self.Out      = Dense(size_in=self.size, size_out=self.size)
        self.params   = self.Embed.params + self.Encdec.params + self.Out.params

    def __call__(self, inp, out_prev):
        return softmax3d(self.Unembed(self.Out(self.Encdec(self.Embed(inp), self.Embed(out_prev)))))

class Model(object):
    """Trainable encoder-decoder model."""
    def __init__(self, size_vocab, size, depth):
        self.size = size
        self.size_vocab = size_vocab
        self.depth = depth
        self.network = EncoderDecoder(self.size_vocab, self.size, self.depth)
        self.input       = T.imatrix()
        self.output_prev = T.imatrix()
        self.output      = T.imatrix()
        OH = OneHot(size_in=self.size_vocab)
        self.output_oh   = OH(self.output)
        self.output_pred = self.network(self.input, self.output_prev)
        self.cost = CrossEntropy(self.output_oh, self.output_pred)
        self.updater = Adam()
        self.updates = self.updater.get_updates(self.network.params, self.cost)
        self.train = theano.function([self.input, self.output_prev, self.output ],
                                      self.cost, updates=self.updates)
        self.predict = theano.function([self.input, self.output_prev], self.output_pred)
        # Like train, but no updates
        self.loss = theano.function([self.input, self.output_prev, self.output ], self.cost)

def pad(xss, padding):
    max_len = max((len(xs) for xs in xss))
    def pad_one(xs):
        return xs + [ padding for _ in range(0,(max_len-len(xs))) ]
    return [ pad_one(xs) for xs in xss ]

def grouper(iterable, n):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3) --> ABC DEF G
    args = [iter(iterable)] * n
    return itertools.izip(*args)

def batch(item, BEG, END):
    """Prepare minibatch."""
    mb = numpy.array(pad([[BEG]+s+[END] for s in item], END), dtype='int32')
    inp = mb[:,1:]
    out = mb[:,1:]
    out_prev = mb[:,0:-1]
    return (inp, out_prev, out)

def valid_loss(model, valid, BEG, END):
    costs = 0.0; N = 0
    for _j, item in enumerate(grouper(valid, 256)):
        j = _j + 1
        inp, out_prev, out = batch(item, BEG, END)
        costs = costs + model.loss(inp, out_prev, out) ; N = N + 1
    return costs / N

def shuffled(x):
    y = copy.copy(x)
    random.shuffle(y)
    return y

def main():
    parser = argparse.ArgumentParser(description="Stacked recurrent autoencoder for sentences.")
    parser.add_argument('--size',   type=int, default=512,       help="Size of embeddings and hidden layers")
    parser.add_argument('--depth',  type=int, default=2,         help="Number of hidden layers")
    parser.add_argument('--epochs', type=int, default=1,         help="Number of training epochs")
    parser.add_argument('--seed',   type=int, default=None,      help="Random seed")
    parser.add_argument('--log',    type=str, default='log.txt', help="Path to log file")
    parser.add_argument('train',    type=str,                    help="Path to training data")
    parser.add_argument('valid',    type=str,                    help="Path to validation data")
    args = parser.parse_args()
    if args.seed is not None:
        random.seed(args.seed)
    mapper = util.IdMapper(min_df=10)
    sents = shuffled(list(mapper.fit_transform([ line.split() for line in open(args.train) ])))
    sents_valid = list(mapper.transform([line.split() for line in open(args.valid) ]))
    mb_size = 128
    model = Model(size_vocab=mapper.size(), size=args.size, depth=args.depth)
    with open(args.log,'w') as log:
        for epoch in range(1,args.epochs + 1):
            costs = 0 ; N = 0
            for _j, item in enumerate(grouper(sents, 128)):
                j = _j + 1
                inp, out_prev, out = batch(item, mapper.BEG_ID, mapper.END_ID)
                costs = costs + model.train(inp, out_prev, out) ; N = N + 1
                print epoch, j, "train", costs / N
                if j % 500 == 0:
                    cost_valid = valid_loss(model, sents_valid, mapper.BEG_ID, mapper.END_ID)
                    print epoch, j, "valid", cost_valid
                if j % 100 == 0:
                    pred = model.predict(inp, out_prev)
                    for i in range(len(pred)):
                        orig = [ w for w in list(mapper.inverse_transform([inp[i]]))[0]
                                 if w != mapper.END ]
                        res =  [ w for w in list(mapper.inverse_transform([numpy.argmax(pred, axis=2)[i]]))[0]
                                 if w != mapper.END ]
                        log.write("{}".format(' '.join(orig)))
                        log.write("\n")
                        log.write("{}".format(' '.join(res)))
                        log.write("\n")
                    log.flush()
            pickle.dump(model, gzip.open('model.{0}.pkl.gz'.format(epoch),'w'))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import tkinter as tk
from tkinter import ttk
import sys

class GUI(object):
    def __init__(self, master, cmd):
        self.master = master
        self.cmd = cmd

        # Create elements
        self.frame = ttk.Frame(self.master)
        self.button = ttk.Button(self.frame, text='Test', command=self.cmd['button'])

        # If you need to pass arguments to the function, use lambda
        # self.button = ttk.Button(self.frame, text='Test', command=lambda: self.cmd['nothing'](5))

        # Pack elements
        self.frame.pack()
        self.button.pack()

def cmd_button():
    print('Button command')

def cmd_nothing(number):
    print('What? Nothing! Or maybe {}!'.format(number))

def main():
    root = tk.Tk()

    # Assing functions to dictionary
    commands = {'button': cmd_button,
                'nothing': cmd_nothing}

    gui = GUI(root, commands)
    root.mainloop()

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Fixed typo<commit_after>import tkinter as tk
from tkinter import ttk
import sys

class GUI(object):
    def __init__(self, master, cmd):
        self.master = master
        self.cmd = cmd

        # Create elements
        self.frame = ttk.Frame(self.master)
        self.button = ttk.Button(self.frame, text='Test', command=self.cmd['button'])

        # If you need to pass arguments to the function, use lambda
        # self.button = ttk.Button(self.frame, text='Test', command=lambda: self.cmd['nothing'](5))

        # Pack elements
        self.frame.pack()
        self.button.pack()

def cmd_button():
    print('Button command')

def cmd_nothing(number):
    print('What? Nothing! Or maybe {}!'.format(number))

def main():
    root = tk.Tk()

    # Assign functions to dictionary
    commands = {'button': cmd_button,
                'nothing': cmd_nothing}

    gui = GUI(root, commands)
    root.mainloop()

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python2

from datetime import datetime
from itertools import combinations

from timetabler.scheduler import Scheduler
from timetabler.ssc.course import Lecture, Discussion
from timetabler import sort, util
from timetabler.sort import earliest_start  # Helper function (should probably be in util)

COMMUTE_HOURS = 1.75
SESSION = "2015W"
TERMS = (1, 2)

def main():
    required = (
        "CPEN 321",
        "CPEN 421",
        "CPEN 422",
        "APSC 486",
        # "CPEN 492",
        "CPEN 481",
        "APSC 450"
    )
    opt = [
        "CPEN 442",
        "CPSC 312"
    ]
    num_required_from_opt = 2
    combs = combinations(opt, r=num_required_from_opt)

    schedules = []
    for courses in [required + comb for comb in combs]:
        s = Scheduler(courses, session=SESSION, terms=TERMS, refresh=False)
        # I don't want any classes that start before 9:00AM
        s.add_constraint(lambda sched: earliest_start(sched.activities) >= 9)
        # Add GEOG122 constraints if we need to
        if "GEOG 122" in courses:
            # STTs are for Vantage College students
            s.courses["GEOG 122"].add_constraint(
                lambda acts: all(a.status not in [u"STT"] for a in acts)
            )
            # Default sections contained a Tutorial but that is for Vantage
            # students, so removing that and only setting Lecture and Discussion
            s.courses["GEOG 122"].num_section_constraints = [
                (Lecture, 1), (Discussion, 1)
            ]

        # Add statuses for courses that shouldn't be considered
        bad_statuses = (
            "Full",
            # "Blocked",
        )
        schedules.extend(s.generate_schedules(bad_statuses=bad_statuses))
    return schedules

if __name__ == '__main__':
    # Setup logging
    util.setup_root_logger()

    # Get schedules (time operation)
    start_time = datetime.now()
    scheds = main()
    print("There were {} valid schedules found.".format(len(scheds)))
    print("This took {} to calculate.".format(
        datetime.now() - start_time
    ))
    # Sort and draw
    scheds = sort.free_days(scheds)
    scheds = sort.even_time_per_day(scheds, commute_hrs=COMMUTE_HOURS)
    scheds = sort.sum_latest_daily_morning(scheds)
    scheds = sort.least_time_at_school(scheds, commute_hrs=COMMUTE_HOURS)
    for i, sched in enumerate(scheds):
        sched.draw(terms=TERMS, draw_location="terminal")
        if i < len(scheds) - 1:
            raw_input("Press ENTER to display the next schedule...")
<commit_msg>Add capstone hack<commit_after>#!/usr/bin/env python2

from datetime import datetime
from itertools import combinations

from timetabler.scheduler import Scheduler
from timetabler.ssc.course import Lecture, Discussion, Lab
from timetabler import sort, util
from timetabler.sort import earliest_start  # Helper function (should probably be in util)

COMMUTE_HOURS = 1.75
SESSION = "2015W"
TERMS = (1, 2)

def main():
    required = (
        "CPEN 321",
        "CPEN 421",
        "CPEN 422",
        "APSC 486",
        # "CPEN 492",
        "CPEN 481",
        "APSC 450"
    )
    opt = [
        "CPEN 442",
        "CPSC 312"
    ]
    num_required_from_opt = 2
    combs = list(combinations(opt, r=num_required_from_opt))

    schedules = []
    for courses in [required + comb for comb in combs]:
        s = Scheduler(courses, session=SESSION, terms=TERMS, refresh=False)
        # I don't want any classes that start before 9:00AM
        s.add_constraint(lambda sched: earliest_start(sched.activities) >= 9)
        # Add GEOG122 constraints if we need to
        if "GEOG 122" in courses:
            # STTs are for Vantage College students
            s.courses["GEOG 122"].add_constraint(
                lambda acts: all(a.status not in [u"STT"] for a in acts)
            )
            # Default sections contained a Tutorial but that is for Vantage
            # students, so removing that and only setting Lecture and Discussion
            s.courses["GEOG 122"].num_section_constraints = [
                (Lecture, 1), (Discussion, 1)
            ]
        # Capstone double-books lab and lecture section in both terms at the
        # same time... so we need to throw in a hack for it
        if "CPEN 492" in courses:
            s.courses["CPEN 492"].num_section_constraints = [
                (Lecture, 1), (Lab, 1)
            ]

        # Add statuses for courses that shouldn't be considered
        bad_statuses = (
            "Full",
            # "Blocked",
        )
        schedules.extend(s.generate_schedules(bad_statuses=bad_statuses))
    return schedules

if __name__ == '__main__':
    # Setup logging
    util.setup_root_logger()

    # Get schedules (time operation)
    start_time = datetime.now()
    scheds = main()
    print("There were {} valid schedules found.".format(len(scheds)))
    print("This took {} to calculate.".format(
        datetime.now() - start_time
    ))
    # Sort and draw
    scheds = sort.free_days(scheds)
    scheds = sort.even_time_per_day(scheds, commute_hrs=COMMUTE_HOURS)
    scheds = sort.sum_latest_daily_morning(scheds)
    scheds = sort.least_time_at_school(scheds, commute_hrs=COMMUTE_HOURS)
    for i, sched in enumerate(scheds):
        sched.draw(terms=TERMS, draw_location="terminal")
        if i < len(scheds) - 1:
            raw_input("Press ENTER to display the next schedule...")
<|endoftext|>


<commit_before>"""An example of how to use the library so far."""
# Standard library imports
from itertools import izip

# Third-party imports
import numpy
import theano
from theano import tensor

# Local imports
from costs import MeanSquaredError
from corruption import GaussianCorruptor
from autoencoder import DenoisingAutoencoder, DATrainer, StackedDA

if __name__ == "__main__":
    conf = {
        'corruption_level': 0.1,
        'n_hid': 20,
        'n_vis': 15,
        'lr_anneal_start': 100,
        'base_lr': 0.01,
        'tied_weights': True,
        'act_enc': 'tanh',
        'act_dec': None,
        #'lr_hb': 0.10,
        #'lr_vb': 0.10,
        'irange': 0.001,
    }

    # A symbolic input representing your minibatch.
    minibatch = tensor.dmatrix()

    # Allocate a denoising autoencoder with binomial noise corruption.
    corruptor = GaussianCorruptor.alloc(conf)
    da = DenoisingAutoencoder.alloc(corruptor, conf)

    # Allocate a trainer, which tells us how to update our model.
    cost_fn = MeanSquaredError.alloc(conf, da)
    trainer = DATrainer.alloc(da, cost_fn, minibatch, conf)

    # Finally, build a Theano function out of all this.
    train_fn = trainer.function(minibatch)

    # Simulate some fake data.
    data = numpy.random.normal(size=(1000, 15))

    # Suppose we want minibatches of size 10
    batchsize = 10

    # Here's a manual training loop. I hope to have some classes that
    # automate this a litle bit.
    for epoch in xrange(5):
        for offset in xrange(0, data.shape[0], batchsize):
            minibatch_err = train_fn(data[offset:(offset + batchsize)])
            print "epoch %d, batch %d-%d: %f" % \
                    (epoch, offset, offset + batchsize - 1, minibatch_err)

    # Suppose you then want to use the representation for something.
    transform = theano.function([minibatch], da([minibatch])[0])

    print "Transformed data:"
    print numpy.histogram(transform(data))

    # We'll now create a stacked denoising autoencoder. First, we change
    # the number of hidden units to be a list. This tells the StackedDA
    # class how many layers to make.
    sda_conf = conf.copy()
    sda_conf['n_hid'] = [20, 20, 10]
    sda = StackedDA.alloc(corruptor, sda_conf)

    # To pretrain it, we'll use a DATrainer for each layer.
    trainers = []
    thislayer_input = [minibatch]
    for layer in sda.layers():
        cost_fn = MeanSquaredError.alloc(sda_conf, layer)
        trainer = DATrainer.alloc(layer, cost_fn, thislayer_input[0], sda_conf)
        trainers.append(trainer)
        # Retrieve a Theano function for training this layer.
        thislayer_train_fn = trainer.function(minibatch)

        # Train as before.
        for epoch in xrange(10):
            for offset in xrange(0, data.shape[0], batchsize):
                minibatch_err = thislayer_train_fn(
                    data[offset:(offset + batchsize)]
                )
                print "epoch %d, batch %d-%d: %f" % \
                        (epoch, offset, offset + batchsize - 1, minibatch_err)

        # Now, get a symbolic input for the next layer.
        thislayer_input = layer(thislayer_input)
<commit_msg>Set conf['n_vis'] to data.shape[1] rather than hardcoded value.<commit_after>"""An example of how to use the library so far."""
# Standard library imports
from itertools import izip

# Third-party imports
import numpy
import theano
from theano import tensor

# Local imports
from costs import MeanSquaredError
from corruption import GaussianCorruptor
from autoencoder import DenoisingAutoencoder, DATrainer, StackedDA

if __name__ == "__main__":
    # Simulate some fake data.
    data = numpy.random.normal(size=(1000, 15))

    conf = {
        'corruption_level': 0.1,
        'n_hid': 20,
        'n_vis': data.shape[1],
        'lr_anneal_start': 100,
        'base_lr': 0.01,
        'tied_weights': True,
        'act_enc': 'tanh',
        'act_dec': None,
        #'lr_hb': 0.10,
        #'lr_vb': 0.10,
        'irange': 0.001,
    }

    # A symbolic input representing your minibatch.
    minibatch = tensor.dmatrix()

    # Allocate a denoising autoencoder with binomial noise corruption.
    corruptor = GaussianCorruptor.alloc(conf)
    da = DenoisingAutoencoder.alloc(corruptor, conf)

    # Allocate a trainer, which tells us how to update our model.
    cost_fn = MeanSquaredError.alloc(conf, da)
    trainer = DATrainer.alloc(da, cost_fn, minibatch, conf)

    # Finally, build a Theano function out of all this.
    train_fn = trainer.function(minibatch)

    # Suppose we want minibatches of size 10
    batchsize = 10

    # Here's a manual training loop. I hope to have some classes that
    # automate this a litle bit.
    for epoch in xrange(5):
        for offset in xrange(0, data.shape[0], batchsize):
            minibatch_err = train_fn(data[offset:(offset + batchsize)])
            print "epoch %d, batch %d-%d: %f" % \
                    (epoch, offset, offset + batchsize - 1, minibatch_err)

    # Suppose you then want to use the representation for something.
    transform = theano.function([minibatch], da([minibatch])[0])

    print "Transformed data:"
    print numpy.histogram(transform(data))

    # We'll now create a stacked denoising autoencoder. First, we change
    # the number of hidden units to be a list. This tells the StackedDA
    # class how many layers to make.
    sda_conf = conf.copy()
    sda_conf['n_hid'] = [20, 20, 10]
    sda = StackedDA.alloc(corruptor, sda_conf)

    # To pretrain it, we'll use a DATrainer for each layer.
    trainers = []
    thislayer_input = [minibatch]
    for layer in sda.layers():
        cost_fn = MeanSquaredError.alloc(sda_conf, layer)
        trainer = DATrainer.alloc(layer, cost_fn, thislayer_input[0], sda_conf)
        trainers.append(trainer)
        # Retrieve a Theano function for training this layer.
        thislayer_train_fn = trainer.function(minibatch)

        # Train as before.
        for epoch in xrange(10):
            for offset in xrange(0, data.shape[0], batchsize):
                minibatch_err = thislayer_train_fn(
                    data[offset:(offset + batchsize)]
                )
                print "epoch %d, batch %d-%d: %f" % \
                        (epoch, offset, offset + batchsize - 1, minibatch_err)

        # Now, get a symbolic input for the next layer.
        thislayer_input = layer(thislayer_input)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
from urllib.parse import urljoin, urlparse
import yaml
import requests
from bs4 import BeautifulSoup

download_url = 'https://swift.org/download/'

def parse_url(url):
    if url.endswith('.tar.gz') or url.endswith('.pkg'):
        parse = urlparse(url)
        name = parse.path.split('/')[-1]
        (software, rest) = name.split('-', 1)
        assert(software == 'swift')

        rest = rest.replace('.tar.gz', '').replace('.pkg', '')
        (rest, platform) = rest.rsplit('-', 1)
        version = rest.replace('-RELEASE', '').replace('-osx', '')

        if platform == 'symbols':
            # macOS Debugging Symbol link
            return (None, None)

        return (version, platform)

    return (None, None)

if __name__ == '__main__':
    request = requests.get(download_url)
    soup = BeautifulSoup(request.text, 'html.parser')
    releases = soup.find_all('a')

    versions = {}

    for a in releases:
        url = urljoin(download_url, a['href'])
        version, platform = parse_url(url)

        if version and platform:
            if version in versions:
                versions[version][platform] = url
            else:
                versions[version] = {platform: url}

    for version in versions:
        path = 'versions/{}.yaml'.format(version)

        if not os.path.exists(path):
            print('Add {}'.format(version))

            with open(path, 'w') as fp:
                content = yaml.dump({'binaries': versions[version]}, default_flow_style=False)
                fp.write(content)

<commit_msg>feat(extract): Inform when mismatched data<commit_after>#!/usr/bin/env python

import os
from urllib.parse import urljoin, urlparse
import yaml
import requests
from bs4 import BeautifulSoup

download_url = 'https://swift.org/download/'

def parse_url(url):
    if url.endswith('.tar.gz') or url.endswith('.pkg'):
        parse = urlparse(url)
        name = parse.path.split('/')[-1]
        (software, rest) = name.split('-', 1)
        assert(software == 'swift')

        rest = rest.replace('.tar.gz', '').replace('.pkg', '')
        (rest, platform) = rest.rsplit('-', 1)
        version = rest.replace('-RELEASE', '').replace('-osx', '')

        if platform == 'symbols':
            # macOS Debugging Symbol link
            return (None, None)

        return (version, platform)

    return (None, None)

def save(path, binaries):
    with open(path, 'w') as fp:
        content = yaml.dump({'binaries': binaries}, default_flow_style=False)
        fp.write(content)

def load(path):
    with open(path, 'r') as fp:
        return yaml.load(fp)['binaries']

if __name__ == '__main__':
    request = requests.get(download_url)
    soup = BeautifulSoup(request.text, 'html.parser')
    releases = soup.find_all('a')

    versions = {}

    for a in releases:
        url = urljoin(download_url, a['href'])
        version, platform = parse_url(url)

        if version and platform:
            if version in versions:
                versions[version][platform] = url
            else:
                versions[version] = {platform: url}

    for version in versions:
        path = 'versions/{}.yaml'.format(version)
        binaries = versions[version]

        if os.path.exists(path):
            existing_binaries = load(path)

            if binaries != existing_binaries:
                print('Mismatched Data: {}'.format(version))
        else:
            print('Add {}'.format(version))
            save(path, binaries)

<|endoftext|>