<commit_before>#!/usr/bin/env python3

import os
import sys
import time
import select
import logging
import argparse
import threading
import subprocess

ETCD_IMAGE = "quay.io/coreos/etcd:v3.3.5"

LOG_LEVELS = {
    "critical": logging.CRITICAL,
    "error": logging.ERROR,
    "warning": logging.WARNING,
    "info": logging.INFO,
    "debug": logging.DEBUG,
}

LOG_COLORS = {
    "critical": "\x1b[31;1m",
    "error": "\x1b[31;1m",
    "warning": "\x1b[33;1m",
}

log = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(levelname)s:%(message)s"))
log.addHandler(handler)

class ExcThread(threading.Thread):
    def __init__(self, target):
        super().__init__(target=target)
        self.error = None

    def run(self):
        try:
            self._target()
        except Exception as e:
            self.error = e

def join(*targets):
    threads = []

    for target in targets:
        t = ExcThread(target)
        t.start()
        threads.append(t)

    for t in threads:
        t.join()
    for t in threads:
        if t.error is not None:
            raise Exception("Thread error") from t.error

class Output:
    def __init__(self, pipe, level):
        self.pipe = pipe
        self.level = level
        self.lines = []

class ProcessResult:
    def __init__(self, rc, stdout, stderr):
        self.rc = rc
        self.stdout = stdout
        self.stderr = stderr

class DefaultDriver:
    def available(self):
        return True

    def clear(self):
        if run("yes | pachctl delete-all", shell=True, raise_on_error=False).rc != 0:
            log.error("could not call `pachctl delete-all`; most likely this just means that a pachyderm cluster hasn't been setup, but may indicate a bad state")

    def start(self):
        pass

    def inspect(self):
        pass

    def push_images(self, deploy_version, dash_image):
        pass

    def wait(self):
        while suppress("pachctl", "version") != 0:
            print("Waiting for pachyderm to come up...")
            time.sleep(1)

class MinikubeDriver(DefaultDriver):
    def available(self):
        return run("which", "minikube", raise_on_error=False).rc == 0

    def clear(self):
        run("minikube", "delete")

    def start(self):
        run("minikube", "start")

        while suppress("minikube", "status") != 0:
            print("Waiting for minikube to come up...")
            time.sleep(1)

    def push_images(self, deploy_version, dash_image):
        run("./etc/kube/push-to-minikube.sh", "pachyderm/pachd:{}".format(deploy_version))
        run("./etc/kube/push-to-minikube.sh", "pachyderm/worker:{}".format(deploy_version))
        run("./etc/kube/push-to-minikube.sh", ETCD_IMAGE)
        run("./etc/kube/push-to-minikube.sh", dash_image)

class MicroK8sDriver(DefaultDriver):
    def available(self):
        return run("which", "microk8s.kubectl", raise_on_error=False).rc == 0

    def clear(self):
        # `microk8s.reset` doesn't clear out cluster pods, so we'll go ahead
        # and do that through pachctl functionality if possible
        if run("yes | pachctl delete-all --no-port-forwarding", shell=True, raise_on_error=False).rc != 0:
            log.error("could not call `pachctl delete-all`; most likely this just means that a pachyderm cluster hasn't been setup, but may indicate a bad state")

        run("microk8s.stop")

    def start(self):
        # starting microk8s immediately after stopping it can fail, so try a
        # few times
        for i in range(5):
            if run("microk8s.start", raise_on_error=False).rc == 0:
                break
            time.sleep(1)

        # `microk8s.reset` has a couple of issues:
        # 1) it can fail when called immediately after `microk8s.start`
        # 2) it doesn't always output a proper return code when there's an error
        stderr = None
        for i in range(5):
            stderr = run("microk8s.reset").stderr
            if len(stderr) == 0:
                break
            time.sleep(1)
        if len(stderr) > 0:
            raise Exception("reset failed")

        while suppress("microk8s.status") != 0:
            print("Waiting for microk8s to come up...")
            time.sleep(1)

    def inspect(self):
        # get output of `microk8s.inspect`, as it may include a warning about
        # firewall rules that need to be changed in order for it to work
        run("microk8s.inspect")

    def push_images(self, deploy_version, dash_image):
        run("./etc/kube/push-to-microk8s.sh", "pachyderm/pachd:{}".format(deploy_version))
        run("./etc/kube/push-to-microk8s.sh", "pachyderm/worker:{}".format(deploy_version))
        run("./etc/kube/push-to-microk8s.sh", ETCD_IMAGE)
        run("./etc/kube/push-to-microk8s.sh", dash_image)

    def wait(self):
        while suppress("pachctl", "version", "--no-port-forwarding") != 0:
            print("Waiting for pachyderm to come up...")
            time.sleep(1)

def parse_log_level(s):
    try:
        return LOG_LEVELS[s]
    except KeyError:
        raise Exception("Unknown log level: {}".format(s))

def redirect_to_logger(stdout, stderr):
    for io in select.select([stdout.pipe, stderr.pipe], [], [], 1000)[0]:
        line = io.readline().decode().rstrip()

        if line == "":
            continue

        dest = stdout if io == stdout.pipe else stderr
        log.log(LOG_LEVELS[dest.level], "{}{}\x1b[0m".format(LOG_COLORS.get(dest.level, ""), line))
        dest.lines.append(line)

def run(cmd, *args, raise_on_error=True, shell=False, stdout_log_level="info", stderr_log_level="error"):
    log.debug("Running `%s %s`", cmd, " ".join(args))

    proc = subprocess.Popen([cmd, *args], shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout = Output(proc.stdout, stdout_log_level)
    stderr = Output(proc.stderr, stderr_log_level)

    while proc.poll() is None:
        redirect_to_logger(stdout, stderr)
    redirect_to_logger(stdout, stderr)

    rc = proc.wait()

    if raise_on_error and rc != 0:
        raise Exception("Unexpected return code for `{} {}`: {}".format(cmd, " ".join(args), rc))

    return ProcessResult(rc, "\n".join(stdout.lines), "\n".join(stderr.lines))

def capture(cmd, *args, shell=False):
    return run(cmd, *args, shell=shell, stdout_log_level="debug").stdout

def suppress(cmd, *args):
    return run(cmd, *args, stdout_log_level="debug", stderr_log_level="debug", raise_on_error=False).rc

def get_pachyderm(deploy_version):
    print("Deploying pachd:{}".format(deploy_version))

    should_download = suppress("which", "pachctl") != 0 \
        or capture("pachctl", "version", "--client-only") != deploy_version

    if should_download:
        release_url = "https://github.com/pachyderm/pachyderm/releases/download/v{}/pachctl_{}_linux_amd64.tar.gz".format(deploy_version, deploy_version)
        outpath = os.path.join(os.environ["GOPATH"], "bin")
        filepath = "pachctl_{}_linux_amd64/pachctl".format(deploy_version)
        run("curl -L {} | tar -C \"{}\" --strip-components=1 -xzf - {}".format(release_url, outpath, filepath), shell=True)

    run("docker", "pull", "pachyderm/pachd:{}".format(deploy_version))
    run("docker", "pull", "pachyderm/worker:{}".format(deploy_version))

def main():
    parser = argparse.ArgumentParser(description="Resets the pachyderm cluster and tools.")
    parser.add_argument("--no-deploy", default=False, action="store_true", help="Disables deployment")
    parser.add_argument("--deploy-args", default="", help="Arguments to be passed into `pachctl deploy`")
    parser.add_argument("--deploy-version", default="local", help="Sets the deployment version")
    parser.add_argument("--log-level", default="info", type=parse_log_level, help="Sets the log level; defaults to 'info', other options include 'critical', 'error', 'warning', and 'debug'")
    args = parser.parse_args()

    log.setLevel(args.log_level)

    if "GOPATH" not in os.environ:
        print("Must set GOPATH")
        sys.exit(1)
    if not args.no_deploy and "PACH_CA_CERTS" in os.environ:
        print("Must unset PACH_CA_CERTS\nRun:\nunset PACH_CA_CERTS", file=sys.stderr)
        sys.exit(1)
    if args.deploy_version == "local" and not os.getcwd().startswith(os.path.join(os.environ["GOPATH"], "src", "github.com", "pachyderm", "pachyderm")):
        print("Must be in a Pachyderm client", file=sys.stderr)
        sys.exit(1)

    if MinikubeDriver().available():
        log.info("using the minikube driver")
        driver = MinikubeDriver()
    elif MicroK8sDriver().available():
        log.info("using the microk8s driver")
        driver = MicroK8sDriver()
    else:
        log.info("using the k8s for docker driver")
        log.warn("with this driver, it's not possible to fully reset the cluster")
        driver = DefaultDriver()

    driver.clear()

    gopath = os.environ["GOPATH"]

    if args.deploy_version == "local":
        os.chdir(os.path.join(gopath, "src", "github.com", "pachyderm", "pachyderm"))

        try:
            os.remove(os.path.join(gopath, "bin", "pachctl"))
        except:
            pass

        join(
            driver.start,
            lambda: run("make", "install"),
            lambda: run("make", "docker-build"),
        )
    else:
        join(
            driver.start,
            lambda: get_pachyderm(args.deploy_version),
        )

    driver.inspect()

    version = capture("pachctl", "version", "--client-only")
    print("Deploy pachyderm version v{}".format(version))

    while suppress("pachctl", "version", "--client-only") != 0:
        print("Waiting for pachctl to build...")
        time.sleep(1)

    run("which", "pachctl")

    dash_image = capture("pachctl deploy local -d --dry-run | jq -r '.. | select(.name? == \"dash\" and has(\"image\")).image'", shell=True)
    grpc_proxy_image = capture("pachctl deploy local -d --dry-run | jq -r '.. | select(.name? == \"grpc-proxy\").image'", shell=True)

    run("docker", "pull", dash_image)
    run("docker", "pull", grpc_proxy_image)
    run("docker", "pull", ETCD_IMAGE)
    driver.push_images(args.deploy_version, dash_image)

    if not args.no_deploy:
        if args.deploy_version == "local":
            run("pachctl deploy local -d {}".format(args.deploy_args), shell=True)
        else:
            run("pachctl deploy local -d {} --dry-run | sed \"s/:local/:{}/g\" | kubectl create -f -".format(args.deploy_args, args.deploy_version), shell=True)

        driver.wait()

    run("killall", "kubectl", raise_on_error=False)

if __name__ == "__main__":
    main()
<commit_msg>Fixed use of deprecated method<commit_after>#!/usr/bin/env python3

import os
import sys
import time
import select
import logging
import argparse
import threading
import subprocess

ETCD_IMAGE = "quay.io/coreos/etcd:v3.3.5"

LOG_LEVELS = {
    "critical": logging.CRITICAL,
    "error": logging.ERROR,
    "warning": logging.WARNING,
    "info": logging.INFO,
    "debug": logging.DEBUG,
}

LOG_COLORS = {
    "critical": "\x1b[31;1m",
    "error": "\x1b[31;1m",
    "warning": "\x1b[33;1m",
}

log = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(levelname)s:%(message)s"))
log.addHandler(handler)

class ExcThread(threading.Thread):
    def __init__(self, target):
        super().__init__(target=target)
        self.error = None

    def run(self):
        try:
            self._target()
        except Exception as e:
            self.error = e

def join(*targets):
    threads = []

    for target in targets:
        t = ExcThread(target)
        t.start()
        threads.append(t)

    for t in threads:
        t.join()
    for t in threads:
        if t.error is not None:
            raise Exception("Thread error") from t.error

class Output:
    def __init__(self, pipe, level):
        self.pipe = pipe
        self.level = level
        self.lines = []

class ProcessResult:
    def __init__(self, rc, stdout, stderr):
        self.rc = rc
        self.stdout = stdout
        self.stderr = stderr

class DefaultDriver:
    def available(self):
        return True

    def clear(self):
        if run("yes | pachctl delete-all", shell=True, raise_on_error=False).rc != 0:
            log.error("could not call `pachctl delete-all`; most likely this just means that a pachyderm cluster hasn't been setup, but may indicate a bad state")

    def start(self):
        pass

    def inspect(self):
        pass

    def push_images(self, deploy_version, dash_image):
        pass

    def wait(self):
        while suppress("pachctl", "version") != 0:
            print("Waiting for pachyderm to come up...")
            time.sleep(1)

class MinikubeDriver(DefaultDriver):
    def available(self):
        return run("which", "minikube", raise_on_error=False).rc == 0

    def clear(self):
        run("minikube", "delete")

    def start(self):
        run("minikube", "start")

        while suppress("minikube", "status") != 0:
            print("Waiting for minikube to come up...")
            time.sleep(1)

    def push_images(self, deploy_version, dash_image):
        run("./etc/kube/push-to-minikube.sh", "pachyderm/pachd:{}".format(deploy_version))
        run("./etc/kube/push-to-minikube.sh", "pachyderm/worker:{}".format(deploy_version))
        run("./etc/kube/push-to-minikube.sh", ETCD_IMAGE)
        run("./etc/kube/push-to-minikube.sh", dash_image)

class MicroK8sDriver(DefaultDriver):
    def available(self):
        return run("which", "microk8s.kubectl", raise_on_error=False).rc == 0

    def clear(self):
        # `microk8s.reset` doesn't clear out cluster pods, so we'll go ahead
        # and do that through pachctl functionality if possible
        if run("yes | pachctl delete-all --no-port-forwarding", shell=True, raise_on_error=False).rc != 0:
            log.error("could not call `pachctl delete-all`; most likely this just means that a pachyderm cluster hasn't been setup, but may indicate a bad state")

        run("microk8s.stop")

    def start(self):
        # starting microk8s immediately after stopping it can fail, so try a
        # few times
        for i in range(5):
            if run("microk8s.start", raise_on_error=False).rc == 0:
                break
            time.sleep(1)

        # `microk8s.reset` has a couple of issues:
        # 1) it can fail when called immediately after `microk8s.start`
        # 2) it doesn't always output a proper return code when there's an error
        stderr = None
        for i in range(5):
            stderr = run("microk8s.reset").stderr
            if len(stderr) == 0:
                break
            time.sleep(1)
        if len(stderr) > 0:
            raise Exception("reset failed")

        while suppress("microk8s.status") != 0:
            print("Waiting for microk8s to come up...")
            time.sleep(1)

    def inspect(self):
        # get output of `microk8s.inspect`, as it may include a warning about
        # firewall rules that need to be changed in order for it to work
        run("microk8s.inspect")

    def push_images(self, deploy_version, dash_image):
        run("./etc/kube/push-to-microk8s.sh", "pachyderm/pachd:{}".format(deploy_version))
        run("./etc/kube/push-to-microk8s.sh", "pachyderm/worker:{}".format(deploy_version))
        run("./etc/kube/push-to-microk8s.sh", ETCD_IMAGE)
        run("./etc/kube/push-to-microk8s.sh", dash_image)

    def wait(self):
        while suppress("pachctl", "version", "--no-port-forwarding") != 0:
            print("Waiting for pachyderm to come up...")
            time.sleep(1)

def parse_log_level(s):
    try:
        return LOG_LEVELS[s]
    except KeyError:
        raise Exception("Unknown log level: {}".format(s))

def redirect_to_logger(stdout, stderr):
    for io in select.select([stdout.pipe, stderr.pipe], [], [], 1000)[0]:
        line = io.readline().decode().rstrip()

        if line == "":
            continue

        dest = stdout if io == stdout.pipe else stderr
        log.log(LOG_LEVELS[dest.level], "{}{}\x1b[0m".format(LOG_COLORS.get(dest.level, ""), line))
        dest.lines.append(line)

def run(cmd, *args, raise_on_error=True, shell=False, stdout_log_level="info", stderr_log_level="error"):
    log.debug("Running `%s %s`", cmd, " ".join(args))

    proc = subprocess.Popen([cmd, *args], shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout = Output(proc.stdout, stdout_log_level)
    stderr = Output(proc.stderr, stderr_log_level)

    while proc.poll() is None:
        redirect_to_logger(stdout, stderr)
    redirect_to_logger(stdout, stderr)

    rc = proc.wait()

    if raise_on_error and rc != 0:
        raise Exception("Unexpected return code for `{} {}`: {}".format(cmd, " ".join(args), rc))

    return ProcessResult(rc, "\n".join(stdout.lines), "\n".join(stderr.lines))

def capture(cmd, *args, shell=False):
    return run(cmd, *args, shell=shell, stdout_log_level="debug").stdout

def suppress(cmd, *args):
    return run(cmd, *args, stdout_log_level="debug", stderr_log_level="debug", raise_on_error=False).rc

def get_pachyderm(deploy_version):
    print("Deploying pachd:{}".format(deploy_version))

    should_download = suppress("which", "pachctl") != 0 \
        or capture("pachctl", "version", "--client-only") != deploy_version

    if should_download:
        release_url = "https://github.com/pachyderm/pachyderm/releases/download/v{}/pachctl_{}_linux_amd64.tar.gz".format(deploy_version, deploy_version)
        outpath = os.path.join(os.environ["GOPATH"], "bin")
        filepath = "pachctl_{}_linux_amd64/pachctl".format(deploy_version)
        run("curl -L {} | tar -C \"{}\" --strip-components=1 -xzf - {}".format(release_url, outpath, filepath), shell=True)

    run("docker", "pull", "pachyderm/pachd:{}".format(deploy_version))
    run("docker", "pull", "pachyderm/worker:{}".format(deploy_version))

def main():
    parser = argparse.ArgumentParser(description="Resets the pachyderm cluster and tools.")
    parser.add_argument("--no-deploy", default=False, action="store_true", help="Disables deployment")
    parser.add_argument("--deploy-args", default="", help="Arguments to be passed into `pachctl deploy`")
    parser.add_argument("--deploy-version", default="local", help="Sets the deployment version")
    parser.add_argument("--log-level", default="info", type=parse_log_level, help="Sets the log level; defaults to 'info', other options include 'critical', 'error', 'warning', and 'debug'")
    args = parser.parse_args()

    log.setLevel(args.log_level)

    if "GOPATH" not in os.environ:
        print("Must set GOPATH")
        sys.exit(1)
    if not args.no_deploy and "PACH_CA_CERTS" in os.environ:
        print("Must unset PACH_CA_CERTS\nRun:\nunset PACH_CA_CERTS", file=sys.stderr)
        sys.exit(1)
    if args.deploy_version == "local" and not os.getcwd().startswith(os.path.join(os.environ["GOPATH"], "src", "github.com", "pachyderm", "pachyderm")):
        print("Must be in a Pachyderm client", file=sys.stderr)
        sys.exit(1)

    if MinikubeDriver().available():
        log.info("using the minikube driver")
        driver = MinikubeDriver()
    elif MicroK8sDriver().available():
        log.info("using the microk8s driver")
        driver = MicroK8sDriver()
    else:
        log.info("using the k8s for docker driver")
        log.warning("with this driver, it's not possible to fully reset the cluster")
        driver = DefaultDriver()

    driver.clear()

    gopath = os.environ["GOPATH"]

    if args.deploy_version == "local":
        os.chdir(os.path.join(gopath, "src", "github.com", "pachyderm", "pachyderm"))

        try:
            os.remove(os.path.join(gopath, "bin", "pachctl"))
        except:
            pass

        join(
            driver.start,
            lambda: run("make", "install"),
            lambda: run("make", "docker-build"),
        )
    else:
        join(
            driver.start,
            lambda: get_pachyderm(args.deploy_version),
        )

    driver.inspect()

    version = capture("pachctl", "version", "--client-only")
    print("Deploy pachyderm version v{}".format(version))

    while suppress("pachctl", "version", "--client-only") != 0:
        print("Waiting for pachctl to build...")
        time.sleep(1)

    run("which", "pachctl")

    dash_image = capture("pachctl deploy local -d --dry-run | jq -r '.. | select(.name? == \"dash\" and has(\"image\")).image'", shell=True)
    grpc_proxy_image = capture("pachctl deploy local -d --dry-run | jq -r '.. | select(.name? == \"grpc-proxy\").image'", shell=True)

    run("docker", "pull", dash_image)
    run("docker", "pull", grpc_proxy_image)
    run("docker", "pull", ETCD_IMAGE)
    driver.push_images(args.deploy_version, dash_image)

    if not args.no_deploy:
        if args.deploy_version == "local":
            run("pachctl deploy local -d {}".format(args.deploy_args), shell=True)
        else:
            run("pachctl deploy local -d {} --dry-run | sed \"s/:local/:{}/g\" | kubectl create -f -".format(args.deploy_args, args.deploy_version), shell=True)

        driver.wait()

    run("killall", "kubectl", raise_on_error=False)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import json
import os

from django.conf import settings
from django.http import HttpResponse
from django.views.decorators.cache import cache_page

from .wordcloud import popular_words

@cache_page(60*60*4)
def wordcloud(request, max_entries=30):
    """ Return tag cloud JSON results"""
    cache_path = settings.WORDCLOUD_CACHE_PATH
    if os.path.exists(cache_path):
        response = HttpResponse()
        response['Content-Type'] = 'application/json'
        response['X-Sendfile'] = cache_path.encode('utf-8')
        return response

    content = json.dumps(popular_words(max_entries=max_entries))

    return HttpResponse(
        content,
        mimetype='application/json',
        )
<commit_msg>Fix a deprecation warning; content_type replaces mimetype<commit_after>import json
import os

from django.conf import settings
from django.http import HttpResponse
from django.views.decorators.cache import cache_page

from .wordcloud import popular_words

@cache_page(60*60*4)
def wordcloud(request, max_entries=30):
    """ Return tag cloud JSON results"""
    cache_path = settings.WORDCLOUD_CACHE_PATH
    if os.path.exists(cache_path):
        response = HttpResponse()
        response['Content-Type'] = 'application/json'
        response['X-Sendfile'] = cache_path.encode('utf-8')
        return response

    content = json.dumps(popular_words(max_entries=max_entries))

    return HttpResponse(
        content,
        content_type='application/json',
    )
<|endoftext|>


<commit_before>from django.conf import settings
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.views.generic import list_detail, date_based
from wordpress.models import Post
import urllib

PER_PAGE = getattr(settings, 'WP_PER_PAGE', 10)

TAXONOMIES = {
    'term': 'post_tag',
    'category': 'category',
    'link_category': 'link_category',
}

def object_detail(request, year, month, day, slug):
    slug = urllib.quote(slug.encode('utf-8')).lower()
    return date_based.object_detail(request, queryset=Post.objects.published(),
        date_field='post_date', year=year, month=month, month_format="%m",
        day=day, slug=slug, template_object_name='post', allow_future=True,
        extra_context={'post_url': request.build_absolute_uri(request.path)})

def archive_day(request, year, month, day):
    return date_based.archive_day(request, queryset=Post.objects.published(),
        date_field='post_date', year=year, month=month, month_format="%m",
        day=day, template_object_name='post')

def archive_month(request, year, month):
    return date_based.archive_month(request, queryset=Post.objects.published(),
        date_field='post_date', year=year, month=month, month_format="%m",
        template_object_name='post')

def archive_year(request, year):
    return date_based.archive_year(request, queryset=Post.objects.published(),
        date_field='post_date', year=year)

def archive_index(request):
    p = request.GET.get('p', None)
    if p:
        post = Post.objects.get(pk=p)
        return render_to_response('wordpress/post_detail.html', {'post': post})
    posts = Post.objects.published().select_related()
    return list_detail.object_list(request, queryset=posts,
        paginate_by=10, template_name='wordpress/post_archive.html',
        template_object_name='post', allow_empty=True)

def taxonomy(request, taxonomy, term):
    taxonomy = TAXONOMIES.get(taxonomy, None)
    if taxonomy:
        posts = Post.objects.term(term, taxonomy=taxonomy).select_related()
        return list_detail.object_list(request, queryset=posts,
            paginate_by=10, template_name='wordpress/post_term.html',
            template_object_name='post', allow_empty=True,
            extra_context={'term': term})<commit_msg>add preview view<commit_after>from django.conf import settings
from django.http import HttpResponseRedirect
from django.views.generic import list_detail, date_based
from wordpress.models import Post
import urllib

PER_PAGE = getattr(settings, 'WP_PER_PAGE', 10)

TAXONOMIES = {
    'term': 'post_tag',
    'category': 'category',
    'link_category': 'link_category',
}

def preview(request, post_id):
    return list_detail.object_detail(
        request,
        queryset=Post.objects.all(),
        object_id=post_id,
        template_object_name='post',
        extra_context={
            'preview': True,
        }
    )

def object_detail(request, year, month, day, slug):
    slug = urllib.quote(slug.encode('utf-8')).lower()
    return date_based.object_detail(request, queryset=Post.objects.published(),
        date_field='post_date', year=year, month=month, month_format="%m",
        day=day, slug=slug, template_object_name='post', allow_future=True,
        extra_context={'post_url': request.build_absolute_uri(request.path)})

def archive_day(request, year, month, day):
    return date_based.archive_day(request, queryset=Post.objects.published(),
        date_field='post_date', year=year, month=month, month_format="%m",
        day=day, template_object_name='post')

def archive_month(request, year, month):
    return date_based.archive_month(request, queryset=Post.objects.published(),
        date_field='post_date', year=year, month=month, month_format="%m",
        template_object_name='post')

def archive_year(request, year):
    return date_based.archive_year(request, queryset=Post.objects.published(),
        date_field='post_date', year=year)

def archive_index(request):
    p = request.GET.get('p', None)
    if p:
        return preview(request, p)
    posts = Post.objects.published().select_related()
    return list_detail.object_list(request, queryset=posts,
        paginate_by=10, template_name='wordpress/post_archive.html',
        template_object_name='post', allow_empty=True)

def taxonomy(request, taxonomy, term):
    taxonomy = TAXONOMIES.get(taxonomy, None)
    if taxonomy:
        posts = Post.objects.term(term, taxonomy=taxonomy).select_related()
        return list_detail.object_list(request, queryset=posts,
            paginate_by=10, template_name='wordpress/post_term.html',
            template_object_name='post', allow_empty=True,
            extra_context={'term': term})<|endoftext|>


<commit_before>import numpy as np

from astropy.tests.helper import pytest
from stingray.simulator import simulator, models
from scipy.stats import chisquare

class TestSimulator(object):

    @classmethod
    def setup_class(self):
        self.simulator = simulator.Simulator(N=1024)

    def test_periodogram_without_lc(self):
        """
        Create a periodogram without light curve.
        """
        self.simulator.periodogram(self.simulator.lc)

    def test_simulate_with_seed(self):
        """
        Simulate with a random seed value.
        """
        self.simulator = simulator.Simulator(N=1024, seed=12)
        assert len(self.simulator.simulate(2).counts), 1024

    def test_simulate_powerlaw(self):
        """
        Simulate light curve from power law spectrum.
        """
        assert len(self.simulator.simulate(2).counts), 1024

    def test_compare_powerlaw(self):
        """
        Compare simulated power spectrum with actual one.
        """
        B, N, red_noise, dt = 2, 1024, 10, 1

        self.simulator = simulator.Simulator(N=N, dt=dt, mean=5, rms=1,
                                             red_noise=red_noise)
        lc = [self.simulator.simulate(B) for i in range(1,30)]
        simulated = self.simulator.periodogram(lc, lc[0].tseg)

        w = np.fft.rfftfreq(N, d=dt)[1:]
        actual = np.power((1/w), B/2)[:-1]

        actual_prob = actual/float(sum(actual))
        simulated_prob = simulated/float(sum(simulated))

        assert np.all(np.abs(actual_prob - simulated_prob) < 3
                      * np.sqrt(actual_prob))

    def test_simulate_powerspectrum(self):
        """
        Simulate light curve from any power spectrum.
        """
        s = np.random.rand(1024)
        assert len(self.simulator.simulate(s)), 1024

    def test_simulate_lorenzian(self):
        """
        Simulate light curve using lorenzian model.
        """
        assert len(self.simulator.simulate('lorenzian',[1,2,3,4])), 1024

    def test_compare_lorenzian(self):
        """
        Compare simulated lorenzian spectrum with original spectrum.
        """
        N, red_noise, dt = 1024, 10, 1

        self.simulator = simulator.Simulator(N=N, dt=dt, mean=0.1,
                                             rms=0.4, red_noise=red_noise)
        lc = [self.simulator.simulate('lorenzian',[0.3, 0.9, 0.6, 0.5])
              for i in range(1,30)]
        simulated = self.simulator.periodogram(lc, lc[0].tseg)

        w = np.fft.rfftfreq(N, d=dt)[1:]
        actual = models.lorenzian(w,[0.3, 0.9, 0.6, 0.5])[:-1]

        actual_prob = actual/float(sum(actual))
        simulated_prob = simulated/float(sum(simulated))

        assert np.all(np.abs(actual_prob - simulated_prob) < 3
                      * np.sqrt(actual_prob))

    def test_simulate_smoothbknpo(self):
        """
        Simulate light curve using smooth broken power law model.
        """
        assert len(self.simulator.simulate('smoothbknpo',[1,2,3,4])), 1024

    def test_compare_smoothbknpo(self):
        """
        Compare simulated smooth broken power law spectrum with original
        spectrum.
        """
        N, red_noise, dt = 1024, 10, 1

        self.simulator = simulator.Simulator(N=N, dt=dt, mean=0.1, rms=0.7,
                                             red_noise=red_noise)
        lc = [self.simulator.simulate('smoothbknpo',[0.6, 0.2, 0.6, 0.5])
              for i in range(1,30)]

        simulated = self.simulator.periodogram(lc, lc[0].tseg)

        w = np.fft.rfftfreq(N, d=dt)[1:]
        actual = models.smoothbknpo(w,[0.6, 0.2, 0.6, 0.5])[:-1]

        actual_prob = actual/float(sum(actual))
        simulated_prob = simulated/float(sum(simulated))

        assert np.all(np.abs(actual_prob - simulated_prob) < 3
                      * np.sqrt(actual_prob))

    def test_simulate_impulse(self):
        """
        Simulate light curve from impulse response.
        """
        self.simulator.simulate([],[])

    def test_periodogram_with_lc(self):
        """
        Create a periodogram from light curve.
        """
        self.simulator.simulate(2)
        print(np.max(self.simulator.lc.counts))
        self.simulator.periodogram(self.simulator.lc)

<commit_msg>Removed superfluous tesdt<commit_after>import numpy as np

from astropy.tests.helper import pytest
from stingray.simulator import simulator, models
from scipy.stats import chisquare

class TestSimulator(object):

    @classmethod
    def setup_class(self):
        self.simulator = simulator.Simulator(N=1024)

    def test_simulate_with_seed(self):
        """
        Simulate with a random seed value.
        """
        self.simulator = simulator.Simulator(N=1024, seed=12)
        assert len(self.simulator.simulate(2).counts), 1024

    def test_simulate_powerlaw(self):
        """
        Simulate light curve from power law spectrum.
        """
        assert len(self.simulator.simulate(2).counts), 1024

    def test_compare_powerlaw(self):
        """
        Compare simulated power spectrum with actual one.
        """
        B, N, red_noise, dt = 2, 1024, 10, 1

        self.simulator = simulator.Simulator(N=N, dt=dt, mean=5, rms=1,
                                             red_noise=red_noise)
        lc = [self.simulator.simulate(B) for i in range(1,30)]
        simulated = self.simulator.periodogram(lc, lc[0].tseg)

        w = np.fft.rfftfreq(N, d=dt)[1:]
        actual = np.power((1/w), B/2)[:-1]

        actual_prob = actual/float(sum(actual))
        simulated_prob = simulated/float(sum(simulated))

        assert np.all(np.abs(actual_prob - simulated_prob) < 3
                      * np.sqrt(actual_prob))

    def test_simulate_powerspectrum(self):
        """
        Simulate light curve from any power spectrum.
        """
        s = np.random.rand(1024)
        assert len(self.simulator.simulate(s)), 1024

    def test_simulate_lorenzian(self):
        """
        Simulate light curve using lorenzian model.
        """
        assert len(self.simulator.simulate('lorenzian',[1,2,3,4])), 1024

    def test_compare_lorenzian(self):
        """
        Compare simulated lorenzian spectrum with original spectrum.
        """
        N, red_noise, dt = 1024, 10, 1

        self.simulator = simulator.Simulator(N=N, dt=dt, mean=0.1,
                                             rms=0.4, red_noise=red_noise)
        lc = [self.simulator.simulate('lorenzian',[0.3, 0.9, 0.6, 0.5])
              for i in range(1,30)]
        simulated = self.simulator.periodogram(lc, lc[0].tseg)

        w = np.fft.rfftfreq(N, d=dt)[1:]
        actual = models.lorenzian(w,[0.3, 0.9, 0.6, 0.5])[:-1]

        actual_prob = actual/float(sum(actual))
        simulated_prob = simulated/float(sum(simulated))

        assert np.all(np.abs(actual_prob - simulated_prob) < 3
                      * np.sqrt(actual_prob))

    def test_simulate_smoothbknpo(self):
        """
        Simulate light curve using smooth broken power law model.
        """
        assert len(self.simulator.simulate('smoothbknpo',[1,2,3,4])), 1024

    def test_compare_smoothbknpo(self):
        """
        Compare simulated smooth broken power law spectrum with original
        spectrum.
        """
        N, red_noise, dt = 1024, 10, 1

        self.simulator = simulator.Simulator(N=N, dt=dt, mean=0.1, rms=0.7,
                                             red_noise=red_noise)
        lc = [self.simulator.simulate('smoothbknpo',[0.6, 0.2, 0.6, 0.5])
              for i in range(1,30)]

        simulated = self.simulator.periodogram(lc, lc[0].tseg)

        w = np.fft.rfftfreq(N, d=dt)[1:]
        actual = models.smoothbknpo(w,[0.6, 0.2, 0.6, 0.5])[:-1]

        actual_prob = actual/float(sum(actual))
        simulated_prob = simulated/float(sum(simulated))

        assert np.all(np.abs(actual_prob - simulated_prob) < 3
                      * np.sqrt(actual_prob))

    def test_simulate_impulse(self):
        """
        Simulate light curve from impulse response.
        """
        self.simulator.simulate([],[])

    def test_periodogram_with_lc(self):
        """
        Create a periodogram from light curve.
        """
        self.simulator.simulate(2)
        print(np.max(self.simulator.lc.counts))
        self.simulator.periodogram(self.simulator.lc)

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json

from gimme.api import GimmeRequest, GimmeCall
from gimme.auth import GimmeAuth

class Gimme(object):

    def __init__(self, auth_token):
        self.auth_token = auth_token

    def request(self, check_sucsess=True, **kwargs):
        req = GimmeRequest(self.auth_token, **kwargs)
        return req

    def follow_user(self, username):
        kwargs = {'cmd': 'set_favorite_celebrity_insert',
                  'celebrityCode': username}
        return self.request(**kwargs)

    def unfollow_user(self, username):
        kwargs = {'cmd': 'set_favorite_celebrity_delete',
                  'celebrityCode': username}
        return self.request(**kwargs)

    def timeline(self, username=False, type=0, max_timestamp=False):
        kwargs = {}
        if username:
            kwargs['cmd'] = 'get_celebrity_message_list_for_fan'
            kwargs['code'] = username
        else:
            kwargs['cmd'] = 'get_celebrity_timeline_list'

        kwargs['type'] = str(type)

        if max_timestamp:
            kwargs['createdAt'] = max_timestamp

        return self.request(**kwargs)

    def itimeline(self, type=0, username=False, max_timestamp=False):
        """Like timeline(), except that it returns an generator and yields
        messages until it hits an empty feed."""

        max_timestamp = max_timestamp
        while True:
            feed = self.timeline(type=type,
                                 username=username,
                                 max_timestamp=max_timestamp)

            if len(feed) == 0:
                break
            else:
                for message in feed:
                    yield message
                max_timestamp = feed[-1]['createdAt']

    @property
    def user_info(self):
        kwargs = {'cmd': 'get_my_profile'}
        response = self.request(**kwargs)
        return response

    @property
    def follows(self):
        kwargs = {'cmd': 'get_favorite_celebrity_list'}
        return self.request(**kwargs)

    @property
    def celebrities(self):
        kwargs = {'cmd': 'get_celebrity_list'}
        return self.request(**kwargs)

if __name__ == "__main__":
    pass
<commit_msg>Switch the order of type and username in itimeline<commit_after>#!/usr/bin/env python3

import json

from gimme.api import GimmeRequest, GimmeCall
from gimme.auth import GimmeAuth

class Gimme(object):

    def __init__(self, auth_token):
        self.auth_token = auth_token

    def request(self, check_sucsess=True, **kwargs):
        req = GimmeRequest(self.auth_token, **kwargs)
        return req

    def follow_user(self, username):
        kwargs = {'cmd': 'set_favorite_celebrity_insert',
                  'celebrityCode': username}
        return self.request(**kwargs)

    def unfollow_user(self, username):
        kwargs = {'cmd': 'set_favorite_celebrity_delete',
                  'celebrityCode': username}
        return self.request(**kwargs)

    def timeline(self, username=False, type=0, max_timestamp=False):
        kwargs = {}
        if username:
            kwargs['cmd'] = 'get_celebrity_message_list_for_fan'
            kwargs['code'] = username
        else:
            kwargs['cmd'] = 'get_celebrity_timeline_list'

        kwargs['type'] = str(type)

        if max_timestamp:
            kwargs['createdAt'] = max_timestamp

        return self.request(**kwargs)

    def itimeline(self, username=False, type=0, max_timestamp=False):
        """Like timeline(), except that it returns an generator and yields
        messages until it hits an empty feed."""

        max_timestamp = max_timestamp
        while True:
            feed = self.timeline(type=type,
                                 username=username,
                                 max_timestamp=max_timestamp)

            if len(feed) == 0:
                break
            else:
                for message in feed:
                    yield message
                max_timestamp = feed[-1]['createdAt']

    @property
    def user_info(self):
        kwargs = {'cmd': 'get_my_profile'}
        response = self.request(**kwargs)
        return response

    @property
    def follows(self):
        kwargs = {'cmd': 'get_favorite_celebrity_list'}
        return self.request(**kwargs)

    @property
    def celebrities(self):
        kwargs = {'cmd': 'get_celebrity_list'}
        return self.request(**kwargs)

if __name__ == "__main__":
    pass
<|endoftext|>


<commit_before><commit_msg>Implement WINFF.FT.S.GRA.23<commit_after><|endoftext|>


<commit_before># Author: Tianqi Chen, Bing Xu
# module for xgboost
import ctypes
import os
# optinally have scipy sparse, though not necessary
import numpy
import sys
import numpy.ctypeslib
import scipy.sparse as scp

# set this line correctly
if os.name == 'nt':
    XGBOOST_PATH = os.path.dirname(__file__)+'/../windows/x64/Release/xgboost_wrapper.dll'
else:
     XGBOOST_PATH = os.path.dirname(__file__)+'/../libxgboostwrapper.so'

# load in xgboost library
xglib = ctypes.cdll.LoadLibrary(XGBOOST_PATH)

xglib.XGDMatrixCreateFromFile.restype = ctypes.c_void_p
xglib.XGDMatrixCreateFromCSR.restype = ctypes.c_void_p
xglib.XGDMatrixCreateFromMat.restype = ctypes.c_void_p
xglib.XGDMatrixSliceDMatrix.restype = ctypes.c_void_p
xglib.XGDMatrixGetFloatInfo.restype = ctypes.POINTER(ctypes.c_float)
xglib.XGDMatrixGetUIntInfo.restype = ctypes.POINTER(ctypes.c_uint)
xglib.XGDMatrixNumRow.restype = ctypes.c_ulong

xglib.XGBoosterCreate.restype = ctypes.c_void_p
xglib.XGBoosterPredict.restype = ctypes.POINTER(ctypes.c_float)
xglib.XGBoosterEvalOneIter.restype = ctypes.c_char_p
xglib.XGBoosterDumpModel.restype = ctypes.POINTER(ctypes.c_char_p)

def ctypes2numpy(cptr, length, dtype):
    # convert a ctypes pointer array to numpy
    assert isinstance(cptr, ctypes.POINTER(ctypes.c_float))
    res = numpy.zeros(length, dtype=dtype)
    assert ctypes.memmove(res.ctypes.data, cptr, length * res.strides[0])
    return res

# data matrix used in xgboost
class DMatrix:
    # constructor
    def __init__(self, data, label=None, missing=0.0, weight = None):
        # force into void_p, mac need to pass things in as void_p
        if data == None:
            self.handle = None
            return
        if isinstance(data, str):
            self.handle = ctypes.c_void_p(
                xglib.XGDMatrixCreateFromFile(ctypes.c_char_p(data.encode('utf-8')), 0))
        elif isinstance(data, scp.csr_matrix):
            self.__init_from_csr(data)
        elif isinstance(data, numpy.ndarray) and len(data.shape) == 2:
            self.__init_from_npy2d(data, missing)
        else:
            try:
                csr = scp.csr_matrix(data)
                self.__init_from_csr(csr)
            except:
                raise Exception("can not intialize DMatrix from"+str(type(data)))
        if label != None:
            self.set_label(label)
        if weight !=None:
            self.set_weight(weight)
    # convert data from csr matrix
    def __init_from_csr(self, csr):
        assert len(csr.indices) == len(csr.data)
        self.handle = ctypes.c_void_p(xglib.XGDMatrixCreateFromCSR(
            (ctypes.c_ulong  * len(csr.indptr))(*csr.indptr),
            (ctypes.c_uint  * len(csr.indices))(*csr.indices),
            (ctypes.c_float * len(csr.data))(*csr.data),
            len(csr.indptr), len(csr.data)))
    # convert data from numpy matrix
    def __init_from_npy2d(self,mat,missing):
        data = numpy.array(mat.reshape(mat.size), dtype='float32')
        self.handle = ctypes.c_void_p(xglib.XGDMatrixCreateFromMat(
            data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            mat.shape[0], mat.shape[1], ctypes.c_float(missing)))
    # destructor
    def __del__(self):
        xglib.XGDMatrixFree(self.handle)
    def get_float_info(self, field):
        length = ctypes.c_ulong()
        ret = xglib.XGDMatrixGetFloatInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                          ctypes.byref(length))
        return ctypes2numpy(ret, length.value, 'float32')
    def get_uint_info(self, field):
        length = ctypes.c_ulong()
        ret = xglib.XGDMatrixGetUIntInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                         ctypes.byref(length))
        return ctypes2numpy(ret, length.value, 'uint32')
    def set_float_info(self, field, data):
        xglib.XGDMatrixSetFloatInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                    (ctypes.c_float*len(data))(*data), len(data))
    def set_uint_info(self, field, data):
        xglib.XGDMatrixSetUIntInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                   (ctypes.c_uint*len(data))(*data), len(data))
    # load data from file
    def save_binary(self, fname, silent=True):
        xglib.XGDMatrixSaveBinary(self.handle, ctypes.c_char_p(fname.encode('utf-8')), int(silent))
    # set label of dmatrix
    def set_label(self, label):
        self.set_float_info('label', label)
    # set weight of each instances
    def set_weight(self, weight):
        self.set_float_info('weight', weight)
    # set initialized margin prediction
    def set_base_margin(self, margin):
        """
        set base margin of booster to start from
        this can be used to specify a prediction value of
        existing model to be base_margin
        However, remember margin is needed, instead of transformed prediction
        e.g. for logistic regression: need to put in value before logistic transformation
        see also example/demo.py
        """
        self.set_float_info('base_margin', margin)
    # set group size of dmatrix, used for rank
    def set_group(self, group):
        xglib.XGDMatrixSetGroup(self.handle, (ctypes.c_uint*len(group))(*group), len(group))
    # get label from dmatrix
    def get_label(self):
        return self.get_float_info('label')
    # get weight from dmatrix
    def get_weight(self):
        return self.get_float_info('weight')
    # get base_margin from dmatrix
    def get_base_margin(self):
        return self.get_float_info('base_margin')
    def num_row(self):
        return xglib.XGDMatrixNumRow(self.handle)
    # slice the DMatrix to return a new DMatrix that only contains rindex
    def slice(self, rindex):
        res = DMatrix(None)
        res.handle = ctypes.c_void_p(xglib.XGDMatrixSliceDMatrix(
            self.handle, (ctypes.c_int*len(rindex))(*rindex), len(rindex)))
        return res

class Booster:
    """learner class """
    def __init__(self, params={}, cache=[], model_file = None):
        """ constructor, param: """
        for d in cache:
            assert isinstance(d, DMatrix)
        dmats = (ctypes.c_void_p  * len(cache))(*[ d.handle for d in cache])
        self.handle = ctypes.c_void_p(xglib.XGBoosterCreate(dmats, len(cache)))
        self.set_param({'seed':0})
        self.set_param(params)
        if model_file != None:
            self.load_model(model_file)
    def __del__(self):
        xglib.XGBoosterFree(self.handle)
    def set_param(self, params, pv=None):
        if isinstance(params, dict):
            for k, v in params.items():
                xglib.XGBoosterSetParam(
                    self.handle, ctypes.c_char_p(k.encode('utf-8')),
                    ctypes.c_char_p(str(v).encode('utf-8')))
        elif isinstance(params,str) and pv != None:
            xglib.XGBoosterSetParam(
                self.handle, ctypes.c_char_p(params.encode('utf-8')),
                ctypes.c_char_p(str(pv).encode('utf-8')))
        else:
            for k, v in params:
                xglib.XGBoosterSetParam(
                    self.handle, ctypes.c_char_p(k.encode('utf-8')),
                    ctypes.c_char_p(str(v).encode('utf-8')))
    def update(self, dtrain, it):
        """
        update
          dtrain: the training DMatrix
          it: current iteration number
        """
        assert isinstance(dtrain, DMatrix)
        xglib.XGBoosterUpdateOneIter(self.handle, it, dtrain.handle)
    def boost(self, dtrain, grad, hess):
        """ update """
        assert len(grad) == len(hess)
        assert isinstance(dtrain, DMatrix)
        xglib.XGBoosterBoostOneIter(self.handle, dtrain.handle,
                                    (ctypes.c_float*len(grad))(*grad),
                                    (ctypes.c_float*len(hess))(*hess),
                                    len(grad))
    def eval_set(self, evals, it = 0):
        for d in evals:
            assert isinstance(d[0], DMatrix)
            assert isinstance(d[1], str)
        dmats = (ctypes.c_void_p * len(evals) )(*[ d[0].handle for d in evals])
        evnames = (ctypes.c_char_p * len(evals))(
            * [ctypes.c_char_p(d[1].encode('utf-8')) for d in evals])
        return xglib.XGBoosterEvalOneIter(self.handle, it, dmats, evnames, len(evals))
    def eval(self, mat, name = 'eval', it = 0):
        return self.eval_set( [(mat,name)], it)
    def predict(self, data, output_margin=False):
        """
        predict with data
            data: the dmatrix storing the input
            output_margin: whether output raw margin value that is untransformed
        """
        length = ctypes.c_ulong()
        preds = xglib.XGBoosterPredict(self.handle, data.handle,
                                       int(output_margin), ctypes.byref(length))
        return ctypes2numpy(preds, length.value, 'float32')
    def save_model(self, fname):
        """ save model to file """
        xglib.XGBoosterSaveModel(self.handle, ctypes.c_char_p(fname.encode('utf-8')))
    def load_model(self, fname):
        """load model from file"""
        xglib.XGBoosterLoadModel( self.handle, ctypes.c_char_p(fname.encode('utf-8')) )
    def dump_model(self, fo, fmap=''):
        """dump model into text file"""
        if isinstance(fo,str):
            fo = open(fo,'w')
            need_close = True
        else:
            need_close = False
        ret = self.get_dump(fmap)
        for i in range(len(ret)):
            fo.write('booster[%d]:\n' %i)
            fo.write( ret[i] )
        if need_close:
            fo.close()
    def get_dump(self, fmap=''):
        """get dump of model as list of strings """
        length = ctypes.c_ulong()
        sarr = xglib.XGBoosterDumpModel(self.handle, ctypes.c_char_p(fmap.encode('utf-8')), ctypes.byref(length))
        res = []
        for i in range(length.value):
            res.append( str(sarr[i]) )
        return res
    def get_fscore(self, fmap=''):
        """ get feature importance of each feature """
        trees = self.get_dump(fmap)
        fmap = {}
        for tree in trees:
            print (tree)
            for l in tree.split('\n'):
                arr = l.split('[')
                if len(arr) == 1:
                    continue
                fid = arr[1].split(']')[0]
                fid = fid.split('<')[0]
                if fid not in fmap:
                    fmap[fid] = 1
                else:
                    fmap[fid]+= 1
        return fmap

def evaluate(bst, evals, it, feval = None):
    """evaluation on eval set"""
    if feval != None:
        res = '[%d]' % it
        for dm, evname in evals:
            name, val = feval(bst.predict(dm), dm)
            res += '\t%s-%s:%f' % (evname, name, val)
    else:
        res = bst.eval_set(evals, it)

    return res

def train(params, dtrain, num_boost_round = 10, evals = [], obj=None, feval=None):
    """ train a booster with given paramaters """
    bst = Booster(params, [dtrain]+[ d[0] for d in evals ] )
    if obj == None:
        for i in range(num_boost_round):
            bst.update( dtrain, i )
            if len(evals) != 0:
                sys.stderr.write(evaluate(bst, evals, i, feval).decode()+'\n')
    else:
        # try customized objective function
        for i in range(num_boost_round):
            pred = bst.predict( dtrain )
            grad, hess = obj( pred, dtrain )
            bst.boost( dtrain, grad, hess )
            if len(evals) != 0:
                sys.stderr.write(evaluate(bst, evals, i, feval)+'\n')
    return bst
<commit_msg>Update xgboost.py<commit_after># Author: Tianqi Chen, Bing Xu
# module for xgboost
import ctypes
import os
# optinally have scipy sparse, though not necessary
import numpy
import sys
import numpy.ctypeslib
import scipy.sparse as scp

# set this line correctly
if os.name == 'nt':
    XGBOOST_PATH = os.path.dirname(__file__)+'/../windows/x64/Release/xgboost_wrapper.dll'
else:
    XGBOOST_PATH = os.path.dirname(__file__)+'/../libxgboostwrapper.so'

# load in xgboost library
xglib = ctypes.cdll.LoadLibrary(XGBOOST_PATH)

xglib.XGDMatrixCreateFromFile.restype = ctypes.c_void_p
xglib.XGDMatrixCreateFromCSR.restype = ctypes.c_void_p
xglib.XGDMatrixCreateFromMat.restype = ctypes.c_void_p
xglib.XGDMatrixSliceDMatrix.restype = ctypes.c_void_p
xglib.XGDMatrixGetFloatInfo.restype = ctypes.POINTER(ctypes.c_float)
xglib.XGDMatrixGetUIntInfo.restype = ctypes.POINTER(ctypes.c_uint)
xglib.XGDMatrixNumRow.restype = ctypes.c_ulong

xglib.XGBoosterCreate.restype = ctypes.c_void_p
xglib.XGBoosterPredict.restype = ctypes.POINTER(ctypes.c_float)
xglib.XGBoosterEvalOneIter.restype = ctypes.c_char_p
xglib.XGBoosterDumpModel.restype = ctypes.POINTER(ctypes.c_char_p)

def ctypes2numpy(cptr, length, dtype):
    # convert a ctypes pointer array to numpy
    assert isinstance(cptr, ctypes.POINTER(ctypes.c_float))
    res = numpy.zeros(length, dtype=dtype)
    assert ctypes.memmove(res.ctypes.data, cptr, length * res.strides[0])
    return res

# data matrix used in xgboost
class DMatrix:
    # constructor
    def __init__(self, data, label=None, missing=0.0, weight = None):
        # force into void_p, mac need to pass things in as void_p
        if data == None:
            self.handle = None
            return
        if isinstance(data, str):
            self.handle = ctypes.c_void_p(
                xglib.XGDMatrixCreateFromFile(ctypes.c_char_p(data.encode('utf-8')), 0))
        elif isinstance(data, scp.csr_matrix):
            self.__init_from_csr(data)
        elif isinstance(data, numpy.ndarray) and len(data.shape) == 2:
            self.__init_from_npy2d(data, missing)
        else:
            try:
                csr = scp.csr_matrix(data)
                self.__init_from_csr(csr)
            except:
                raise Exception("can not intialize DMatrix from"+str(type(data)))
        if label != None:
            self.set_label(label)
        if weight !=None:
            self.set_weight(weight)
    # convert data from csr matrix
    def __init_from_csr(self, csr):
        assert len(csr.indices) == len(csr.data)
        self.handle = ctypes.c_void_p(xglib.XGDMatrixCreateFromCSR(
            (ctypes.c_ulong  * len(csr.indptr))(*csr.indptr),
            (ctypes.c_uint  * len(csr.indices))(*csr.indices),
            (ctypes.c_float * len(csr.data))(*csr.data),
            len(csr.indptr), len(csr.data)))
    # convert data from numpy matrix
    def __init_from_npy2d(self,mat,missing):
        data = numpy.array(mat.reshape(mat.size), dtype='float32')
        self.handle = ctypes.c_void_p(xglib.XGDMatrixCreateFromMat(
            data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            mat.shape[0], mat.shape[1], ctypes.c_float(missing)))
    # destructor
    def __del__(self):
        xglib.XGDMatrixFree(self.handle)
    def get_float_info(self, field):
        length = ctypes.c_ulong()
        ret = xglib.XGDMatrixGetFloatInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                          ctypes.byref(length))
        return ctypes2numpy(ret, length.value, 'float32')
    def get_uint_info(self, field):
        length = ctypes.c_ulong()
        ret = xglib.XGDMatrixGetUIntInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                         ctypes.byref(length))
        return ctypes2numpy(ret, length.value, 'uint32')
    def set_float_info(self, field, data):
        xglib.XGDMatrixSetFloatInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                    (ctypes.c_float*len(data))(*data), len(data))
    def set_uint_info(self, field, data):
        xglib.XGDMatrixSetUIntInfo(self.handle, ctypes.c_char_p(field.encode('utf-8')),
                                   (ctypes.c_uint*len(data))(*data), len(data))
    # load data from file
    def save_binary(self, fname, silent=True):
        xglib.XGDMatrixSaveBinary(self.handle, ctypes.c_char_p(fname.encode('utf-8')), int(silent))
    # set label of dmatrix
    def set_label(self, label):
        self.set_float_info('label', label)
    # set weight of each instances
    def set_weight(self, weight):
        self.set_float_info('weight', weight)
    # set initialized margin prediction
    def set_base_margin(self, margin):
        """
        set base margin of booster to start from
        this can be used to specify a prediction value of
        existing model to be base_margin
        However, remember margin is needed, instead of transformed prediction
        e.g. for logistic regression: need to put in value before logistic transformation
        see also example/demo.py
        """
        self.set_float_info('base_margin', margin)
    # set group size of dmatrix, used for rank
    def set_group(self, group):
        xglib.XGDMatrixSetGroup(self.handle, (ctypes.c_uint*len(group))(*group), len(group))
    # get label from dmatrix
    def get_label(self):
        return self.get_float_info('label')
    # get weight from dmatrix
    def get_weight(self):
        return self.get_float_info('weight')
    # get base_margin from dmatrix
    def get_base_margin(self):
        return self.get_float_info('base_margin')
    def num_row(self):
        return xglib.XGDMatrixNumRow(self.handle)
    # slice the DMatrix to return a new DMatrix that only contains rindex
    def slice(self, rindex):
        res = DMatrix(None)
        res.handle = ctypes.c_void_p(xglib.XGDMatrixSliceDMatrix(
            self.handle, (ctypes.c_int*len(rindex))(*rindex), len(rindex)))
        return res

class Booster:
    """learner class """
    def __init__(self, params={}, cache=[], model_file = None):
        """ constructor, param: """
        for d in cache:
            assert isinstance(d, DMatrix)
        dmats = (ctypes.c_void_p  * len(cache))(*[ d.handle for d in cache])
        self.handle = ctypes.c_void_p(xglib.XGBoosterCreate(dmats, len(cache)))
        self.set_param({'seed':0})
        self.set_param(params)
        if model_file != None:
            self.load_model(model_file)
    def __del__(self):
        xglib.XGBoosterFree(self.handle)
    def set_param(self, params, pv=None):
        if isinstance(params, dict):
            for k, v in params.items():
                xglib.XGBoosterSetParam(
                    self.handle, ctypes.c_char_p(k.encode('utf-8')),
                    ctypes.c_char_p(str(v).encode('utf-8')))
        elif isinstance(params,str) and pv != None:
            xglib.XGBoosterSetParam(
                self.handle, ctypes.c_char_p(params.encode('utf-8')),
                ctypes.c_char_p(str(pv).encode('utf-8')))
        else:
            for k, v in params:
                xglib.XGBoosterSetParam(
                    self.handle, ctypes.c_char_p(k.encode('utf-8')),
                    ctypes.c_char_p(str(v).encode('utf-8')))
    def update(self, dtrain, it):
        """
        update
          dtrain: the training DMatrix
          it: current iteration number
        """
        assert isinstance(dtrain, DMatrix)
        xglib.XGBoosterUpdateOneIter(self.handle, it, dtrain.handle)
    def boost(self, dtrain, grad, hess):
        """ update """
        assert len(grad) == len(hess)
        assert isinstance(dtrain, DMatrix)
        xglib.XGBoosterBoostOneIter(self.handle, dtrain.handle,
                                    (ctypes.c_float*len(grad))(*grad),
                                    (ctypes.c_float*len(hess))(*hess),
                                    len(grad))
    def eval_set(self, evals, it = 0):
        for d in evals:
            assert isinstance(d[0], DMatrix)
            assert isinstance(d[1], str)
        dmats = (ctypes.c_void_p * len(evals) )(*[ d[0].handle for d in evals])
        evnames = (ctypes.c_char_p * len(evals))(
            * [ctypes.c_char_p(d[1].encode('utf-8')) for d in evals])
        return xglib.XGBoosterEvalOneIter(self.handle, it, dmats, evnames, len(evals))
    def eval(self, mat, name = 'eval', it = 0):
        return self.eval_set( [(mat,name)], it)
    def predict(self, data, output_margin=False):
        """
        predict with data
            data: the dmatrix storing the input
            output_margin: whether output raw margin value that is untransformed
        """
        length = ctypes.c_ulong()
        preds = xglib.XGBoosterPredict(self.handle, data.handle,
                                       int(output_margin), ctypes.byref(length))
        return ctypes2numpy(preds, length.value, 'float32')
    def save_model(self, fname):
        """ save model to file """
        xglib.XGBoosterSaveModel(self.handle, ctypes.c_char_p(fname.encode('utf-8')))
    def load_model(self, fname):
        """load model from file"""
        xglib.XGBoosterLoadModel( self.handle, ctypes.c_char_p(fname.encode('utf-8')) )
    def dump_model(self, fo, fmap=''):
        """dump model into text file"""
        if isinstance(fo,str):
            fo = open(fo,'w')
            need_close = True
        else:
            need_close = False
        ret = self.get_dump(fmap)
        for i in range(len(ret)):
            fo.write('booster[%d]:\n' %i)
            fo.write( ret[i] )
        if need_close:
            fo.close()
    def get_dump(self, fmap=''):
        """get dump of model as list of strings """
        length = ctypes.c_ulong()
        sarr = xglib.XGBoosterDumpModel(self.handle, ctypes.c_char_p(fmap.encode('utf-8')), ctypes.byref(length))
        res = []
        for i in range(length.value):
            res.append( str(sarr[i]) )
        return res
    def get_fscore(self, fmap=''):
        """ get feature importance of each feature """
        trees = self.get_dump(fmap)
        fmap = {}
        for tree in trees:
            print (tree)
            for l in tree.split('\n'):
                arr = l.split('[')
                if len(arr) == 1:
                    continue
                fid = arr[1].split(']')[0]
                fid = fid.split('<')[0]
                if fid not in fmap:
                    fmap[fid] = 1
                else:
                    fmap[fid]+= 1
        return fmap

def evaluate(bst, evals, it, feval = None):
    """evaluation on eval set"""
    if feval != None:
        res = '[%d]' % it
        for dm, evname in evals:
            name, val = feval(bst.predict(dm), dm)
            res += '\t%s-%s:%f' % (evname, name, val)
    else:
        res = bst.eval_set(evals, it)

    return res

def train(params, dtrain, num_boost_round = 10, evals = [], obj=None, feval=None):
    """ train a booster with given paramaters """
    bst = Booster(params, [dtrain]+[ d[0] for d in evals ] )
    if obj == None:
        for i in range(num_boost_round):
            bst.update( dtrain, i )
            if len(evals) != 0:
                sys.stderr.write(evaluate(bst, evals, i, feval).decode()+'\n')
    else:
        # try customized objective function
        for i in range(num_boost_round):
            pred = bst.predict( dtrain )
            grad, hess = obj( pred, dtrain )
            bst.boost( dtrain, grad, hess )
            if len(evals) != 0:
                sys.stderr.write(evaluate(bst, evals, i, feval)+'\n')
    return bst
<|endoftext|>


<commit_before>import os
import csv
import urllib2
import urllib
import json
import subprocess
from time import sleep
import glob

def write_data(data, outputpath): #This is used for two different writing functions. Make sure to change the file name and fieldnames
    ordered_fieldnames =  ["Title",	"Contract Type","Tender Number","Winning Bidder","Winning Bidder Address",	"Related affiliate corp","Date Effective","Duration",	"Contract Description",	"Estimated Value",	"Payment Variation Provisions",	"Renegotiation Provisions",	"Proccess/ Assessment Description"]

    if os.path.exists(outputpath):
        with open(outputpath, "a") as datatest:
    #csv.register_dialect("custom", delimiter="", skipinitialspace=True)
            writer = csv.writer(datatest, dialect="excel")
            try:
                writer.writerow(data)
            except UnicodeEncodeError:
                print userdataset
                pass
    else:
        with open(outputpath, "w") as datatest:
        #csv.register_dialect("custom", delimiter="", skipinitialspace=True)
            csv.DictWriter(datatest, dialect="excel", fieldnames=ordered_fieldnames).writeheader()
            writer = csv.writer(datatest, dialect="excel")
            try:
                writer.writerow(data)
            except UnicodeEncodeError:
                print i
                pass

if __name__=='__main__':
    input_dir = "/Users/sunlight/Documents/Sydney-proc/Sydney Contract PDFs"
    output_path = "/Users/sunlight/Documents/Sydney-proc/Sydney_Contracts_2013_Bulk.csv"

    for filename in os.listdir(input_dir):
        filepath = input_dir + "/" + filename
        subprocess.call(['pdftotext','-nopgbrk',"-layout", filepath])
    os.chdir(input_dir)
    for files in glob.glob("*.txt"):
        pdf_filepath = input_dir + "/" + files
        f = open(pdf_filepath, 'rU')
        content = f.readlines()
        contract_type = content[0]
        if "2" in contract_type:
            contract_type_dum = 2
        elif "1" in contract_type:
            contract_type_dum = 1
        else:
            contract_type_dum = "N/A"
        useless = ["2","","1"]
        all_lines = []
        for line in content:
            all_lines.append(line.strip())

        lines = [line for line in all_lines if line not in useless]
        doc_length=len(lines)
        field_dict = {"one":{"begin":0, "end":0}, "two":{"begin":0, "end":0}, "three":{"begin":0, "end":0}, "four":{"begin":0, "end":0}, "five":{"begin":0, "end":0},"six":{"begin":0, "end":0},"seven":{"begin":0, "end":0},"eight":{"begin":0, "end":0},"nine":{"begin":0, "end":0},"ten":{"begin":0, "end":0},"eleven":{"begin":0, "end":0},"twelve":{"begin":0, "end":0},"thirteen":{"begin":0, "end":0},"fourteen":{"begin":0, "end":0},"fifteen":{"begin":0, "end":0},"sixteen": {"begin":0, "end":0}, "seventeen":{"begin":0, "end":0}, "eighteen":{"begin":0, "end":0}}
        line_num=0
        print lines
        for line in lines:
            if "1. Tender Number:" in line:
                field_dict["one"]["begin"]=line_num
            elif "Name and business address" in line:
                field_dict["one"]["end"]=line_num
                field_dict["two"]["begin"]=line_num
            elif "3. Particulars of any related" in line:
                field_dict["two"]["end"]=line_num
            elif "benefit under the contract" in line:
                field_dict["three"]["begin"]=line_num
            elif "4. Date on which the contract" in line:
                field_dict["three"]["end"]=line_num
                field_dict["four"]["begin"]=line_num
            elif "5. Particulars of the project" in line:
                field_dict["four"]["end"]=line_num
            elif "leased under the contract:" in line:
                field_dict["five"]["begin"]=line_num
            elif "6. Estimated amount payable" in line:
                field_dict["five"]["end"]=line_num
                field_dict["six"]["begin"]=line_num
            elif "7. Description of any provisions" in line:
                field_dict["six"]["end"]=line_num
            elif "may be varied:" in line:
                field_dict["seven"]["begin"]=line_num
            elif "8. Description of any provision" in line:
                field_dict["seven"]["end"]=line_num
                field_dict["eight"]["begin"]=line_num
            elif "9. In the case of a contract" in line:
                field_dict["eight"]["end"]=line_num
            elif "assessed:" in line:
                field_dict["nine"]["begin"]=line_num
            elif "10. Description of any provision under which" in line:
                field_dict["nine"]["end"]=line_num
            elif "or maintenance services:" in line:
                field_dict["ten"]["begin"]=line_num
            elif "11. Particulars of future" in line:
                field_dict["ten"]["end"]=line_num
            elif "their proposed transfer:" in line:
                field_dict["eleven"]["begin"]=line_num
            elif "12. Particulars of future transfers" in line:
                field_dict["eleven"]["end"]=line_num
            elif "of the proposed transfer:" in line:
                field_dict["twelve"]["begin"]=line_num
            elif "13. The results of any" in line:
                field_dict["twelve"]["end"]=line_num
            elif "of Sydney:" in line:
                field_dict["thirteen"]["begin"]=line_num
            elif "14. The components and quantum" in line:
                field_dict["thirteen"]["end"]=line_num
                field_dict["fourteen"]["begin"]=line_num
            elif "15. If relevant," in line:
                field_dict["fourteen"]["end"]=line_num
            elif "sage charges):" in line:
                field_dict["fifteen"]["begin"]=line_num
            elif "16. If relevant," in line:
                field_dict["fifteen"]["end"]=line_num
            elif "assumptions involved:" in line:
                field_dict["sixteen"]["begin"]=line_num
            elif "17. Particulars as to any " in line:
                field_dict["sixteen"]["end"]=line_num
            elif "be entered into:" in line:
                field_dict["seventeen"]["begin"]=line_num
            elif "18. Particulars of any other" in line:
                field_dict["seventeen"]["end"]=line_num
                field_dict["eighteen"]["begin"]=line_num
                field_dict["eighteen"]["end"]=doc_length
            else:
                pass
            line_num += 1

        field_loc_dict = {}
        for field in field_dict:
            locations = [i for i in range(field_dict[field]["begin"]+1,field_dict[field]["end"] )]
            field_loc_dict[field] = locations
        results_dict = {}
        for field in field_loc_dict:
            line_indices = field_loc_dict[field]
            print line_indices
            ln = len(line_indices)
            if ln == 0:
                results = "Not found"

            elif field == "two":
                if ln == 2:
                    name = lines[line_indices[0]].replace("Name:", "")
                    address = lines[line_indices[1]].replace("Address:", "")
                    results_dict["name"] = name
                    results_dict["address"] = address
                elif ln < 2:
                    name = lines[line_indices[0]].replace("Name:", "")
                    results_dict["name"] = name
                    results_dict["address"] = "Not Found"
                else:
                    name = lines[line_indices[0]].replace("Name:", "")
                    multiple_line_field= []
                    line_indices.pop(0)
                    for l in line_indices:
                        multiple_line_field.append(lines[l])
                    address = "; ".join(multiple_line_field).replace("Address:", "")
                    results_dict["address"] = address
                    results_dict["name"] = name
            elif field == "four":
                if ln == 2:
                    date = lines[line_indices[0]].replace("Effective Date:", "")
                    duration = lines[line_indices[1]].replace("Duration:", "")
                    results_dict["date"] = date
                    results_dict["duration"] = duration
                elif ln < 2:
                    date = lines[line_indices[0]].replace("Effective Date:", "")
                    print date
                    results_dict["date"] = date
                    results_dict["duration"] = "Not Found"
                else:
                    date = lines[line_indices[0]].replace("Effective Date:", "")
                    multiple_line_field= []
                    line_indices.pop(0)
                    for l in line_indices:
                        multiple_line_field.append(lines[l])
                    duration = "; ".join(multiple_line_field).replace("Duration: ", "")
                    results_dict["date"] = date
                    results_dict["duration"] = duration

            elif ln <2:
                results = lines[line_indices[0]]
            else:
                multiple_line_field= []
                for l in line_indices:
                    multiple_line_field.append(lines[l])
                results = "; ".join(multiple_line_field)
            results_dict[field] = results
        data = (contract_type,contract_type_dum,results_dict["one"],results_dict["name"],results_dict["address"],results_dict["three"],results_dict["date"],results_dict["duration"],results_dict["five"],results_dict["six"],results_dict["seven"],results_dict["eight"],results_dict["nine"],results_dict["ten"],results_dict["eleven"],results_dict["twelve"],results_dict["thirteen"],results_dict["fourteen"],results_dict["fifteen"],results_dict["sixteen"],results_dict["seventeen"],results_dict["eighteen"])
        write_data(data, output_path)<commit_msg>Fixed typo<commit_after>import os
import csv
import urllib2
import urllib
import json
import subprocess
from time import sleep
import glob

def write_data(data, outputpath): #This is used for two different writing functions. Make sure to change the file name and fieldnames
    ordered_fieldnames =  ["Title",	"Contract Type","Tender Number","Winning Bidder","Winning Bidder Address",	"Related affiliate corp","Date Effective","Duration",	"Contract Description",	"Estimated Value",	"Payment Variation Provisions",	"Renegotiation Provisions",	"Process/ Assessment Description"]

    if os.path.exists(outputpath):
        with open(outputpath, "a") as datatest:
    #csv.register_dialect("custom", delimiter="", skipinitialspace=True)
            writer = csv.writer(datatest, dialect="excel")
            try:
                writer.writerow(data)
            except UnicodeEncodeError:
                print userdataset
                pass
    else:
        with open(outputpath, "w") as datatest:
        #csv.register_dialect("custom", delimiter="", skipinitialspace=True)
            csv.DictWriter(datatest, dialect="excel", fieldnames=ordered_fieldnames).writeheader()
            writer = csv.writer(datatest, dialect="excel")
            try:
                writer.writerow(data)
            except UnicodeEncodeError:
                print i
                pass

if __name__=='__main__':
    input_dir = "/Users/sunlight/Documents/Sydney-proc/Sydney Contract PDFs"
    output_path = "/Users/sunlight/Documents/Sydney-proc/Sydney_Contracts_2013_Bulk.csv"

    for filename in os.listdir(input_dir):
        filepath = input_dir + "/" + filename
        subprocess.call(['pdftotext','-nopgbrk',"-layout", filepath])
    os.chdir(input_dir)
    for files in glob.glob("*.txt"):
        pdf_filepath = input_dir + "/" + files
        f = open(pdf_filepath, 'rU')
        content = f.readlines()
        contract_type = content[0]
        if "2" in contract_type:
            contract_type_dum = 2
        elif "1" in contract_type:
            contract_type_dum = 1
        else:
            contract_type_dum = "N/A"
        useless = ["2","","1"]
        all_lines = []
        for line in content:
            all_lines.append(line.strip())

        lines = [line for line in all_lines if line not in useless]
        doc_length=len(lines)
        field_dict = {"one":{"begin":0, "end":0}, "two":{"begin":0, "end":0}, "three":{"begin":0, "end":0}, "four":{"begin":0, "end":0}, "five":{"begin":0, "end":0},"six":{"begin":0, "end":0},"seven":{"begin":0, "end":0},"eight":{"begin":0, "end":0},"nine":{"begin":0, "end":0},"ten":{"begin":0, "end":0},"eleven":{"begin":0, "end":0},"twelve":{"begin":0, "end":0},"thirteen":{"begin":0, "end":0},"fourteen":{"begin":0, "end":0},"fifteen":{"begin":0, "end":0},"sixteen": {"begin":0, "end":0}, "seventeen":{"begin":0, "end":0}, "eighteen":{"begin":0, "end":0}}
        line_num=0
        print lines
        for line in lines:
            if "1. Tender Number:" in line:
                field_dict["one"]["begin"]=line_num
            elif "Name and business address" in line:
                field_dict["one"]["end"]=line_num
                field_dict["two"]["begin"]=line_num
            elif "3. Particulars of any related" in line:
                field_dict["two"]["end"]=line_num
            elif "benefit under the contract" in line:
                field_dict["three"]["begin"]=line_num
            elif "4. Date on which the contract" in line:
                field_dict["three"]["end"]=line_num
                field_dict["four"]["begin"]=line_num
            elif "5. Particulars of the project" in line:
                field_dict["four"]["end"]=line_num
            elif "leased under the contract:" in line:
                field_dict["five"]["begin"]=line_num
            elif "6. Estimated amount payable" in line:
                field_dict["five"]["end"]=line_num
                field_dict["six"]["begin"]=line_num
            elif "7. Description of any provisions" in line:
                field_dict["six"]["end"]=line_num
            elif "may be varied:" in line:
                field_dict["seven"]["begin"]=line_num
            elif "8. Description of any provision" in line:
                field_dict["seven"]["end"]=line_num
                field_dict["eight"]["begin"]=line_num
            elif "9. In the case of a contract" in line:
                field_dict["eight"]["end"]=line_num
            elif "assessed:" in line:
                field_dict["nine"]["begin"]=line_num
            elif "10. Description of any provision under which" in line:
                field_dict["nine"]["end"]=line_num
            elif "or maintenance services:" in line:
                field_dict["ten"]["begin"]=line_num
            elif "11. Particulars of future" in line:
                field_dict["ten"]["end"]=line_num
            elif "their proposed transfer:" in line:
                field_dict["eleven"]["begin"]=line_num
            elif "12. Particulars of future transfers" in line:
                field_dict["eleven"]["end"]=line_num
            elif "of the proposed transfer:" in line:
                field_dict["twelve"]["begin"]=line_num
            elif "13. The results of any" in line:
                field_dict["twelve"]["end"]=line_num
            elif "of Sydney:" in line:
                field_dict["thirteen"]["begin"]=line_num
            elif "14. The components and quantum" in line:
                field_dict["thirteen"]["end"]=line_num
                field_dict["fourteen"]["begin"]=line_num
            elif "15. If relevant," in line:
                field_dict["fourteen"]["end"]=line_num
            elif "sage charges):" in line:
                field_dict["fifteen"]["begin"]=line_num
            elif "16. If relevant," in line:
                field_dict["fifteen"]["end"]=line_num
            elif "assumptions involved:" in line:
                field_dict["sixteen"]["begin"]=line_num
            elif "17. Particulars as to any " in line:
                field_dict["sixteen"]["end"]=line_num
            elif "be entered into:" in line:
                field_dict["seventeen"]["begin"]=line_num
            elif "18. Particulars of any other" in line:
                field_dict["seventeen"]["end"]=line_num
                field_dict["eighteen"]["begin"]=line_num
                field_dict["eighteen"]["end"]=doc_length
            else:
                pass
            line_num += 1

        field_loc_dict = {}
        for field in field_dict:
            locations = [i for i in range(field_dict[field]["begin"]+1,field_dict[field]["end"] )]
            field_loc_dict[field] = locations
        results_dict = {}
        for field in field_loc_dict:
            line_indices = field_loc_dict[field]
            print line_indices
            ln = len(line_indices)
            if ln == 0:
                results = "Not found"

            elif field == "two":
                if ln == 2:
                    name = lines[line_indices[0]].replace("Name:", "")
                    address = lines[line_indices[1]].replace("Address:", "")
                    results_dict["name"] = name
                    results_dict["address"] = address
                elif ln < 2:
                    name = lines[line_indices[0]].replace("Name:", "")
                    results_dict["name"] = name
                    results_dict["address"] = "Not Found"
                else:
                    name = lines[line_indices[0]].replace("Name:", "")
                    multiple_line_field= []
                    line_indices.pop(0)
                    for l in line_indices:
                        multiple_line_field.append(lines[l])
                    address = "; ".join(multiple_line_field).replace("Address:", "")
                    results_dict["address"] = address
                    results_dict["name"] = name
            elif field == "four":
                if ln == 2:
                    date = lines[line_indices[0]].replace("Effective Date:", "")
                    duration = lines[line_indices[1]].replace("Duration:", "")
                    results_dict["date"] = date
                    results_dict["duration"] = duration
                elif ln < 2:
                    date = lines[line_indices[0]].replace("Effective Date:", "")
                    print date
                    results_dict["date"] = date
                    results_dict["duration"] = "Not Found"
                else:
                    date = lines[line_indices[0]].replace("Effective Date:", "")
                    multiple_line_field= []
                    line_indices.pop(0)
                    for l in line_indices:
                        multiple_line_field.append(lines[l])
                    duration = "; ".join(multiple_line_field).replace("Duration: ", "")
                    results_dict["date"] = date
                    results_dict["duration"] = duration

            elif ln <2:
                results = lines[line_indices[0]]
            else:
                multiple_line_field= []
                for l in line_indices:
                    multiple_line_field.append(lines[l])
                results = "; ".join(multiple_line_field)
            results_dict[field] = results
        data = (contract_type,contract_type_dum,results_dict["one"],results_dict["name"],results_dict["address"],results_dict["three"],results_dict["date"],results_dict["duration"],results_dict["five"],results_dict["six"],results_dict["seven"],results_dict["eight"],results_dict["nine"],results_dict["ten"],results_dict["eleven"],results_dict["twelve"],results_dict["thirteen"],results_dict["fourteen"],results_dict["fifteen"],results_dict["sixteen"],results_dict["seventeen"],results_dict["eighteen"])
        write_data(data, output_path)<|endoftext|>