<commit_before>#ffgrep - A Python-based file search utility
import argparse
import sys
import os
import subprocess

items_per_page = 10
line_length = 50

def setup_arg_parser():
    parser = argparse.ArgumentParser(description="Search for a string in the specified file or path.")

    parser.add_argument('pattern', metavar='S', type=str,
                        help='search pattern')

    parser.add_argument('paths', default='.', nargs='*',
                        help='file(s) and/or path(s) to search (default is local directory)')

    parser.add_argument('-i','--ignore-case', action='store_const', const=True,
                        help='ignore case in the search term')

    parser.add_argument('-n','--no-subdir', action='store_const', const=True,
                        help='do not look in sub-directories of the search directories')
    return parser

def get_files(path, include_subs=True):
    file_list = []

    if (include_subs):
        for root, dirs, files in os.walk(path):
            for file in files:
                file_list.append(os.path.join(root,file))
    else:
        for file in os.listdir(path):
            file_list.append(os.path.join(path,file))

    return file_list

def find_matching_lines(filepath, search_term, ignore_case=False):
    f = open(filepath)
    data = f.readlines()

    if (ignore_case):
        matches  = [(num+1,line.strip(),filepath) for num,line in enumerate(data) if
                    line.lower().find(search_term.lower()) >= 0]
    else:
        matches  = [(num+1,line.strip(),filepath) for num,line in enumerate(data) if line.find(search_term) >= 0]

    f.close()
    return matches

def print_results(results, search_term, start_line=1, ignore_case=True):
    for (num,line_match) in enumerate(results,start_line):
        filename = line_match[2]
        if ("\\" in filename):
            filename = filename.split("\\")[-1]

        line_data = str(line_match[1]).strip()

        if (ignore_case):
            item_idx = line_data.lower().find(search_term.lower())
        else:
            item_idx = line_data.lower().find(search_term.lower())

        start_idx = item_idx - (line_length - len(search_term))/2
        end_idx = item_idx + len(search_term) + (line_length - len(search_term))/2

        if (start_idx < 0):
            start_idx = 0
            end_idx = min(line_length, len(line_data))
        elif (end_idx > len(line_data)):
            start_idx -= (line_length - len(line_data))
            end_idx = min(line_length, len(line_data))

        if (start_idx < 0):
            start_idx = 0

        line_data = line_data[start_idx:end_idx]

        print "{3} : {0}[{1}]: {2}".format(filename, line_match[0], line_data, num)

if __name__ == '__main__':
    parser = setup_arg_parser()
    parser.set_defaults()
    args = parser.parse_args()

    search_matches = []

    for path in args.paths:
        print path
        if (os.path.isfile(path)):
            match_set = [find_matching_lines(path, args.pattern, True)]
        elif (os.path.isdir(path)):
            match_set = map(lambda x: find_matching_lines(x, args.pattern, True), get_files(path))
        else:
            match_set = []

        for match in match_set:
            map(lambda x: search_matches.append(x),match)

    current_page = 0
    print_results(search_matches[0:min(items_per_page, len(search_matches))], args.pattern, 1)

    while(True):
        print "Items [{0}:{1} of {2}]. Enter a match number, N for next page or X to exit"\
            .format(current_page*items_per_page+1,min((current_page+1)*items_per_page,
                    len(search_matches)),len(search_matches))

        openLine = sys.stdin.readline()

        if (openLine[0] == "n" or openLine[0] == "N"):
            current_page += 1
            if (len(search_matches) > current_page*items_per_page):
               print_results(search_matches[current_page*items_per_page:min((current_page+1)*items_per_page,
                             len(search_matches))], args.pattern, current_page*items_per_page+1)
            else:
                exit()
        elif (openLine[0] == "x" or openLine[0] == "X"):
            exit()
        else:
            try:
                viewLine = int(openLine)
            except:
                continue

            if (viewLine > 0 and viewLine <= len(search_matches)):
                os.system("\"C:\\Program Files (x86)\\Notepad++\\notepad++.exe\" {0} -n{1}".format(
                    search_matches[viewLine-1][2],search_matches[viewLine-1][0]))
                exit()

<commit_msg>Changed items per page and line length into command line parameters.<commit_after>#ffgrep - A Python-based file search utility
import argparse
import sys
import os

DEFAULT_ITEMS_PER_PAGE = 10
DEFAULT_LINE_LENGTH = 50

items_per_page = DEFAULT_ITEMS_PER_PAGE
line_length = DEFAULT_LINE_LENGTH

def setup_arg_parser():
    parser = argparse.ArgumentParser(description="Search for a string in the specified file or path.")

    parser.add_argument('pattern', metavar='pattern', type=str,
                        help='search pattern')

    parser.add_argument('paths', default='.', nargs='*',
                        help='file(s) and/or path(s) to search (default is local directory)')

    parser.add_argument('-i','--ignore-case', action='store_const', const=True,
                        help='ignore case in the search term')

    parser.add_argument('-n','--no-subdir', action='store_const', const=True,
                        help='do not look in sub-directories of the search directories')

    parser.add_argument('-p', '--items_per_page', nargs='?', type=int,
                        default=DEFAULT_ITEMS_PER_PAGE, help='number of items to list per page')

    parser.add_argument('-l', '--line_length', nargs='?', type=int,
                        default=DEFAULT_ITEMS_PER_PAGE, help='length of each match result line')

    return parser

def get_files(path, include_subs=True):
    file_list = []

    if (include_subs):
        for root, dirs, files in os.walk(path):
            for file in files:
                file_list.append(os.path.join(root,file))
    else:
        for file in os.listdir(path):
            file_list.append(os.path.join(path,file))

    return file_list

def find_matching_lines(filepath, search_term, ignore_case=False):
    f = open(filepath)
    data = f.readlines()

    if (ignore_case):
        matches  = [(num+1,line.strip(),filepath) for num,line in enumerate(data) if
                    line.lower().find(search_term.lower()) >= 0]
    else:
        matches  = [(num+1,line.strip(),filepath) for num,line in enumerate(data) if line.find(search_term) >= 0]

    f.close()
    return matches

def print_results(results, search_term, start_line=1, ignore_case=True):
    for (num,line_match) in enumerate(results,start_line):
        filename = line_match[2]
        if ("\\" in filename):
            filename = filename.split("\\")[-1]

        line_data = str(line_match[1]).strip()

        if (ignore_case):
            item_idx = line_data.lower().find(search_term.lower())
        else:
            item_idx = line_data.lower().find(search_term.lower())

        start_idx = item_idx - (line_length - len(search_term))/2
        end_idx = item_idx + len(search_term) + (line_length - len(search_term))/2

        if (start_idx < 0):
            start_idx = 0
            end_idx = min(line_length, len(line_data))
        elif (end_idx > len(line_data)):
            start_idx -= (line_length - len(line_data))
            end_idx = min(line_length, len(line_data))

        if (start_idx < 0):
            start_idx = 0

        line_data = line_data[start_idx:end_idx]

        print "{3} : {0}[{1}]: {2}".format(filename, line_match[0], line_data, num)

if __name__ == '__main__':
    parser = setup_arg_parser()
    parser.set_defaults()
    args = parser.parse_args()

    if (args.items_per_page == None):
        items_per_page = DEFAULT_ITEMS_PER_PAGE
    else:
        items_per_page = int(args.items_per_page)

    if (args.line_length == None):
        line_length = DEFAULT_LINE_LENGTH
    else:
        line_length = int(line_length)

    search_matches = []

    for path in args.paths:
        print path
        if (os.path.isfile(path)):
            match_set = [find_matching_lines(path, args.pattern, True)]
        elif (os.path.isdir(path)):
            match_set = map(lambda x: find_matching_lines(x, args.pattern, True), get_files(path))
        else:
            match_set = []

        for match in match_set:
            map(lambda x: search_matches.append(x),match)

    current_page = 0
    print_results(search_matches[0:min(items_per_page, len(search_matches))], args.pattern, 1)

    while(True):
        print "Items [{0}:{1} of {2}]. Enter a match number, N for next page or X to exit"\
            .format(current_page*items_per_page+1,min((current_page+1)*items_per_page,
                    len(search_matches)),len(search_matches))

        openLine = sys.stdin.readline()

        if (openLine[0] == "n" or openLine[0] == "N"):
            current_page += 1
            if (len(search_matches) > current_page*items_per_page):
               print_results(search_matches[current_page*items_per_page:min((current_page+1)*items_per_page,
                             len(search_matches))], args.pattern, current_page*items_per_page+1)
            else:
                exit()
        elif (openLine[0] == "x" or openLine[0] == "X"):
            exit()
        else:
            try:
                viewLine = int(openLine)
            except:
                continue

            if (viewLine > 0 and viewLine <= len(search_matches)):
                os.system("\"C:\\Program Files (x86)\\Notepad++\\notepad++.exe\" {0} -n{1}".format(
                    search_matches[viewLine-1][2],search_matches[viewLine-1][0]))
                exit()

<|endoftext|>


<commit_before>from flask import request

class BaseModel:
    def __init__(self):
        """This is the base model for all the models

        used to handle requests and other common stuff
        """
        if request.json:  # check request.json (ajax request) first
            self._option = request.json
        elif request.form:  # fall back to request.form (form request)
            self._option = request.form
        else:  # no data send to the back end
            self._option = None

    @property
    def front_end_data(self) -> dict:
        return self._option

<commit_msg>fix pep8<commit_after>from flask import request

class BaseModel:
    def __init__(self):
        """This is the base model for all the models

        used to handle requests and other common stuff
        """
        if request.json:  # check request.json (ajax request) first
            self._option = request.json
        elif request.form:  # fall back to request.form (form request)
            self._option = request.form
        else:  # no data send to the back end
            self._option = None

    @property
    def front_end_data(self) -> dict:
        return self._option
<|endoftext|>


<commit_before>from IPython.core.magic import (Magics, magics_class, line_magic,
                                cell_magic, line_cell_magic)
from IPython import get_ipython
from IPython.display import publish_display_data, display, Javascript
from IPython.core.formatters import DisplayFormatter

from pprint import pprint

import argparse
import os
import sys
import shlex
import json
from time import sleep

if sys.version.startswith("3"):
    unicode = str

from twisted.internet.error import ReactorAlreadyInstalledError
from zmq.eventloop import ioloop
ioloop.install()
from tornado.ioloop import IOLoop
import tornado.platform.twisted
try:
    tornado.platform.twisted.install()
except ReactorAlreadyInstalledError:
    pass

from twisted.python import log
from twisted.internet import reactor, threads
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred, CancelledError
from autobahn.twisted.wamp import ApplicationSession
from autobahn import wamp
from autobahn.wamp.exception import ApplicationError
from autobahn.websocket.util import parse_url
from autobahn.twisted.util import sleep as absleep

from juno_magic.runner import JunoRunner

from collections import deque

from sh import wampify

import requests
import re

JUNO_KERNEL_URI = "https://juno.timbr.io/api/kernels/list"

def publish_to_display(obj):
    output, _ = DisplayFormatter().format(obj)
    publish_display_data(output)

def build_display_data(obj):
    output = {"text/plain": repr(obj)}
    methods = dir(obj)
    if "_repr_html_" in methods:
        output["text/html"] = obj._repr_html_()
    if "_repr_javascript_" in methods:
        output["application/javascript"] = obj._repr_javascript_()
    return output

def build_bridge_class(magics_instance):
    class WampConnectionComponent(ApplicationSession):
        _wamp_prefix = ""
        _ipython = get_ipython()
        _msg_id_lut = deque(maxlen=10)
        _machine_callbacks = []
        _iopub_sub = None
        _machine_sub = None

        @inlineCallbacks
        def reset_prefix(self):
            if self._iopub_sub:
                yield self._iopub_sub.unsubscribe()
                self._iopub_sub = None
            if self._machine_sub:
                yield self._machine_sub.unsubscribe()
                self._machine_sub = None
            returnValue(None)

        @inlineCallbacks
        def set_prefix(self, prefix):
            yield self.reset_prefix()
            self._wamp_prefix = unicode(prefix)
            self._iopub_sub = yield self.subscribe(self.on_iopub, u".".join([self._wamp_prefix, u"iopub"]))
            # try:
            #     yield self.subscribe(self.on_machine, u".".join([self._wamp_prefix, u"machine"]))
            # except:
            #     # whatever
            #     pass

        def on_iopub(self, msg):
            if msg["msg_type"] == "error":
                publish_display_data({"text/plain": "{} - {}\n{}".format(msg["content"]["ename"],
                                                                         msg["content"]["evalue"],
                                                                         "\n".join(msg["content"]["traceback"]))},
                                     metadata={"echo": True})
            elif msg["msg_type"] == "stream":
                publish_display_data({"text/plain": msg["content"]["text"]}, metadata={"echo": True})
            elif msg["msg_type"] == "display_data":
                publish_display_data(msg["content"]["data"], metadata={"echo": True})
            elif msg["msg_type"] == "execute_result":
                publish_display_data(msg["content"]["data"], metadata={"echo": True})
            elif msg["msg_type"] == "status":
                display(Javascript('setJunoStatus("{}")'.format(msg["content"]["execution_state"])))
            elif msg["msg_type"] in ["execute_input", "execution_state", "status"]:
                pass
            else:
                pprint(msg)

        def on_machine(self, msg):
            # print("[on_machine] {}".format(msg))
            # print("[on_machine] {}".format(self._machine_callbacks))
            bad_callbacks = []
            for cb in self._machine_callbacks:
                try:
                    callback = self._ipython.user_ns[cb]
                    # print(callback)
                except (IndexError, KeyError) as ie:
                    bad_callbacks.append(cb)
                try:
                    callback(msg)
                except Exception as e:
                    log.msg("Exception in callback '{}'".format(cb))
                    log.msg(str(e))
            for cb in bad_callbacks:
                self._machine_callbacks.remove(cb)

        def add_machine_callback(self, cb_str):
            if cb_str not in self._machine_callbacks:
                self._machine_callbacks.append(cb_str)

        @inlineCallbacks
        def execute(self, *args, **kwargs):
            result = yield self.call(".".join([self._wamp_prefix, u"execute"]), *args, **kwargs)
            # result is the remote execute_request msg_id,
            returnValue(result)

        @inlineCallbacks
        def onJoin(self, details):
            log.msg("[onJoin] Registering RPC methods...")
            yield self.register(self)
            log.msg("[onJoin] ...done.")
            log.msg("[onJoin] Checking in with Magics class")
            magics_instance.set_connection(self)
            log.msg("[onJoin] ...done.")
            print("Successfully connected to {}".format(magics_instance._router_url))
            if magics_instance._kernel_prefix:
                print("Attempting to reconnect to {}".format(magics_instance._kernel_prefix))
                yield self.set_prefix(magics_instance._kernel_prefix)
                print("Reconnected to kernel prefix {}".format(magics_instance._kernel_prefix))
            returnValue(None)

        def onLeave(self, details):
            log.msg("[WampConnectionComponent] onLeave()")
            log.msg("details: {}".format(str(details)))
            super(self.__class__, self).onLeave(details)

        @inlineCallbacks
        def onDisconnect(self):
            # onDisconnect we should just set the connection to None so that we know to reconnect
            # next time connect is called
            magics_instance.set_connection(None)

    return WampConnectionComponent

@magics_class
class JunoMagics(Magics):
    def __init__(self, shell):
        super(JunoMagics, self).__init__(shell)
        self._router_url = os.environ.get("JUPYTER_WAMP_ROUTER", "wss://juno.timbr.io/wamp/route")
        self._realm = os.environ.get("JUPYTER_WAMP_REALM", "jupyter")
        self._wamp = None
        self._wamp_runner = None
        self._kernel_prefix = None
        # NOTE: this strategy only seems to work in kernels launched by the notebook server
        self._connection_file = get_ipython().config["IPKernelApp"]["connection_file"]
        self._token = os.environ.get("JUNO_AUTH_TOKEN")
        self._sp = None
        self._connected = None

        # set local kernel key
        with open(self._connection_file) as f:
            config = json.load(f)
            self._kernel_key = config["key"]

        self._parser = self.generate_parser()

    def set_connection(self, wamp_connection):
        log.msg("[set_connection] Connection component set.")
        self._wamp = wamp_connection
        if wamp_connection is not None:
            self._connected.callback(wamp_connection)

    def generate_parser(self):
        parser = argparse.ArgumentParser(prog="juno")

        subparsers = parser.add_subparsers()
        token_parser = subparsers.add_parser("token", help="Set an OAuth token for WAMP router access")
        token_parser.add_argument("token", help="OAuth token for WAMP router access", nargs="?")
        token_parser.set_defaults(fn=self.token)
        connect_parser = subparsers.add_parser("connect", help="[Re]-connect to the WAMP router")
        connect_parser.add_argument("wamp_url", help="WAMP router url to join", default=self._router_url, nargs="?")
        connect_parser.add_argument("--reconnect", help="Force reconnection even if we don't need to.", action="store_true")
        connect_parser.set_defaults(fn=self.connect)
        list_parser = subparsers.add_parser("list", help="List registered kernel prefixes")
        list_parser.add_argument("--details", help="Display detailed information about existing kernels", action="store_true")
        list_parser.add_argument("--raw", help="Display raw kernel and prefix information about existing kernels", action="store_true")
        list_parser.set_defaults(fn=self.list)
        select_parser = subparsers.add_parser("select", help="Select a remote kernel to make active")
        select_parser.add_argument("kernel", help="Kernel name or prefix id for accessing the remote kernel", nargs="?")
        select_parser.set_defaults(fn=self.select)
        start_bridge_parser = subparsers.add_parser("start_bridge", help="Expose this kernel over WAMP for remote access")
        start_bridge_parser.add_argument("wamp_url", help="WAMP router url to expose over", default=self._router_url, nargs="?")
        start_bridge_parser.add_argument("--wamp-realm", help="WAMP realm", default="jupyter")
        start_bridge_parser.add_argument("--token", help="Authentication token", default=self._token, nargs="?")
        start_bridge_parser.set_defaults(fn=self.start_bridge)
        stop_bridge_parser = subparsers.add_parser("stop_bridge", help="Stop exposing this kernel over WAMP")
        stop_bridge_parser.set_defaults(fn=self.stop_bridge)
        subscribe_parser = subparsers.add_parser("subscribe", help="Register a callback callable in the user namespace that gets called in response to timbr-machine messages")
        subscribe_parser.add_argument("callback", help="Name of the callback function")
        subscribe_parser.set_defaults(fn=self.subscribe)
        execute_parser = subparsers.add_parser("execute", help="Evaluate code on remote kernel")
        execute_parser.add_argument("prefix", help="Prefix for accessing the remote kernel", nargs="?")
        execute_parser.set_defaults(fn=self.execute)
        return parser

    @line_cell_magic
    def juno(self, line, cell=None):
        try:
            input_args = shlex.split(line)
            if cell is not None:
                input_args.insert(0, "execute")
            args, extra = self._parser.parse_known_args(input_args)
            output = args.fn(cell=cell, **vars(args))
            if isinstance(output, Deferred):
                output.addCallback(lambda x: publish_to_display(x) if x is not None else "[muted]")
            else:
                return output
        except SystemExit:
            pass

    def token(self, token, **kwargs):
        self._token = token

    def connect(self, wamp_url, reconnect=False, **kwargs):
        # NOTE: we would like connect to return immediately if there is an active connection, disconnect and
        # connect if the connection url has changed, or reconnect if the connection has dropped
        if (wamp_url != self._router_url) or reconnect:
            # NOTE: this means that connect was called with either a new url or we are forcibly reconnecting
            try:
                self._wamp_runner.cancel()
                self.set_connection(None)
            except (CancelledError, AttributeError):
                # NOTE: this means self._wamp_runner was already set to None, or already cancelled.
                pass
        if self._wamp is None:
            # NOTE: this means we have dropped the connection (ie onDisconnect has been called), so we'll make
            # a new one.
            self._connected = Deferred() # allocate a new deferred
            self._router_url = wamp_url
            _wamp_application_runner = JunoRunner(url=unicode(self._router_url), realm=unicode(self._realm), headers={"Authorization": "Bearer {}".format(self._token)})
            self._wamp_runner = _wamp_application_runner.run(build_bridge_class(self), start_reactor=False) # -> returns a deferred
            log.msg("Connecting to router: {}".format(self._router_url))
            log.msg("  Project Realm: {}".format(self._realm))
        return self._connected # either the new or the old deferred, depending on if we have reconnected or not

    def start_bridge(self, wamp_url, wamp_realm="jupyter", token=None, **kwargs):
        self.stop_bridge()
        if token is None:
            token = self._token
        self._sp = wampify(self._connection_file, "--wamp-url", wamp_url, "--token", token, _bg=True)
        sleep(1)
        if self._sp.process.is_alive():
            print("Bridge Running")
        else:
            print(self._sp.stderr)

    def stop_bridge(self, **kwargs):
        try:
            self._sp.process.terminate()
            print("WAMP bridge exposure process terminated successfully")
        except AttributeError as ae:
            pass
        self._sp = None

    @inlineCallbacks
    def list(self, raw=False, **kwargs):
        yield self.connect(self._router_url)
        try:
            output = yield self._wamp.call(u"io.timbr.kernel.list")
            try:
                output.remove(self._kernel_key)
            except ValueError:
                # kernel key doesn't exist in the list
                pass
        except ApplicationError:
            output = []
        if raw is not True:
            prefix_map = yield threads.deferToThread(self._get_kernel_names, output, details=kwargs.get('details'))
            returnValue(prefix_map)
        else:
            returnValue(output)

    @inlineCallbacks
    def select(self, kernel, **kwargs):
        yield self.connect(self._router_url)
        @inlineCallbacks
        def _select(prefix):
            yield self._wamp.reset_prefix()
            if self._kernel_prefix:
                print("Successfully unsubscribed from prefix {}".format(self._kernel_prefix))
            else:
                print("No previous subscriptions")
            self._kernel_prefix = prefix
            yield self._wamp.set_prefix(prefix)
            print("Kernel selected [{}]".format(prefix))

        prefix_list = yield self.list(raw=True)
        is_kernel_key = lambda t: re.match(r'io\.timbr\.kernel\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', t)
        if not is_kernel_key(kernel):
            prefix_map = yield threads.deferToThread(self._get_kernel_names, prefix_list, details=True)
            if kernel not in prefix_map:
                print("Kernel not in prefix map")
                returnValue(None)
            else:
                kernel = prefix_map[kernel]
        if kernel in prefix_list:
            if kernel != self._kernel_prefix:
                yield _select(kernel)
            else:
                print("Kernel already selected")
        else:
            print("Kernel not available")

    @inlineCallbacks
    def subscribe(self, callback, **kwargs):
        # NOTE: callback is a string
        yield self.connect(self._router_url)
        yield self._wamp.add_machine_callback(callback)

    @inlineCallbacks
    def execute(self, cell, prefix=None, **kwargs):
        yield self.connect(self._router_url)
        if prefix is not None:
            output = yield self._wamp.call(".".join([prefix, "execute"]), cell)
        output = yield self._wamp.call(".".join([self._kernel_prefix, "execute"]), cell)

    def _get_kernel_names(self, prefix_list, details=False):
        headers = {"Authorization": "Bearer {}".format(self._token)}
        payload = {"addresses": [prefix.split(".")[-1] for prefix in prefix_list]}
        r = requests.post(JUNO_KERNEL_URI, headers=headers, data=payload)
        if details:
            prefix_map = {str(v): ".".join(['io.timbr.kernel', str(k)]) for k, v in r.json().iteritems()}
        else:
            prefix_map = [str(v) for k, v in r.json().iteritems()]
        return prefix_map
<commit_msg>remove status from 2nd logic check<commit_after>from IPython.core.magic import (Magics, magics_class, line_magic,
                                cell_magic, line_cell_magic)
from IPython import get_ipython
from IPython.display import publish_display_data, display, Javascript
from IPython.core.formatters import DisplayFormatter

from pprint import pprint

import argparse
import os
import sys
import shlex
import json
from time import sleep

if sys.version.startswith("3"):
    unicode = str

from twisted.internet.error import ReactorAlreadyInstalledError
from zmq.eventloop import ioloop
ioloop.install()
from tornado.ioloop import IOLoop
import tornado.platform.twisted
try:
    tornado.platform.twisted.install()
except ReactorAlreadyInstalledError:
    pass

from twisted.python import log
from twisted.internet import reactor, threads
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred, CancelledError
from autobahn.twisted.wamp import ApplicationSession
from autobahn import wamp
from autobahn.wamp.exception import ApplicationError
from autobahn.websocket.util import parse_url
from autobahn.twisted.util import sleep as absleep

from juno_magic.runner import JunoRunner

from collections import deque

from sh import wampify

import requests
import re

JUNO_KERNEL_URI = "https://juno.timbr.io/api/kernels/list"

def publish_to_display(obj):
    output, _ = DisplayFormatter().format(obj)
    publish_display_data(output)

def build_display_data(obj):
    output = {"text/plain": repr(obj)}
    methods = dir(obj)
    if "_repr_html_" in methods:
        output["text/html"] = obj._repr_html_()
    if "_repr_javascript_" in methods:
        output["application/javascript"] = obj._repr_javascript_()
    return output

def build_bridge_class(magics_instance):
    class WampConnectionComponent(ApplicationSession):
        _wamp_prefix = ""
        _ipython = get_ipython()
        _msg_id_lut = deque(maxlen=10)
        _machine_callbacks = []
        _iopub_sub = None
        _machine_sub = None

        @inlineCallbacks
        def reset_prefix(self):
            if self._iopub_sub:
                yield self._iopub_sub.unsubscribe()
                self._iopub_sub = None
            if self._machine_sub:
                yield self._machine_sub.unsubscribe()
                self._machine_sub = None
            returnValue(None)

        @inlineCallbacks
        def set_prefix(self, prefix):
            yield self.reset_prefix()
            self._wamp_prefix = unicode(prefix)
            self._iopub_sub = yield self.subscribe(self.on_iopub, u".".join([self._wamp_prefix, u"iopub"]))
            # try:
            #     yield self.subscribe(self.on_machine, u".".join([self._wamp_prefix, u"machine"]))
            # except:
            #     # whatever
            #     pass

        def on_iopub(self, msg):
            if msg["msg_type"] == "error":
                publish_display_data({"text/plain": "{} - {}\n{}".format(msg["content"]["ename"],
                                                                         msg["content"]["evalue"],
                                                                         "\n".join(msg["content"]["traceback"]))},
                                     metadata={"echo": True})
            elif msg["msg_type"] == "stream":
                publish_display_data({"text/plain": msg["content"]["text"]}, metadata={"echo": True})
            elif msg["msg_type"] == "display_data":
                publish_display_data(msg["content"]["data"], metadata={"echo": True})
            elif msg["msg_type"] == "execute_result":
                publish_display_data(msg["content"]["data"], metadata={"echo": True})
            elif msg["msg_type"] == "status":
                display(Javascript('setJunoStatus("{}")'.format(msg["content"]["execution_state"])))
            elif msg["msg_type"] in ["execute_input", "execution_state"]:
                pass
            else:
                pprint(msg)

        def on_machine(self, msg):
            # print("[on_machine] {}".format(msg))
            # print("[on_machine] {}".format(self._machine_callbacks))
            bad_callbacks = []
            for cb in self._machine_callbacks:
                try:
                    callback = self._ipython.user_ns[cb]
                    # print(callback)
                except (IndexError, KeyError) as ie:
                    bad_callbacks.append(cb)
                try:
                    callback(msg)
                except Exception as e:
                    log.msg("Exception in callback '{}'".format(cb))
                    log.msg(str(e))
            for cb in bad_callbacks:
                self._machine_callbacks.remove(cb)

        def add_machine_callback(self, cb_str):
            if cb_str not in self._machine_callbacks:
                self._machine_callbacks.append(cb_str)

        @inlineCallbacks
        def execute(self, *args, **kwargs):
            result = yield self.call(".".join([self._wamp_prefix, u"execute"]), *args, **kwargs)
            # result is the remote execute_request msg_id,
            returnValue(result)

        @inlineCallbacks
        def onJoin(self, details):
            log.msg("[onJoin] Registering RPC methods...")
            yield self.register(self)
            log.msg("[onJoin] ...done.")
            log.msg("[onJoin] Checking in with Magics class")
            magics_instance.set_connection(self)
            log.msg("[onJoin] ...done.")
            print("Successfully connected to {}".format(magics_instance._router_url))
            if magics_instance._kernel_prefix:
                print("Attempting to reconnect to {}".format(magics_instance._kernel_prefix))
                yield self.set_prefix(magics_instance._kernel_prefix)
                print("Reconnected to kernel prefix {}".format(magics_instance._kernel_prefix))
            returnValue(None)

        def onLeave(self, details):
            log.msg("[WampConnectionComponent] onLeave()")
            log.msg("details: {}".format(str(details)))
            super(self.__class__, self).onLeave(details)

        @inlineCallbacks
        def onDisconnect(self):
            # onDisconnect we should just set the connection to None so that we know to reconnect
            # next time connect is called
            magics_instance.set_connection(None)

    return WampConnectionComponent

@magics_class
class JunoMagics(Magics):
    def __init__(self, shell):
        super(JunoMagics, self).__init__(shell)
        self._router_url = os.environ.get("JUPYTER_WAMP_ROUTER", "wss://juno.timbr.io/wamp/route")
        self._realm = os.environ.get("JUPYTER_WAMP_REALM", "jupyter")
        self._wamp = None
        self._wamp_runner = None
        self._kernel_prefix = None
        # NOTE: this strategy only seems to work in kernels launched by the notebook server
        self._connection_file = get_ipython().config["IPKernelApp"]["connection_file"]
        self._token = os.environ.get("JUNO_AUTH_TOKEN")
        self._sp = None
        self._connected = None

        # set local kernel key
        with open(self._connection_file) as f:
            config = json.load(f)
            self._kernel_key = config["key"]

        self._parser = self.generate_parser()

    def set_connection(self, wamp_connection):
        log.msg("[set_connection] Connection component set.")
        self._wamp = wamp_connection
        if wamp_connection is not None:
            self._connected.callback(wamp_connection)

    def generate_parser(self):
        parser = argparse.ArgumentParser(prog="juno")

        subparsers = parser.add_subparsers()
        token_parser = subparsers.add_parser("token", help="Set an OAuth token for WAMP router access")
        token_parser.add_argument("token", help="OAuth token for WAMP router access", nargs="?")
        token_parser.set_defaults(fn=self.token)
        connect_parser = subparsers.add_parser("connect", help="[Re]-connect to the WAMP router")
        connect_parser.add_argument("wamp_url", help="WAMP router url to join", default=self._router_url, nargs="?")
        connect_parser.add_argument("--reconnect", help="Force reconnection even if we don't need to.", action="store_true")
        connect_parser.set_defaults(fn=self.connect)
        list_parser = subparsers.add_parser("list", help="List registered kernel prefixes")
        list_parser.add_argument("--details", help="Display detailed information about existing kernels", action="store_true")
        list_parser.add_argument("--raw", help="Display raw kernel and prefix information about existing kernels", action="store_true")
        list_parser.set_defaults(fn=self.list)
        select_parser = subparsers.add_parser("select", help="Select a remote kernel to make active")
        select_parser.add_argument("kernel", help="Kernel name or prefix id for accessing the remote kernel", nargs="?")
        select_parser.set_defaults(fn=self.select)
        start_bridge_parser = subparsers.add_parser("start_bridge", help="Expose this kernel over WAMP for remote access")
        start_bridge_parser.add_argument("wamp_url", help="WAMP router url to expose over", default=self._router_url, nargs="?")
        start_bridge_parser.add_argument("--wamp-realm", help="WAMP realm", default="jupyter")
        start_bridge_parser.add_argument("--token", help="Authentication token", default=self._token, nargs="?")
        start_bridge_parser.set_defaults(fn=self.start_bridge)
        stop_bridge_parser = subparsers.add_parser("stop_bridge", help="Stop exposing this kernel over WAMP")
        stop_bridge_parser.set_defaults(fn=self.stop_bridge)
        subscribe_parser = subparsers.add_parser("subscribe", help="Register a callback callable in the user namespace that gets called in response to timbr-machine messages")
        subscribe_parser.add_argument("callback", help="Name of the callback function")
        subscribe_parser.set_defaults(fn=self.subscribe)
        execute_parser = subparsers.add_parser("execute", help="Evaluate code on remote kernel")
        execute_parser.add_argument("prefix", help="Prefix for accessing the remote kernel", nargs="?")
        execute_parser.set_defaults(fn=self.execute)
        return parser

    @line_cell_magic
    def juno(self, line, cell=None):
        try:
            input_args = shlex.split(line)
            if cell is not None:
                input_args.insert(0, "execute")
            args, extra = self._parser.parse_known_args(input_args)
            output = args.fn(cell=cell, **vars(args))
            if isinstance(output, Deferred):
                output.addCallback(lambda x: publish_to_display(x) if x is not None else "[muted]")
            else:
                return output
        except SystemExit:
            pass

    def token(self, token, **kwargs):
        self._token = token

    def connect(self, wamp_url, reconnect=False, **kwargs):
        # NOTE: we would like connect to return immediately if there is an active connection, disconnect and
        # connect if the connection url has changed, or reconnect if the connection has dropped
        if (wamp_url != self._router_url) or reconnect:
            # NOTE: this means that connect was called with either a new url or we are forcibly reconnecting
            try:
                self._wamp_runner.cancel()
                self.set_connection(None)
            except (CancelledError, AttributeError):
                # NOTE: this means self._wamp_runner was already set to None, or already cancelled.
                pass
        if self._wamp is None:
            # NOTE: this means we have dropped the connection (ie onDisconnect has been called), so we'll make
            # a new one.
            self._connected = Deferred() # allocate a new deferred
            self._router_url = wamp_url
            _wamp_application_runner = JunoRunner(url=unicode(self._router_url), realm=unicode(self._realm), headers={"Authorization": "Bearer {}".format(self._token)})
            self._wamp_runner = _wamp_application_runner.run(build_bridge_class(self), start_reactor=False) # -> returns a deferred
            log.msg("Connecting to router: {}".format(self._router_url))
            log.msg("  Project Realm: {}".format(self._realm))
        return self._connected # either the new or the old deferred, depending on if we have reconnected or not

    def start_bridge(self, wamp_url, wamp_realm="jupyter", token=None, **kwargs):
        self.stop_bridge()
        if token is None:
            token = self._token
        self._sp = wampify(self._connection_file, "--wamp-url", wamp_url, "--token", token, _bg=True)
        sleep(1)
        if self._sp.process.is_alive():
            print("Bridge Running")
        else:
            print(self._sp.stderr)

    def stop_bridge(self, **kwargs):
        try:
            self._sp.process.terminate()
            print("WAMP bridge exposure process terminated successfully")
        except AttributeError as ae:
            pass
        self._sp = None

    @inlineCallbacks
    def list(self, raw=False, **kwargs):
        yield self.connect(self._router_url)
        try:
            output = yield self._wamp.call(u"io.timbr.kernel.list")
            try:
                output.remove(self._kernel_key)
            except ValueError:
                # kernel key doesn't exist in the list
                pass
        except ApplicationError:
            output = []
        if raw is not True:
            prefix_map = yield threads.deferToThread(self._get_kernel_names, output, details=kwargs.get('details'))
            returnValue(prefix_map)
        else:
            returnValue(output)

    @inlineCallbacks
    def select(self, kernel, **kwargs):
        yield self.connect(self._router_url)
        @inlineCallbacks
        def _select(prefix):
            yield self._wamp.reset_prefix()
            if self._kernel_prefix:
                print("Successfully unsubscribed from prefix {}".format(self._kernel_prefix))
            else:
                print("No previous subscriptions")
            self._kernel_prefix = prefix
            yield self._wamp.set_prefix(prefix)
            print("Kernel selected [{}]".format(prefix))

        prefix_list = yield self.list(raw=True)
        is_kernel_key = lambda t: re.match(r'io\.timbr\.kernel\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', t)
        if not is_kernel_key(kernel):
            prefix_map = yield threads.deferToThread(self._get_kernel_names, prefix_list, details=True)
            if kernel not in prefix_map:
                print("Kernel not in prefix map")
                returnValue(None)
            else:
                kernel = prefix_map[kernel]
        if kernel in prefix_list:
            if kernel != self._kernel_prefix:
                yield _select(kernel)
            else:
                print("Kernel already selected")
        else:
            print("Kernel not available")

    @inlineCallbacks
    def subscribe(self, callback, **kwargs):
        # NOTE: callback is a string
        yield self.connect(self._router_url)
        yield self._wamp.add_machine_callback(callback)

    @inlineCallbacks
    def execute(self, cell, prefix=None, **kwargs):
        yield self.connect(self._router_url)
        if prefix is not None:
            output = yield self._wamp.call(".".join([prefix, "execute"]), cell)
        output = yield self._wamp.call(".".join([self._kernel_prefix, "execute"]), cell)

    def _get_kernel_names(self, prefix_list, details=False):
        headers = {"Authorization": "Bearer {}".format(self._token)}
        payload = {"addresses": [prefix.split(".")[-1] for prefix in prefix_list]}
        r = requests.post(JUNO_KERNEL_URI, headers=headers, data=payload)
        if details:
            prefix_map = {str(v): ".".join(['io.timbr.kernel', str(k)]) for k, v in r.json().iteritems()}
        else:
            prefix_map = [str(v) for k, v in r.json().iteritems()]
        return prefix_map
<|endoftext|>


<commit_before>"""Next gen sequence alignments with Bowtie2.

http://bowtie-bio.sourceforge.net/bowtie2/index.shtml
"""
import os
from itertools import ifilter, imap
import pysam
import sys

from bcbio.pipeline import config_utils
from bcbio.utils import file_exists, compose
from bcbio.distributed.transaction import file_transaction
from bcbio.provenance import do
from bcbio import bam

def _bowtie2_args_from_config(config):
    """Configurable high level options for bowtie2.
    """
    qual_format = config["algorithm"].get("quality_format", "")
    if qual_format.lower() == "illumina":
        qual_flags = ["--phred64-quals"]
    else:
        qual_flags = []
    num_cores = config["algorithm"].get("num_cores", 1)
    core_flags = ["-p", str(num_cores)] if num_cores > 1 else []
    return core_flags + qual_flags

def align(fastq_file, pair_file, ref_file, names, align_dir, data,
          extra_args=None):
    """Alignment with bowtie2.
    """
    config = data["config"]
    analysis_config = ANALYSIS.get(data["analysis"])
    assert analysis_config, "Analysis %s is not supported by bowtie2" % (data["analysis"])
    out_file = os.path.join(align_dir, "%s.sam" % names["lane"])
    if not file_exists(out_file):
        with file_transaction(out_file) as tx_out_file:
            cl = [config_utils.get_program("bowtie2", config)]
            cl += _bowtie2_args_from_config(config)
            cl += extra_args if extra_args is not None else []
            cl += ["-q",
                   "-x", ref_file]
            cl += analysis_config.get("params", [])
            if pair_file:
                cl += ["-1", fastq_file, "-2", pair_file]
            else:
                cl += ["-U", fastq_file]
            cl += ["-S", tx_out_file]
            cl = [str(i) for i in cl]
            do.run(cl, "Aligning %s and %s with Bowtie2." % (fastq_file, pair_file),
                   None)
    return out_file

# Optional galaxy location file. Falls back on remap_index_fn if not found
galaxy_location_file = "bowtie2_indices.loc"

def remap_index_fn(ref_file):
    """Map sequence references to equivalent bowtie2 indexes.
    """
    return os.path.splitext(ref_file)[0].replace("/seq/", "/bowtie2/")

def filter_multimappers(align_file):
    """
    It does not seem like bowtie2 has a corollary to the -m 1 flag in bowtie,
    there are some options that are close but don't do the same thing. Bowtie2
    sets the XS flag for reads mapping in more than one place, so we can just
    filter on that. This will not work for other aligners.
    """
    type_flag = "b" if bam.is_bam(align_file) else ""
    base, ext = os.path.splitext(align_file)
    align_handle = pysam.Samfile(align_file, "r" + type_flag)
    tmp_out_file = os.path.splitext(align_file)[0] + ".tmp"
    def keep_fn(read):
        return _is_properly_mapped(read) and _is_unique(read)
    keep = ifilter(keep_fn, align_handle)
    with pysam.Samfile(tmp_out_file, "w" + type_flag, template=align_handle) as out_handle:
        for read in keep:
            out_handle.write(read)
    align_handle.close()
    out_handle.close()
    os.rename(tmp_out_file, align_file)
    return align_file

def _is_properly_mapped(read):
    if read.is_paired and not read.is_proper_pair:
        return False
    if read.is_unmapped:
        return False
    return True

def _is_unique(read):
    tags = [x[0] for x in read.tags]
    return "XS" not in tags

ANALYSIS = {"chip-seq": {"params": ["-X", 2000]},
            "RNA-seq": {"params": ["--sensitive", "-X", 2000]}}
<commit_msg>Added proper readgroup information in bowtie2 alignment. Closes #552.<commit_after>"""Next gen sequence alignments with Bowtie2.

http://bowtie-bio.sourceforge.net/bowtie2/index.shtml
"""
import os
from itertools import ifilter, imap
import pysam
import sys

from bcbio.pipeline import config_utils
from bcbio.utils import file_exists, compose
from bcbio.distributed.transaction import file_transaction
from bcbio.provenance import do
from bcbio import bam

def _bowtie2_args_from_config(config):
    """Configurable high level options for bowtie2.
    """
    qual_format = config["algorithm"].get("quality_format", "")
    if qual_format.lower() == "illumina":
        qual_flags = ["--phred64-quals"]
    else:
        qual_flags = []
    num_cores = config["algorithm"].get("num_cores", 1)
    core_flags = ["-p", str(num_cores)] if num_cores > 1 else []
    return core_flags + qual_flags

def align(fastq_file, pair_file, ref_file, names, align_dir, data,
          extra_args=None):
    """Alignment with bowtie2.
    """
    config = data["config"]
    analysis_config = ANALYSIS.get(data["analysis"])
    assert analysis_config, "Analysis %s is not supported by bowtie2" % (data["analysis"])
    out_file = os.path.join(align_dir, "%s.sam" % names["lane"])
    if not file_exists(out_file):
        with file_transaction(out_file) as tx_out_file:
            cl = [config_utils.get_program("bowtie2", config)]
            cl += _bowtie2_args_from_config(config)
            cl += extra_args if extra_args is not None else []
            cl += ["-q",
                   "-x", ref_file]
            cl += analysis_config.get("params", [])
            if pair_file:
                cl += ["-1", fastq_file, "-2", pair_file]
            else:
                cl += ["-U", fastq_file]
            cl += ["-S", tx_out_file]
            if names and names.has_key("rg"):
                cl += ["--rg-id", names["rg"]]
                for key, tag in [("sample", "SM"), ("pl", "PL"), ("pu", "PU")]:
                    if names.has_key(key):
                        cl += ["--rg", "%s:%s" % (tag, names[key])]
            cl = [str(i) for i in cl]
            do.run(cl, "Aligning %s and %s with Bowtie2." % (fastq_file, pair_file),
                   None)
    return out_file

# Optional galaxy location file. Falls back on remap_index_fn if not found
galaxy_location_file = "bowtie2_indices.loc"

def remap_index_fn(ref_file):
    """Map sequence references to equivalent bowtie2 indexes.
    """
    return os.path.splitext(ref_file)[0].replace("/seq/", "/bowtie2/")

def filter_multimappers(align_file):
    """
    It does not seem like bowtie2 has a corollary to the -m 1 flag in bowtie,
    there are some options that are close but don't do the same thing. Bowtie2
    sets the XS flag for reads mapping in more than one place, so we can just
    filter on that. This will not work for other aligners.
    """
    type_flag = "b" if bam.is_bam(align_file) else ""
    base, ext = os.path.splitext(align_file)
    align_handle = pysam.Samfile(align_file, "r" + type_flag)
    tmp_out_file = os.path.splitext(align_file)[0] + ".tmp"
    def keep_fn(read):
        return _is_properly_mapped(read) and _is_unique(read)
    keep = ifilter(keep_fn, align_handle)
    with pysam.Samfile(tmp_out_file, "w" + type_flag, template=align_handle) as out_handle:
        for read in keep:
            out_handle.write(read)
    align_handle.close()
    out_handle.close()
    os.rename(tmp_out_file, align_file)
    return align_file

def _is_properly_mapped(read):
    if read.is_paired and not read.is_proper_pair:
        return False
    if read.is_unmapped:
        return False
    return True

def _is_unique(read):
    tags = [x[0] for x in read.tags]
    return "XS" not in tags

ANALYSIS = {"chip-seq": {"params": ["-X", 2000]},
            "RNA-seq": {"params": ["--sensitive", "-X", 2000]}}
<|endoftext|>


<commit_before>import datetime

from django.conf import settings
from django.utils.translation import ugettext_lazy as _

import mongoengine

GENDER_CHOICES = (
    ('male', _('male')),
    ('female', _('female'))
)

def limit_date(value, lower_limit, upper_limit, error):
    if not value:
        return

    if upper_limit:
        tmp_value = value
        tmp_upper_limit = upper_limit
        if not isinstance(value, datetime.datetime) or not isinstance(upper_limit, datetime.datetime):
            if isinstance(upper_limit, datetime.datetime):
                tmp_upper_limit = upper_limit.date()
            elif isinstance(tmp_value, datetime.datetime):
                tmp_value = value.date()
        if tmp_value > tmp_upper_limit:
            error()

    if lower_limit:
        tmp_value = value
        tmp_lower_limit = lower_limit
        if not isinstance(value, datetime.datetime) or not isinstance(lower_limit, datetime.datetime):
            if isinstance(lower_limit, datetime.datetime):
                tmp_lower_limit = lower_limit.date()
            elif isinstance(tmp_value, datetime.datetime):
                tmp_value = value.date()
        if tmp_value < tmp_lower_limit:
            error()

def get_initial_language(request=None):
    return settings.LANGUAGE_CODE

class LanguageField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 5)
        kwargs.setdefault('choices', settings.LANGUAGES)
        kwargs.setdefault('default', get_initial_language)

        super(LanguageField, self).__init__(*args, **kwargs)

class GenderField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 6)
        kwargs.setdefault('choices', GENDER_CHOICES)
        kwargs.setdefault('default', GENDER_CHOICES[1][0])

        super(GenderField, self).__init__(*args, **kwargs)

class LimitedDateTimeField(mongoengine.DateTimeField):
    def __init__(self, upper_limit=None, lower_limit=None, *args, **kwargs):
        self.upper_limit = upper_limit
        self.lower_limit = lower_limit

        if self.upper_limit and not callable(self.upper_limit) and not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
            self.error(u"Invalid upper_limit argument.")
        if self.lower_limit and not callable(self.upper_limit) and not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
            self.error(u"Invalid lower_limit argument.")

        super(LimitedDateTimeField, self).__init__(*args, **kwargs)

    def validate(self, value):
        super(LimitedDateTimeField, self).validate(value)

        def error():
            self.error(u"Value is out of bounds.")

        if callable(self.upper_limit):
            upper_limit = self.upper_limit()
            if not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
                raise Exception("Callable did not return datetime.date or datetime.datetime object.")
        else:
            upper_limit = self.upper_limit

        if callable(self.lower_limit):
            lower_limit = self.lower_limit()
            if not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
                raise Exception("Callable did not return datetime.date or datetime.datetime object.")
        else:
            lower_limit = self.lower_limit

        limit_date(value, lower_limit, upper_limit, error)<commit_msg>Fixed error when checking lower_limit for callable.<commit_after>import datetime

from django.conf import settings
from django.utils.translation import ugettext_lazy as _

import mongoengine

GENDER_CHOICES = (
    ('male', _('male')),
    ('female', _('female'))
)

def limit_date(value, lower_limit, upper_limit, error):
    if not value:
        return

    if upper_limit:
        tmp_value = value
        tmp_upper_limit = upper_limit
        if not isinstance(value, datetime.datetime) or not isinstance(upper_limit, datetime.datetime):
            if isinstance(upper_limit, datetime.datetime):
                tmp_upper_limit = upper_limit.date()
            elif isinstance(tmp_value, datetime.datetime):
                tmp_value = value.date()
        if tmp_value > tmp_upper_limit:
            error()

    if lower_limit:
        tmp_value = value
        tmp_lower_limit = lower_limit
        if not isinstance(value, datetime.datetime) or not isinstance(lower_limit, datetime.datetime):
            if isinstance(lower_limit, datetime.datetime):
                tmp_lower_limit = lower_limit.date()
            elif isinstance(tmp_value, datetime.datetime):
                tmp_value = value.date()
        if tmp_value < tmp_lower_limit:
            error()

def get_initial_language(request=None):
    return settings.LANGUAGE_CODE

class LanguageField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 5)
        kwargs.setdefault('choices', settings.LANGUAGES)
        kwargs.setdefault('default', get_initial_language)

        super(LanguageField, self).__init__(*args, **kwargs)

class GenderField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 6)
        kwargs.setdefault('choices', GENDER_CHOICES)
        kwargs.setdefault('default', GENDER_CHOICES[1][0])

        super(GenderField, self).__init__(*args, **kwargs)

class LimitedDateTimeField(mongoengine.DateTimeField):
    def __init__(self, upper_limit=None, lower_limit=None, *args, **kwargs):
        self.upper_limit = upper_limit
        self.lower_limit = lower_limit

        if self.upper_limit and not callable(self.upper_limit) and not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
            self.error(u"Invalid upper_limit argument.")
        if self.lower_limit and not callable(self.lower_limit) and not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
            self.error(u"Invalid lower_limit argument.")

        super(LimitedDateTimeField, self).__init__(*args, **kwargs)

    def validate(self, value):
        super(LimitedDateTimeField, self).validate(value)

        def error():
            self.error(u"Value is out of bounds.")

        if callable(self.upper_limit):
            upper_limit = self.upper_limit()
            if not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
                raise Exception("Callable did not return datetime.date or datetime.datetime object.")
        else:
            upper_limit = self.upper_limit

        if callable(self.lower_limit):
            lower_limit = self.lower_limit()
            if not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
                raise Exception("Callable did not return datetime.date or datetime.datetime object.")
        else:
            lower_limit = self.lower_limit

        limit_date(value, lower_limit, upper_limit, error)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import csv
import sys
from django.db import transaction
from load.isocode import *
from load.environmental import *
from load.language import *
from load.society_ea import *
from load.society_binford import *
from load.geographic import *
from load.tree import *
from load.variables import *
from load.values import *
from load.sources import *
from load.glottocode import *

#iso = load isocodes
#env_vals = load environmental values
#langs =
#soc_lat_long = load locations for societies
#ea_soc = load EA societies
#bf_soc =
#bf_vals =
#vars = load (EA or BF) variables
#ea_stacked = load EA stacked

LOAD_BY_ROW=('iso', 'env_vals',
             'langs','soc_lat_long',
             'ea_soc','bf_soc', 'bf_vals',
             'vars', 'ea_vals', 'glotto', 'glotto_iso',
             'xd_lang', 'ea_refs')

def run(file_name=None, mode=None):
    if mode == 'geo':
        load_regions(file_name)
    elif mode == 'tree':
        load_tree(file_name)
    elif mode == 'glottotree':
        load_glotto_tree(file_name)
    else:
        # read the csv file
        with open(file_name, 'rb') as csvfile:
            if mode in LOAD_BY_ROW:
                csv_reader = csv.DictReader(csvfile)
                for dict_row in csv_reader:
                    if mode == 'iso':
                        load_isocode(dict_row)
                    elif mode == 'vars':
                        load_vars(dict_row)
                    elif mode == 'ea_soc':
                        load_ea_society(dict_row)
                    elif mode == 'soc_lat_long':
                        society_locations(dict_row)
                    elif mode == 'env_vals':
                        load_environmental(dict_row)
                    elif mode == 'ea_vals':
                        load_data(dict_row)
                    elif mode == 'bf_vals':
                        load_data(dict_row)
                    elif mode == 'ea_refs':
                        load_val_references(dict_row)
                    elif mode == 'langs':
                        load_lang(dict_row)
                    elif mode == 'bf_soc':
                        load_bf_society(dict_row)
                    elif mode == 'glotto':
                        load_glottocode(dict_row)
                    elif mode == 'glotto_iso':
                        map_isocodes(dict_row)
                    elif mode == 'xd_lang':
                        xd_to_language(dict_row)
            elif mode == 'refs': #load references
                load_references(csvfile)
            elif mode == 'codes': #load codes for variables
                load_codes(csvfile)
        if len(MISSING_CODES) > 0:
            print "Missing ISO Codes:"
            print '\n'.join(MISSING_CODES)
        elif mode == 'env_vars':
            create_environmental_variables()
        elif mode == 'langs' or mode == 'xd_lang':
            update_language_counts()

if __name__ == '__main__':
    if len(sys.argv) != 3:
        print "\nUsage: %s source_file mode" % sys.argv[0]
        print "You should run load_all_datasets.sh instead of this script directly."
        print
    else:
        try:
            transaction.enter_transaction_management()
            transaction.managed(True)
            run(sys.argv[1], sys.argv[2])
        finally:
            transaction.commit()
            <commit_msg>upgraded load script<commit_after># -*- coding: utf-8 -*-
import csv
import sys

import django
from django.db import transaction
from load.isocode import *
from load.environmental import *
from load.language import *
from load.society_ea import *
from load.society_binford import *
from load.geographic import *
from load.tree import *
from load.variables import *
from load.values import *
from load.sources import *
from load.glottocode import *

#iso = load isocodes
#env_vals = load environmental values
#langs =
#soc_lat_long = load locations for societies
#ea_soc = load EA societies
#bf_soc =
#bf_vals =
#vars = load (EA or BF) variables
#ea_stacked = load EA stacked

LOAD_BY_ROW=('iso', 'env_vals',
             'langs','soc_lat_long',
             'ea_soc','bf_soc', 'bf_vals',
             'vars', 'ea_vals', 'glotto', 'glotto_iso',
             'xd_lang', 'ea_refs')

def run(file_name=None, mode=None):
    if mode == 'geo':
        load_regions(file_name)
    elif mode == 'tree':
        load_tree(file_name)
    elif mode == 'glottotree':
        load_glotto_tree(file_name)
    else:
        # read the csv file
        with open(file_name, 'rb') as csvfile:
            if mode in LOAD_BY_ROW:
                csv_reader = csv.DictReader(csvfile)
                for dict_row in csv_reader:
                    if mode == 'iso':
                        load_isocode(dict_row)
                    elif mode == 'vars':
                        load_vars(dict_row)
                    elif mode == 'ea_soc':
                        load_ea_society(dict_row)
                    elif mode == 'soc_lat_long':
                        society_locations(dict_row)
                    elif mode == 'env_vals':
                        load_environmental(dict_row)
                    elif mode == 'ea_vals':
                        load_data(dict_row)
                    elif mode == 'bf_vals':
                        load_data(dict_row)
                    elif mode == 'ea_refs':
                        load_val_references(dict_row)
                    elif mode == 'langs':
                        load_lang(dict_row)
                    elif mode == 'bf_soc':
                        load_bf_society(dict_row)
                    elif mode == 'glotto':
                        load_glottocode(dict_row)
                    elif mode == 'glotto_iso':
                        map_isocodes(dict_row)
                    elif mode == 'xd_lang':
                        xd_to_language(dict_row)
            elif mode == 'refs': #load references
                load_references(csvfile)
            elif mode == 'codes': #load codes for variables
                load_codes(csvfile)
        if len(MISSING_CODES) > 0:
            print "Missing ISO Codes:"
            print '\n'.join(MISSING_CODES)
        elif mode == 'env_vars':
            create_environmental_variables()
        elif mode == 'langs' or mode == 'xd_lang':
            update_language_counts()

if __name__ == '__main__':
    if len(sys.argv) != 3:
        print "\nUsage: %s source_file mode" % sys.argv[0]
        print "You should run load_all_datasets.sh instead of this script directly."
        print
    else:
        django.setup()
        with transaction.atomic():
            run(sys.argv[1], sys.argv[2])

<|endoftext|>


<commit_before>
import csv
from io import StringIO
import itertools as it
import json
import logging
from pathlib import Path

from ._models import ContactDetails, Identifier, MP, MultilingualField, OtherName
from .questions import pair_name
from ..crawling import Task
from ..text_utils import (clean_spaces, parse_long_date,
                          translit_elGrek2Latn, translit_el2tr)

logger = logging.getLogger(__name__)

class MPs(Task):

    async def __call__(self):
        listing_urls = ('http://www.parliament.cy/easyconsole.cfm/id/186',
                        'http://www.parliament.cy/easyconsole.cfm/id/2004')

        mp_urls = await self.c.gather(map(self.process_multi_page_listing,
                                          listing_urls))
        mp_urls = tuple(it.chain.from_iterable(mp_urls))
        return zip(mp_urls,
                   await self.c.gather(self.c.get_html(url + '/lang/el')
                                       for url in mp_urls),
                   await self.c.gather(self.c.get_html(url + '/lang/en')
                                       for url in mp_urls))

    async def process_multi_page_listing(self, url):
        html = await self.c.get_html(url)
        pages = html.xpath('//a[contains(@class, "pagingStyle")]/@href')
        if pages:
            pages = {self.process_multi_page_page(
                      url, form_data={'page': ''.join(filter(str.isdigit, p))})
                     for p in pages}
            return it.chain.from_iterable(await self.c.gather(pages))
        else:
            return await self.process_multi_page_page(url)

    async def process_multi_page_page(self, url, form_data=None):
        html = await self.c.get_html(url, form_data=form_data,
                                     request_method='post')
        return html.xpath('//a[@class="h3Style"]/@href')

    def after(output):
        for url, html_el, html_en in output:
            parse_mp(url, html_el, html_en)

class WikidataIds(Task):

    async def __call__(self):
        url = ('https://cdn.rawgit.com/everypolitician/everypolitician-data'
               '/master/data/Cyprus/House_of_Representatives/ep-popolo-v1.0.json')
        return await self.c.get_payload(url)

    def after(ep_data):
        ep_persons = (p for p in json.loads(ep_data.decode())['persons']
                      if extract_id(p, 'wikidata'))
        for p in ep_persons:
            wd = Identifier(identifier=extract_id(p, 'wikidata'),
                            scheme='http://www.wikidata.org/entity/')
            mp = MP(_id=extract_id(p, 'openpatata'), identifiers=[wd])
            logger.info('Updating identifiers of {!r} with {!r}'
                        .format(mp._id, mp.data['identifiers']))
            mp.insert(merge=mp.exists)

def extract_id(person, scheme):
    return next((i for i in person['identifiers'] if i['scheme'] == scheme),
                {}).get('identifier')

class ContactInfo(Task):

    with (Path(__file__).parent.parent/'data'/'contact_names.csv').open() \
            as file:
        NAMES = dict(it.islice(csv.reader(file), 1, None))

    async def __call__(self):
        url = 'http://www.parliament.cy/easyconsole.cfm/id/185'
        return await self.c.get_html(url)

    def after(output):
        for mp_name, _, voice, email in extract_contact_rows(output):
            if not ContactInfo.NAMES.get(mp_name):
                logger.warning('{!r} not found in `NAMES`; skipping'
                               .format(mp_name))
                continue
            voice = ''.join(voice.split())
            voice = '(+357) {} {}'.format(voice[:2], voice[2:])
            mp = MP(_id=ContactInfo.NAMES[mp_name],
                    _sources=['http://www.parliament.cy/easyconsole.cfm/id/185'],
                    contact_details=[ContactDetails(type='email', value=email),
                                     ContactDetails(type='voice', value=voice)],
                    email=email)
            try:
                mp.insert(mp.exists)
            except mp.InsertError:
                logger.warning('Unable to insert ' + repr(mp))

class ReconcileContactNames(ContactInfo):

    def after(output):
        names_and_ids = {i['_id']: i['name']['el'] for i in MP.collection.find()}
        names = tuple(r[0] for r in extract_contact_rows(output))
        output = StringIO()
        csv_writer = csv.writer(output)
        csv_writer.writerow(('name', 'id'))
        csv_writer.writerows(pair_name(n, names_and_ids, ContactInfo.NAMES)
                             for n in names)
        print(output.getvalue())

def extract_contact_rows(rows):
    rows = rows.xpath('//div[@class = "articleBox"]/div[2]/table[1]//tr')
    rows = (tuple(clean_spaces(cell.text_content()) for cell in row)
            for row in rows)
    rows = (row[1:] for row in rows if row[0].isnumeric())
    return rows

LABELS = {
    'el': {
        'foreign languages': 'Ξένες γλώσσες',
        'pob dob': 'Τόπος και ημερομηνία γέννησης',
        'poo dob': 'Τόπος καταγωγής και ημερομηνία γέννησης',
        'poo': 'Τόπος καταγωγής',
        'profession': 'Επάγγελμα',
        'studies': 'Σπουδές',},
    'en': {
        'foreign languages': 'Foreign languages',
        'pob dob': 'Place of origin and date of birth',
        'poo dob': 'Place and date of birth',
        'poo': 'Place of origin',
        'profession': 'Profession',
        'studies': 'Studies',}}

def extract_item(el, en):
    html = locals()

    def inner(lang, *items):
        for item in items:
            item = html[lang].xpath('//p[contains(strong/text(), "{}")]/text()'
                                    .format(LABELS[lang][item]))
            if item:
                yield item
    return inner

def extract_birth_date_place(url, html):
    extractor = extract_item(html, None)
    birth_date, birth_place = None, None
    try:
        birth_things = next(extractor('el', 'poo dob', 'pob dob', 'poo'))
    except StopIteration:
        logger.error('No date and place of birth found in ' + repr(url))
    else:
        birth_things = clean_spaces(''.join(birth_things).strip(':.'),
                                    medial_newlines=True)
        try:
            birth_place, birth_date = birth_things.split(',')
        except ValueError:
            logger.error('No date of birth found in ' + repr(url))
            birth_place = birth_things
        else:
            birth_date = parse_long_date(birth_date)
    return birth_date, birth_place

def extract_contact_details(url, lang, html):
    heading = {'el': 'Στοιχεία επικοινωνίας', 'en': 'Contact info'}[lang]
    try:
        contact_details = next(iter(html.xpath(
            '//p[contains(strong/text(), "{}")]/following-sibling::p'
            .format(heading))))
    except StopIteration:
        logger.error("Could not extract contact details in '{}/lang/{}'"
                     .format(url, lang))
        return {}
    else:
        contact_details = dict(zip(
            (clean_spaces(i.strip(': '), True)
             for i in contact_details.xpath('./strong/text()')),
            (clean_spaces(i.strip(': '), True)
             for i in ''.join(i.xpath('string()') if hasattr(i, 'xpath') else i
                              for i in contact_details.xpath('./node()[not(self::strong)]')
                              ).splitlines()
             )))
        return contact_details

def extract_images(html):
    images = tuple(it.chain(html.xpath('//a[@class = "lightview"]/@href'),
                            html.xpath('//a[contains(@href, "/assets/image/imageoriginal")]'
                                       '/@href')))
    images = sorted(set(images), key=images.index)
    return images

def parse_mp(url, html_el, html_en):
    extractor = extract_contact_details(url, 'el', html_el)
    contact_details = []
    email = extractor.get('Ηλεκτρονικό ταχυδρομείο')
    if email:
        contact_details = email.split(', ')
        email = contact_details[0]
        contact_details = [ContactDetails(type='email', value=v)
                           for v in contact_details]
    images = extract_images(html_el)
    name = clean_spaces(html_el.xpath('string(//h1)'))
    name = MultilingualField(el=name,
                             en=translit_elGrek2Latn(name),
                             tr=translit_el2tr(name))
    other_names = [OtherName(name=clean_spaces(html_en.xpath('string(//h1)')),
                             note="Official spelling in the 'en' locale;"
                                  " possibly anglicised")]

    mp = MP(_sources=[url],
            birth_date=extract_birth_date_place(url, html_el)[0],
            contact_details=contact_details,
            email=email,
            image=[*images, None][0],
            images=images,
            name=name,
            other_names=other_names)
    mp.insert(merge=mp.exists)
<commit_msg>Extract more contact info from MP pages<commit_after>
import csv
from io import StringIO
import itertools as it
import json
import logging
from pathlib import Path

from ._models import (ContactDetails, Identifier, Link,
                      MP, MultilingualField, OtherName)
from .questions import pair_name
from ..crawling import Task
from ..text_utils import (clean_spaces, parse_long_date,
                          translit_elGrek2Latn, translit_el2tr)

logger = logging.getLogger(__name__)

class MPs(Task):

    async def __call__(self):
        listing_urls = ('http://www.parliament.cy/easyconsole.cfm/id/186',
                        'http://www.parliament.cy/easyconsole.cfm/id/2004')

        mp_urls = await self.c.gather(map(self.process_multi_page_listing,
                                          listing_urls))
        mp_urls = tuple(it.chain.from_iterable(mp_urls))
        return zip(
            mp_urls,
            await self.c.gather(self.c.get_html(u + '/lang/el') for u in mp_urls),
            await self.c.gather(self.c.get_html(u + '/lang/en') for u in mp_urls))

    async def process_multi_page_listing(self, url):
        html = await self.c.get_html(url)
        pages = html.xpath('//a[contains(@class, "pagingStyle")]/@href')
        if pages:
            pages = {self.process_multi_page_page(
                      url, form_data={'page': ''.join(filter(str.isdigit, p))})
                     for p in pages}
            return it.chain.from_iterable(await self.c.gather(pages))
        else:
            return await self.process_multi_page_page(url)

    async def process_multi_page_page(self, url, form_data=None):
        html = await self.c.get_html(url, form_data=form_data,
                                     request_method='post')
        return html.xpath('//a[@class="h3Style"]/@href')

    def after(output):
        for url, html_el, html_en in output:
            extractor = extract_contact_details(url, 'el', html_el)
            contact_details = [ContactDetails(type=t, value=f(i))
                                for l, t, d, f in CONTACT_DETAIL_SUBS
                                if extractor.get(l)
                                for i in extractor[l].split(d)]
            email = next((i['value'] for i in contact_details if i['type'] == 'email'),
                         None)
            images = extract_images(html_el)
            links = [Link(note=MultilingualField(**n), url=f(extractor[l], html_el))
                                for l, n, d, f in LINK_SUBS if extractor.get(l)
                                for i in extractor[l].split(d)]
            name = clean_spaces(html_el.xpath('string(//h1)'))
            name = MultilingualField(el=name, en=translit_elGrek2Latn(name),
                                     tr=translit_el2tr(name))
            other_names = [OtherName(name=clean_spaces(html_en.xpath('string(//h1)')),
                                     note="Official spelling in the 'en' locale;"
                                          " possibly anglicised")]

            mp = MP(_sources=[url],
                    birth_date=extract_birth_date_place(url, html_el)[0],
                    contact_details=contact_details,
                    email=email,
                    image=[*images, None][0],
                    images=images,
                    links=links,
                    name=name,
                    other_names=other_names)
            mp.insert(merge=mp.exists)

class WikidataIds(Task):

    async def __call__(self):
        url = ('https://cdn.rawgit.com/everypolitician/everypolitician-data'
               '/master/data/Cyprus/House_of_Representatives/ep-popolo-v1.0.json')
        return await self.c.get_payload(url)

    def after(ep_data):
        ep_persons = (p for p in json.loads(ep_data.decode())['persons']
                      if extract_id(p, 'wikidata'))
        for p in ep_persons:
            wd = Identifier(identifier=extract_id(p, 'wikidata'),
                            scheme='http://www.wikidata.org/entity/')
            mp = MP(_id=extract_id(p, 'openpatata'), identifiers=[wd])
            logger.info('Updating identifiers of {!r} with {!r}'
                        .format(mp._id, mp.data['identifiers']))
            mp.insert(merge=mp.exists)

def extract_id(person, scheme):
    return next((i for i in person['identifiers'] if i['scheme'] == scheme),
                {}).get('identifier')

class ContactInfo(Task):

    with (Path(__file__).parent.parent/'data'/'contact_names.csv').open() \
            as file:
        NAMES = dict(it.islice(csv.reader(file), 1, None))

    async def __call__(self):
        url = 'http://www.parliament.cy/easyconsole.cfm/id/185'
        return await self.c.get_html(url)

    def after(output):
        for mp_name, _, voice, email in extract_contact_rows(output):
            if not ContactInfo.NAMES.get(mp_name):
                logger.warning('{!r} not found in `NAMES`; skipping'
                               .format(mp_name))
                continue
            voice = ''.join(voice.split())
            voice = '(+357) {} {}'.format(voice[:2], voice[2:])
            mp = MP(_id=ContactInfo.NAMES[mp_name],
                    _sources=['http://www.parliament.cy/easyconsole.cfm/id/185'],
                    contact_details=[ContactDetails(type='email', value=email),
                                     ContactDetails(type='voice', value=voice)],
                    email=email)
            try:
                mp.insert(mp.exists)
            except mp.InsertError:
                logger.warning('Unable to insert ' + repr(mp))

class ReconcileContactNames(ContactInfo):

    def after(output):
        names_and_ids = {i['_id']: i['name']['el'] for i in MP.collection.find()}
        names = tuple(r[0] for r in extract_contact_rows(output))
        output = StringIO()
        csv_writer = csv.writer(output)
        csv_writer.writerow(('name', 'id'))
        csv_writer.writerows(pair_name(n, names_and_ids, ContactInfo.NAMES)
                             for n in names)
        print(output.getvalue())

def extract_contact_rows(rows):
    rows = rows.xpath('//div[@class = "articleBox"]/div[2]/table[1]//tr')
    rows = (tuple(clean_spaces(cell.text_content()) for cell in row)
            for row in rows)
    rows = (row[1:] for row in rows if row[0].isnumeric())
    return rows

LABELS = {
    'el': {
        'foreign languages': 'Ξένες γλώσσες',
        'pob dob': 'Τόπος και ημερομηνία γέννησης',
        'poo dob': 'Τόπος καταγωγής και ημερομηνία γέννησης',
        'poo': 'Τόπος καταγωγής',
        'profession': 'Επάγγελμα',
        'studies': 'Σπουδές',},
    'en': {
        'foreign languages': 'Foreign languages',
        'pob dob': 'Place of origin and date of birth',
        'poo dob': 'Place and date of birth',
        'poo': 'Place of origin',
        'profession': 'Profession',
        'studies': 'Studies',}}

def extract_item(el, en):
    html = locals()

    def inner(lang, *items):
        for item in items:
            item = html[lang].xpath('//p[contains(strong/text(), "{}")]/text()'
                                    .format(LABELS[lang][item]))
            if item:
                yield item
    return inner

def extract_birth_date_place(url, html):
    extractor = extract_item(html, None)
    birth_date, birth_place = None, None
    try:
        birth_things = next(extractor('el', 'poo dob', 'pob dob', 'poo'))
    except StopIteration:
        logger.error('No date and place of birth found in ' + repr(url))
    else:
        birth_things = clean_spaces(''.join(birth_things).strip(':.'),
                                    medial_newlines=True)
        try:
            birth_place, birth_date = birth_things.split(',')
        except ValueError:
            logger.error('No date of birth found in ' + repr(url))
            birth_place = birth_things
        else:
            birth_date = parse_long_date(birth_date)
    return birth_date, birth_place

def extract_contact_details(url, lang, html):
    heading = {'el': 'Στοιχεία επικοινωνίας', 'en': 'Contact info'}[lang]
    try:
        contact_details = next(iter(html.xpath(
            '//p[contains(strong/text(), "{}")]/following-sibling::p'
            .format(heading))))
    except StopIteration:
        logger.error("Could not extract contact details in '{}/lang/{}'"
                     .format(url, lang))
        return {}
    else:
        contact_details = dict(zip(
            (clean_spaces(i.strip(': '), True)
             for i in contact_details.xpath('./strong/text()')),
            (clean_spaces(i.strip(': '), True)
             for i in ''.join(i.xpath('string()') if hasattr(i, 'xpath') else i
                              for i in contact_details.xpath('./node()[not(self::strong)]')
                              ).splitlines()
             )))
        return contact_details

def extract_images(html):
    images = tuple(it.chain(html.xpath('//a[@class = "lightview"]/@href'),
                            html.xpath('//a[contains(@href, "/assets/image/imageoriginal")]'
                                       '/@href')))
    images = sorted(set(images), key=images.index)
    return images

def extract_homepage(url, html):
    try:
        return html.xpath('//a[contains(string(.), "{}")]/@href'.format(url))[0]
    except IndexError:
        if url.startswith('http'):
            return url
        return 'http://{}/'.format(url.rstrip('/'))

CONTACT_DETAIL_SUBS = (
    ('Διεύθυνση', 'address', ' / ', lambda v: v),
    ('Τηλέφωνο', 'voice', ', ', lambda v: '(+357) ' + v.replace('(+357) ', '')),
    ('Τηλεομοιότυπο', 'fax', ', ', lambda v: '(+357) ' + v.replace('(+357) ', '')),
    ('Ηλεκτρονικό ταχυδρομείο', 'email', ', ', lambda v: v),)

LINK_SUBS = (
    ('Διαδικτυακός τόπος',
     {'el': 'Προσωπική ιστοσελίδα', 'en': 'Personal website', 'tr': 'Kişisel websitesi'},
     ', ',
     extract_homepage),
    ('Λογαριασμός στο Twitter',
     {'el': 'twitter', 'en': 'twitter', 'tr': 'twitter'},
     ', ',
     lambda v, _: 'https://twitter.com/' + v.lstrip('@')),)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

# Disable:
# - 'Maximum number of public methods for a class'
# - 'Missing docstring' (because of all of the model Meta)
# pylint: disable=R0904,C0111

'''
Model representations of the OpenQuake DB tables.
'''
import os
from datetime import datetime

from openquake.commonlib import datastore
from openquake.commonlib.oqvalidation import OqParam, RISK_CALCULATORS
import django
if hasattr(django, 'setup'):
    django.setup()  # for Django >= 1.7
from django.db import models as djm
from django.core.exceptions import ObjectDoesNotExist
from django.db import connection

#: Kind of supported curve statistics
STAT_CHOICES = (
    (u'mean', u'Mean'),
    (u'quantile', u'Quantile'))

#: System Reference ID used for geometry objects
DEFAULT_SRID = 4326

VS30_TYPE_CHOICES = (
    (u"measured", u"Value obtained from on-site measurements"),
    (u"inferred", u"Estimated value"),
)

IMT_CHOICES = (
    (u'PGA', u'Peak Ground Acceleration'),
    (u'PGV', u'Peak Ground Velocity'),
    (u'PGD', u'Peak Ground Displacement'),
    (u'SA', u'Spectral Acceleration'),
    (u'IA', u'Arias Intensity'),
    (u'RSD', u'Relative Significant Duration'),
    (u'MMI', u'Modified Mercalli Intensity'),
)

#: Minimum value for a seed number
MIN_SINT_32 = -(2 ** 31)
#: Maximum value for a seed number
MAX_SINT_32 = (2 ** 31) - 1

#: Kind of supported type of loss outputs
LOSS_TYPES = ["structural", "nonstructural", "fatalities", "contents"]

#: relative tolerance to consider two risk outputs (almost) equal
RISK_RTOL = 0.05

#: absolute tolerance to consider two risk outputs (almost) equal
RISK_ATOL = 0.01

# TODO: these want to be dictionaries
INPUT_TYPE_CHOICES = (
    (u'unknown', u'Unknown'),
    (u'source', u'Source Model'),
    (u'source_model_logic_tree', u'Source Model Logic Tree'),
    (u'gsim_logic_tree', u'Ground Shaking Intensity Model Logic Tree'),
    (u'exposure', u'Exposure'),
    (u'fragility', u'Fragility'),
    (u'site_model', u'Site Model'),
    (u'rupture_model', u'Rupture Model'),

    # vulnerability models
    (u'structural_vulnerability', u'Structural Vulnerability'),
    (u'nonstructural_vulnerability', u'Non Structural Vulnerability'),
    (u'contents_vulnerability', u'Contents Vulnerability'),
    (u'business_interruption_vulnerability',
     u'Business Interruption Vulnerability'),
    (u'occupants_vulnerability', u'Occupants Vulnerability'),
    (u'structural_vulnerability_retrofitted',
     u'Structural Vulnerability Retrofitted'))

# Tables in the 'uiapi' schema.

class OqJob(djm.Model):
    '''
    An OpenQuake engine run started by the user
    '''
    description = djm.TextField()
    user_name = djm.TextField()
    calculation_mode = djm.TextField()
    hazard_calculation = djm.ForeignKey('OqJob', null=True)
    STATUS_CHOICES = (
        (u'created', u'Created'),
        (u'executing', u'Executing'),
        (u'post_executing', u'Post-Executing'),
        (u'post_processing', u'Post-Processing'),
        (u'export', u'Exporting results'),
        (u'clean_up', u'Cleaning up'),
        (u'complete', u'Complete'),
    )
    status = djm.TextField(choices=STATUS_CHOICES, default='executing')
    is_running = djm.BooleanField(default=True)
    start_time = djm.DateTimeField(editable=False, default=datetime.utcnow)
    stop_time = djm.DateTimeField(editable=False)
    relevant = djm.BooleanField(null=False, default=True)
    ds_calc_dir = djm.TextField(null=False, blank=True)  # datastore calc_dir

    class Meta:
        db_table = 'job'

    @property
    def job_type(self):
        """
        'hazard' or 'risk'
        """
        # the calculation mode can be unknown if the job parameters
        # have not been written on the database yet
        return ('risk' if self.calculation_mode in RISK_CALCULATORS
                else 'hazard')

    def get_or_create_output(self, display_name, ds_key):
        """
        :param disp_name: display name of the output
        :returns: an Output instance
        """
        try:
            output = Output.objects.get(
                oq_job=self, display_name=display_name)
        except ObjectDoesNotExist:
            output = Output.objects.create_output(
                self, display_name, ds_key)
        return output

    def get_oqparam(self):
        """
        Return an OqParam object as read from the database
        """
        datadir = os.path.dirname(self.ds_calc_dir)
        dstore = datastore.read(self.id, datadir=datadir)
        oqparam = OqParam.from_(dstore.attrs)
        return oqparam

    def __repr__(self):
        return '<%s %d, %s>' % (self.__class__.__name__,
                                self.id, self.job_type)

def oqparam(job_id):
    """
    :param job_id: ID of :class:`openquake.server.db.models.OqJob`
    :returns: instance of :class:`openquake.commonlib.oqvalidation.OqParam`
    """
    return OqJob.objects.get(pk=job_id).get_oqparam()

class Performance(djm.Model):
    '''
    Contains performance information about a job.
    '''
    job = djm.ForeignKey('OqJob')
    operation = djm.TextField(null=False)
    time_sec = djm.FloatField(null=False)
    memory_mb = djm.FloatField(null=False)
    counts = djm.IntegerField(null=False)

    class Meta:
        db_table = 'performance'

class Log(djm.Model):
    '''
    Log table for calculations
    '''
    job = djm.ForeignKey('OqJob', null=True)
    timestamp = djm.DateTimeField(editable=False, default=datetime.utcnow)
    LOG_LEVEL_CHOICES = (
        (u'debug', u'Debug'),
        (u'info', u'Info'),
        (u'progress', u'Progress'),
        (u'warn', u'Warn'),
        (u'error', u'Error'),
        (u'critical', u'Critical'),
    )
    level = djm.TextField(choices=LOG_LEVEL_CHOICES)
    process = djm.TextField(null=False)
    message = djm.TextField(null=False)

    class Meta:
        db_table = 'log'
        ordering = ['id']

def extract_from(objlist, attr):
    """
    Extract an attribute from a list of Django objects, by scanning
    them in order until a not None attribute is found. If nothing is
    found, or if an exception ObjectDoesNotExist is raised, return None.

    :param objlist: the list of Django objects
    :param str attr: the name of the attribute to look for
    """
    for obj in objlist:
        try:
            value = getattr(obj, attr, None)
        except ObjectDoesNotExist:
            value = None
        if value is not None:
            return value

class OutputManager(djm.Manager):
    """
    Manager class to filter and create Output objects
    """
    def create_output(self, job, display_name, ds_key):
        """
        Create an output for the given `job`, `display_name` and
        `ds_key`
        """
        return self.create(oq_job=job,
                           display_name=display_name,
                           ds_key=ds_key)

class Output(djm.Model):
    '''
    A single artifact which is a result of an OpenQuake job.
    The data may reside in a file or in the database.
    '''
    oq_job = djm.ForeignKey('OqJob', null=False)
    display_name = djm.TextField()
    last_update = djm.DateTimeField(editable=False, default=datetime.utcnow)
    ds_key = djm.TextField(null=False, blank=True)  # datastore key

    objects = OutputManager()

    def __str__(self):
        return "%d||%s" % (self.id, self.display_name)

    class Meta:
        db_table = 'output'
        ordering = ['id']
<commit_msg>Remove 'progress' log level server db model<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

# Disable:
# - 'Maximum number of public methods for a class'
# - 'Missing docstring' (because of all of the model Meta)
# pylint: disable=R0904,C0111

'''
Model representations of the OpenQuake DB tables.
'''
import os
from datetime import datetime

from openquake.commonlib import datastore
from openquake.commonlib.oqvalidation import OqParam, RISK_CALCULATORS
import django
if hasattr(django, 'setup'):
    django.setup()  # for Django >= 1.7
from django.db import models as djm
from django.core.exceptions import ObjectDoesNotExist
from django.db import connection

#: Kind of supported curve statistics
STAT_CHOICES = (
    (u'mean', u'Mean'),
    (u'quantile', u'Quantile'))

#: System Reference ID used for geometry objects
DEFAULT_SRID = 4326

VS30_TYPE_CHOICES = (
    (u"measured", u"Value obtained from on-site measurements"),
    (u"inferred", u"Estimated value"),
)

IMT_CHOICES = (
    (u'PGA', u'Peak Ground Acceleration'),
    (u'PGV', u'Peak Ground Velocity'),
    (u'PGD', u'Peak Ground Displacement'),
    (u'SA', u'Spectral Acceleration'),
    (u'IA', u'Arias Intensity'),
    (u'RSD', u'Relative Significant Duration'),
    (u'MMI', u'Modified Mercalli Intensity'),
)

#: Minimum value for a seed number
MIN_SINT_32 = -(2 ** 31)
#: Maximum value for a seed number
MAX_SINT_32 = (2 ** 31) - 1

#: Kind of supported type of loss outputs
LOSS_TYPES = ["structural", "nonstructural", "fatalities", "contents"]

#: relative tolerance to consider two risk outputs (almost) equal
RISK_RTOL = 0.05

#: absolute tolerance to consider two risk outputs (almost) equal
RISK_ATOL = 0.01

# TODO: these want to be dictionaries
INPUT_TYPE_CHOICES = (
    (u'unknown', u'Unknown'),
    (u'source', u'Source Model'),
    (u'source_model_logic_tree', u'Source Model Logic Tree'),
    (u'gsim_logic_tree', u'Ground Shaking Intensity Model Logic Tree'),
    (u'exposure', u'Exposure'),
    (u'fragility', u'Fragility'),
    (u'site_model', u'Site Model'),
    (u'rupture_model', u'Rupture Model'),

    # vulnerability models
    (u'structural_vulnerability', u'Structural Vulnerability'),
    (u'nonstructural_vulnerability', u'Non Structural Vulnerability'),
    (u'contents_vulnerability', u'Contents Vulnerability'),
    (u'business_interruption_vulnerability',
     u'Business Interruption Vulnerability'),
    (u'occupants_vulnerability', u'Occupants Vulnerability'),
    (u'structural_vulnerability_retrofitted',
     u'Structural Vulnerability Retrofitted'))

# Tables in the 'uiapi' schema.

class OqJob(djm.Model):
    '''
    An OpenQuake engine run started by the user
    '''
    description = djm.TextField()
    user_name = djm.TextField()
    calculation_mode = djm.TextField()
    hazard_calculation = djm.ForeignKey('OqJob', null=True)
    STATUS_CHOICES = (
        (u'created', u'Created'),
        (u'executing', u'Executing'),
        (u'post_executing', u'Post-Executing'),
        (u'post_processing', u'Post-Processing'),
        (u'export', u'Exporting results'),
        (u'clean_up', u'Cleaning up'),
        (u'complete', u'Complete'),
    )
    status = djm.TextField(choices=STATUS_CHOICES, default='executing')
    is_running = djm.BooleanField(default=True)
    start_time = djm.DateTimeField(editable=False, default=datetime.utcnow)
    stop_time = djm.DateTimeField(editable=False)
    relevant = djm.BooleanField(null=False, default=True)
    ds_calc_dir = djm.TextField(null=False, blank=True)  # datastore calc_dir

    class Meta:
        db_table = 'job'

    @property
    def job_type(self):
        """
        'hazard' or 'risk'
        """
        # the calculation mode can be unknown if the job parameters
        # have not been written on the database yet
        return ('risk' if self.calculation_mode in RISK_CALCULATORS
                else 'hazard')

    def get_or_create_output(self, display_name, ds_key):
        """
        :param disp_name: display name of the output
        :returns: an Output instance
        """
        try:
            output = Output.objects.get(
                oq_job=self, display_name=display_name)
        except ObjectDoesNotExist:
            output = Output.objects.create_output(
                self, display_name, ds_key)
        return output

    def get_oqparam(self):
        """
        Return an OqParam object as read from the database
        """
        datadir = os.path.dirname(self.ds_calc_dir)
        dstore = datastore.read(self.id, datadir=datadir)
        oqparam = OqParam.from_(dstore.attrs)
        return oqparam

    def __repr__(self):
        return '<%s %d, %s>' % (self.__class__.__name__,
                                self.id, self.job_type)

def oqparam(job_id):
    """
    :param job_id: ID of :class:`openquake.server.db.models.OqJob`
    :returns: instance of :class:`openquake.commonlib.oqvalidation.OqParam`
    """
    return OqJob.objects.get(pk=job_id).get_oqparam()

class Performance(djm.Model):
    '''
    Contains performance information about a job.
    '''
    job = djm.ForeignKey('OqJob')
    operation = djm.TextField(null=False)
    time_sec = djm.FloatField(null=False)
    memory_mb = djm.FloatField(null=False)
    counts = djm.IntegerField(null=False)

    class Meta:
        db_table = 'performance'

class Log(djm.Model):
    '''
    Log table for calculations
    '''
    job = djm.ForeignKey('OqJob', null=True)
    timestamp = djm.DateTimeField(editable=False, default=datetime.utcnow)
    LOG_LEVEL_CHOICES = (
        (u'debug', u'Debug'),
        (u'info', u'Info'),
        (u'warn', u'Warn'),
        (u'error', u'Error'),
        (u'critical', u'Critical'),
    )
    level = djm.TextField(choices=LOG_LEVEL_CHOICES)
    process = djm.TextField(null=False)
    message = djm.TextField(null=False)

    class Meta:
        db_table = 'log'
        ordering = ['id']

def extract_from(objlist, attr):
    """
    Extract an attribute from a list of Django objects, by scanning
    them in order until a not None attribute is found. If nothing is
    found, or if an exception ObjectDoesNotExist is raised, return None.

    :param objlist: the list of Django objects
    :param str attr: the name of the attribute to look for
    """
    for obj in objlist:
        try:
            value = getattr(obj, attr, None)
        except ObjectDoesNotExist:
            value = None
        if value is not None:
            return value

class OutputManager(djm.Manager):
    """
    Manager class to filter and create Output objects
    """
    def create_output(self, job, display_name, ds_key):
        """
        Create an output for the given `job`, `display_name` and
        `ds_key`
        """
        return self.create(oq_job=job,
                           display_name=display_name,
                           ds_key=ds_key)

class Output(djm.Model):
    '''
    A single artifact which is a result of an OpenQuake job.
    The data may reside in a file or in the database.
    '''
    oq_job = djm.ForeignKey('OqJob', null=False)
    display_name = djm.TextField()
    last_update = djm.DateTimeField(editable=False, default=datetime.utcnow)
    ds_key = djm.TextField(null=False, blank=True)  # datastore key

    objects = OutputManager()

    def __str__(self):
        return "%d||%s" % (self.id, self.display_name)

    class Meta:
        db_table = 'output'
        ordering = ['id']
<|endoftext|>