<commit_before>import os
import socket
import subprocess
import urllib2
import filecmp
import datetime
import logging
log = logging.getLogger(__file__)

from ott.utils.config_util import ConfigUtil
from ott.utils import exe_utils
from ott.utils import file_utils
from ott.utils import web_utils

# constants
DEF_NAME   = "prod"
DEF_PORT   = "55555"
DEF_SSL_PORT = "55551"
GRAPH_NAME = "Graph.obj"
OTP_NAME   = "otp.jar"
VLOG_NAME  = "otp.v"
OTP_DOWNLOAD_URL = "http://maven.conveyal.com.s3.amazonaws.com/org/opentripplanner/otp/0.20.0/otp-0.20.0-shaded.jar"

def restart_call(self, call_db_path="call_center/db/call_db.tar.gz", call_runner="call_center/run.sh"):
    """ retstart call-center app
        basically, if we see 'call' a database export (backup) in the call_center folder, we'll assume that call runs
        here and we'll restart the call server from a shell script (all pretty TriMet specific)
    """
    if os.path.isfile(call_db_path):
        subprocess.call([call_runner])

def get_test_urls_from_config(section='otp', port=None, hostname=None, ws_path=None, map_path=None):
    """ return the OTP map and ws urls from
    """
    config = ConfigUtil(section=section)

    if not hostname:
        hostname = config.get('host', def_val=web_utils.get_hostname())
    if not port:
        port = DEF_PORT

    if ws_path is None:
        ws_path = config.get('ws_url_path', def_val="/otp/routers/default/plan")
    ws_url = "http://{}:{}{}".format(hostname, port, ws_path)

    if map_path is None:
        map_path = config.get('map_url_path', def_val="")
    map_url = "http://{}:{}{}".format(hostname, port, map_path)

    return ws_url, map_url

def call_planner_svc(url, accept='application/xml'):
    """ make a call to the OTP web service
    """
    ret_val = None
    try:
        socket.setdefaulttimeout(2000)
        log.debug("call_otp: OTP output for " + url)
        req = urllib2.Request(url, None, {'Accept':accept})
        res = urllib2.urlopen(req)
        log.debug("call_otp: OTP output for " + url)
        ret_val = res.read()
        res.close()
    except:
        log.warn('ERROR: could not get data from url (timeout?): {0}'.format(url))
    return ret_val

def run_otp_server(dir_path=None, port=DEF_PORT, ssl=DEF_SSL_PORT, otp_name=OTP_NAME, java_mem=None, **kwargs):
    """ launch the server in a separate process
    """
    file_utils.cd(dir_path)
    otp_path = os.path.join(dir_path, otp_name)
    cmd='-server -jar {} --port {} --securePort {} --router "" --graphs {}'.format(otp_path, port, ssl, dir)
    ret_val = exe_utils.run_java(cmd, fork=True, big_xmx=java_mem, pid_file="pid.txt")
    return ret_val

def run_graph_builder(graph_dir, graph_name=GRAPH_NAME, otp_name=OTP_NAME, java_mem=None):
    """ run OTP graph builder
    """
    log.info("building the graph")
    graph_path = os.path.join(graph_dir, graph_name)
    otp_path = os.path.join(graph_dir, otp_name)
    file_utils.rm(graph_path)
    file_utils.cd(graph_dir)
    cmd='-jar {} --build {} --cache {}'.format(otp_path, graph_dir, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def vizualize_graph(graph_dir, java_mem=None, otp_name=OTP_NAME):
    otp_path = os.path.join(graph_dir, otp_name)
    file_utils.cd(graph_dir)
    cmd='-jar {} --visualize --router "" --graphs {}'.format(otp_path, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def send_build_test_email(to, build_status=True, test_status=True, server_status=True):
    """ utility to make the graph dir, copy OTP config files into the graph directory, etc...
    """
    #name = graph_config.get('name', DEF_NAME)
    web_utils.simple_email("msg", to)

def config_graph_dir(graph_config, base_dir, overwrite=False):
    """ utility to make the graph dir, copy OTP config files into the graph directory, etc...
    """
    name = graph_config.get('name', DEF_NAME)
    dir  = graph_config.get('dir',  name)     # optional 'dir' name overrides graph name

    # step 1: mkdir (makes the dir if it doesn't exist)
    graph_dir = os.path.join(base_dir, dir)
    file_utils.mkdir(graph_dir)
    graph_config['dir'] = graph_dir  # save off the full graph dir back struct

    # step 2: copy OTP config files
    config_dir = os.path.join(base_dir, "config")
    file_utils.copy_contents(config_dir, graph_dir, overwrite)

    # step 3: check OTP jar exists in config dir
    check_otp_jar(graph_dir)
    return graph_dir

def get_initial_arg_parser():
    """ make the initial cli argparse for OTP graph building and other fun things
    """
    import argparse
    parser = argparse.ArgumentParser(prog='otp-build', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('name', default="all", nargs='?', help="Name of OTP graph folder in the 'cache' build (e.g., 'all', 'prod', 'test' or 'call')")
    parser.add_argument('--test_suite', '-ts', help="regex name of test suites to run (e.g., 'rail', 'bus|rail', etc...)")
    return parser

def get_graphs(cache):
    """ routine that both returns the list of graphs, but also (the main
        purpose) add the 'dir' for each graph folder based on graph name
    """
    return get_graphs_from_config(cache.config, cache.this_module_dir)

def get_graphs_from_config(config=None, graph_root_dir='.'):
    """ return the OTP graph info from config --  from
    """
    if config is None:
        config = ConfigUtil(section='otp')

    graphs = config.get_json('graphs')
    for g in graphs:
        dir_path = os.path.join(graph_root_dir, g['name'])
        g['dir'] = dir_path
    return graphs

def find_graph(graphs, find_name):
    """ will build and test each of the graphs we have in self.graphs
    """
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = get_graph_details(None)
    else:
        for g in graphs:
            if find_name in g['name']:
                ret_val = g
                break
    return ret_val

def get_graph_details(graphs, index=0):
    """ utility function to find a graph config (e.g., graph folder name, web port, etc...) from self.graphs
        @see [otp] section in config/app.ini
    """
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = {"name":DEF_NAME, "port":DEF_PORT, "ssl":DEF_SSL_PORT}
        log.warn("graphs config was NIL, using default 'prod' graph info")
    else:
        if index >= len(graphs):
            index = 0
            log.warn("graph index of {} exceeds list length, so defaulting to index 0".format(index))
        ret_val = graphs[index]
    return ret_val

def check_otp_jar(graph_dir, jar=OTP_NAME, expected_size=50000000, download_url=OTP_DOWNLOAD_URL):
    """ utility to make sure otp.jar exists in the particular graph dir...
        if not, download it
        :return full-path to otp.jar
    """
    jar_path = os.path.join(graph_dir, jar)
    exists = os.path.exists(jar_path)
    if not exists or file_utils.file_size(jar_path) < expected_size:
        log.info("we don't see OTP {} in {}, so will download {} now".format(jar, graph_dir, download_url))
        web_utils.wget(download_url, jar_path)
    return jar_path

def get_vlog_file_path(dir_path, vlog_name=VLOG_NAME):
    """ build otp.vlog path """
    # import pdb; pdb.set_trace()
    vlog_path = os.path.join(dir_path, vlog_name)
    return vlog_path

def append_vlog_file(dir_path, feed_msg=None, vlog_name=VLOG_NAME):
    """ print out gtfs feed(s) version numbers and dates to the otp.v log file
    """
    msg = "\nUpdated graph on {} with GTFS feed(s):\n".format(datetime.datetime.now().strftime("%B %d, %Y @ %I:%M %p"))

    # add any specific feeds messages
    if feed_msg and len(feed_msg) > 1:
        msg = "{}{}\n".format(msg, feed_msg)

    # write message to vlog file
    vlog_path = get_vlog_file_path(dir_path, vlog_name)
    f = open(vlog_path, 'a')
    f.write(msg)
    f.flush()
    f.close()

def diff_vlog_files(svr, dir_path, vlog_name=VLOG_NAME):
    """ return True if the files are different and need to be redeployed ...

        - grab vlog from remote server that builds new OTP graphs
        - compare it to our local vlog
        - send email if we can't find remote vlog...
    """
    ret_val = False

    # step 1: grab otp.v from build server
    url = "{}/{}".format(svr, vlog_name)
    vlog_path = get_vlog_file_path(dir_path, vlog_name)
    tmp_vlog_path = vlog_path + ".tmp"
    ok = web_utils.wget(url, tmp_vlog_path, 10)

    if not ok:
        # step 2: remote server doesn't have otp.v exposed ... send an error email...
        msg = "No vlog available at {0}".format(url)
        web_utils.email(msg, msg)
        ret_val = False
    else:
        # step 3: make sure the otp.v we just downloaded has content ... if not, send an error email
        if not file_utils.is_min_sized(tmp_vlog_path, 20):
            msg = "vlog file {0} (grabbed from {1}) isn't right ".format(tmp_vlog_path, url)
            email(msg, msg)
            ret_val = False
        else:
            # step 4a: we currently don't have a vlog, so assume we don't have an existing OTP ... let's deploy new download...
            if not file_utils.is_min_sized(vlog_path, 20):
                ret_val = True
                logging.info("{0} doesn't exist ... try to grab new OTP from {1} and deploy".format(vlog_path, svr))
            else:
                # step 4b: check if the vlog files are different...if so, we'll assume the remote is newer and start a new deploy...
                if filecmp.cmp(tmp_vlog_path, vlog_path):
                    logging.info("{0} == {1} ... we're done, going to keep the current graph running".format(tmp_vlog_path, vlog_path))
                else:
                    ret_val = True
                    logging.info("{0} != {1} ... will try to grab new OTP from {2} and deploy".format(tmp_vlog_path, vlog_path, svr))
    return ret_val

def deploy_new_otp_graph(dir_path, graph_name=GRAPH_NAME, vlog_name=VLOG_NAME, otp_name=OTP_NAME):
    """ go thru steps of backing up old graph and moving new graph into place
    """
    ret_val = False

    new_graph = file_utils.make_new_path(dir_path, graph_name)
    new_vlog = file_utils.make_new_path(dir_path, vlog_name)
    new_otp = file_utils.make_new_path(dir_path, otp_name)

    # step 1: check if new OTP GRAPH and VLOG exist ... if both do, proceed
    if file_utils.is_min_sized(new_graph, quiet=True) and file_utils.is_min_sized(new_vlog, 20, quiet=True):
        new_otp_exists = file_utils.is_min_sized(new_otp, quiet=True)

        # step 2: current paths
        curr_graph = os.path.join(dir_path, graph_name)
        curr_vlog = os.path.join(dir_path, vlog_name)
        curr_otp = os.path.join(dir_path, otp_name)

        # step 3: create OLD folder and build old paths
        old_path = file_utils.make_old_dir(dir_path)
        old_graph = os.path.join(old_path, graph_name)
        old_vlog = os.path.join(old_path, vlog_name)
        old_otp = os.path.join(old_path, otp_name)

        # step 4: mv current stuff to the OLD directory
        file_utils.mv(curr_graph, old_graph)
        if new_otp_exists:
            file_utils.mv(curr_otp, old_otp)

        # step 5: make sure we moved old stuff out of the way ... if not, we have to exit
        if file_utils.is_min_sized(curr_graph, quiet=True) or (new_otp_exists and file_utils.is_min_sized(curr_otp, quiet=True)):
            # @todo this should be an email in addtion to a log message
            log.error("in trying to deploy new graph, I wasn't able to mv old {} (or {}) out of the way".format(curr_graph, curr_otp))
        else:
            # step 6: ok, we could move the graph (and maybe otp) to OLD dir ... now let's back those files up (rename with date stamp)
            file_utils.mv(curr_vlog, old_vlog)
            file_utils.bkup(old_vlog)
            file_utils.bkup(old_graph)
            if new_otp_exists:
                file_utils.bkup(old_otp)

            # step 7: move new stuff into the 'current' position
            file_utils.mv(new_graph, curr_graph)
            file_utils.mv(new_vlog, curr_vlog)
            if new_otp_exists:
                file_utils.mv(new_otp, curr_otp)

            # step 8: last check to make sure we did move things around properly
            if file_utils.is_min_sized(curr_graph) and file_utils.is_min_sized(curr_otp):
                ret_val = True
            else:
                # @todo this should be an email in addtion to a log message
                log.error("ruh roh: after trying to deploy a new graph, I don't see either {} or {}".format(curr_graph, curr_otp))

    return ret_val<commit_msg>-new for otp.jar support<commit_after>import os
import socket
import subprocess
import urllib2
import filecmp
import datetime
import logging
log = logging.getLogger(__file__)

from ott.utils.config_util import ConfigUtil
from ott.utils import exe_utils
from ott.utils import file_utils
from ott.utils import web_utils

# constants
DEF_NAME   = "prod"
DEF_PORT   = "55555"
DEF_SSL_PORT = "55551"
GRAPH_NAME = "Graph.obj"
OTP_NAME   = "otp.jar"
VLOG_NAME  = "otp.v"
OTP_DOWNLOAD_URL = "http://maven.conveyal.com.s3.amazonaws.com/org/opentripplanner/otp/0.20.0/otp-0.20.0-shaded.jar"

def restart_call(self, call_db_path="call_center/db/call_db.tar.gz", call_runner="call_center/run.sh"):
    """ retstart call-center app
        basically, if we see 'call' a database export (backup) in the call_center folder, we'll assume that call runs
        here and we'll restart the call server from a shell script (all pretty TriMet specific)
    """
    if os.path.isfile(call_db_path):
        subprocess.call([call_runner])

def get_graph_path(dir_path=None, graph_name=GRAPH_NAME):
    graph_path = os.path.join(dir_path, graph_name)
    return graph_path

def get_otp_path(dir_path=None, otp_name=OTP_NAME):
    """"return otp.jar path"""
    otp_path = os.path.join(dir_path, otp_name)
    return otp_path

def get_vlog_file_path(dir_path, vlog_name=VLOG_NAME):
    """return otp.vlog directory path"""
    # import pdb; pdb.set_trace()
    vlog_path = os.path.join(dir_path, vlog_name)
    return vlog_path

def get_test_urls_from_config(section='otp', port=None, hostname=None, ws_path=None, map_path=None):
    """ return the OTP map and ws urls from
    """
    config = ConfigUtil(section=section)

    if not hostname:
        hostname = config.get('host', def_val=web_utils.get_hostname())
    if not port:
        port = DEF_PORT

    if ws_path is None:
        ws_path = config.get('ws_url_path', def_val="/otp/routers/default/plan")
    ws_url = "http://{}:{}{}".format(hostname, port, ws_path)

    if map_path is None:
        map_path = config.get('map_url_path', def_val="")
    map_url = "http://{}:{}{}".format(hostname, port, map_path)

    return ws_url, map_url

def call_planner_svc(url, accept='application/xml'):
    """ make a call to the OTP web service
    """
    ret_val = None
    try:
        socket.setdefaulttimeout(2000)
        log.debug("call_otp: OTP output for " + url)
        req = urllib2.Request(url, None, {'Accept':accept})
        res = urllib2.urlopen(req)
        log.debug("call_otp: OTP output for " + url)
        ret_val = res.read()
        res.close()
    except:
        log.warn('ERROR: could not get data from url (timeout?): {0}'.format(url))
    return ret_val

def run_otp_server(dir_path=None, port=DEF_PORT, ssl=DEF_SSL_PORT, otp_name=OTP_NAME, java_mem=None, **kwargs):
    """ launch the server in a separate process
    """
    file_utils.cd(dir_path)
    otp_path = get_otp_path(dir_path, otp_name)
    cmd='-server -jar {} --port {} --securePort {} --router "" --graphs {}'.format(otp_path, port, ssl, dir)
    ret_val = exe_utils.run_java(cmd, fork=True, big_xmx=java_mem, pid_file="pid.txt")
    return ret_val

def run_graph_builder(graph_dir, graph_name=GRAPH_NAME, otp_name=OTP_NAME, java_mem=None):
    """ run OTP graph builder
    """
    log.info("building the graph")
    graph_path = get_graph_path(graph_dir, graph_name)
    otp_path = get_otp_path(graph_dir, otp_name)
    file_utils.rm(graph_path)
    file_utils.cd(graph_dir)
    cmd='-jar {} --build {} --cache {}'.format(otp_path, graph_dir, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def vizualize_graph(graph_dir, java_mem=None, otp_name=OTP_NAME):
    otp_path = os.path.join(graph_dir, otp_name)
    file_utils.cd(graph_dir)
    cmd='-jar {} --visualize --router "" --graphs {}'.format(otp_path, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def send_build_test_email(to, build_status=True, test_status=True, server_status=True):
    """ utility to make the graph dir, copy OTP config files into the graph directory, etc...
    """
    #name = graph_config.get('name', DEF_NAME)
    web_utils.simple_email("msg", to)

def config_graph_dir(graph_config, base_dir, overwrite=False):
    """ utility to make the graph dir, copy OTP config files into the graph directory, etc...
    """
    name = graph_config.get('name', DEF_NAME)
    dir  = graph_config.get('dir',  name)     # optional 'dir' name overrides graph name

    # step 1: mkdir (makes the dir if it doesn't exist)
    graph_dir = os.path.join(base_dir, dir)
    file_utils.mkdir(graph_dir)
    graph_config['dir'] = graph_dir  # save off the full graph dir back struct

    # step 2: copy OTP config files
    config_dir = os.path.join(base_dir, "config")
    file_utils.copy_contents(config_dir, graph_dir, overwrite)

    # step 3: check OTP jar exists in config dir
    check_otp_jar(graph_dir)
    return graph_dir

def get_initial_arg_parser():
    """ make the initial cli argparse for OTP graph building and other fun things
    """
    import argparse
    parser = argparse.ArgumentParser(prog='otp-build', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('name', default="all", nargs='?', help="Name of OTP graph folder in the 'cache' build (e.g., 'all', 'prod', 'test' or 'call')")
    parser.add_argument('--test_suite', '-ts', help="regex name of test suites to run (e.g., 'rail', 'bus|rail', etc...)")
    return parser

def get_graphs(cache):
    """ routine that both returns the list of graphs, but also (the main
        purpose) add the 'dir' for each graph folder based on graph name
    """
    return get_graphs_from_config(cache.config, cache.this_module_dir)

def get_graphs_from_config(config=None, graph_root_dir='.'):
    """ return the OTP graph info from config --  from
    """
    if config is None:
        config = ConfigUtil(section='otp')

    graphs = config.get_json('graphs')
    for g in graphs:
        dir_path = os.path.join(graph_root_dir, g['name'])
        g['dir'] = dir_path
    return graphs

def find_graph(graphs, find_name):
    """ will build and test each of the graphs we have in self.graphs
    """
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = get_graph_details(None)
    else:
        for g in graphs:
            if find_name in g['name']:
                ret_val = g
                break
    return ret_val

def get_graph_details(graphs, index=0):
    """ utility function to find a graph config (e.g., graph folder name, web port, etc...) from self.graphs
        @see [otp] section in config/app.ini
    """
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = {"name":DEF_NAME, "port":DEF_PORT, "ssl":DEF_SSL_PORT}
        log.warn("graphs config was NIL, using default 'prod' graph info")
    else:
        if index >= len(graphs):
            index = 0
            log.warn("graph index of {} exceeds list length, so defaulting to index 0".format(index))
        ret_val = graphs[index]
    return ret_val

def check_otp_jar(graph_dir, jar=OTP_NAME, expected_size=50000000, download_url=OTP_DOWNLOAD_URL):
    """ utility to make sure otp.jar exists in the particular graph dir...
        if not, download it
        :return full-path to otp.jar
    """
    jar_path = os.path.join(graph_dir, jar)
    exists = os.path.exists(jar_path)
    if not exists or file_utils.file_size(jar_path) < expected_size:
        log.info("we don't see OTP {} in {}, so will download {} now".format(jar, graph_dir, download_url))
        web_utils.wget(download_url, jar_path)
    return jar_path

def append_vlog_file(dir_path, feed_msg=None, vlog_name=VLOG_NAME):
    """ print out gtfs feed(s) version numbers and dates to the otp.v log file
    """
    msg = "\nUpdated graph on {} with GTFS feed(s):\n".format(datetime.datetime.now().strftime("%B %d, %Y @ %I:%M %p"))

    # add any specific feeds messages
    if feed_msg and len(feed_msg) > 1:
        msg = "{}{}\n".format(msg, feed_msg)

    # write message to vlog file
    vlog_path = get_vlog_file_path(dir_path, vlog_name)
    f = open(vlog_path, 'a')
    f.write(msg)
    f.flush()
    f.close()

def diff_vlog_files(svr, dir_path, vlog_name=VLOG_NAME):
    """ return True if the files are different and need to be redeployed ...

        - grab vlog from remote server that builds new OTP graphs
        - compare it to our local vlog
        - send email if we can't find remote vlog...
    """
    ret_val = False

    # step 1: grab otp.v from build server
    url = "{}/{}".format(svr, vlog_name)
    vlog_path = get_vlog_file_path(dir_path, vlog_name)
    tmp_vlog_path = vlog_path + ".tmp"
    ok = web_utils.wget(url, tmp_vlog_path, 10)

    if not ok:
        # step 2: remote server doesn't have otp.v exposed ... send an error email...
        msg = "No vlog available at {0}".format(url)
        web_utils.email(msg, msg)
        ret_val = False
    else:
        # step 3: make sure the otp.v we just downloaded has content ... if not, send an error email
        if not file_utils.is_min_sized(tmp_vlog_path, 20):
            msg = "vlog file {0} (grabbed from {1}) isn't right ".format(tmp_vlog_path, url)
            email(msg, msg)
            ret_val = False
        else:
            # step 4a: we currently don't have a vlog, so assume we don't have an existing OTP ... let's deploy new download...
            if not file_utils.is_min_sized(vlog_path, 20):
                ret_val = True
                logging.info("{0} doesn't exist ... try to grab new OTP from {1} and deploy".format(vlog_path, svr))
            else:
                # step 4b: check if the vlog files are different...if so, we'll assume the remote is newer and start a new deploy...
                if filecmp.cmp(tmp_vlog_path, vlog_path):
                    logging.info("{0} == {1} ... we're done, going to keep the current graph running".format(tmp_vlog_path, vlog_path))
                else:
                    ret_val = True
                    logging.info("{0} != {1} ... will try to grab new OTP from {2} and deploy".format(tmp_vlog_path, vlog_path, svr))
    return ret_val

def deploy_new_otp_graph(dir_path, graph_name=GRAPH_NAME, vlog_name=VLOG_NAME, otp_name=OTP_NAME):
    """ go thru steps of backing up old graph and moving new graph into place
    """
    ret_val = False

    new_graph = file_utils.make_new_path(dir_path, graph_name)
    new_vlog = file_utils.make_new_path(dir_path, vlog_name)
    new_otp = file_utils.make_new_path(dir_path, otp_name)

    # step 1: check if new OTP GRAPH and VLOG exist ... if both do, proceed
    if file_utils.is_min_sized(new_graph, quiet=True) and file_utils.is_min_sized(new_vlog, 20, quiet=True):
        new_otp_exists = file_utils.is_min_sized(new_otp, quiet=True)

        # step 2: current paths
        curr_graph = os.path.join(dir_path, graph_name)
        curr_vlog = os.path.join(dir_path, vlog_name)
        curr_otp = os.path.join(dir_path, otp_name)

        # step 3: create OLD folder and build old paths
        old_path = file_utils.make_old_dir(dir_path)
        old_graph = os.path.join(old_path, graph_name)
        old_vlog = os.path.join(old_path, vlog_name)
        old_otp = os.path.join(old_path, otp_name)

        # step 4: mv current stuff to the OLD directory
        file_utils.mv(curr_graph, old_graph)
        if new_otp_exists:
            file_utils.mv(curr_otp, old_otp)

        # step 5: make sure we moved old stuff out of the way ... if not, we have to exit
        if file_utils.is_min_sized(curr_graph, quiet=True) or (new_otp_exists and file_utils.is_min_sized(curr_otp, quiet=True)):
            # @todo this should be an email in addtion to a log message
            log.error("in trying to deploy new graph, I wasn't able to mv old {} (or {}) out of the way".format(curr_graph, curr_otp))
        else:
            # step 6: ok, we could move the graph (and maybe otp) to OLD dir ... now let's back those files up (rename with date stamp)
            file_utils.mv(curr_vlog, old_vlog)
            file_utils.bkup(old_vlog)
            file_utils.bkup(old_graph)
            if new_otp_exists:
                file_utils.bkup(old_otp)

            # step 7: move new stuff into the 'current' position
            file_utils.mv(new_graph, curr_graph)
            file_utils.mv(new_vlog, curr_vlog)
            if new_otp_exists:
                file_utils.mv(new_otp, curr_otp)

            # step 8: last check to make sure we did move things around properly
            if file_utils.is_min_sized(curr_graph) and file_utils.is_min_sized(curr_otp):
                ret_val = True
            else:
                # @todo this should be an email in addtion to a log message
                log.error("ruh roh: after trying to deploy a new graph, I don't see either {} or {}".format(curr_graph, curr_otp))

    return ret_val<|endoftext|>


<commit_before>import os
import socket
import urllib2
import filecmp
import datetime
import logging
log = logging.getLogger(__file__)

from ott.utils.config_util import ConfigUtil
from ott.utils import exe_utils
from ott.utils import file_utils
from ott.utils import web_utils

# constants
DEF_NAME   = "prod"
DEF_PORT   = "55555"
DEF_SSL_PORT = "55551"
GRAPH_NAME = "Graph.obj"
OTP_NAME   = "otp.jar"
VLOG_NAME  = "otp.v"
OTP_DOWNLOAD_URL = "http://maven.conveyal.com.s3.amazonaws.com/org/opentripplanner/otp/0.20.0/otp-0.20.0-shaded.jar"

def get_test_urls_from_config(section='otp', port=DEF_PORT, host=None):
    ''' return the OTP map and ws urls from
    '''
    config = ConfigUtil(section=section)

    if not host:
        host = config.get('host', def_val=web_utils.get_hostname())

    ws_path = config.get('ws_url_path', def_val="/otp/routers/default/plan")
    ws_url = "http://{}:{}{}".format(host, port, ws_path)

    map_path = config.get('map_url_path', def_val="")
    map_url = "http://{}:{}{}".format(host, port, map_path)

    return ws_url, map_url

def call_planner_svc(url, accept='application/xml'):
    ''' make a call to the OTP web service
    '''
    ret_val = None
    try:
        socket.setdefaulttimeout(2000)
        log.debug("call_otp: OTP output for " + url)
        req = urllib2.Request(url, None, {'Accept':accept})
        res = urllib2.urlopen(req)
        log.debug("call_otp: OTP output for " + url)
        ret_val = res.read()
        res.close()
    except:
        log.warn('ERROR: could not get data from url (timeout?): {0}'.format(url))
    return ret_val

def run_otp_server(dir=None, port=DEF_PORT, ssl=DEF_SSL_PORT, otp_name=OTP_NAME, java_mem=None, **kwargs):
    ''' launch the server in a separate process
    '''
    #import pdb; pdb.set_trace()
    file_utils.cd(dir)
    otp_path = os.path.join(dir, otp_name)
    cmd='-server -jar {} --port {} --securePort {} --router "" --graphs {}'.format(otp_path, port, ssl, dir)
    ret_val = exe_utils.run_java(cmd, fork=True, big_xmx=java_mem, pid_file="pid.txt")
    return ret_val

def run_graph_builder(graph_dir, graph_name=GRAPH_NAME, otp_name=OTP_NAME, java_mem=None):
    ''' run OTP graph builder
    '''
    log.info("building the graph")
    graph_path = os.path.join(graph_dir, graph_name)
    otp_path = os.path.join(graph_dir, otp_name)
    file_utils.rm(graph_path)
    file_utils.cd(graph_dir)
    cmd='-jar {} --build {} --cache {}'.format(otp_path, graph_dir, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def vizualize_graph(graph_dir, java_mem=None, otp_name=OTP_NAME):
    otp_path = os.path.join(graph_dir, otp_name)
    file_utils.cd(graph_dir)
    cmd='-jar {} --visualize --router "" --graphs {}'.format(otp_path, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def send_build_test_email(to, build_status=True, test_status=True, server_status=True):
    ''' utility to make the graph dir, copy OTP config files into the graph directory, etc...
    '''
    #name = graph_config.get('name', DEF_NAME)
    web_utils.simple_email("msg", to)

def config_graph_dir(graph_config, base_dir, overwrite=False):
    ''' utility to make the graph dir, copy OTP config files into the graph directory, etc...
    '''
    name = graph_config.get('name', DEF_NAME)
    dir  = graph_config.get('dir',  name)     # optional 'dir' name overrides graph name

    # step 1: mkdir (makes the dir if it doesn't exist)
    graph_dir = os.path.join(base_dir, dir)
    file_utils.mkdir(graph_dir)
    graph_config['dir'] = graph_dir  # save off the full graph dir back struct

    # step 2: copy OTP config files
    config_dir = os.path.join(base_dir, "config")
    file_utils.copy_contents(config_dir, graph_dir, overwrite)

    # step 3: check OTP jar exists in config dir
    check_otp_jar(graph_dir)
    return graph_dir

def get_graphs(cache):
    ''' routine that both returns the list of graphs, but also (the main
        purpose) add the 'dir' for each graph folder based on graph name
    '''
    graphs = cache.config.get_json('graphs')
    for g in graphs:
        dir = os.path.join(cache.this_module_dir, g['name'])
        g['dir'] = dir
    return graphs

def find_graph(graphs, find_name):
    ''' will build and test each of the graphs we have in self.graphs
    '''
    #import pdb; pdb.set_trace()
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = get_graph_details(None)
    else:
        for g in graphs:
            if find_name in g['name']:
                ret_val = g
                break
    return ret_val

def get_graph_details(graphs, index=0):
    ''' utility function to find a graph config (e.g., graph folder name, web port, etc...) from self.graphs
        @see [otp] section in config/app.ini
    '''
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = {"name":DEF_NAME, "port":DEF_PORT, "ssl":DEF_SSL_PORT}
        log.warn("graphs config was NIL, using default 'prod' graph info")
    else:
        if index >= len(graphs):
            index = 0
            log.warn("graph index of {} exceeds list length, so defaulting to index 0".format(index))
        ret_val = graphs[index]
    return ret_val

def check_otp_jar(graph_dir, jar=OTP_NAME, expected_size=50000000, download_url=OTP_DOWNLOAD_URL):
    """ utility to make sure otp.jar exists in the particular graph dir...
        if not, download it
        :return full-path to otp.jar
    """
    jar_path = os.path.join(graph_dir, jar)
    exists = os.path.exists(jar_path)
    if not exists or file_utils.file_size(jar_path) < expected_size:
        log.info("we don't see OTP {} in {}, so will download {} now".format(jar, graph_dir, download_url))
        web_utils.wget(download_url, jar_path)
    return jar_path

def append_vlog_file(dir, feed_msg=None, vlog_name=VLOG_NAME):
    """ print out gtfs feed(s) version numbers and dates to the otp.v log file
    """
    #
    msg = "\nUpdated graph on {} with GTFS feed(s):\n".format(datetime.datetime.now().strftime("%B %d, %Y @ %I:%M %p"))

    # add any specific feeds messages
    if feed_msg and len(feed_msg) > 1:
        msg = "{}{}\n".format(msg, feed_msg)

    # write message to vlog file
    vlog = os.path.join(dir, vlog_name)
    f = open(vlog, 'a')
    f.write(msg)
    f.flush()
    f.close()

def diff_vlog_files(svr, dir, vlog_name=VLOG_NAME):
    """ return True if the files are different and need to be redeployed ...

        - grab vlog from remote server that builds new OTP graphs
        - compare it to our local vlog
        - send email if we can't find remote vlog...
    """
    ret_val = False

    # step 1: grab otp.v from build server
    url = "{}/{}".format(svr, vlog_name)
    vlog_path = os.path.join(dir, vlog_name)
    tmp_vlog_path = vlog_path + ".tmp"
    ok = web_utils.wget(url, tmp_vlog_path, 10)

    if not ok:
        # step 2: remote server doesn't have otp.v exposed ... send an error email...
        msg = "No vlog available at {0}".format(url)
        web_utils.email(msg, msg)
        ret_val = False
    else:
        # step 3: make sure the otp.v we just downloaded has content ... if note, send an error email
        if not exists_and_sized(tmp_vlog_path, 20):
            msg = "vlog file {0} (grabbed from {1}) isn't right ".format(tmp_vlog_path, url)
            email(msg, msg)
            ret_val = False
        else:
            # step 4a: we currently don't have a vlog, so assume we don't have an existing OTP ... let's deploy new download...
            if not exists_and_sized(vlog_path, 20):
                ret_val = True
                logging.info("{0} doesn't exist ... try to grab new OTP from {1} and deploy".format(vlog_path, svr))
            else:
                # step 4b: check if the vlog files are different...if so, we'll assume the remote is newer and start a new deploy...
                if filecmp.cmp(tmp_vlog_path, vlog_path):
                    logging.info("{0} == {1} ... we're done, going to keep the current graph running".format(tmp_vlog_path, vlog_path))
                else:
                    ret_val = True
                    logging.info("{0} != {1} ... will try to grab new OTP from {2} and deploy".format(tmp_vlog_path, vlog_path, svr))
    return ret_val

<commit_msg>better default port number<commit_after>import os
import socket
import urllib2
import filecmp
import datetime
import logging
log = logging.getLogger(__file__)

from ott.utils.config_util import ConfigUtil
from ott.utils import exe_utils
from ott.utils import file_utils
from ott.utils import web_utils

# constants
DEF_NAME   = "prod"
DEF_PORT   = "55555"
DEF_SSL_PORT = "55551"
GRAPH_NAME = "Graph.obj"
OTP_NAME   = "otp.jar"
VLOG_NAME  = "otp.v"
OTP_DOWNLOAD_URL = "http://maven.conveyal.com.s3.amazonaws.com/org/opentripplanner/otp/0.20.0/otp-0.20.0-shaded.jar"

def get_test_urls_from_config(section='otp', port=None, host=None):
    ''' return the OTP map and ws urls from
    '''
    #import pdb; pdb.set_trace()
    config = ConfigUtil(section=section)

    if not host:
        host = config.get('host', def_val=web_utils.get_hostname())
    if not port:
        port = DEF_PORT

    ws_path = config.get('ws_url_path', def_val="/otp/routers/default/plan")
    ws_url = "http://{}:{}{}".format(host, port, ws_path)

    map_path = config.get('map_url_path', def_val="")
    map_url = "http://{}:{}{}".format(host, port, map_path)

    return ws_url, map_url

def call_planner_svc(url, accept='application/xml'):
    ''' make a call to the OTP web service
    '''
    ret_val = None
    try:
        socket.setdefaulttimeout(2000)
        log.debug("call_otp: OTP output for " + url)
        req = urllib2.Request(url, None, {'Accept':accept})
        res = urllib2.urlopen(req)
        log.debug("call_otp: OTP output for " + url)
        ret_val = res.read()
        res.close()
    except:
        log.warn('ERROR: could not get data from url (timeout?): {0}'.format(url))
    return ret_val

def run_otp_server(dir=None, port=DEF_PORT, ssl=DEF_SSL_PORT, otp_name=OTP_NAME, java_mem=None, **kwargs):
    ''' launch the server in a separate process
    '''
    file_utils.cd(dir)
    otp_path = os.path.join(dir, otp_name)
    cmd='-server -jar {} --port {} --securePort {} --router "" --graphs {}'.format(otp_path, port, ssl, dir)
    ret_val = exe_utils.run_java(cmd, fork=True, big_xmx=java_mem, pid_file="pid.txt")
    return ret_val

def run_graph_builder(graph_dir, graph_name=GRAPH_NAME, otp_name=OTP_NAME, java_mem=None):
    ''' run OTP graph builder
    '''
    log.info("building the graph")
    graph_path = os.path.join(graph_dir, graph_name)
    otp_path = os.path.join(graph_dir, otp_name)
    file_utils.rm(graph_path)
    file_utils.cd(graph_dir)
    cmd='-jar {} --build {} --cache {}'.format(otp_path, graph_dir, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def vizualize_graph(graph_dir, java_mem=None, otp_name=OTP_NAME):
    otp_path = os.path.join(graph_dir, otp_name)
    file_utils.cd(graph_dir)
    cmd='-jar {} --visualize --router "" --graphs {}'.format(otp_path, graph_dir)
    ret_val = exe_utils.run_java(cmd, big_xmx=java_mem)
    return ret_val

def send_build_test_email(to, build_status=True, test_status=True, server_status=True):
    ''' utility to make the graph dir, copy OTP config files into the graph directory, etc...
    '''
    #name = graph_config.get('name', DEF_NAME)
    web_utils.simple_email("msg", to)

def config_graph_dir(graph_config, base_dir, overwrite=False):
    ''' utility to make the graph dir, copy OTP config files into the graph directory, etc...
    '''
    name = graph_config.get('name', DEF_NAME)
    dir  = graph_config.get('dir',  name)     # optional 'dir' name overrides graph name

    # step 1: mkdir (makes the dir if it doesn't exist)
    graph_dir = os.path.join(base_dir, dir)
    file_utils.mkdir(graph_dir)
    graph_config['dir'] = graph_dir  # save off the full graph dir back struct

    # step 2: copy OTP config files
    config_dir = os.path.join(base_dir, "config")
    file_utils.copy_contents(config_dir, graph_dir, overwrite)

    # step 3: check OTP jar exists in config dir
    check_otp_jar(graph_dir)
    return graph_dir

def get_graphs(cache):
    ''' routine that both returns the list of graphs, but also (the main
        purpose) add the 'dir' for each graph folder based on graph name
    '''
    graphs = cache.config.get_json('graphs')
    for g in graphs:
        dir = os.path.join(cache.this_module_dir, g['name'])
        g['dir'] = dir
    return graphs

def find_graph(graphs, find_name):
    ''' will build and test each of the graphs we have in self.graphs
    '''
    #import pdb; pdb.set_trace()
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = get_graph_details(None)
    else:
        for g in graphs:
            if find_name in g['name']:
                ret_val = g
                break
    return ret_val

def get_graph_details(graphs, index=0):
    ''' utility function to find a graph config (e.g., graph folder name, web port, etc...) from self.graphs
        @see [otp] section in config/app.ini
    '''
    ret_val = None
    if graphs is None or len(graphs) < 1:
        ret_val = {"name":DEF_NAME, "port":DEF_PORT, "ssl":DEF_SSL_PORT}
        log.warn("graphs config was NIL, using default 'prod' graph info")
    else:
        if index >= len(graphs):
            index = 0
            log.warn("graph index of {} exceeds list length, so defaulting to index 0".format(index))
        ret_val = graphs[index]
    return ret_val

def check_otp_jar(graph_dir, jar=OTP_NAME, expected_size=50000000, download_url=OTP_DOWNLOAD_URL):
    """ utility to make sure otp.jar exists in the particular graph dir...
        if not, download it
        :return full-path to otp.jar
    """
    jar_path = os.path.join(graph_dir, jar)
    exists = os.path.exists(jar_path)
    if not exists or file_utils.file_size(jar_path) < expected_size:
        log.info("we don't see OTP {} in {}, so will download {} now".format(jar, graph_dir, download_url))
        web_utils.wget(download_url, jar_path)
    return jar_path

def append_vlog_file(dir, feed_msg=None, vlog_name=VLOG_NAME):
    """ print out gtfs feed(s) version numbers and dates to the otp.v log file
    """
    #
    msg = "\nUpdated graph on {} with GTFS feed(s):\n".format(datetime.datetime.now().strftime("%B %d, %Y @ %I:%M %p"))

    # add any specific feeds messages
    if feed_msg and len(feed_msg) > 1:
        msg = "{}{}\n".format(msg, feed_msg)

    # write message to vlog file
    vlog = os.path.join(dir, vlog_name)
    f = open(vlog, 'a')
    f.write(msg)
    f.flush()
    f.close()

def diff_vlog_files(svr, dir, vlog_name=VLOG_NAME):
    """ return True if the files are different and need to be redeployed ...

        - grab vlog from remote server that builds new OTP graphs
        - compare it to our local vlog
        - send email if we can't find remote vlog...
    """
    ret_val = False

    # step 1: grab otp.v from build server
    url = "{}/{}".format(svr, vlog_name)
    vlog_path = os.path.join(dir, vlog_name)
    tmp_vlog_path = vlog_path + ".tmp"
    ok = web_utils.wget(url, tmp_vlog_path, 10)

    if not ok:
        # step 2: remote server doesn't have otp.v exposed ... send an error email...
        msg = "No vlog available at {0}".format(url)
        web_utils.email(msg, msg)
        ret_val = False
    else:
        # step 3: make sure the otp.v we just downloaded has content ... if note, send an error email
        if not exists_and_sized(tmp_vlog_path, 20):
            msg = "vlog file {0} (grabbed from {1}) isn't right ".format(tmp_vlog_path, url)
            email(msg, msg)
            ret_val = False
        else:
            # step 4a: we currently don't have a vlog, so assume we don't have an existing OTP ... let's deploy new download...
            if not exists_and_sized(vlog_path, 20):
                ret_val = True
                logging.info("{0} doesn't exist ... try to grab new OTP from {1} and deploy".format(vlog_path, svr))
            else:
                # step 4b: check if the vlog files are different...if so, we'll assume the remote is newer and start a new deploy...
                if filecmp.cmp(tmp_vlog_path, vlog_path):
                    logging.info("{0} == {1} ... we're done, going to keep the current graph running".format(tmp_vlog_path, vlog_path))
                else:
                    ret_val = True
                    logging.info("{0} != {1} ... will try to grab new OTP from {2} and deploy".format(tmp_vlog_path, vlog_path, svr))
    return ret_val

<|endoftext|>


<commit_before>"""Base class object for superclasses."""

from json import dumps, loads
from os.path import getmtime
from time import time

from pybliometrics.scopus.exception import ScopusQueryError
from pybliometrics.scopus.utils import get_content, print_progress

class Base:
    def __init__(self, fname, refresh, params, url, download=None,
                 max_entries=None, verbose=False, *args, **kwds):
        """Class intended as base class for superclasses.

        Parameters
        ----------
        fname : str
            The filename (including path) of the cache object.

        refresh : bool or int
            Whether to refresh the cached file if it exists or not.  If int
            is passed, cached file will be refreshed if the number of days
            since last modification exceeds that value.

        params : dict
            Dictionary used as header during the API request.

        url : str
            The URL to be accessed.

        download : bool (optional, default=None)
            Whether to download the query or not.  Has no effect for
            retrieval requests.

        max_entries : int (optional, default=None)
            Raise error when the number of results is beyond this number.
            Has no effect for retrieval requests.

        verbose : bool (optional, default=False)
            Whether to print a progress bar for multip-page requests.

        *args, **kwds
            Arguments and key-value pairings to be passed on
            to `get_content()`.

        Raises
        ------
        ScopusQueryError
            If `refresh` is neither boolean nor numeric.

        ValueError
            If `refresh` is neither boolean nor numeric.
        """
        # Compare age of file to test whether we refresh
        refresh, exists, mod_ts = _check_file_age(fname, refresh)

        # Read or dowload eventually with caching
        search_request = "query" in params
        if exists and not refresh:
            self._mdate = mod_ts
            if search_request:
                with open(fname, 'rb') as f:
                    self._json = [loads(line) for line in f.readlines()]
                self._n = len(self._json)
            else:
                with open(fname, 'rb') as f:
                    self._json = loads(f.read().decode('utf-8'))
        else:
            if search_request:
                # Download results
                res = get_content(url, params, *args, **kwds).json()
                n = int(res['search-results'].get('opensearch:totalResults', 0))
                self._n = n
                cursor_false = "cursor" in params and not params["cursor"]
                if cursor_false and n > max_entries:
                    # Stop if there are too many results
                    text = (f'Found {n} matches. Set max_entries to a higher '
                            f'number, change your query ({query}) or set '
                            'subscription=True')
                    raise ScopusQueryError(text)
                if download:
                    self._json = _parse(res, n, url, params, verbose,
                                        *args, **kwds)
                    with open(fname, 'wb') as f:
                        for item in self._json:
                            f.write(f'{dumps(item)}\n'.encode('utf-8'))
                else:
                    # Assures that properties will not result in an error
                    self._json = []
            else:
                content = get_content(url, params, *args, **kwds).text.encode('utf-8')
                self._json = loads(content)
                with open(fname, 'wb') as f:
                    f.write(content)
            self._mdate = time()

    def get_cache_file_age(self):
        """Return the age of the cached file in days."""
        diff = time() - self._mdate
        return int(diff / 86400)

    def get_cache_file_mdate(self):
        """Return the modification date of the cached file as
        datetime object.
        """
        from datetime import datetime
        return datetime.fromtimestamp(self._mdate)

def _check_file_age(fname, refresh):
    """Check whether a file needs to be refreshed based on its age."""
    exists = None
    try:
        mod_ts = getmtime(fname)
        exists = True
        if not isinstance(refresh, bool):
            diff = time() - mod_ts
            days = int(diff / 86400) + 1
            try:
                allowed_age = int(refresh)
            except ValueError:
                msg = "Parameter refresh needs to be numeric or boolean."
                raise ValueError(msg)
            refresh = allowed_age < days
    except FileNotFoundError:
        exists = False
        refresh = True
        mod_ts = None
    return refresh, exists, mod_ts

def _parse(res, n, url, params, verbose, *args, **kwds):
    """Auxiliary function to download results and parse json."""
    cursor = "cursor" in params
    if not cursor:
        start = params["start"]
    if n == 0:
        return ""
    _json = res.get('search-results', {}).get('entry', [])
    if verbose:
        chunk = 1
        # Roundup + 1 for the final iteration
        chunks = int(n/params['count']) + (n % params['count'] > 0) + 1
        print(f'Downloading results for query "{params["query"]}":')
        print_progress(chunk, chunks)
    # Download the remaining information in chunks
    while n > 0:
        n -= params["count"]
        if cursor:
            pointer = res['search-results']['cursor'].get('@next')
            params.update({'cursor': pointer})
        else:
            start += params["count"]
            params.update({'start': start})
        res = get_content(url, params, *args, **kwds).json()
        _json.extend(res.get('search-results', {}).get('entry', []))
        if verbose:
            chunk += 1
            print_progress(chunk, chunks)
    return _json
<commit_msg>Methods for key reset and remaining quota (close #129)<commit_after>"""Base class object for superclasses."""

from json import dumps, loads
from os.path import getmtime
from time import localtime, strftime, time

from pybliometrics.scopus.exception import ScopusQueryError
from pybliometrics.scopus.utils import get_content, print_progress

class Base:
    def __init__(self, fname, refresh, params, url, download=None,
                 max_entries=None, verbose=False, *args, **kwds):
        """Class intended as base class for superclasses.

        Parameters
        ----------
        fname : str
            The filename (including path) of the cache object.

        refresh : bool or int
            Whether to refresh the cached file if it exists or not.  If int
            is passed, cached file will be refreshed if the number of days
            since last modification exceeds that value.

        params : dict
            Dictionary used as header during the API request.

        url : str
            The URL to be accessed.

        download : bool (optional, default=None)
            Whether to download the query or not.  Has no effect for
            retrieval requests.

        max_entries : int (optional, default=None)
            Raise error when the number of results is beyond this number.
            Has no effect for retrieval requests.

        verbose : bool (optional, default=False)
            Whether to print a progress bar for multip-page requests.

        *args, **kwds
            Arguments and key-value pairings to be passed on
            to `get_content()`.

        Raises
        ------
        ScopusQueryError
            If `refresh` is neither boolean nor numeric.

        ValueError
            If `refresh` is neither boolean nor numeric.
        """
        # Compare age of file to test whether we refresh
        refresh, exists, mod_ts = _check_file_age(fname, refresh)

        # Read or dowload eventually with caching
        search_request = "query" in params
        if exists and not refresh:
            self._mdate = mod_ts
            if search_request:
                with open(fname, 'rb') as f:
                    self._json = [loads(line) for line in f.readlines()]
                self._n = len(self._json)
            else:
                with open(fname, 'rb') as f:
                    self._json = loads(f.read().decode('utf-8'))
        else:
            resp = get_content(url, params, *args, **kwds)
            if search_request:
                # Download results
                res = resp.json()
                n = int(res['search-results'].get('opensearch:totalResults', 0))
                self._n = n
                cursor_false = "cursor" in params and not params["cursor"]
                if cursor_false and n > max_entries:
                    # Stop if there are too many results
                    text = (f'Found {n} matches. Set max_entries to a higher '
                            f'number, change your query ({query}) or set '
                            'subscription=True')
                    raise ScopusQueryError(text)
                if download:
                    data, header = _parse(res, n, url, params, verbose,
                                          *args, **kwds)
                    self._json = data
                    self._header = header
                    with open(fname, 'wb') as f:
                        for item in self._json:
                            f.write(f'{dumps(item)}\n'.encode('utf-8'))
                else:
                    # Assures that properties will not result in an error
                    self._json = []
            else:
                content = resp.text.encode('utf-8')
                self._json = loads(content)
                with open(fname, 'wb') as f:
                    f.write(content)
                self._header = resp.headers
            self._mdate = time()

    def get_cache_file_age(self):
        """Return the age of the cached file in days."""
        diff = time() - self._mdate
        return int(diff / 86400)

    def get_cache_file_mdate(self):
        """Return the modification date of the cached file."""
        return strftime('%Y-%m-%d %H:%M:%S', localtime(self._mdate))

    def get_key_remaining_quota(self):
        """Return number of remaining requests for the current key and the
        current API (relative on last actual request).
        """
        try:
            return self._header['X-RateLimit-Remaining']
        except AttributeError:
            return None

    def get_key_reset_time(self):
        """Return time when current key is reset (relative on last
        actual request).
        """
        try:
            date = int(self._header['X-RateLimit-Reset'])/1000
            return strftime('%Y-%m-%d %H:%M:%S', localtime(date))
        except AttributeError:
            return None

def _check_file_age(fname, refresh):
    """Check whether a file needs to be refreshed based on its age."""
    exists = None
    try:
        mod_ts = getmtime(fname)
        exists = True
        if not isinstance(refresh, bool):
            diff = time() - mod_ts
            days = int(diff / 86400) + 1
            try:
                allowed_age = int(refresh)
            except ValueError:
                msg = "Parameter refresh needs to be numeric or boolean."
                raise ValueError(msg)
            refresh = allowed_age < days
    except FileNotFoundError:
        exists = False
        refresh = True
        mod_ts = None
    return refresh, exists, mod_ts

def _parse(res, n, url, params, verbose, *args, **kwds):
    """Auxiliary function to download results and parse json."""
    cursor = "cursor" in params
    if not cursor:
        start = params["start"]
    if n == 0:
        return ""
    _json = res.get('search-results', {}).get('entry', [])
    if verbose:
        chunk = 1
        # Roundup + 1 for the final iteration
        chunks = int(n/params['count']) + (n % params['count'] > 0) + 1
        print(f'Downloading results for query "{params["query"]}":')
        print_progress(chunk, chunks)
    # Download the remaining information in chunks
    while n > 0:
        n -= params["count"]
        if cursor:
            pointer = res['search-results']['cursor'].get('@next')
            params.update({'cursor': pointer})
        else:
            start += params["count"]
            params.update({'start': start})
        resp = get_content(url, params, *args, **kwds)
        data = resp.json()
        _json.extend(data.get('search-results', {}).get('entry', []))
        if verbose:
            chunk += 1
            print_progress(chunk, chunks)
    return _json, resp.headers
<|endoftext|>


<commit_before>from boadata import __version__, load
from boadata.core import DataObject

import click

@click.command()
@click.version_option(__version__)
@click.argument("from_uri")
@click.argument("to_uri")
@click.option("-t", "--type", default=None, help="What type should be the destination object.")
def run_app(from_uri, to_uri, **kwargs):
    kwargs = {key: value for key, value in kwargs.items() if value is not None}
    type = kwargs.get("type")

    do = load(from_uri)
    if not do:
        print("URI not understood: {0}").format(from_uri)
        sys.exit(-1)
    if not type:
        for conversion in do.allowed_conversions:
            type_candidate = DataObject.registered_types[conversion[1]]
            if type_candidate.accepts_uri(to_uri):
                type = conversion[1]
                break
    if not type:
        print("No suitable output type found.")
        exit(-1)

    do.convert(type, uri=to_uri)<commit_msg>boaconvert without output uri<commit_after>from boadata import __version__, load
from boadata.core import DataObject

import click

@click.command()
@click.version_option(__version__)
@click.argument("from_uri")
@click.argument("to_uri", required=False, default=None)
@click.option("-t", "--type", default=None, help="What type should be the destination object.")
def run_app(from_uri, to_uri, **kwargs):
    kwargs = {key: value for key, value in kwargs.items() if value is not None}
    type = kwargs.get("type")

    do = load(from_uri)
    if not do:
        print("URI not understood: {0}").format(from_uri)
        sys.exit(-1)

    if to_uri:
        if not type:
            for conversion in do.allowed_conversions:
                type_candidate = DataObject.registered_types[conversion[1]]
                if type_candidate.accepts_uri(to_uri):
                    type = conversion[1]
                    break
        if not type:
            print("No suitable output type found.")
            exit(-1)

        do.convert(type, uri=to_uri)
    else:
        print("Allowed conversions")
        print("-------------------")
        for conversion in do.allowed_conversions:
            print(conversion[1])<|endoftext|>


<commit_before>import re
import argparse
import os

def check_parenthesis(file):
    open_parenthesis = re.compile('[\(]')
    close_parenthesis = re.compile('[\)]')
    quotation_marks = re.compile('[\"]')
    opened = 0
    closed = 0
    quote = 0
    for match in open_parenthesis.finditer(file):
        opened += 1
    for match in close_parenthesis.finditer(file):
        closed += 1
    for march in quotation_marks.finditer(file):
        quote += 1
    if opened != closed:
        raise Exception("Problem with TBL file parentheses (%d opening for %d closing parentheses)" % (opened, closed))
    if quote % 2 != 0:
        raise Exception("Problem with TBL file, odd number of quotation marks (%d quotation marks)" % quote)

def validate_tbl(restraints, pcs = False):
    output = ""
    parentmatch = re.compile('[\(\)]')
    # Global mode is activated outside any assign statement
    mode = "global"
    # Remove any carriage return/new line caracters
    lines = restraints.replace('\r','').split("\n")
    # Line number
    lnr = 0
    # Temporary line storage for future output
    tmp_output = None

    for l in lines:
        lnr+=1
        # Take everything which is before putative "!" (comment) caracter
        if l.find("!") > -1:
            l = l[:l.find("!")]
        # Remove whitespaces
        l = l.strip()
        # End of line
        if not len(l):
            continue
        # Check if all "" are closed
        if l.count('"') % 2 > 0:
            raise Exception('Unclosed " at line %d for line: %s' % (lnr,l))
        if mode in ("global", "postglobal"):
            # Start of an assign statement
            if l.lower().startswith("assi"):
                if mode == "postglobal":
                    output += "\n!\n"
                    output += tmp_output
                mode = "assign"
                selections = []
                # assign is the only caracter of the line, check following lines
                if l.find("(") == -1:
                    continue
                # Reset temporary buffer
                tmp_output = None
            # Check for "OR" restraint
            elif mode == "postglobal" and l.lower().startswith("or"):
                mode = "postassign"
                selections = []
                l = l[len("or"):]
                # Case where "OR" statement is the only one on the line (rest of selection at the next line)
                if l == "":
                    continue
            # We are not treating an assign params (postglobal) or at the end (global)
            # and no "OR" restraint is present -> ERROR
            else:
                raise Exception("Invalid TBL file: Unknown statement (line %d): %s" % (lnr, l))

        matched = True
        while matched:
            matched = False
            # We are looking for parenthesis as start of the selections
            if mode in ("assign", "postassign"):
                pos = l.find("(")
                if pos != -1:
                    matched = True
                    l = l[pos+1:]
                    # We look for an "OR" selection
                    if mode == "postassign":
                        mode = "postsel"
                    else:
                        mode = "sel"
                    lastassign = lnr
                    s = ""
                    level = 1
            # Get the structural selections
            if mode in ("sel", "postsel"):
                # print mode
                for match in parentmatch.finditer(l):
                    if match.group() == "(":
                        level += 1
                    else:
                        level -= 1
                    # End of the parenthesis content
                    if level == 0:
                        if mode == "postsel":
                            mode = "postassign"
                        else:
                            mode = "assign"
                        matched = True
                        # Get back the parentheses content
                        s += l[:match.start()]
                        selections.append(s)
                        s = None
                        # Get the rest of the line
                        l = l[match.end():]
                        # Go to the process of the selection
                        break
                # No parenthesis, we get the whole line
                if level > 0:
                    if l != "":
                        s += l + "\n\t"
                    else:
                        # To avoid multiple blank lines
                        if repr(s) == "''":
                            # print repr(l), repr(s), selections
                            s += "\n\t"
                        # else:
                        #     s += "SALUT"
        # TBD
        if mode in ("sel", "postsel"):
            continue
        # Selection parsed for "OR" line
        if mode == "postassign":
            if len(selections) != postselections:
                raise Exception("Invalid TBL file: wrong number of selections: in-term %d, cross-term %d (stopped at line %d)" % (postselections, len(selections), lnr))
            tmp_output +=" or"
            for s in selections:
                tmp_output += "\t(%s)\n" % s
            # output += tmp_output
            # tmp_output = None
            # We let the possibility for other "OR"
            mode = "postglobal"
        if len(l) == 0:
            continue
        # Define the distance restraints type to adapt the parsing
        if mode == "assign":
            mode = "numbers"
            if pcs:
                if len(selections) == 5:
                    types = (" %.3f", " %.3f")
                else:
                    raise Exception("Invalid TBL file: wrong number of selections (must be 5 in PCS mode)")
            else:
                if len(selections) == 2:
                    types = (" %.3f", " %.3f", " %.3f")
                elif len(selections) == 4:
                    types = (" %.3f", " %.3f", " %.3f", " %d")
                elif len(selections) == 5:
                    raise Exception("Invalid TBL file: wrong number of selections (can be 5 only in PCS mode)")
                elif len(selections) == 6:
                    types = (" %.3f", " %.3f")
                else:
                    check_parenthesis(restraints)
                    raise Exception("Invalid TBL file: wrong number of selections (must be 2,4 or 6)")
            postselections = len(selections)
            numbers = []
        # Distance restraints parsing
        if mode == "numbers":
            ll = l.split()
            for num in ll:
                if len(numbers) == len(types):
                    break
                numbers.append(float(num))
            if len(numbers) == len(types):
                tmp_output = "assign "
                for s in selections:
                    tmp_output += "\t(%s)\n" % s
                tmp_output = tmp_output[:-len("\n")]
                for n,t in zip(numbers,types):
                    tmp_output += t % n
                tmp_output += "\n"
                mode = "postglobal"
    # "OR" lines have been parsed, we store the selections
    if mode == "postglobal":
        output += "!\n"
        output += tmp_output
        tmp_output = None
        mode = "global"
    # If mode is not back to global, something has not been processed properly
    if mode != "global":
        raise Exception("Invalid TBL file: Malformed ASSIGN statement (line %d), use --quick to check for putative syntax issues" % lastassign)
    if not len(output.strip()):
        raise Exception("Invalid or empty TBL file")

    # Remove extra lines before each assign statement
    output = output.replace("\n\n", "\n")
    # Remove extra line at the beginning of the file
    if output.startswith("\n"):
        output = output.replace("\n", "", 1)
    return output

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="This script validates a restraint file (*.tbl).\n")

    parser.add_argument("file",
      help="TBL file to be validated")

    parser.add_argument("--pcs", action='store_true',
      help="PCS mode")

    parser.add_argument("--quick", action='store_true',
        help="Check global formatting before going line by line (opening/closing parenthesis and quotation marks")

    args = parser.parse_args()

    if (args.quick):
        tbldata = open(args.file).read()
        # Check the parenthesis and quotation marks opening/closure
        check_parenthesis(tbldata)

    if os.path.exists(args.file):
        tbldata = open(args.file).read()
        # Parse and process the restraints
        print validate_tbl(tbldata, args.pcs)
    else:
        raise Exception("TBL file %s does not exist, check the path" % args.file)

<commit_msg>Add multiple selection capability<commit_after>import re
import argparse
import os

def check_parenthesis(file):
    open_parenthesis = re.compile('[\(]')
    close_parenthesis = re.compile('[\)]')
    quotation_marks = re.compile('[\"]')
    opened = 0
    closed = 0
    quote = 0
    for match in open_parenthesis.finditer(file):
        opened += 1
    for match in close_parenthesis.finditer(file):
        closed += 1
    for march in quotation_marks.finditer(file):
        quote += 1
    if opened != closed:
        raise Exception("Problem with TBL file parentheses (%d opening for %d closing parentheses)" % (opened, closed))
    if quote % 2 != 0:
        raise Exception("Problem with TBL file, odd number of quotation marks (%d quotation marks)" % quote)

def validate_tbl(restraints, pcs = False):
    output = ""
    parentmatch = re.compile('[\(\)]')
    # Global mode is activated outside any assign statement
    mode = "global"
    # Remove any carriage return/new line caracters
    lines = restraints.replace('\r','').split("\n")
    # Line number
    lnr = 0
    # Temporary line storage for future output
    tmp_output = None

    for l in lines:
        lnr+=1
        # Take everything which is before putative "!" (comment) caracter
        if l.find("!") > -1:
            l = l[:l.find("!")]
        # Remove whitespaces and merge with previous line if in OR statement for AIR restraints
        if mode != "format":
            l = l.strip()
        else:
            l = tmp + l.strip()
            mode = "postassign"
        # End of line
        if not len(l):
            continue
        # Check if all "" are closed
        if l.count('"') % 2 > 0:
            raise Exception('Unclosed " at line %d for line: %s' % (lnr,l))
        if mode in ("global", "postglobal"):
            # Start of an assign statement
            if l.lower().startswith("assi"):
                if mode == "postglobal":
                    output += "\n!\n"
                    output += tmp_output
                mode = "assign"
                selections = []
                # assign is the only caracter of the line, check following lines
                if l.find("(") == -1:
                    continue
                # Reset temporary buffer
                tmp_output = None
            # Check for "OR" restraint
            elif mode == "postglobal" and l.lower().startswith("or"):
                mode = "postassign"
                selections = []
                l = l[len("or"):]
                # Case where "OR" statement is the only one on the line (rest of selection at the next line)
                if l == "":
                    continue
            # We are not treating an assign params (postglobal) or at the end (global)
            # and no "OR" restraint is present -> ERROR
            else:
                raise Exception("Invalid TBL file: Unknown statement (line %d): %s" % (lnr, l))
        # We check if the selection is made over two lines (thanks to "segid" keyword)
        if mode == "postassign":
            if l.count("segid") == 1:
                mode = "format"
                tmp = l
                continue

        matched = True
        while matched:
            matched = False
            # We are looking for parenthesis as start of the selections
            if mode in ("assign", "postassign"):
                # Ambiguous restraint for two different pairs of atoms (ex: THR 1 B <-> ALA 2 A OR GLY 2 B <-> ASP 10 A )
                pos = l.find("(")
                if pos != -1:
                    matched = True
                    l = l[pos+1:]
                    # We look for an "OR" selection
                    if mode == "postassign":
                        mode = "postsel"
                    else:
                        mode = "sel"
                    lastassign = lnr
                    s = ""
                    level = 1
            # Get the structural selections
            if mode in ("sel", "postsel"):
                # Detect opening and closing parenthesis
                for match in parentmatch.finditer(l):
                    if match.group() == "(":
                        level += 1
                    else:
                        level -= 1
                    # End of the parenthesis content
                    if level == 0:
                        if mode == "postsel":
                            mode = "postassign"
                        else:
                            mode = "assign"
                        matched = True
                        # Get back the parentheses content
                        s += l[:match.start()]
                        selections.append(s)
                        s = None
                        # Get the rest of the line
                        l = l[match.end():]
                        # Go to the process of the selection
                        break
                # No parenthesis, we get the whole line
                if level > 0:
                    if l != "":
                        s += l + "\n\t"
                    else:
                        # To avoid multiple blank lines
                        if repr(s) == "''":
                            # print repr(l), repr(s), selections
                            s += "\n\t"
        # TBD
        if mode in ("sel", "postsel"):
            continue
        # Selection parsed for "OR" line
        if mode == "postassign":
            if len(selections) != postselections:
                raise Exception("Invalid TBL file: wrong number of selections: in-term %d, cross-term %d (stopped at line %d)" % (postselections, len(selections), lnr))
            tmp_output +=" or"
            for s in selections:
                tmp_output += "\t(%s)\n" % s
            # We let the possibility for other "OR"
            mode = "postglobal"
        if len(l) == 0:
            continue
        # Define the distance restraints type to adapt the parsing
        if mode == "assign":
            mode = "numbers"
            if pcs:
                if len(selections) == 5:
                    types = (" %.3f", " %.3f")
                else:
                    raise Exception("Invalid TBL file: wrong number of selections (must be 5 in PCS mode)")
            else:
                if len(selections) == 2:
                    types = (" %.3f", " %.3f", " %.3f")
                elif len(selections) == 4:
                    types = (" %.3f", " %.3f", " %.3f", " %d")
                elif len(selections) == 5:
                    raise Exception("Invalid TBL file: wrong number of selections (can be 5 only in PCS mode)")
                elif len(selections) == 6:
                    types = (" %.3f", " %.3f")
                else:
                    check_parenthesis(restraints)
                    raise Exception("Invalid TBL file: wrong number of selections (must be 2,4 or 6)")
            postselections = len(selections)
            numbers = []
        # Distance restraints parsing
        if mode == "numbers":
            ll = l.split()
            for num in ll:
                if len(numbers) == len(types):
                    break
                numbers.append(float(num))
            if len(numbers) == len(types):
                tmp_output = "assign "
                for s in selections:
                    tmp_output += "\t(%s)\n" % s
                tmp_output = tmp_output[:-len("\n")]
                for n,t in zip(numbers,types):
                    tmp_output += t % n
                tmp_output += "\n"
                mode = "postglobal"
    # "OR" lines have been parsed, we store the selections
    if mode == "postglobal":
        output += "!\n"
        output += tmp_output
        tmp_output = None
        mode = "global"
    # If mode is not back to global, something has not been processed properly
    if mode != "global":
        raise Exception("Invalid TBL file: Malformed ASSIGN statement (line %d), use --quick to check for putative syntax issues" % lastassign)
    if not len(output.strip()):
        raise Exception("Invalid or empty TBL file")

    # Remove extra lines before each assign statement
    output = output.replace("\n\n", "\n")
    # Remove extra line at the beginning of the file
    if output.startswith("\n"):
        output = output.replace("\n", "", 1)
    return output

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="This script validates a restraint file (*.tbl).\n")

    parser.add_argument("file",
      help="TBL file to be validated")

    parser.add_argument("--pcs", action='store_true',
      help="PCS mode")

    parser.add_argument("--quick", action='store_true',
        help="Check global formatting before going line by line (opening/closing parenthesis and quotation marks")

    args = parser.parse_args()

    if (args.quick):
        tbldata = open(args.file).read()
        # Check the parenthesis and quotation marks opening/closure
        check_parenthesis(tbldata)

    if os.path.exists(args.file):
        tbldata = open(args.file).read()
        # Parse and process the restraints
        print validate_tbl(tbldata, args.pcs)
    else:
        raise Exception("TBL file %s does not exist, check the path" % args.file)

<|endoftext|>


<commit_before>from haystack import indexes
from books.models import Book, Series

class BookIndex(indexes.SearchIndex, indexes.Indexable):

    text = indexes.CharField(document=True, use_template=True)
    library = indexes.IntegerField(model_attr="library_id")

    def get_model(self):
        return Book

    def index_queryset(self, using=None):
        return self.get_model().objects.all()

class SeriesIndex(indexes.SearchIndex, indexes.Indexable):

    text = indexes.CharField(document=True, use_template=True)

    def get_model(self):
        return Series

    def index_queryset(self, using=None):
        return self.get_model().objects.all()<commit_msg>Add library property to series search index<commit_after>from haystack import indexes
from books.models import Book, Series

class BookIndex(indexes.SearchIndex, indexes.Indexable):

    text = indexes.CharField(document=True, use_template=True)
    library = indexes.IntegerField(model_attr="library_id")

    def get_model(self):
        return Book

    def index_queryset(self, using=None):
        return self.get_model().objects.all()

class SeriesIndex(indexes.SearchIndex, indexes.Indexable):

    text = indexes.CharField(document=True, use_template=True)
    library = indexes.IntegerField(model_attr="library_id")

    def get_model(self):
        return Series

    def index_queryset(self, using=None):
        return self.get_model().objects.all()<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
import skimage.feature
from skimage.measure import regionprops

def ComputeHaralickFeatures(I, Label, Offset=[[0, 1]], NumLevels=256,
                            MaxGray=255, MinGray=0):
    """
    Calculates 26 Haralick features from an intensity image with the labels.

    Parameters
    ----------
    I : array_like
        M x N intensity image.
    Label : array_like
        M x N label image.
    Offset : array_like
        Specifies common angles for a 2D image, given the pixel distacne D.
        Defalut value = [[0,1]]
        AngleXY     OFFSET
        -------     ------
        0           [0 D]
        45          [-D D]
        90          [-D 0]
        135         [-D -D]
    NumLevels : integer
        Specifies the number of gray level. NumLevels determines the size of
        the gray-level co-occurrence matrix.
        Default value = 256.
    MaxGray : integer
        Specifies a maximum grayscale value in I. Defalut value = 255
    MinGray : integer
        Specifies a minimum grayscale value in I. Defalut value = 0

    Returns
    -------
    df : 2-dimensional labeled data structure, float64
        Pandas data frame.

    References
    ----------
    .. [1] Haralick, et al. "Textural features for image classification,"
    IEEE Transactions on Systems, Man, and Cybernatics, vol. 6, pp: 610-621,
    1973.
    .. [2] Luis Pedro Coelho. "Mahotas: Open source software for scriptable
    computer vision," Journal of Open Research Software, vol 1, 2013.
    """

    # check for consistent shapes between 'I' and 'Label'
    if I.shape != Label.shape:
        raise ValueError("Inputs 'I' and 'Label' must have same shape")

    # determine image type
    if np.issubclass_(I.dtype.type, np.float_):
        if I.max() < 1.0:
            I = I * 255
            I = I.astype(np.uint8)
    elif np.issubclass_(I.dtype.type, np.bool_):
        I = I.astype(np.uint8)
        I = I * 255

    # determine if image is grayscale or RGB
    if len(I.shape) != 2:  # color image
        raise ValueError("Inputs 'I' should be a grayscale image")

    # check for Offset
    offsetShape = np.asarray(Offset).shape
    if (len(offsetShape) != 2):
        raise ValueError("Shape of Offset should be an numOffsets by 2")

    # restict the range of intensity from MinGray to MaxGray
    I = np.clip(I, MinGray, MaxGray)

    # initialize feature names
    featureList = [
        'ASM', 'Contrast', 'Correlation', 'SumofSquar',
        'IDM', 'SumAverage', 'SumVariance', 'SumEntropy', 'Entropy',
        'Variance', 'DifferenceEntropy', 'IMC1', 'IMC2'
    ]

    # initialize panda dataframe
    columnList = []
    for i in range(0, len(featureList)):
        columnList.append(featureList[i] + 'Mean')
        columnList.append(featureList[i] + 'Range')
    df = pd.DataFrame(columns=columnList)

    # gets total number of regions
    numofLabels = Label.max()

    # decodes angels and distances
    numOffsets = offsetShape[0]
    arrayOffset = np.asarray(Offset)
    listAngles = []
    listDistances = []

    for i in range(0, len(arrayOffset)):
        # angle is 0
        if (arrayOffset[i][0] == 0) & (arrayOffset[i][1] > 0):
            listAngles = np.append(listAngles, 0)
            listDistances = np.append(listDistances, arrayOffset[i][1])
        # angle is np.pi/2
        elif (arrayOffset[i][0] < 0) & (arrayOffset[i][1] == 0):
            listAngles = np.append(listAngles, np.pi/2)
            listDistances = np.append(listDistances, abs(arrayOffset[i][0]))
        elif abs(arrayOffset[i][0]) == abs(arrayOffset[i][1]):
            # angle is np.pi/4
            if (arrayOffset[i][0] < 0) & (arrayOffset[i][1] > 0):
                listAngles = np.append(listAngles, np.pi/4)
                listDistances = np.append(listDistances, arrayOffset[i][1])
            # angle is 3*np.pi/4
            elif (arrayOffset[i][0] < 0) & (arrayOffset[i][1] < 0):
                listAngles = np.append(listAngles, 3*np.pi/4)
                listDistances = np.append(listDistances, abs(arrayOffset[i][0]))
            else:
                raise ValueError("Current Offset format is not availabe.")
        else:
            raise ValueError("Current Offset format is not availabe.")

    # initialize sub panda dataframe
    subDataframe = np.zeros((numofLabels, len(featureList)*2))

    # extract feature information
    for region in regionprops(Label):
        # get bounds of an intensity image
        box = region.bbox
        # grab nucleus mask
        subImage = I[box[0]:box[2], box[1]:box[3]].astype(np.uint8)

        # gets GLCM or gray-tone spatial dependence matrix
        arrayGLCM = skimage.feature.greycomatrix(
            subImage, listDistances, listAngles,
            symmetric=True, levels=NumLevels
        )
        # gets size x and y of GLCM
        sizeX = arrayGLCM.shape[0]
        sizeY = arrayGLCM.shape[1]
        offsetsGLCM = np.zeros((sizeX, sizeY, numOffsets))
        # initialize a feature set for 13 features
        f = np.zeros((13, numOffsets))
        # initialize H for 26 features
        H = np.zeros(26)
        for r in range(0, numOffsets):
            # get each offset GLCM
            offsetsGLCM[:, :, r] = arrayGLCM[:, :, r, r]
            # normalize GLCM
            R = np.sum(offsetsGLCM[:, :, r], dtype=np.float)
            nGLCM = offsetsGLCM[:, :, r]/R
            # get marginal-probability matrix summing the rows
            px = np.sum(nGLCM, axis=1)
            py = np.sum(nGLCM, axis=0)
            # initialize marginal-probability matrix sets
            # summing normalizedGLCM such that i+j = k or i-j = k
            pxPlusy = np.zeros(2*NumLevels-1)
            pxMinusy = np.zeros(NumLevels)
            # arbitarily small positive constant to avoid log 0
            e = 0.00001
            for i in range(0, NumLevels):
                for j in range(0, NumLevels):
                    # gets marginal-probability matrix
                    pxPlusy[i+j] = pxPlusy[i+j] + nGLCM[i, j]
                    pxMinusy[abs(i-j)] = pxMinusy[abs(i-j)] + nGLCM[i, j]
            # f0: computes angular second moment
            f[0, r] = np.sum(np.square(nGLCM))
            # f1: computes contrast
            n_Minus = np.arange(NumLevels)
            f[1, r] = np.dot(np.square(n_Minus), pxMinusy)
            # f2: computes correlation
            # gets weighted mean and standard deviation of px and py
            meanx = np.dot(n_Minus, px)
            variance = np.dot(px, np.square(n_Minus)) - np.square(meanx)
            nGLCMr = np.ravel(nGLCM)
            i, j = np.mgrid[0:NumLevels, 0:NumLevels]
            ij = i*j
            f[2, r] = (np.dot(np.ravel(ij), nGLCMr) - np.square(meanx)) / \
                variance
            # f3: computes sum of squares : variance
            f[3, r] = variance
            # f4: computes inverse difference moment
            ij_IDM = 1. / (1+np.square(i-j))
            f[4, r] = np.dot(np.ravel(ij_IDM), nGLCMr)
            # f5: computes sum average
            n_Plus = np.arange(2*NumLevels-1)
            f[5, r] = np.dot(n_Plus, pxPlusy)
            # f6: computes sum variance
            # [1] uses sum entropy, but we use sum average
            f[6, r] = np.dot(np.square(n_Plus), pxPlusy) - \
                np.square(f[5, r])
            # f7: computes sum entropy
            f[7, r] = -np.dot(pxPlusy, np.log2(pxPlusy+e))
            # f8: computes entropy
            f[8, r] = -np.dot(nGLCMr, np.log2(nGLCMr+e))
            # f9: computes variance px-y
            f[9, r] = np.var(pxMinusy)
            # f10: computes difference entropy px-y
            f[10, r] = -np.dot(pxMinusy, np.log2(pxMinusy+e))
            # f11: computes information measures of correlation
            # gets entropies of px and py
            HX = -np.dot(px, np.log2(px+e))
            HY = -np.dot(py, np.log2(py+e))
            HXY = f[8, r]
            pxy_ij = np.outer(px, py)
            pxy_ijr = np.ravel(pxy_ij)
            HXY1 = -np.dot(nGLCMr, np.log2(pxy_ijr+e))
            HXY2 = -np.dot(pxy_ijr, np.log2(pxy_ijr+e))
            f[11, r] = (HXY-HXY1)/max(HX, HY)
            # f12: computes information measures of correlation
            f[12, r] = np.sqrt(1 - np.exp(-2.0*(HXY2-HXY)))
        # computes means and ranges of the features
        H[:13] = np.mean(f, axis=1)
        H[13:26] = np.ptp(f, axis=1)

        subDataframe[region.label-1, :] = H

    for i in range(0, len(featureList)):
        df[featureList[i] + 'Mean'] = subDataframe[:, i]
        df[featureList[i] + 'Range'] = subDataframe[:, i+len(featureList)]

    return df
<commit_msg>Modified ComputeHaralickFeatures.py - three parameters: offsets, graylimits, and numlevels, are updated.<commit_after>import numpy as np
import pandas as pd
import skimage.feature
from skimage.measure import regionprops

def ComputeHaralickFeatures(I, Label, Offset=[[0, 1]], NumLevels=256,
                            MaxGray=255, MinGray=0):  # noqa: C901
    """
    Calculates 26 Haralick features from an intensity image with the labels.

    Parameters
    ----------
    I : array_like
        M x N intensity image.
    Label : array_like
        M x N label image.
    Offset : array_like
        Specifies common angles for a 2D image, given the pixel distacne D.
        Defalut value = [[0,1]]
        AngleXY     OFFSET
        -------     ------
        0           [0 D]
        45          [-D D]
        90          [-D 0]
        135         [-D -D]
    NumLevels : integer
        Specifies the number of gray level. NumLevels determines the size of
        the gray-level co-occurrence matrix.
        Default value = 256.
    MaxGray : integer
        Specifies a maximum grayscale value in I. Defalut value = 255
    MinGray : integer
        Specifies a minimum grayscale value in I. Defalut value = 0

    Returns
    -------
    df : 2-dimensional labeled data structure, float64
        Pandas data frame.

    References
    ----------
    .. [1] Haralick, et al. "Textural features for image classification,"
    IEEE Transactions on Systems, Man, and Cybernatics, vol. 6, pp: 610-621,
    1973.
    .. [2] Luis Pedro Coelho. "Mahotas: Open source software for scriptable
    computer vision," Journal of Open Research Software, vol 1, 2013.
    """

    # check for consistent shapes between 'I' and 'Label'
    if I.shape != Label.shape:
        raise ValueError("Inputs 'I' and 'Label' must have same shape")

    # determine image type
    if np.issubclass_(I.dtype.type, np.float_):
        if I.max() < 1.0:
            I = I * 255
            I = I.astype(np.uint8)
    elif np.issubclass_(I.dtype.type, np.bool_):
        I = I.astype(np.uint8)
        I = I * 255

    # determine if image is grayscale or RGB
    if len(I.shape) != 2:  # color image
        raise ValueError("Inputs 'I' should be a grayscale image")

    # check for Offset
    offsetShape = np.asarray(Offset).shape
    if (len(offsetShape) != 2):
        raise ValueError("Shape of Offset should be an numOffsets by 2")

    # restict the range of intensity from MinGray to MaxGray
    I = np.clip(I, MinGray, MaxGray)

    # initialize feature names
    featureList = [
        'ASM', 'Contrast', 'Correlation', 'SumofSquar',
        'IDM', 'SumAverage', 'SumVariance', 'SumEntropy', 'Entropy',
        'Variance', 'DifferenceEntropy', 'IMC1', 'IMC2'
    ]

    # initialize panda dataframe
    columnList = []
    for i in range(0, len(featureList)):
        columnList.append(featureList[i] + 'Mean')
        columnList.append(featureList[i] + 'Range')
    df = pd.DataFrame(columns=columnList)

    # gets total number of regions
    numofLabels = Label.max()

    # decodes angels and distances
    numOffsets = offsetShape[0]
    arrayOffset = np.asarray(Offset)
    listAngles = []
    listDistances = []

    for i in range(0, len(arrayOffset)):
        # angle is 0
        if (arrayOffset[i][0] == 0) & (arrayOffset[i][1] > 0):
            listAngles = np.append(listAngles, 0)
            listDistances = np.append(listDistances, arrayOffset[i][1])
        # angle is np.pi/2
        elif (arrayOffset[i][0] < 0) & (arrayOffset[i][1] == 0):
            listAngles = np.append(listAngles, np.pi/2)
            listDistances = np.append(listDistances, abs(arrayOffset[i][0]))
        elif abs(arrayOffset[i][0]) == abs(arrayOffset[i][1]):
            # angle is np.pi/4
            if (arrayOffset[i][0] < 0) & (arrayOffset[i][1] > 0):
                listAngles = np.append(listAngles, np.pi/4)
                listDistances = np.append(listDistances, arrayOffset[i][1])
            # angle is 3*np.pi/4
            elif (arrayOffset[i][0] < 0) & (arrayOffset[i][1] < 0):
                listAngles = np.append(listAngles, 3*np.pi/4)
                listDistances = np.append(listDistances, abs(arrayOffset[i][0]))
            else:
                raise ValueError("Current Offset format is not availabe.")
        else:
            raise ValueError("Current Offset format is not availabe.")

    # initialize sub panda dataframe
    subDataframe = np.zeros((numofLabels, len(featureList)*2))

    # extract feature information
    for region in regionprops(Label):
        # get bounds of an intensity image
        box = region.bbox
        # grab nucleus mask
        subImage = I[box[0]:box[2], box[1]:box[3]].astype(np.uint8)

        # gets GLCM or gray-tone spatial dependence matrix
        arrayGLCM = skimage.feature.greycomatrix(
            subImage, listDistances, listAngles,
            symmetric=True, levels=NumLevels
        )
        # gets size x and y of GLCM
        sizeX = arrayGLCM.shape[0]
        sizeY = arrayGLCM.shape[1]
        offsetsGLCM = np.zeros((sizeX, sizeY, numOffsets))
        # initialize a feature set for 13 features
        f = np.zeros((13, numOffsets))
        # initialize H for 26 features
        H = np.zeros(26)
        for r in range(0, numOffsets):
            # get each offset GLCM
            offsetsGLCM[:, :, r] = arrayGLCM[:, :, r, r]
            # normalize GLCM
            R = np.sum(offsetsGLCM[:, :, r], dtype=np.float)
            nGLCM = offsetsGLCM[:, :, r]/R
            # get marginal-probability matrix summing the rows
            px = np.sum(nGLCM, axis=1)
            py = np.sum(nGLCM, axis=0)
            # initialize marginal-probability matrix sets
            # summing normalizedGLCM such that i+j = k or i-j = k
            pxPlusy = np.zeros(2*NumLevels-1)
            pxMinusy = np.zeros(NumLevels)
            # arbitarily small positive constant to avoid log 0
            e = 0.00001
            for i in range(0, NumLevels):
                for j in range(0, NumLevels):
                    # gets marginal-probability matrix
                    pxPlusy[i+j] = pxPlusy[i+j] + nGLCM[i, j]
                    pxMinusy[abs(i-j)] = pxMinusy[abs(i-j)] + nGLCM[i, j]
            # f0: computes angular second moment
            f[0, r] = np.sum(np.square(nGLCM))
            # f1: computes contrast
            n_Minus = np.arange(NumLevels)
            f[1, r] = np.dot(np.square(n_Minus), pxMinusy)
            # f2: computes correlation
            # gets weighted mean and standard deviation of px and py
            meanx = np.dot(n_Minus, px)
            variance = np.dot(px, np.square(n_Minus)) - np.square(meanx)
            nGLCMr = np.ravel(nGLCM)
            i, j = np.mgrid[0:NumLevels, 0:NumLevels]
            ij = i*j
            f[2, r] = (np.dot(np.ravel(ij), nGLCMr) - np.square(meanx)) / \
                variance
            # f3: computes sum of squares : variance
            f[3, r] = variance
            # f4: computes inverse difference moment
            ij_IDM = 1. / (1+np.square(i-j))
            f[4, r] = np.dot(np.ravel(ij_IDM), nGLCMr)
            # f5: computes sum average
            n_Plus = np.arange(2*NumLevels-1)
            f[5, r] = np.dot(n_Plus, pxPlusy)
            # f6: computes sum variance
            # [1] uses sum entropy, but we use sum average
            f[6, r] = np.dot(np.square(n_Plus), pxPlusy) - \
                np.square(f[5, r])
            # f7: computes sum entropy
            f[7, r] = -np.dot(pxPlusy, np.log2(pxPlusy+e))
            # f8: computes entropy
            f[8, r] = -np.dot(nGLCMr, np.log2(nGLCMr+e))
            # f9: computes variance px-y
            f[9, r] = np.var(pxMinusy)
            # f10: computes difference entropy px-y
            f[10, r] = -np.dot(pxMinusy, np.log2(pxMinusy+e))
            # f11: computes information measures of correlation
            # gets entropies of px and py
            HX = -np.dot(px, np.log2(px+e))
            HY = -np.dot(py, np.log2(py+e))
            HXY = f[8, r]
            pxy_ij = np.outer(px, py)
            pxy_ijr = np.ravel(pxy_ij)
            HXY1 = -np.dot(nGLCMr, np.log2(pxy_ijr+e))
            HXY2 = -np.dot(pxy_ijr, np.log2(pxy_ijr+e))
            f[11, r] = (HXY-HXY1)/max(HX, HY)
            # f12: computes information measures of correlation
            f[12, r] = np.sqrt(1 - np.exp(-2.0*(HXY2-HXY)))
        # computes means and ranges of the features
        H[:13] = np.mean(f, axis=1)
        H[13:26] = np.ptp(f, axis=1)

        subDataframe[region.label-1, :] = H

    for i in range(0, len(featureList)):
        df[featureList[i] + 'Mean'] = subDataframe[:, i]
        df[featureList[i] + 'Range'] = subDataframe[:, i+len(featureList)]

    return df
<|endoftext|>