<commit_before>"""Amalgate json-cpp library sources into a single source and header file.

Requires Python 2.6

Example of invocation (must be invoked from json-cpp top directory):
python amalgate.py
"""
import os
import os.path
import sys

class AmalgamationFile:
    def __init__(self, top_dir):
        self.top_dir = top_dir
        self.blocks = []

    def add_text(self, text):
        if not text.endswith("\n"):
            text += "\n"
        self.blocks.append(text)

    def add_file(self, relative_input_path, wrap_in_comment=False):
        def add_marker(prefix):
            self.add_text("")
            self.add_text("// " + "/"*70)
            self.add_text("// %s of content of file: %s" % (prefix, relative_input_path.replace("\\","/")))
            self.add_text("// " + "/"*70)
            self.add_text("")
        add_marker("Beginning")
        f = open(os.path.join(self.top_dir, relative_input_path), "rt")
        content = f.read()
        if wrap_in_comment:
            content = "/*\n" + content + "\n*/"
        self.add_text(content)
        f.close()
        add_marker("End")
        self.add_text("\n\n\n\n")

    def get_value(self):
        return "".join(self.blocks).replace("\r\n","\n")

    def write_to(self, output_path):
        output_dir = os.path.dirname(output_path)
        if output_dir and not os.path.isdir(output_dir):
            os.makedirs(output_dir)
        f = open(output_path, "wb")
        f.write(str.encode(self.get_value(), 'UTF-8'))
        f.close()

def amalgamate_source(source_top_dir=None,
                       target_source_path=None,
                       header_include_path=None):
    """Produces amalgated source.
       Parameters:
           source_top_dir: top-directory
           target_source_path: output .cpp path
           header_include_path: generated header path relative to target_source_path.
    """
    print("Amalgating header...")
    header = AmalgamationFile(source_top_dir)
    header.add_text("/// Json-cpp amalgated header (http://jsoncpp.sourceforge.net/).")
    header.add_text('/// It is intended to be used with #include "%s"' % header_include_path)
    header.add_file("LICENSE", wrap_in_comment=True)
    header.add_text("#ifndef JSON_AMALGATED_H_INCLUDED")
    header.add_text("# define JSON_AMALGATED_H_INCLUDED")
    header.add_text("/// If defined, indicates that the source file is amalgated")
    header.add_text("/// to prevent private header inclusion.")
    header.add_text("#define JSON_IS_AMALGAMATION")
    header.add_file("include/json/version.h")
    header.add_file("include/json/config.h")
    header.add_file("include/json/forwards.h")
    header.add_file("include/json/features.h")
    header.add_file("include/json/value.h")
    header.add_file("include/json/reader.h")
    header.add_file("include/json/writer.h")
    header.add_file("include/json/assertions.h")
    header.add_text("#endif //ifndef JSON_AMALGATED_H_INCLUDED")

    target_header_path = os.path.join(os.path.dirname(target_source_path), header_include_path)
    print("Writing amalgated header to %r" % target_header_path)
    header.write_to(target_header_path)

    base, ext = os.path.splitext(header_include_path)
    forward_header_include_path = base + "-forwards" + ext
    print("Amalgating forward header...")
    header = AmalgamationFile(source_top_dir)
    header.add_text("/// Json-cpp amalgated forward header (http://jsoncpp.sourceforge.net/).")
    header.add_text('/// It is intended to be used with #include "%s"' % forward_header_include_path)
    header.add_text("/// This header provides forward declaration for all JsonCpp types.")
    header.add_file("LICENSE", wrap_in_comment=True)
    header.add_text("#ifndef JSON_FORWARD_AMALGATED_H_INCLUDED")
    header.add_text("# define JSON_FORWARD_AMALGATED_H_INCLUDED")
    header.add_text("/// If defined, indicates that the source file is amalgated")
    header.add_text("/// to prevent private header inclusion.")
    header.add_text("#define JSON_IS_AMALGAMATION")
    header.add_file("include/json/config.h")
    header.add_file("include/json/forwards.h")
    header.add_text("#endif //ifndef JSON_FORWARD_AMALGATED_H_INCLUDED")

    target_forward_header_path = os.path.join(os.path.dirname(target_source_path),
                                               forward_header_include_path)
    print("Writing amalgated forward header to %r" % target_forward_header_path)
    header.write_to(target_forward_header_path)

    print("Amalgating source...")
    source = AmalgamationFile(source_top_dir)
    source.add_text("/// Json-cpp amalgated source (http://jsoncpp.sourceforge.net/).")
    source.add_text('/// It is intended to be used with #include "%s"' % header_include_path)
    source.add_file("LICENSE", wrap_in_comment=True)
    source.add_text("")
    source.add_text('#include "%s"' % header_include_path)
    source.add_text("""
#ifndef JSON_IS_AMALGAMATED
#error "Compile with -I PATH_TO_JSON_DIRECTORY"
#endif
""")
    source.add_text("")
    lib_json = "src/lib_json"
    source.add_file(os.path.join(lib_json, "json_tool.h"))
    source.add_file(os.path.join(lib_json, "json_reader.cpp"))
    source.add_file(os.path.join(lib_json, "json_batchallocator.h"))
    source.add_file(os.path.join(lib_json, "json_valueiterator.inl"))
    source.add_file(os.path.join(lib_json, "json_value.cpp"))
    source.add_file(os.path.join(lib_json, "json_writer.cpp"))

    print("Writing amalgated source to %r" % target_source_path)
    source.write_to(target_source_path)

def main():
    usage = """%prog [options]
Generate a single amalgated source and header file from the sources.
"""
    from optparse import OptionParser
    parser = OptionParser(usage=usage)
    parser.allow_interspersed_args = False
    parser.add_option("-s", "--source", dest="target_source_path", action="store", default="dist/jsoncpp.cpp",
        help="""Output .cpp source path. [Default: %default]""")
    parser.add_option("-i", "--include", dest="header_include_path", action="store", default="json/json.h",
        help="""Header include path. Used to include the header from the amalgated source file. [Default: %default]""")
    parser.add_option("-t", "--top-dir", dest="top_dir", action="store", default=os.getcwd(),
        help="""Source top-directory. [Default: %default]""")
    parser.enable_interspersed_args()
    options, args = parser.parse_args()

    msg = amalgamate_source(source_top_dir=options.top_dir,
                             target_source_path=options.target_source_path,
                             header_include_path=options.header_include_path)
    if msg:
        sys.stderr.write(msg + "\n")
        sys.exit(1)
    else:
        print("Source succesfully amalagated")

if __name__ == "__main__":
    main()
<commit_msg>fix typo: issue #177<commit_after>"""Amalgate json-cpp library sources into a single source and header file.

Requires Python 2.6

Example of invocation (must be invoked from json-cpp top directory):
python amalgate.py
"""
import os
import os.path
import sys

class AmalgamationFile:
    def __init__(self, top_dir):
        self.top_dir = top_dir
        self.blocks = []

    def add_text(self, text):
        if not text.endswith("\n"):
            text += "\n"
        self.blocks.append(text)

    def add_file(self, relative_input_path, wrap_in_comment=False):
        def add_marker(prefix):
            self.add_text("")
            self.add_text("// " + "/"*70)
            self.add_text("// %s of content of file: %s" % (prefix, relative_input_path.replace("\\","/")))
            self.add_text("// " + "/"*70)
            self.add_text("")
        add_marker("Beginning")
        f = open(os.path.join(self.top_dir, relative_input_path), "rt")
        content = f.read()
        if wrap_in_comment:
            content = "/*\n" + content + "\n*/"
        self.add_text(content)
        f.close()
        add_marker("End")
        self.add_text("\n\n\n\n")

    def get_value(self):
        return "".join(self.blocks).replace("\r\n","\n")

    def write_to(self, output_path):
        output_dir = os.path.dirname(output_path)
        if output_dir and not os.path.isdir(output_dir):
            os.makedirs(output_dir)
        f = open(output_path, "wb")
        f.write(str.encode(self.get_value(), 'UTF-8'))
        f.close()

def amalgamate_source(source_top_dir=None,
                       target_source_path=None,
                       header_include_path=None):
    """Produces amalgated source.
       Parameters:
           source_top_dir: top-directory
           target_source_path: output .cpp path
           header_include_path: generated header path relative to target_source_path.
    """
    print("Amalgating header...")
    header = AmalgamationFile(source_top_dir)
    header.add_text("/// Json-cpp amalgated header (http://jsoncpp.sourceforge.net/).")
    header.add_text('/// It is intended to be used with #include "%s"' % header_include_path)
    header.add_file("LICENSE", wrap_in_comment=True)
    header.add_text("#ifndef JSON_AMALGATED_H_INCLUDED")
    header.add_text("# define JSON_AMALGATED_H_INCLUDED")
    header.add_text("/// If defined, indicates that the source file is amalgated")
    header.add_text("/// to prevent private header inclusion.")
    header.add_text("#define JSON_IS_AMALGAMATION")
    header.add_file("include/json/version.h")
    header.add_file("include/json/config.h")
    header.add_file("include/json/forwards.h")
    header.add_file("include/json/features.h")
    header.add_file("include/json/value.h")
    header.add_file("include/json/reader.h")
    header.add_file("include/json/writer.h")
    header.add_file("include/json/assertions.h")
    header.add_text("#endif //ifndef JSON_AMALGATED_H_INCLUDED")

    target_header_path = os.path.join(os.path.dirname(target_source_path), header_include_path)
    print("Writing amalgated header to %r" % target_header_path)
    header.write_to(target_header_path)

    base, ext = os.path.splitext(header_include_path)
    forward_header_include_path = base + "-forwards" + ext
    print("Amalgating forward header...")
    header = AmalgamationFile(source_top_dir)
    header.add_text("/// Json-cpp amalgated forward header (http://jsoncpp.sourceforge.net/).")
    header.add_text('/// It is intended to be used with #include "%s"' % forward_header_include_path)
    header.add_text("/// This header provides forward declaration for all JsonCpp types.")
    header.add_file("LICENSE", wrap_in_comment=True)
    header.add_text("#ifndef JSON_FORWARD_AMALGATED_H_INCLUDED")
    header.add_text("# define JSON_FORWARD_AMALGATED_H_INCLUDED")
    header.add_text("/// If defined, indicates that the source file is amalgated")
    header.add_text("/// to prevent private header inclusion.")
    header.add_text("#define JSON_IS_AMALGAMATION")
    header.add_file("include/json/config.h")
    header.add_file("include/json/forwards.h")
    header.add_text("#endif //ifndef JSON_FORWARD_AMALGATED_H_INCLUDED")

    target_forward_header_path = os.path.join(os.path.dirname(target_source_path),
                                               forward_header_include_path)
    print("Writing amalgated forward header to %r" % target_forward_header_path)
    header.write_to(target_forward_header_path)

    print("Amalgating source...")
    source = AmalgamationFile(source_top_dir)
    source.add_text("/// Json-cpp amalgated source (http://jsoncpp.sourceforge.net/).")
    source.add_text('/// It is intended to be used with #include "%s"' % header_include_path)
    source.add_file("LICENSE", wrap_in_comment=True)
    source.add_text("")
    source.add_text('#include "%s"' % header_include_path)
    source.add_text("""
#ifndef JSON_IS_AMALGAMATION
#error "Compile with -I PATH_TO_JSON_DIRECTORY"
#endif
""")
    source.add_text("")
    lib_json = "src/lib_json"
    source.add_file(os.path.join(lib_json, "json_tool.h"))
    source.add_file(os.path.join(lib_json, "json_reader.cpp"))
    source.add_file(os.path.join(lib_json, "json_batchallocator.h"))
    source.add_file(os.path.join(lib_json, "json_valueiterator.inl"))
    source.add_file(os.path.join(lib_json, "json_value.cpp"))
    source.add_file(os.path.join(lib_json, "json_writer.cpp"))

    print("Writing amalgated source to %r" % target_source_path)
    source.write_to(target_source_path)

def main():
    usage = """%prog [options]
Generate a single amalgated source and header file from the sources.
"""
    from optparse import OptionParser
    parser = OptionParser(usage=usage)
    parser.allow_interspersed_args = False
    parser.add_option("-s", "--source", dest="target_source_path", action="store", default="dist/jsoncpp.cpp",
        help="""Output .cpp source path. [Default: %default]""")
    parser.add_option("-i", "--include", dest="header_include_path", action="store", default="json/json.h",
        help="""Header include path. Used to include the header from the amalgated source file. [Default: %default]""")
    parser.add_option("-t", "--top-dir", dest="top_dir", action="store", default=os.getcwd(),
        help="""Source top-directory. [Default: %default]""")
    parser.enable_interspersed_args()
    options, args = parser.parse_args()

    msg = amalgamate_source(source_top_dir=options.top_dir,
                             target_source_path=options.target_source_path,
                             header_include_path=options.header_include_path)
    if msg:
        sys.stderr.write(msg + "\n")
        sys.exit(1)
    else:
        print("Source succesfully amalagated")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import json
import os
import time

from klein import Klein
from twisted.web.static import File
import zmq

from db import Database
from docgetter import get_docs
from ipgetter import get_ip
import msg
from servicegetters import got_cluster, got_slave, got_notebook
import settings

app = Klein()
slave_db = Database()

def zocket_send(**kwargs):
    zcontext = zmq.Context()
    socket = zcontext.socket(zmq.PUSH)
    socket.bind("ipc:///tmp/sock")
    socket.send_json(kwargs)

@app.route('/', branch=True)
def root(request):
    return File("./static/")

@app.route('/api/getdocs')
def all_the_docs(reques):
    return json.dumps({'allthedocs':get_docs()})

@app.route('/api/status')
def status(request):

    result = {}

    result['virtual'] = json.loads(os.environ.get('VBOX','false'))

    ip = get_ip()
    if ip:
        result['network'] = True
        result['ip'] = ip.split('.')
        result['slave_ip'] = got_slave(ip)
        result['master_owner'] = got_cluster(ip)
        result['pysparknotebook'] = got_notebook()
        if result['master_owner'] or result['slave_ip']:
            result['status'] = 'active'
        else:
            result['status'] = 'dormant'
    else:
        result['status'] = 'error'

    return json.dumps(result)

def valid_ip(ip):
    try:
        chunks = ip.split('.')
        return len(chunks) == 4 and all([c.isnumeric and 0 <= int(c) <= 255 for c in chunks])
    except:
        return False

@app.route('/api/startcluster')
def start_master(request):
    result = {'okay':False}
    ip = get_ip()
    if not got_cluster(ip):
        if not got_slave(ip):
            zocket_send(msg=msg.STARTMASTER,ip=ip)
            slave_db.create_db()
            result = {'okay': True, 'ip':ip}
        else:
            result['error'] = "This node is already a Spark slave."
    else:
        result['error'] = "A Spark master is already running."
    return json.dumps(result)

@app.route('/api/joincluster')
def start_slave(request):
    result = {'okay':False}
    master_ip = request.args.get('ip', False)
    if valid_ip(master_ip):
        slave_ip = get_ip()
        if got_cluster(master_ip):
            if not got_slave(slave_ip):
                zocket_send(msg=msg.STARTSLAVE, master_ip=master_ip, slave_ip=slave_ip)
                result['okay'] = True
            else:
                result['error'] = "This node is already a Spark slave."
        else:
            result['error'] = "Can't find the Spark master."
    else:
        result['error'] = 'Missing or invalid IP address.'
    return json.dumps(result)

@app.route('/api/startclusternotebook')
def start_cluster_notebook(request):
    result = {'okay':False}
    ip = request.args.get('ip', False)
    if valid_ip(ip):
        if got_cluster(ip):
            if not got_notebook():
                slave_db.purge_slaves()
                zocket_send(msg=msg.STARTPYSPARKNOTEBOOK, ip=ip)
                result['okay'] = True
            else:
                 result['error'] = "A notebook server was already started."
        else:
            result['error'] = "Can't find the Spark master."
    else:
        result['error'] = 'Missing or invalid IP address.'
    return json.dumps(result)

@app.route('/api/startsinglenotebook')
def start_single_notebook(request):
    result = {'okay':False}
    if not got_notebook():
        zocket_send(msg=msg.STARTSINGLENOTEBOOK)
    else:
        result['error'] = "A notebook server was already started."
    return json.dumps(result)

@app.route('/api/stopclusternotebook')
def stop_cluster_notebook(request):
    result = {'okay':False}
    if got_notebook():
        zocket_send(msg=msg.KILLPYSPARKNOTEBOOK)
        result['okay'] = True
    else:
        result['error'] = "No notebook server is active."
    return json.dumps(result)

@app.route('/api/stopsinglenotebook')
def stop_single_notebook(request):
    result = {'okay':False}
    if got_notebook():
        zocket_send(msg=msg.KILLSINGLENOTEBOOK)
        result['okay'] = True
    else:
        result['error'] = "No notebook server is active."
    return json.dumps(result)

@app.route('/api/stopcluster')
def stop_master(request):
    result = {'okay':False}
    if not got_notebook():
        if got_cluster(get_ip()):
            zocket_send(msg=msg.KILLMASTER)
            slave_db.purge_slaves()
            result['okay'] = True
        else:
            result['error'] = "No Mesos master is active."
    else:
        result['error'] = "A notebook server is still active."
    return json.dumps(result)

@app.route('/api/leavecluster')
def stop_slave(request):
    result = {'okay':False}
    if not got_notebook():
        if got_slave(get_ip()):
            zocket_send(msg=msg.KILLSLAVE)
            result['okay'] = True
        else:
            result['error'] = "No Spark slave is active."
    else:
        result['error'] = "A notebook server is still active."
    return json.dumps(result)

@app.route('/api/reportslave', methods=['GET'])
def report_slave(request):
    result = {'okay':False}
    try:
        slave_ip = request.args.get(b'ip', False)[0].decode('utf-8')
    except:
        slave_ip = False
    if valid_ip(slave_ip):
        print("Slave reported: "+slave_ip)
        slave_db.insert_slave(slave_ip)
        result['okay'] = True
    else:
        result['error'] = 'Missing or invalid IP address.'
    return json.dumps(result)

@app.route('/api/starthdfs')
def start_hdfs(request):
    result = {'okay':False}
    ip = get_ip()
    if ip:
        slave_db.get_slaves().addCallback(dump_slaves, ip=ip)
        result = {'okay':True}
    return json.dumps(result)

def dump_slaves(slaves,ip=False):
    slave_path = os.environ['HADOOP_HOME'] + '/etc/hadoop/slaves'
    with open(slave_path, 'w') as slave_file:
        for s in slaves:
            slave_file.write(s[0]+'\n')
    zocket_send(msg=msg.STARTHDFS, ip=ip, slave_count=len(slaves))

@app.route('/api/ping')
def ping(request):
    zocket_send(ping="pong")
    return json.dumps({'ping':'pong'})

<commit_msg>Testing... 	modified:   ananke/app.py<commit_after>import json
import os
import time

from klein import Klein
from twisted.web.static import File
import zmq

from db import Database
from docgetter import get_docs
from ipgetter import get_ip
import msg
from servicegetters import got_cluster, got_slave, got_notebook
import settings

app = Klein()
slave_db = Database()

def zocket_send(**kwargs):
    zcontext = zmq.Context()
    socket = zcontext.socket(zmq.PUSH)
    socket.bind("ipc:///tmp/sock")
    socket.send_json(kwargs)

@app.route('/', branch=True)
def root(request):
    return File("./static/")

@app.route('/api/getdocs')
def all_the_docs(reques):
    return json.dumps({'allthedocs':get_docs()})

@app.route('/api/status')
def status(request):

    result = {}

    result['virtual'] = json.loads(os.environ.get('VBOX','false'))

    ip = get_ip()
    if ip:
        result['network'] = True
        result['ip'] = ip.split('.')
        result['slave_ip'] = got_slave(ip)
        result['master_owner'] = got_cluster(ip)
        result['pysparknotebook'] = got_notebook()
        if result['master_owner'] or result['slave_ip']:
            result['status'] = 'active'
        else:
            result['status'] = 'dormant'
    else:
        result['status'] = 'error'

    return json.dumps(result)

def valid_ip(ip):
    try:
        chunks = ip.split('.')
        return len(chunks) == 4 and all([c.isnumeric and 0 <= int(c) <= 255 for c in chunks])
    except:
        return False

@app.route('/api/startcluster')
def start_master(request):
    result = {'okay':False}
    ip = get_ip()
    if not got_cluster(ip):
        if not got_slave(ip):
            zocket_send(msg=msg.STARTMASTER,ip=ip)
            slave_db.create_db()
            result = {'okay': True, 'ip':ip}
        else:
            result['error'] = "This node is already a Spark slave."
    else:
        result['error'] = "A Spark master is already running."
    return json.dumps(result)

@app.route('/api/joincluster')
def start_slave(request):
    result = {'okay':False}
    master_ip = request.args.get(b'ip', False)[0].decode('utf-8')
    print(" MASTER IP ",master_ip)
    if valid_ip(master_ip):
        slave_ip = get_ip()
        if got_cluster(master_ip):
            if not got_slave(slave_ip):
                zocket_send(msg=msg.STARTSLAVE, master_ip=master_ip, slave_ip=slave_ip)
                result['okay'] = True
            else:
                result['error'] = "This node is already a Spark slave."
        else:
            result['error'] = "Can't find the Spark master."
    else:
        result['error'] = 'Missing or invalid IP address.'
    return json.dumps(result)

@app.route('/api/startclusternotebook')
def start_cluster_notebook(request):
    result = {'okay':False}
    ip = request.args.get('ip', False)
    if valid_ip(ip):
        if got_cluster(ip):
            if not got_notebook():
                slave_db.purge_slaves()
                zocket_send(msg=msg.STARTPYSPARKNOTEBOOK, ip=ip)
                result['okay'] = True
            else:
                 result['error'] = "A notebook server was already started."
        else:
            result['error'] = "Can't find the Spark master."
    else:
        result['error'] = 'Missing or invalid IP address.'
    return json.dumps(result)

@app.route('/api/startsinglenotebook')
def start_single_notebook(request):
    result = {'okay':False}
    if not got_notebook():
        zocket_send(msg=msg.STARTSINGLENOTEBOOK)
    else:
        result['error'] = "A notebook server was already started."
    return json.dumps(result)

@app.route('/api/stopclusternotebook')
def stop_cluster_notebook(request):
    result = {'okay':False}
    if got_notebook():
        zocket_send(msg=msg.KILLPYSPARKNOTEBOOK)
        result['okay'] = True
    else:
        result['error'] = "No notebook server is active."
    return json.dumps(result)

@app.route('/api/stopsinglenotebook')
def stop_single_notebook(request):
    result = {'okay':False}
    if got_notebook():
        zocket_send(msg=msg.KILLSINGLENOTEBOOK)
        result['okay'] = True
    else:
        result['error'] = "No notebook server is active."
    return json.dumps(result)

@app.route('/api/stopcluster')
def stop_master(request):
    result = {'okay':False}
    if not got_notebook():
        if got_cluster(get_ip()):
            zocket_send(msg=msg.KILLMASTER)
            slave_db.purge_slaves()
            result['okay'] = True
        else:
            result['error'] = "No Mesos master is active."
    else:
        result['error'] = "A notebook server is still active."
    return json.dumps(result)

@app.route('/api/leavecluster')
def stop_slave(request):
    result = {'okay':False}
    if not got_notebook():
        if got_slave(get_ip()):
            zocket_send(msg=msg.KILLSLAVE)
            result['okay'] = True
        else:
            result['error'] = "No Spark slave is active."
    else:
        result['error'] = "A notebook server is still active."
    return json.dumps(result)

@app.route('/api/reportslave', methods=['GET'])
def report_slave(request):
    result = {'okay':False}
    try:
        slave_ip = request.args.get(b'ip', False)[0].decode('utf-8')
    except:
        slave_ip = False
    if valid_ip(slave_ip):
        print("Slave reported: "+slave_ip)
        slave_db.insert_slave(slave_ip)
        result['okay'] = True
    else:
        result['error'] = 'Missing or invalid IP address.'
    return json.dumps(result)

@app.route('/api/writeslaves')
def write_slaves(request):
    slave_db.get_slaves().addCallback(export_slaves)
    return json.dumps({'okay':True})

def export_slaves(slaves):
    slave_path = os.environ['HADOOP_HOME'] + '/etc/hadoop/slaves'
    with open(slave_path, 'w') as slave_file:
        for s in slaves:
            slave_file.write(s[0]+'\n')

@app.route('/api/starthdfs')
def start_hdfs(request):
    result = {'okay':False}
    ip = get_ip()
    if ip:
        slave_db.get_slaves().addCallback(dump_slaves, ip=ip)
        result = {'okay':True}
    return json.dumps(result)

def dump_slaves(slaves,ip=False):
    slave_path = os.environ['HADOOP_HOME'] + '/etc/hadoop/slaves'
    with open(slave_path, 'w') as slave_file:
        for s in slaves:
            slave_file.write(s[0]+'\n')
    zocket_send(msg=msg.STARTHDFS, ip=ip, slave_count=len(slaves))

@app.route('/api/ping')
def ping(request):
    zocket_send(ping="pong")
    return json.dumps({'ping':'pong'})

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright: Damien Elmes <anki@ichi2.net>
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html

import os, shutil, re, urllib, unicodedata, \
    sys, zipfile
from cStringIO import StringIO
from anki.utils import checksum, isWin, isMac, json
from anki.db import DB
from anki.consts import *
from anki.latex import mungeQA

class MediaManager(object):

    # other code depends on this order, so don't reorder
    regexps = ("(?i)(\[sound:([^]]+)\])",
               "(?i)(<img[^>]+src=[\"']?([^\"'>]+)[\"']?[^>]*>)")

    def __init__(self, col, server):
        self.col = col
        if server:
            self._dir = None
            return
        # media directory
        self._dir = re.sub("(?i)\.(anki2)$", ".media", self.col.path)
        # convert dir to unicode if it's not already
        if isinstance(self._dir, str):
            self._dir = unicode(self._dir, sys.getfilesystemencoding())
        if not os.path.exists(self._dir):
            os.makedirs(self._dir)
        try:
            self._oldcwd = os.getcwd()
        except OSError:
            # cwd doesn't exist
            self._oldcwd = None
        os.chdir(self._dir)
        # change database
        self.connect()

    def connect(self):
        if self.col.server:
            return
        path = self.dir()+".db"
        create = not os.path.exists(path)
        os.chdir(self._dir)
        self.db = DB(path)
        if create:
            self._initDB()

    def close(self):
        if self.col.server:
            return
        self.db.close()
        self.db = None
        # change cwd back to old location
        if self._oldcwd:
            try:
                os.chdir(self._oldcwd)
            except:
                # may have been deleted
                pass

    def dir(self):
        return self._dir

    # Adding media
    ##########################################################################

    def addFile(self, opath):
        """Copy PATH to MEDIADIR, and return new filename.
If the same name exists, compare checksums."""
        mdir = self.dir()
        # remove any dangerous characters
        base = re.sub(r"[][<>:/\\&?\"\|]", "", os.path.basename(opath))
        dst = os.path.join(mdir, base)
        # if it doesn't exist, copy it directly
        if not os.path.exists(dst):
            shutil.copyfile(opath, dst)
            return base
        # if it's identical, reuse
        if self.filesIdentical(opath, dst):
            return base
        # otherwise, find a unique name
        (root, ext) = os.path.splitext(base)
        def repl(match):
            n = int(match.group(1))
            return " (%d)" % (n+1)
        while True:
            path = os.path.join(mdir, root + ext)
            if not os.path.exists(path):
                break
            reg = " \((\d+)\)$"
            if not re.search(reg, root):
                root = root + " (1)"
            else:
                root = re.sub(reg, repl, root)
        # copy and return
        shutil.copyfile(opath, path)
        return os.path.basename(os.path.basename(path))

    def filesIdentical(self, path1, path2):
        "True if files are the same."
        return (checksum(open(path1, "rb").read()) ==
                checksum(open(path2, "rb").read()))

    # String manipulation
    ##########################################################################

    def filesInStr(self, mid, string, includeRemote=False):
        l = []
        model = self.col.models.get(mid)
        strings = []
        if model['type'] == MODEL_CLOZE and "{{c" in string:
            # if the field has clozes in it, we'll need to expand the
            # possibilities so we can render latex
            strings = self._expandClozes(string)
        else:
            strings = [string]
        for string in strings:
            # handle latex
            string = mungeQA(string, None, None, model, None, self.col)
            # extract filenames
            for reg in self.regexps:
                for (full, fname) in re.findall(reg, string):
                    isLocal = not re.match("(https?|ftp)://", fname.lower())
                    if isLocal or includeRemote:
                        l.append(fname)
        return l

    def _expandClozes(self, string):
        ords = set(re.findall("{{c(\d+)::.+?}}", string))
        strings = []
        from anki.template.template import clozeReg
        def qrepl(m):
            if m.group(3):
                return "[%s]" % m.group(3)
            else:
                return "[...]"
        def arepl(m):
            return m.group(1)
        for ord in ords:
            s = re.sub(clozeReg%ord, qrepl, string)
            s = re.sub(clozeReg%".+?", "\\1", s)
            strings.append(s)
        strings.append(re.sub(clozeReg%".+?", arepl, string))
        return strings

    def transformNames(self, txt, func):
        for reg in self.regexps:
            txt = re.sub(reg, func, txt)
        return txt

    def strip(self, txt):
        for reg in self.regexps:
            txt = re.sub(reg, "", txt)
        return txt

    def escapeImages(self, string):
        # Feeding webkit unicode can result in it not finding images, so on
        # linux/osx we percent escape the image paths as utf8. On Windows the
        # problem is more complicated - if we percent-escape as utf8 it fixes
        # some images but breaks others. When filenames are normalized by
        # dropbox they become unreadable if we escape them.
        if isWin:
            return string
        def repl(match):
            tag = match.group(1)
            fname = match.group(2)
            if re.match("(https?|ftp)://", fname):
                return tag
            return tag.replace(
                fname, urllib.quote(fname.encode("utf-8")))
        return re.sub(self.regexps[1], repl, string)

    # Rebuilding DB
    ##########################################################################

    def check(self, local=None):
        "Return (missingFiles, unusedFiles)."
        mdir = self.dir()
        # generate card q/a and look through all references
        normrefs = {}
        def norm(s):
            if isinstance(s, unicode) and isMac:
                return unicodedata.normalize('NFD', s)
            return s
        for f in self.allMedia():
            normrefs[norm(f)] = True
        # loop through directory and find unused & missing media
        unused = []
        if local is None:
            files = os.listdir(mdir)
        else:
            files = local
        for file in files:
            if not local:
                path = os.path.join(mdir, file)
                if not os.path.isfile(path):
                    # ignore directories
                    continue
                if file.startswith("_"):
                    # leading _ says to ignore file
                    continue
            nfile = norm(file)
            if nfile not in normrefs:
                unused.append(file)
            else:
                del normrefs[nfile]
        nohave = normrefs.keys()
        return (nohave, unused)

    def allMedia(self):
        "Return a set of all referenced filenames."
        files = set()
        for mid, flds in self.col.db.execute("select mid, flds from notes"):
            for f in self.filesInStr(mid, flds):
                files.add(f)
        return files

    # Copying on import
    ##########################################################################

    def have(self, fname):
        return os.path.exists(os.path.join(self.dir(), fname))

    # Media syncing - changes and removal
    ##########################################################################

    def hasChanged(self):
        return self.db.scalar("select 1 from log limit 1")

    def removed(self):
        return self.db.list("select * from log where type = ?", MEDIA_REM)

    def syncRemove(self, fnames):
        # remove provided deletions
        for f in fnames:
            if os.path.exists(f):
                os.unlink(f)
            self.db.execute("delete from log where fname = ?", f)
            self.db.execute("delete from media where fname = ?", f)
        # and all locally-logged deletions, as server has acked them
        self.db.execute("delete from log where type = ?", MEDIA_REM)
        self.db.commit()

    # Media syncing - unbundling zip files from server
    ##########################################################################

    def syncAdd(self, zipData):
        "Extract zip data; true if finished."
        f = StringIO(zipData)
        z = zipfile.ZipFile(f, "r")
        finished = False
        meta = None
        media = []
        sizecnt = 0
        # get meta info first
        assert z.getinfo("_meta").file_size < 100000
        meta = json.loads(z.read("_meta"))
        nextUsn = int(z.read("_usn"))
        # then loop through all files
        for i in z.infolist():
            # check for zip bombs
            sizecnt += i.file_size
            assert sizecnt < 100*1024*1024
            if i.filename == "_meta" or i.filename == "_usn":
                # ignore previously-retrieved meta
                continue
            elif i.filename == "_finished":
                # last zip in set
                finished = True
            else:
                data = z.read(i)
                csum = checksum(data)
                name = meta[i.filename]
                # can we store the file on this system?
                if self.illegal(name):
                    continue
                # save file
                open(name, "wb").write(data)
                # update db
                media.append((name, csum, self._mtime(name)))
                # remove entries from local log
                self.db.execute("delete from log where fname = ?", name)
        # update media db and note new starting usn
        if media:
            self.db.executemany(
                "insert or replace into media values (?,?,?)", media)
        self.setUsn(nextUsn) # commits
        # if we have finished adding, we need to record the new folder mtime
        # so that we don't trigger a needless scan
        if finished:
            self.syncMod()
        return finished

    def illegal(self, f):
        if isWin:
            for c in f:
                if c in "<>:\"/\\|?*^":
                    return True
        elif isMac:
            for c in f:
                if c in ":\\/":
                    return True

    # Media syncing - bundling zip files to send to server
    ##########################################################################
    # Because there's no standard filename encoding for zips, and because not
    # all zip clients support retrieving mtime, we store the files as ascii
    # and place a json file in the zip with the necessary information.

    def zipAdded(self):
        "Add files to a zip until over SYNC_ZIP_SIZE/COUNT. Return zip data."
        f = StringIO()
        z = zipfile.ZipFile(f, "w", compression=zipfile.ZIP_DEFLATED)
        sz = 0
        cnt = 0
        files = {}
        cur = self.db.execute(
            "select fname from log where type = ?", MEDIA_ADD)
        fnames = []
        while 1:
            fname = cur.fetchone()
            if not fname:
                # add a flag so the server knows it can clean up
                z.writestr("_finished", "")
                break
            fname = fname[0]
            fnames.append([fname])
            z.write(fname, str(cnt))
            files[str(cnt)] = fname
            sz += os.path.getsize(fname)
            if sz > SYNC_ZIP_SIZE or cnt > SYNC_ZIP_COUNT:
                break
            cnt += 1
        z.writestr("_meta", json.dumps(files))
        z.close()
        return f.getvalue(), fnames

    def forgetAdded(self, fnames):
        if not fnames:
            return
        self.db.executemany("delete from log where fname = ?", fnames)
        self.db.commit()

    # Tracking changes (private)
    ##########################################################################

    def _initDB(self):
        self.db.executescript("""
create table media (fname text primary key, csum text, mod int);
create table meta (dirMod int, usn int); insert into meta values (0, 0);
create table log (fname text primary key, type int);
""")

    def _mtime(self, path):
        return int(os.stat(path).st_mtime)

    def _checksum(self, path):
        return checksum(open(path, "rb").read())

    def usn(self):
        return self.db.scalar("select usn from meta")

    def setUsn(self, usn):
        self.db.execute("update meta set usn = ?", usn)
        self.db.commit()

    def syncMod(self):
        self.db.execute("update meta set dirMod = ?", self._mtime(self.dir()))
        self.db.commit()

    def _changed(self):
        "Return dir mtime if it has changed since the last findChanges()"
        # doesn't track edits, but user can add or remove a file to update
        mod = self.db.scalar("select dirMod from meta")
        mtime = self._mtime(self.dir())
        if mod and mod == mtime:
            return False
        return mtime

    def findChanges(self):
        "Scan the media folder if it's changed, and note any changes."
        if self._changed():
            self._logChanges()

    def _logChanges(self):
        (added, removed) = self._changes()
        log = []
        media = []
        mediaRem = []
        for f in added:
            mt = self._mtime(f)
            media.append((f, self._checksum(f), mt))
            log.append((f, MEDIA_ADD))
        for f in removed:
            mediaRem.append((f,))
            log.append((f, MEDIA_REM))
        # update media db
        self.db.executemany("insert or replace into media values (?,?,?)",
                            media)
        if mediaRem:
            self.db.executemany("delete from media where fname = ?",
                                mediaRem)
        self.db.execute("update meta set dirMod = ?", self._mtime(self.dir()))
        # and logs
        self.db.executemany("insert or replace into log values (?,?)", log)
        self.db.commit()

    def _changes(self):
        self.cache = {}
        for (name, csum, mod) in self.db.execute(
            "select * from media"):
            self.cache[name] = [csum, mod, False]
        added = []
        removed = []
        # loop through on-disk files
        for f in os.listdir(self.dir()):
            # ignore folders and thumbs.db
            if os.path.isdir(f):
                continue
            if f.lower() == "thumbs.db":
                continue
            # and files with invalid chars
            bad = False
            for c in "\0", "/", "\\", ":":
                if c in f:
                    bad = True
                    break
            if bad:
                continue
            # empty files are invalid; clean them up and continue
            if not os.path.getsize(f):
                os.unlink(f)
                continue
            # newly added?
            if f not in self.cache:
                added.append(f)
            else:
                # modified since last time?
                if self._mtime(f) != self.cache[f][1]:
                    # and has different checksum?
                    if self._checksum(f) != self.cache[f][0]:
                        added.append(f)
                # mark as used
                self.cache[f][2] = True
        # look for any entries in the cache that no longer exist on disk
        for (k, v) in self.cache.items():
            if not v[2]:
                removed.append(k)
        return added, removed

    def sanityCheck(self):
        assert not self.db.scalar("select count() from log")
        cnt = self.db.scalar("select count() from media")
        return cnt
<commit_msg>check checksum of all filename alternatives when adding media<commit_after># -*- coding: utf-8 -*-
# Copyright: Damien Elmes <anki@ichi2.net>
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html

import os, shutil, re, urllib, unicodedata, \
    sys, zipfile
from cStringIO import StringIO
from anki.utils import checksum, isWin, isMac, json
from anki.db import DB
from anki.consts import *
from anki.latex import mungeQA

class MediaManager(object):

    # other code depends on this order, so don't reorder
    regexps = ("(?i)(\[sound:([^]]+)\])",
               "(?i)(<img[^>]+src=[\"']?([^\"'>]+)[\"']?[^>]*>)")

    def __init__(self, col, server):
        self.col = col
        if server:
            self._dir = None
            return
        # media directory
        self._dir = re.sub("(?i)\.(anki2)$", ".media", self.col.path)
        # convert dir to unicode if it's not already
        if isinstance(self._dir, str):
            self._dir = unicode(self._dir, sys.getfilesystemencoding())
        if not os.path.exists(self._dir):
            os.makedirs(self._dir)
        try:
            self._oldcwd = os.getcwd()
        except OSError:
            # cwd doesn't exist
            self._oldcwd = None
        os.chdir(self._dir)
        # change database
        self.connect()

    def connect(self):
        if self.col.server:
            return
        path = self.dir()+".db"
        create = not os.path.exists(path)
        os.chdir(self._dir)
        self.db = DB(path)
        if create:
            self._initDB()

    def close(self):
        if self.col.server:
            return
        self.db.close()
        self.db = None
        # change cwd back to old location
        if self._oldcwd:
            try:
                os.chdir(self._oldcwd)
            except:
                # may have been deleted
                pass

    def dir(self):
        return self._dir

    # Adding media
    ##########################################################################

    def addFile(self, opath):
        """Copy PATH to MEDIADIR, and return new filename.
If the same name exists, compare checksums."""
        mdir = self.dir()
        # remove any dangerous characters
        base = re.sub(r"[][<>:/\\&?\"\|]", "", os.path.basename(opath))
        (root, ext) = os.path.splitext(base)
        def repl(match):
            n = int(match.group(1))
            return " (%d)" % (n+1)
        # find the first available name
        while True:
            path = os.path.join(mdir, root + ext)
            # if it doesn't exist, copy it directly
            if not os.path.exists(path):
                shutil.copyfile(opath, path)
                return os.path.basename(os.path.basename(path))
            # if it's identical, reuse
            if self.filesIdentical(opath, path):
                return os.path.basename(path)
            # otherwise, increment the index in the filename
            reg = " \((\d+)\)$"
            if not re.search(reg, root):
                root = root + " (1)"
            else:
                root = re.sub(reg, repl, root)

    def filesIdentical(self, path1, path2):
        "True if files are the same."
        return (checksum(open(path1, "rb").read()) ==
                checksum(open(path2, "rb").read()))

    # String manipulation
    ##########################################################################

    def filesInStr(self, mid, string, includeRemote=False):
        l = []
        model = self.col.models.get(mid)
        strings = []
        if model['type'] == MODEL_CLOZE and "{{c" in string:
            # if the field has clozes in it, we'll need to expand the
            # possibilities so we can render latex
            strings = self._expandClozes(string)
        else:
            strings = [string]
        for string in strings:
            # handle latex
            string = mungeQA(string, None, None, model, None, self.col)
            # extract filenames
            for reg in self.regexps:
                for (full, fname) in re.findall(reg, string):
                    isLocal = not re.match("(https?|ftp)://", fname.lower())
                    if isLocal or includeRemote:
                        l.append(fname)
        return l

    def _expandClozes(self, string):
        ords = set(re.findall("{{c(\d+)::.+?}}", string))
        strings = []
        from anki.template.template import clozeReg
        def qrepl(m):
            if m.group(3):
                return "[%s]" % m.group(3)
            else:
                return "[...]"
        def arepl(m):
            return m.group(1)
        for ord in ords:
            s = re.sub(clozeReg%ord, qrepl, string)
            s = re.sub(clozeReg%".+?", "\\1", s)
            strings.append(s)
        strings.append(re.sub(clozeReg%".+?", arepl, string))
        return strings

    def transformNames(self, txt, func):
        for reg in self.regexps:
            txt = re.sub(reg, func, txt)
        return txt

    def strip(self, txt):
        for reg in self.regexps:
            txt = re.sub(reg, "", txt)
        return txt

    def escapeImages(self, string):
        # Feeding webkit unicode can result in it not finding images, so on
        # linux/osx we percent escape the image paths as utf8. On Windows the
        # problem is more complicated - if we percent-escape as utf8 it fixes
        # some images but breaks others. When filenames are normalized by
        # dropbox they become unreadable if we escape them.
        if isWin:
            return string
        def repl(match):
            tag = match.group(1)
            fname = match.group(2)
            if re.match("(https?|ftp)://", fname):
                return tag
            return tag.replace(
                fname, urllib.quote(fname.encode("utf-8")))
        return re.sub(self.regexps[1], repl, string)

    # Rebuilding DB
    ##########################################################################

    def check(self, local=None):
        "Return (missingFiles, unusedFiles)."
        mdir = self.dir()
        # generate card q/a and look through all references
        normrefs = {}
        def norm(s):
            if isinstance(s, unicode) and isMac:
                return unicodedata.normalize('NFD', s)
            return s
        for f in self.allMedia():
            normrefs[norm(f)] = True
        # loop through directory and find unused & missing media
        unused = []
        if local is None:
            files = os.listdir(mdir)
        else:
            files = local
        for file in files:
            if not local:
                path = os.path.join(mdir, file)
                if not os.path.isfile(path):
                    # ignore directories
                    continue
                if file.startswith("_"):
                    # leading _ says to ignore file
                    continue
            nfile = norm(file)
            if nfile not in normrefs:
                unused.append(file)
            else:
                del normrefs[nfile]
        nohave = normrefs.keys()
        return (nohave, unused)

    def allMedia(self):
        "Return a set of all referenced filenames."
        files = set()
        for mid, flds in self.col.db.execute("select mid, flds from notes"):
            for f in self.filesInStr(mid, flds):
                files.add(f)
        return files

    # Copying on import
    ##########################################################################

    def have(self, fname):
        return os.path.exists(os.path.join(self.dir(), fname))

    # Media syncing - changes and removal
    ##########################################################################

    def hasChanged(self):
        return self.db.scalar("select 1 from log limit 1")

    def removed(self):
        return self.db.list("select * from log where type = ?", MEDIA_REM)

    def syncRemove(self, fnames):
        # remove provided deletions
        for f in fnames:
            if os.path.exists(f):
                os.unlink(f)
            self.db.execute("delete from log where fname = ?", f)
            self.db.execute("delete from media where fname = ?", f)
        # and all locally-logged deletions, as server has acked them
        self.db.execute("delete from log where type = ?", MEDIA_REM)
        self.db.commit()

    # Media syncing - unbundling zip files from server
    ##########################################################################

    def syncAdd(self, zipData):
        "Extract zip data; true if finished."
        f = StringIO(zipData)
        z = zipfile.ZipFile(f, "r")
        finished = False
        meta = None
        media = []
        sizecnt = 0
        # get meta info first
        assert z.getinfo("_meta").file_size < 100000
        meta = json.loads(z.read("_meta"))
        nextUsn = int(z.read("_usn"))
        # then loop through all files
        for i in z.infolist():
            # check for zip bombs
            sizecnt += i.file_size
            assert sizecnt < 100*1024*1024
            if i.filename == "_meta" or i.filename == "_usn":
                # ignore previously-retrieved meta
                continue
            elif i.filename == "_finished":
                # last zip in set
                finished = True
            else:
                data = z.read(i)
                csum = checksum(data)
                name = meta[i.filename]
                # can we store the file on this system?
                if self.illegal(name):
                    continue
                # save file
                open(name, "wb").write(data)
                # update db
                media.append((name, csum, self._mtime(name)))
                # remove entries from local log
                self.db.execute("delete from log where fname = ?", name)
        # update media db and note new starting usn
        if media:
            self.db.executemany(
                "insert or replace into media values (?,?,?)", media)
        self.setUsn(nextUsn) # commits
        # if we have finished adding, we need to record the new folder mtime
        # so that we don't trigger a needless scan
        if finished:
            self.syncMod()
        return finished

    def illegal(self, f):
        if isWin:
            for c in f:
                if c in "<>:\"/\\|?*^":
                    return True
        elif isMac:
            for c in f:
                if c in ":\\/":
                    return True

    # Media syncing - bundling zip files to send to server
    ##########################################################################
    # Because there's no standard filename encoding for zips, and because not
    # all zip clients support retrieving mtime, we store the files as ascii
    # and place a json file in the zip with the necessary information.

    def zipAdded(self):
        "Add files to a zip until over SYNC_ZIP_SIZE/COUNT. Return zip data."
        f = StringIO()
        z = zipfile.ZipFile(f, "w", compression=zipfile.ZIP_DEFLATED)
        sz = 0
        cnt = 0
        files = {}
        cur = self.db.execute(
            "select fname from log where type = ?", MEDIA_ADD)
        fnames = []
        while 1:
            fname = cur.fetchone()
            if not fname:
                # add a flag so the server knows it can clean up
                z.writestr("_finished", "")
                break
            fname = fname[0]
            fnames.append([fname])
            z.write(fname, str(cnt))
            files[str(cnt)] = fname
            sz += os.path.getsize(fname)
            if sz > SYNC_ZIP_SIZE or cnt > SYNC_ZIP_COUNT:
                break
            cnt += 1
        z.writestr("_meta", json.dumps(files))
        z.close()
        return f.getvalue(), fnames

    def forgetAdded(self, fnames):
        if not fnames:
            return
        self.db.executemany("delete from log where fname = ?", fnames)
        self.db.commit()

    # Tracking changes (private)
    ##########################################################################

    def _initDB(self):
        self.db.executescript("""
create table media (fname text primary key, csum text, mod int);
create table meta (dirMod int, usn int); insert into meta values (0, 0);
create table log (fname text primary key, type int);
""")

    def _mtime(self, path):
        return int(os.stat(path).st_mtime)

    def _checksum(self, path):
        return checksum(open(path, "rb").read())

    def usn(self):
        return self.db.scalar("select usn from meta")

    def setUsn(self, usn):
        self.db.execute("update meta set usn = ?", usn)
        self.db.commit()

    def syncMod(self):
        self.db.execute("update meta set dirMod = ?", self._mtime(self.dir()))
        self.db.commit()

    def _changed(self):
        "Return dir mtime if it has changed since the last findChanges()"
        # doesn't track edits, but user can add or remove a file to update
        mod = self.db.scalar("select dirMod from meta")
        mtime = self._mtime(self.dir())
        if mod and mod == mtime:
            return False
        return mtime

    def findChanges(self):
        "Scan the media folder if it's changed, and note any changes."
        if self._changed():
            self._logChanges()

    def _logChanges(self):
        (added, removed) = self._changes()
        log = []
        media = []
        mediaRem = []
        for f in added:
            mt = self._mtime(f)
            media.append((f, self._checksum(f), mt))
            log.append((f, MEDIA_ADD))
        for f in removed:
            mediaRem.append((f,))
            log.append((f, MEDIA_REM))
        # update media db
        self.db.executemany("insert or replace into media values (?,?,?)",
                            media)
        if mediaRem:
            self.db.executemany("delete from media where fname = ?",
                                mediaRem)
        self.db.execute("update meta set dirMod = ?", self._mtime(self.dir()))
        # and logs
        self.db.executemany("insert or replace into log values (?,?)", log)
        self.db.commit()

    def _changes(self):
        self.cache = {}
        for (name, csum, mod) in self.db.execute(
            "select * from media"):
            self.cache[name] = [csum, mod, False]
        added = []
        removed = []
        # loop through on-disk files
        for f in os.listdir(self.dir()):
            # ignore folders and thumbs.db
            if os.path.isdir(f):
                continue
            if f.lower() == "thumbs.db":
                continue
            # and files with invalid chars
            bad = False
            for c in "\0", "/", "\\", ":":
                if c in f:
                    bad = True
                    break
            if bad:
                continue
            # empty files are invalid; clean them up and continue
            if not os.path.getsize(f):
                os.unlink(f)
                continue
            # newly added?
            if f not in self.cache:
                added.append(f)
            else:
                # modified since last time?
                if self._mtime(f) != self.cache[f][1]:
                    # and has different checksum?
                    if self._checksum(f) != self.cache[f][0]:
                        added.append(f)
                # mark as used
                self.cache[f][2] = True
        # look for any entries in the cache that no longer exist on disk
        for (k, v) in self.cache.items():
            if not v[2]:
                removed.append(k)
        return added, removed

    def sanityCheck(self):
        assert not self.db.scalar("select count() from log")
        cnt = self.db.scalar("select count() from media")
        return cnt
<|endoftext|>


<commit_before>
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import re
import yaml
import json

from oslo.config import cfg

from heat.common import exception
from heat.openstack.common.gettextutils import _

cfg.CONF.import_opt('max_template_size', 'heat.common.config')

HEAT_VERSIONS = (u'2012-12-12',)
CFN_VERSIONS = (u'2010-09-09',)

if hasattr(yaml, 'CSafeLoader'):
    yaml_loader = yaml.CSafeLoader
else:
    yaml_loader = yaml.SafeLoader

if hasattr(yaml, 'CSafeDumper'):
    yaml_dumper = yaml.CSafeDumper
else:
    yaml_dumper = yaml.SafeDumper

def _construct_yaml_str(self, node):
    # Override the default string handling function
    # to always return unicode objects
    return self.construct_scalar(node)
yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
# datetime.data which causes problems in API layer when being processed by
# openstack.common.jsonutils. Therefore, make unicode string out of timestamps
# until jsonutils can handle dates.
yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
                            _construct_yaml_str)

def simple_parse(tmpl_str):
    try:
        tpl = json.loads(tmpl_str)
    except ValueError:
        try:
            tpl = yaml.load(tmpl_str, Loader=yaml_loader)
        except yaml.YAMLError as yea:
            raise ValueError(yea)
        else:
            if tpl is None:
                tpl = {}
    return tpl

def parse(tmpl_str):
    '''
    Takes a string and returns a dict containing the parsed structure.
    This includes determination of whether the string is using the
    JSON or YAML format.
    '''
    if len(tmpl_str) > cfg.CONF.max_template_size:
        msg = (_('Template exceeds maximum allowed size (%s bytes)') %
               cfg.CONF.max_template_size)
        raise exception.RequestLimitExceeded(message=msg)
    tpl = simple_parse(tmpl_str)
    if not isinstance(tpl, dict):
        raise ValueError(_('The template is not a JSON object '
                           'or YAML mapping.'))
    # Looking for supported version keys in the loaded template
    if not ('HeatTemplateFormatVersion' in tpl
            or 'heat_template_version' in tpl
            or 'AWSTemplateFormatVersion' in tpl):
        raise ValueError(_("Template format version not found."))
    return tpl

def convert_json_to_yaml(json_str):
    '''Convert a string containing the AWS JSON template format
    to an equivalent string containing the Heat YAML format.
    '''

    global key_order
    # Replace AWS format version with Heat format version
    json_str = re.sub('"AWSTemplateFormatVersion"\s*:\s*"[^"]+"\s*,',
                      '', json_str)

    # insert a sortable order into the key to preserve file ordering
    key_order = 0

    def order_key(matchobj):
        global key_order
        key = '%s"__%05d__order__%s" :' % (
            matchobj.group(1),
            key_order,
            matchobj.group(2))
        key_order = key_order + 1
        return key
    key_re = re.compile('^(\s*)"([^"]+)"\s*:', re.M)
    json_str = key_re.sub(order_key, json_str)

    # parse the string as json to a python structure
    tpl = yaml.load(json_str, Loader=yaml_loader)

    # dump python structure to yaml
    tpl["HeatTemplateFormatVersion"] = '2012-12-12'
    yml = yaml.dump(tpl, Dumper=yaml_dumper)

    # remove ordering from key names
    yml = re.sub('__\d*__order__', '', yml)
    return yml
<commit_msg>Remove unused versions define in template_format.py<commit_after>
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import re
import yaml
import json

from oslo.config import cfg

from heat.common import exception
from heat.openstack.common.gettextutils import _

cfg.CONF.import_opt('max_template_size', 'heat.common.config')

if hasattr(yaml, 'CSafeLoader'):
    yaml_loader = yaml.CSafeLoader
else:
    yaml_loader = yaml.SafeLoader

if hasattr(yaml, 'CSafeDumper'):
    yaml_dumper = yaml.CSafeDumper
else:
    yaml_dumper = yaml.SafeDumper

def _construct_yaml_str(self, node):
    # Override the default string handling function
    # to always return unicode objects
    return self.construct_scalar(node)
yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
# datetime.data which causes problems in API layer when being processed by
# openstack.common.jsonutils. Therefore, make unicode string out of timestamps
# until jsonutils can handle dates.
yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
                            _construct_yaml_str)

def simple_parse(tmpl_str):
    try:
        tpl = json.loads(tmpl_str)
    except ValueError:
        try:
            tpl = yaml.load(tmpl_str, Loader=yaml_loader)
        except yaml.YAMLError as yea:
            raise ValueError(yea)
        else:
            if tpl is None:
                tpl = {}
    return tpl

def parse(tmpl_str):
    '''
    Takes a string and returns a dict containing the parsed structure.
    This includes determination of whether the string is using the
    JSON or YAML format.
    '''
    if len(tmpl_str) > cfg.CONF.max_template_size:
        msg = (_('Template exceeds maximum allowed size (%s bytes)') %
               cfg.CONF.max_template_size)
        raise exception.RequestLimitExceeded(message=msg)
    tpl = simple_parse(tmpl_str)
    if not isinstance(tpl, dict):
        raise ValueError(_('The template is not a JSON object '
                           'or YAML mapping.'))
    # Looking for supported version keys in the loaded template
    if not ('HeatTemplateFormatVersion' in tpl
            or 'heat_template_version' in tpl
            or 'AWSTemplateFormatVersion' in tpl):
        raise ValueError(_("Template format version not found."))
    return tpl

def convert_json_to_yaml(json_str):
    '''Convert a string containing the AWS JSON template format
    to an equivalent string containing the Heat YAML format.
    '''

    global key_order
    # Replace AWS format version with Heat format version
    json_str = re.sub('"AWSTemplateFormatVersion"\s*:\s*"[^"]+"\s*,',
                      '', json_str)

    # insert a sortable order into the key to preserve file ordering
    key_order = 0

    def order_key(matchobj):
        global key_order
        key = '%s"__%05d__order__%s" :' % (
            matchobj.group(1),
            key_order,
            matchobj.group(2))
        key_order = key_order + 1
        return key
    key_re = re.compile('^(\s*)"([^"]+)"\s*:', re.M)
    json_str = key_re.sub(order_key, json_str)

    # parse the string as json to a python structure
    tpl = yaml.load(json_str, Loader=yaml_loader)

    # dump python structure to yaml
    tpl["HeatTemplateFormatVersion"] = '2012-12-12'
    yml = yaml.dump(tpl, Dumper=yaml_dumper)

    # remove ordering from key names
    yml = re.sub('__\d*__order__', '', yml)
    return yml
<|endoftext|>


<commit_before>import pprint
import urllib
from uuid import uuid4
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.internet.threads import deferToThread
from twisted.web.resource import Resource
from twisted.internet import reactor
from twisted.web import server
from .cassandra import CassandraServer
from .base import LOGGER
from ..resources import InterfaceResource
from ..aws import sdb_now
from ..evaluateboolean import evaluateBoolean
import zlib

PRETTYPRINTER = pprint.PrettyPrinter(indent=4)

class InterfaceServer(CassandraServer):

    name = "HiiSpider Interface Server UUID: %s" % str(uuid4())

    def __init__(self,
            aws_access_key_id=None,
            aws_secret_access_key=None,
            cassandra_server=None,
            cassandra_keyspace=None,
            cassandra_cf_cache=None,
            cassandra_cf_content=None,
            cassandra_content=None,
            cassandra_http=None,
            cassandra_headers=None,
            scheduler_server=None,
            scheduler_server_port=5001,
            max_simultaneous_requests=50,
            max_requests_per_host_per_second=1,
            max_simultaneous_requests_per_host=5,
            port=5000,
            log_file='interfaceserver.log',
            log_directory=None,
            log_level="debug"):
        # Cassandra
        self.cassandra_server=cassandra_server
        self.cassandra_keyspace=cassandra_keyspace
        self.cassandra_cf_cache=cassandra_cf_cache
        self.cassandra_cf_content=cassandra_cf_content
        self.cassandra_http=cassandra_http
        self.cassandra_headers=cassandra_headers
        self.cassandra_content=cassandra_content
        self.aws_access_key_id=aws_access_key_id
        self.aws_secret_access_key=aws_secret_access_key
        resource = Resource()
        interface_resource = InterfaceResource(self)
        resource.putChild("interface", interface_resource)
        self.function_resource = Resource()
        resource.putChild("function", self.function_resource)
        self.site_port = reactor.listenTCP(port, server.Site(resource))
        CassandraServer.__init__(
            self,
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            cassandra_server=cassandra_server,
            cassandra_keyspace=cassandra_keyspace,
            cassandra_cf_cache=cassandra_cf_cache,
            cassandra_cf_content=cassandra_cf_content,
            cassandra_content=cassandra_content,
            cassandra_http=cassandra_http,
            cassandra_headers=cassandra_headers,
            scheduler_server=scheduler_server,
            scheduler_server_port=scheduler_server_port,
            max_simultaneous_requests=max_simultaneous_requests,
            max_requests_per_host_per_second=max_requests_per_host_per_second,
            max_simultaneous_requests_per_host=max_simultaneous_requests_per_host,
            log_file=log_file,
            log_directory=log_directory,
            log_level=log_level,
            port=port)

    def start(self):
        reactor.callWhenRunning(self._start)
        return self.start_deferred

    def _start(self):
        deferreds = []
        d = DeferredList(deferreds, consumeErrors=True)
        d.addCallback(self._startCallback)

    def _startCallback(self, data):
        for row in data:
            if row[0] == False:
                d = self.shutdown()
                d.addCallback(self._startHandleError, row[1])
                return d
        d = CassandraServer.start(self)

    def shutdown(self):
        deferreds = []
        LOGGER.debug("%s stopping on main HTTP interface." % self.name)
        d = self.site_port.stopListening()
        if isinstance(d, Deferred):
            deferreds.append(d)
        if len(deferreds) > 0:
            d = DeferredList(deferreds)
            d.addCallback(self._shutdownCallback)
            return d
        else:
            return self._shutdownCallback(None)

    def _shutdownCallback(self, data):
        return CassandraServer.shutdown(self)

    def enqueueUUID(self, uuid):
        if uuid and self.scheduler_server is not None:
            parameters = {'uuid': uuid}
            query_string = urllib.urlencode(parameters)
            url = 'http://%s:%s/function/schedulerserver/enqueueuuid?%s' % (self.scheduler_server, self.scheduler_server_port, query_string)
            LOGGER.info('Sending UUID to scheduler to be queued: %s' % url)
            d = self.rq.getPage(url=url)
            d.addCallback(self._enqueueCallback, uuid)
            d.addErrback(self._enqueueErrback, uuid)
            return d
        return None

    def _enqueueCallback(self, data, uuid):
        LOGGER.info("%s is added to spider queue." % uuid)

    def _enqueueErrback(self, error, uuid):
        LOGGER.info("Could not enqueue %s." % uuid)

<commit_msg>Removed AWS reference<commit_after>import pprint
import urllib
from uuid import uuid4
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.internet.threads import deferToThread
from twisted.web.resource import Resource
from twisted.internet import reactor
from twisted.web import server
from .cassandra import CassandraServer
from .base import LOGGER
from ..resources import InterfaceResource
from ..evaluateboolean import evaluateBoolean
import zlib

PRETTYPRINTER = pprint.PrettyPrinter(indent=4)

class InterfaceServer(CassandraServer):

    name = "HiiSpider Interface Server UUID: %s" % str(uuid4())

    def __init__(self,
            aws_access_key_id=None,
            aws_secret_access_key=None,
            cassandra_server=None,
            cassandra_keyspace=None,
            cassandra_cf_cache=None,
            cassandra_cf_content=None,
            cassandra_content=None,
            cassandra_http=None,
            cassandra_headers=None,
            scheduler_server=None,
            scheduler_server_port=5001,
            max_simultaneous_requests=50,
            max_requests_per_host_per_second=1,
            max_simultaneous_requests_per_host=5,
            port=5000,
            log_file='interfaceserver.log',
            log_directory=None,
            log_level="debug"):
        # Cassandra
        self.cassandra_server=cassandra_server
        self.cassandra_keyspace=cassandra_keyspace
        self.cassandra_cf_cache=cassandra_cf_cache
        self.cassandra_cf_content=cassandra_cf_content
        self.cassandra_http=cassandra_http
        self.cassandra_headers=cassandra_headers
        self.cassandra_content=cassandra_content
        self.aws_access_key_id=aws_access_key_id
        self.aws_secret_access_key=aws_secret_access_key
        resource = Resource()
        interface_resource = InterfaceResource(self)
        resource.putChild("interface", interface_resource)
        self.function_resource = Resource()
        resource.putChild("function", self.function_resource)
        self.site_port = reactor.listenTCP(port, server.Site(resource))
        CassandraServer.__init__(
            self,
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            cassandra_server=cassandra_server,
            cassandra_keyspace=cassandra_keyspace,
            cassandra_cf_cache=cassandra_cf_cache,
            cassandra_cf_content=cassandra_cf_content,
            cassandra_content=cassandra_content,
            cassandra_http=cassandra_http,
            cassandra_headers=cassandra_headers,
            scheduler_server=scheduler_server,
            scheduler_server_port=scheduler_server_port,
            max_simultaneous_requests=max_simultaneous_requests,
            max_requests_per_host_per_second=max_requests_per_host_per_second,
            max_simultaneous_requests_per_host=max_simultaneous_requests_per_host,
            log_file=log_file,
            log_directory=log_directory,
            log_level=log_level,
            port=port)

    def start(self):
        reactor.callWhenRunning(self._start)
        return self.start_deferred

    def _start(self):
        deferreds = []
        d = DeferredList(deferreds, consumeErrors=True)
        d.addCallback(self._startCallback)

    def _startCallback(self, data):
        for row in data:
            if row[0] == False:
                d = self.shutdown()
                d.addCallback(self._startHandleError, row[1])
                return d
        d = CassandraServer.start(self)

    def shutdown(self):
        deferreds = []
        LOGGER.debug("%s stopping on main HTTP interface." % self.name)
        d = self.site_port.stopListening()
        if isinstance(d, Deferred):
            deferreds.append(d)
        if len(deferreds) > 0:
            d = DeferredList(deferreds)
            d.addCallback(self._shutdownCallback)
            return d
        else:
            return self._shutdownCallback(None)

    def _shutdownCallback(self, data):
        return CassandraServer.shutdown(self)

    def enqueueUUID(self, uuid):
        if uuid and self.scheduler_server is not None:
            parameters = {'uuid': uuid}
            query_string = urllib.urlencode(parameters)
            url = 'http://%s:%s/function/schedulerserver/enqueueuuid?%s' % (self.scheduler_server, self.scheduler_server_port, query_string)
            LOGGER.info('Sending UUID to scheduler to be queued: %s' % url)
            d = self.rq.getPage(url=url)
            d.addCallback(self._enqueueCallback, uuid)
            d.addErrback(self._enqueueErrback, uuid)
            return d
        return None

    def _enqueueCallback(self, data, uuid):
        LOGGER.info("%s is added to spider queue." % uuid)

    def _enqueueErrback(self, error, uuid):
        LOGGER.info("Could not enqueue %s." % uuid)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.contrib.syndication.views import Feed
from django_cal.views import Events
from datetime import datetime, date
import django_cal
import locale

from hat.models import Strike, Region

locale.setlocale(locale.LC_ALL, "pt_PT.UTF-8")

class RssFeed(Feed):
    """Generate an RSS of the strikes"""
    title = u'Hoje há greve?'
    link = ''
    description = u'Veja se consegue chegar ao trabalho. Lembre-se que as informações podem estar desactualizadas.'

    def items(self):
        return Strike.objects.filter(start_date__gte=datetime.today().date()).order_by('start_date')[:10]

    def item_summary(self, strike):
        return strike.description

    def item_title(self, strike):
        return strike.company.name + ' - ' + strike.region.name

    def item_description(self, strike):
        return 'Greve da empresa ' + strike.company.name + '
' + 'De ' + str(strike.start_date) + ' a ' + str(strike.end_date) + '
' + strike.description

    def item_link(self, strike):
        return 'http://hagreve.com'

    def item_pubdate(self, strike):
        return strike.start_date

class IcsFeed(Events):
    def cal_name(self):
        return u'Hoje há greve?'

    def cal_desc(self):
        return u'Veja se consegue chegar ao trabalho. Lembre-se que as informações podem estar desactualizadas.'

    def items(self):
        return Strike.objects.filter(start_date__gte=datetime.today().date()).order_by('start_date')[:10]

    def item_summary(self, strike):
        return strike.description

    def item_title(self, strike):
        return strike.company.name + ' - ' + strike.region.name

    def item_start(self, strike):
        return strike.start_date

    def item_end(self, strike):
        return strike.end_date

    def item_description(self, strike):
        return 'Greve da empresa ' + strike.company.name + '\n' + 'De ' + str(strike.start_date) + ' a ' + str(strike.end_date.strftime) + '\n' + strike.description

    def item_link(self, strike):
        return 'https://hagreve.com'

<commit_msg>o br não está a funcionar no rss, revert it\!<commit_after># -*- coding: utf-8 -*-
from django.contrib.syndication.views import Feed
from django_cal.views import Events
from datetime import datetime, date
import django_cal
import locale

from hat.models import Strike, Region

locale.setlocale(locale.LC_ALL, "pt_PT.UTF-8")

class RssFeed(Feed):
    """Generate an RSS of the strikes"""
    title = u'Hoje há greve?'
    link = ''
    description = u'Veja se consegue chegar ao trabalho. Lembre-se que as informações podem estar desactualizadas.'

    def items(self):
        return Strike.objects.filter(start_date__gte=datetime.today().date()).order_by('start_date')[:10]

    def item_summary(self, strike):
        return strike.description

    def item_title(self, strike):
        return strike.company.name + ' - ' + strike.region.name

    def item_description(self, strike):
        return 'Greve da empresa ' + strike.company.name + '\n' + 'De ' + str(strike.start_date) + ' a ' + str(strike.end_date) + '\n\n' + strike.description

    def item_link(self, strike):
        return 'http://hagreve.com'

    def item_pubdate(self, strike):
        return strike.start_date

class IcsFeed(Events):
    def cal_name(self):
        return u'Hoje há greve?'

    def cal_desc(self):
        return u'Veja se consegue chegar ao trabalho. Lembre-se que as informações podem estar desactualizadas.'

    def items(self):
        return Strike.objects.filter(start_date__gte=datetime.today().date()).order_by('start_date')[:10]

    def item_summary(self, strike):
        return strike.description

    def item_title(self, strike):
        return strike.company.name + ' - ' + strike.region.name

    def item_start(self, strike):
        return strike.start_date

    def item_end(self, strike):
        return strike.end_date

    def item_description(self, strike):
        return 'Greve da empresa ' + strike.company.name + '\n' + 'De ' + str(strike.start_date) + ' a ' + str(strike.end_date.strftime) + '\n' + strike.description

    def item_link(self, strike):
        return 'https://hagreve.com'

<|endoftext|>


<commit_before>"""
The holoviews seaborn interface wraps around a wide range
of seaborn plot types including time series, kernel density
estimates, distributions and regression plots.
"""

from __future__ import absolute_import

import numpy as np

import param

from ..core import Dimension, ViewMap, Layer, Overlay
from ..views import DataView, Scatter
from ..core.options import options, StyleOpts, Cycle
from .pandas import DFrame as PandasDFrame

class TimeSeries(Layer):
    """
    TimeSeries is a container for any set of curves, which the
    seaborn interface combines into a confidence interval, error
    bar or overlaid plot.

    The curves should be supplied as an NxM dimensional array,
    x-values may also be supplied and must be of length N or M.

    Alternatively a HoloMap or Overlay of Curve objects may be
    supplied.
    """

    def __init__(self, data, xvals=None, **params):
        if isinstance(data, ViewMap):
            self.xdata = data.values()[0].data[:,0]
            data = np.array([dv.data[:, 1] for dv in data.values()])
        elif isinstance(data, Overlay):
            self.xdata = data.data[0].data[:,0]
            data = np.array([dv.data[:, 1] for dv in data.data])
        else:
            self.xdata = np.array(range(len(data[0, :]))) if xvals is None\
                else xvals
        super(TimeSeries, self).__init__(data, **params)

    def __getitem__(self, slc):
        if slc is ():
            return self
        else:
            raise NotImplementedError('Slicing and indexing of'
                                      'TimeSeries currently not '
                                      'implemented.')

    def sample(self, **samples):
        raise NotImplementedError('Cannot sample a TimeSeries.')

    def reduce(self, **dimreduce_map):
        raise NotImplementedError('Reduction of TimeSeries not '
                                  'implemented.')

    @property
    def xlim(self):
        if self._xlim:
            return self._xlim
        elif self.cyclic_range is not None:
            return (0, self.cyclic_range)
        else:
            x_vals = self.xdata
            return (float(min(x_vals)), float(max(x_vals)))

    @property
    def ylim(self):
        if self._ylim:
            return self._ylim
        else:
            return (np.min(self.data), np.max(self.data))

class Bivariate(DataView):
    """
    Bivariate Views are containers for two dimensional data,
    which is to be visualized as a kernel density estimate. The
    data should be supplied as an Nx2 array, containing the x-
    and y-data.
    """

    @property
    def xlabel(self):
        return str(self.dimensions[0])

    @property
    def ylabel(self):
        return str(self.dimensions[1])

class Distribution(DataView):
    """
    Distribution Views provide a container for data to be
    visualized as a one-dimensional distribution. The data should
    be supplied as a simple one-dimensional array or
    list. Internally it uses seaborn to make all the conversions.
    """

    value = param.ClassSelector(class_=Dimension,
                                default=Dimension('Frequency'))

    @property
    def xlim(self):
        if self._xlim:
            return self._xlim
        elif self.cyclic_range is not None:
            return (0, self.cyclic_range)
        else:
            return (np.min(self.data), np.max(self.data))

    @property
    def ylim(self):
        return (np.NaN, np.NaN)

    def __getitem__(self, index):
        raise NotImplementedError

class Regression(Scatter):
    """
    Regression is identical to a Scatter plot but is visualized
    using the seaborn regplot interface. This allows it to
    implement linear regressions, confidence intervals and a lot
    more.
    """

    pass

class DFrame(PandasDFrame):
    """
    The SNSFrame is largely the same as a DFrame but can only be
    visualized via seaborn plotting functions. Since most seaborn
    plots are two dimensional, the x and y dimensions can be set
    directly on this class to visualize a particular relationship
    in a multi-dimensional Pandas dframe.
    """

    plot_type = param.ObjectSelector(default=None,
                                     objects=['interact', 'regplot',
                                              'lmplot', 'corrplot',
                                              'plot', 'boxplot',
                                              'hist', 'scatter_matrix',
                                              'autocorrelation_plot',
                                              None],
                                     doc="""Selects which Pandas or Seaborn plot
                                            type to use, when visualizing the plot.""")

    x2 = param.String(doc="""Dimension to visualize along a second
                             dependent axis.""")

    @property
    def ylabel(self):
        return self.x2 if self.x2 else self.y

options.TimeSeries = StyleOpts(color=Cycle())
options.Bivariate = StyleOpts(cmap=Cycle(['Blues', 'Oranges', 'PuBu']))
options.Distribution = StyleOpts(color=Cycle())
options.Regression = StyleOpts(color=Cycle())
<commit_msg>Added correct dimensions and value to Bivariate View<commit_after>"""
The holoviews seaborn interface wraps around a wide range
of seaborn plot types including time series, kernel density
estimates, distributions and regression plots.
"""

from __future__ import absolute_import

import numpy as np

import param

from ..core import Dimension, ViewMap, Layer, Overlay
from ..views import DataView, Scatter
from ..core.options import options, StyleOpts, Cycle
from .pandas import DFrame as PandasDFrame

class TimeSeries(Layer):
    """
    TimeSeries is a container for any set of curves, which the
    seaborn interface combines into a confidence interval, error
    bar or overlaid plot.

    The curves should be supplied as an NxM dimensional array,
    x-values may also be supplied and must be of length N or M.

    Alternatively a HoloMap or Overlay of Curve objects may be
    supplied.
    """

    def __init__(self, data, xvals=None, **params):
        if isinstance(data, ViewMap):
            self.xdata = data.values()[0].data[:,0]
            data = np.array([dv.data[:, 1] for dv in data.values()])
        elif isinstance(data, Overlay):
            self.xdata = data.data[0].data[:,0]
            data = np.array([dv.data[:, 1] for dv in data.data])
        else:
            self.xdata = np.array(range(len(data[0, :]))) if xvals is None\
                else xvals
        super(TimeSeries, self).__init__(data, **params)

    def __getitem__(self, slc):
        if slc is ():
            return self
        else:
            raise NotImplementedError('Slicing and indexing of'
                                      'TimeSeries currently not '
                                      'implemented.')

    def sample(self, **samples):
        raise NotImplementedError('Cannot sample a TimeSeries.')

    def reduce(self, **dimreduce_map):
        raise NotImplementedError('Reduction of TimeSeries not '
                                  'implemented.')

    @property
    def xlim(self):
        if self._xlim:
            return self._xlim
        elif self.cyclic_range is not None:
            return (0, self.cyclic_range)
        else:
            x_vals = self.xdata
            return (float(min(x_vals)), float(max(x_vals)))

    @property
    def ylim(self):
        if self._ylim:
            return self._ylim
        else:
            return (np.min(self.data), np.max(self.data))

class Bivariate(DataView):
    """
    Bivariate Views are containers for two dimensional data,
    which is to be visualized as a kernel density estimate. The
    data should be supplied as an Nx2 array, containing the x-
    and y-data.
    """

    dimensions = param.List(default=[Dimension('X'), Dimension('Y')])

    value = param.ClassSelector(class_=Dimension, default=None)

    @property
    def xlabel(self):
        return str(self.dimensions[0])

    @property
    def ylabel(self):
        return str(self.dimensions[1])

class Distribution(DataView):
    """
    Distribution Views provide a container for data to be
    visualized as a one-dimensional distribution. The data should
    be supplied as a simple one-dimensional array or
    list. Internally it uses seaborn to make all the conversions.
    """

    value = param.ClassSelector(class_=Dimension,
                                default=Dimension('Frequency'))

    @property
    def xlim(self):
        if self._xlim:
            return self._xlim
        elif self.cyclic_range is not None:
            return (0, self.cyclic_range)
        else:
            return (np.min(self.data), np.max(self.data))

    @property
    def ylim(self):
        return (np.NaN, np.NaN)

    def __getitem__(self, index):
        raise NotImplementedError

class Regression(Scatter):
    """
    Regression is identical to a Scatter plot but is visualized
    using the seaborn regplot interface. This allows it to
    implement linear regressions, confidence intervals and a lot
    more.
    """

    pass

class DFrame(PandasDFrame):
    """
    The SNSFrame is largely the same as a DFrame but can only be
    visualized via seaborn plotting functions. Since most seaborn
    plots are two dimensional, the x and y dimensions can be set
    directly on this class to visualize a particular relationship
    in a multi-dimensional Pandas dframe.
    """

    plot_type = param.ObjectSelector(default=None,
                                     objects=['interact', 'regplot',
                                              'lmplot', 'corrplot',
                                              'plot', 'boxplot',
                                              'hist', 'scatter_matrix',
                                              'autocorrelation_plot',
                                              None],
                                     doc="""Selects which Pandas or Seaborn plot
                                            type to use, when visualizing the plot.""")

    x2 = param.String(doc="""Dimension to visualize along a second
                             dependent axis.""")

    @property
    def ylabel(self):
        return self.x2 if self.x2 else self.y

options.TimeSeries = StyleOpts(color=Cycle())
options.Bivariate = StyleOpts(cmap=Cycle(['Blues', 'Oranges', 'PuBu']))
options.Distribution = StyleOpts(color=Cycle())
options.Regression = StyleOpts(color=Cycle())
<|endoftext|>


<commit_before>from app.api.models.LXDModule import LXDModule
from pylxd import Client
import logging

logging = logging.getLogger(__name__)

class LXCProfile(LXDModule):

    def __init__(self, input):
        logging.info('Connecting to LXD')
        self.client = Client()
        self.input = input

    def info(self):
        try:
            return self.client.api.profiles[self.input.get('name')].get().json()['metadata']
        except Exception as e:
            raise ValueError(e)

    def info(self, name):
        try:
            logging.info('Reading profile {} information'.format(name))
            return self.client.api.profiles[name].get().json()['metadata']
        except Exception as e:
            logging.error('Failed to retrieve information for profile {}'.format(name))
            logging.exception(e)
            raise ValueError(e)

    def createProfile(self):
        try:
            logging.info('Creating profile {}'.format(self.input.get('name')))
            self.client.profiles.create(self.input.get('name'), config=self.input.get('config'),
                                        devices=self.input.get('devices'))

            return self.client.api.profiles[self.input.get('name')].get().json()['metadata']
        except Exception as e:
            logging.error('Failed to create container {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)

    def deleteProfile(self):
        try:
            logging.info('Deleting profile {}'.format(self.input.get('name')))
            return self.client.api.profiles[self.input.get('name')].delete(json=self.input).json()
        except Exception as e:
            logging.error('Failed to delete profile {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)

    def updateProfile(self):
        try:
            logging.info('Updating profile {}'.format(self.input.get('name')))
            profile = self.client.profiles.get(self.input.get('name'))
            #self.client.api.profiles[self.input.get('name')].put(json=self.input).json()['metadata']
            profile.config.update(self.input.get('config'))
            profile.devices.update(self.input.get('devices'))
            profile.update()
            if self.input.get('new_name'):
                return self.rename()
            return self.info()
        except Exception as e:
            logging.error('Failed to update profile {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)

    def rename(self):
        try:
            logging.info('Renaming profile {}'.format(self.input.get('name')))
            profile = self.client.profiles.get(self.input.get('name'))
            profile.rename(self.input.get('new_name'))
            return self.info(self.input.get('new_name'))
        except Exception as e:
            logging.error('Failed to rename profile {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)
<commit_msg>Update & Rename Profile - API<commit_after>from app.api.models.LXDModule import LXDModule
from pylxd import Client
import logging

logging = logging.getLogger(__name__)

class LXCProfile(LXDModule):

    def __init__(self, input):
        logging.info('Connecting to LXD')
        self.client = Client()
        self.input = input

    def info(self):
        try:
            return self.client.api.profiles[self.input.get('name')].get().json()['metadata']
        except Exception as e:
            raise ValueError(e)

    def info(self, name):
        try:
            logging.info('Reading profile {} information'.format(name))
            return self.client.api.profiles[name].get().json()['metadata']
        except Exception as e:
            logging.error('Failed to retrieve information for profile {}'.format(name))
            logging.exception(e)
            raise ValueError(e)

    def createProfile(self):
        try:
            logging.info('Creating profile {}'.format(self.input.get('name')))
            self.client.profiles.create(self.input.get('name'), config=self.input.get('config'),
                                        devices=self.input.get('devices'))

            return self.client.api.profiles[self.input.get('name')].get().json()['metadata']
        except Exception as e:
            logging.error('Failed to create container {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)

    def deleteProfile(self):
        try:
            logging.info('Deleting profile {}'.format(self.input.get('name')))
            return self.client.api.profiles[self.input.get('name')].delete(json=self.input).json()
        except Exception as e:
            logging.error('Failed to delete profile {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)

    def updateProfile(self):
        try:
            logging.info('Updating profile {}'.format(self.input.get('name')))
            self.client.api.profiles[self.input.get('name')].put(json={'config': self.input.get('config'), 'devices': self.input.get('devices')})
            if self.input.get('new_name'):
                self.input['name'] = self.input.get('new_name')
                return self.rename()

            return self.info(self.input.get('name'))
        except Exception as e:
            logging.error('Failed to update profile {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)

    def rename(self):
        try:
            logging.info('Renaming profile {}'.format(self.input.get('name')))
            profile = self.client.profiles.get(self.input.get('name'))
            profile.rename(self.input.get('new_name'))
            return self.info(self.input.get('new_name'))
        except Exception as e:
            logging.error('Failed to rename profile {}'.format(self.input.get('name')))
            logging.exception(e)
            raise ValueError(e)
<|endoftext|>