<commit_before>import os
import urlparse
import datetime
import threading
import mimetools
import socket
import errno
import xlog
import sys
import select
import time

logging = xlog.Logger()

class HttpServerHandler():
    default_request_version = "HTTP/1.1"
    MessageClass = mimetools.Message
    rbufsize = -1
    wbufsize = 0

    def __init__(self, sock, client, args):
        self.connection = sock
        self.rfile = socket._fileobject(self.connection, "rb", self.rbufsize)
        self.wfile = socket._fileobject(self.connection, "wb", self.wbufsize)
        self.client_address = client
        self.args = args
        self.setup()

    def setup(self):
        pass

    def handle(self):
        #logging.info('Connected from %r', self.client_address)
        while True:
            try:
                self.close_connection = 1
                self.handle_one_request()
            except Exception as e:
                #logging.warn("handle err:%r close", e)
                self.close_connection = 1

            if self.close_connection:
                break
        self.connection.close()
        #logging.debug("closed from %s:%d", self.client_address[0], self.client_address[1])

    def address_string(self):
        return '%s:%s' % self.client_address[:2]

    def parse_request(self):
        self.command = None  # set in case of error on the first line
        self.request_version = version = self.default_request_version

        requestline = self.raw_requestline
        requestline = requestline.rstrip('\r\n')
        self.requestline = requestline
        words = requestline.split()
        if len(words) == 3:
            command, path, version = words
            if version[:5] != 'HTTP/':
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            try:
                base_version_number = version.split('/', 1)[1]
                version_number = base_version_number.split(".")
                # RFC 2145 section 3.1 says there can be only one "." and
                #   - major and minor numbers MUST be treated as
                #      separate integers;
                #   - HTTP/2.4 is a lower version than HTTP/2.13, which in
                #      turn is lower than HTTP/12.3;
                #   - Leading zeros MUST be ignored by recipients.
                if len(version_number) != 2:
                    raise ValueError
                version_number = int(version_number[0]), int(version_number[1])
            except (ValueError, IndexError):
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            if version_number >= (1, 1):
                self.close_connection = 0
            if version_number >= (2, 0):
                self.send_error(505,
                          "Invalid HTTP Version (%s)" % base_version_number)
                return False
        elif len(words) == 2:
            command, path = words
            self.close_connection = 1
            if command != 'GET':
                self.send_error(400,
                                "Bad HTTP/0.9 request type (%r)" % command)
                return False
        elif not words:
            return False
        else:
            self.send_error(400, "Bad request syntax (%r)" % requestline)
            return False
        self.command, self.path, self.request_version = command, path, version

        # Examine the headers and look for a Connection directive
        self.headers = self.MessageClass(self.rfile, 0)

        conntype = self.headers.get('Connection', "")
        if conntype.lower() == 'close':
            self.close_connection = 1
        elif conntype.lower() == 'keep-alive':
            self.close_connection = 0
        return True

    def handle_one_request(self):
        try:
            try:
                self.raw_requestline = self.rfile.readline(65537)
            except Exception as e:
                #logging.warn("simple server handle except %r", e)
                return

            if len(self.raw_requestline) > 65536:
                #logging.warn("recv command line too large")
                return
            if not self.raw_requestline:
                #logging.warn("closed")
                return

            self.parse_request()

            if self.command == "GET":
                self.do_GET()
            elif self.command == "POST":
                self.do_POST()
            elif self.command == "CONNECT":
                self.do_CONNECT()
            elif self.command == "HEAD":
                self.do_HEAD()
            elif self.command == "DELETE":
                self.do_DELETE()
            elif self.command == "OPTIONS":
                self.do_OPTIONS()
            elif self.command == "PUT":
                self.do_PUT()
            else:
                logging.warn("unhandler cmd:%s", self.command)
                return

            self.wfile.flush() #actually send the response if not already done.
            self.close_connection = 0
        except socket.error as e:
            #logging.warn("socket error:%r", e)
            pass
        except IOError as e:
            if e.errno == errno.EPIPE:
                logging.warn("PIPE error:%r", e)
                pass
            else:
                logging.warn("IOError:%r", e)
                pass
        #except OpenSSL.SSL.SysCallError as e:
        #    logging.warn("socket error:%r", e)
        except Exception as e:
            logging.exception("handler:%r", e)
            pass

    def do_GET(self):
        pass

    def do_POST(self):
        pass

    def do_PUT(self):
        pass

    def do_DELETE(self):
        pass

    def do_OPTIONS(self):
        pass

    def do_HEAD(self):
        pass

    def do_CONNECT(self):
        pass

    def send_not_found(self):
        self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found')

    def send_response(self, mimetype, data):
        no_cache = "Cache-Control: no-cache, no-store, must-revalidate\r\nPragma: no-cache\r\nExpires: 0\r\n"
        try:
            self.wfile.write(('HTTP/1.1 200\r\n%sAccess-Control-Allow-Origin: *\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n'
                 % (no_cache, mimetype, len(data))).encode())
            self.wfile.write(data)
        except:
            pass

    def send_error(self, code, message=None):
        self.wfile.write('HTTP/1.1 %d\r\n' % code)
        self.wfile.write('Connection: close\r\n\r\n')
        if message:
            self.wfile.write(message)

    def send_response_data(self, mimetype="", content="", heads="", status=200):
        data = []
        data.append('HTTP/1.1 %d\r\n' % status)
        if len(mimetype):
            data.append('Content-Type: %s\r\n' % mimetype)
        data.append('Content-Length: %s\r\n' % len(content))
        if len(heads):
            data.append(heads)
        data.append("\r\n")

        if len(content) + len(heads) < 1024:
            data.append(content)
            data_str = "".join(data)
            self.wfile.write(data_str)
        else:
            data_str = "".join(data)
            self.wfile.write(data_str)
            if len(content):
                self.wfile.write(content)

    def send_file(self, filename, mimetype):
        try:
            if not os.path.isfile(filename):
                self.send_not_found()
                return

            file_size = os.path.getsize(filename)
            tme = (datetime.datetime.today()+datetime.timedelta(minutes=330)).strftime('%a, %d %b %Y %H:%M:%S GMT')
            head = 'HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nCache-Control:public, max-age=31536000\r\n'
            head += 'Expires: %s\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (tme, mimetype, file_size)
            self.wfile.write(head.encode())

            with open(filename, 'rb') as fp:
                while True:
                    data = fp.read(65535)
                    if not data:
                        break
                    self.wfile.write(data)
        except:
            pass
            #logging.warn("download broken")

class HTTPServer():
    def __init__(self, address, handler, args=(), use_https=False, cert=""):
        self.running = True
        if isinstance(address, tuple):
            self.server_address = [address]
        else:
            #server can listen multi-port
            self.server_address = address
        self.handler = handler
        self.args = args
        self.use_https = use_https
        self.cert = cert
        self.init_socket()
        #logging.info("server %s:%d started.", address[0], address[1])

    def init_socket(self):
        self.sockets = []
        for addr in self.server_address:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            try:
                sock.bind(addr)
            except Exception as e:
                logging.error("bind to %s:%d fail", addr[0], addr[1])
                raise e

            if self.use_https:
                import OpenSSL
                ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
                #server.pem's location (containing the server private key and the server certificate).
                fpem = self.cert
                ctx.use_privatekey_file(fpem)
                ctx.use_certificate_file(fpem)
                sock = OpenSSL.SSL.Connection(ctx, sock)
            sock.listen(200)
            self.sockets.append(sock)
            logging.info("server %s:%d started.", addr[0], addr[1])

    def serve_forever(self):
        while self.running:
            r, w, e = select.select(self.sockets, [], [], 1)
            for rsock in r:
                try:
                    (sock, address) = rsock.accept()
                except IOError as e:
                    logging.warn("socket accept fail(errno: %s).", e.args[0])
                    if e.args[0] == 10022:
                        logging.info("restart socket server.")
                        self.init_socket()
                    return
                self.process_connect(sock, address)

    def process_connect(self, sock, address):
        #logging.debug("connect from %s:%d", address[0], address[1])
        client_obj = self.handler(sock, address, self.args)
        client_thread = threading.Thread(target=client_obj.handle)
        client_thread.start()

    def shutdown(self):
        self.running = False

    def server_close(self):
        for sock in self.sockets:
            sock.close()

class TestHttpServer(HttpServerHandler):
    def __init__(self, sock, client, args):
        self.data_path = args
        HttpServerHandler.__init__(self, sock, client, args)

    def generate_random_lowercase(self, n):
        min_lc = ord(b'a')
        len_lc = 26
        ba = bytearray(os.urandom(n))
        for i, b in enumerate(ba):
            ba[i] = min_lc + b % len_lc # convert 0..255 to 97..122
        #sys.stdout.buffer.write(ba)
        return ba

    def do_GET(self):
        url_path = urlparse.urlparse(self.path).path
        req = urlparse.urlparse(self.path).query
        reqs = urlparse.parse_qs(req, keep_blank_values=True)

        #logging.debug("GET %s from %s:%d", self.path, self.client_address[0], self.client_address[1])

        if url_path == '/':
            data = "OK\r\n"
            self.wfile.write('HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nContent-Length: %d\r\n\r\n%s' %(len(data), data) )
        elif url_path == '/null':
            mimetype = "application/x-binary"
            if "size" in reqs:
                file_size = int(reqs['size'][0])
            else:
                file_size = 1024 * 1024 * 1024

            self.wfile.write('HTTP/1.1 200\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (mimetype, file_size))
            start = 0
            data = self.generate_random_lowercase(65535)
            while start < file_size:
                left = file_size - start
                send_batch = min(left, 65535)
                self.wfile.write(data[:send_batch])
                start += send_batch
        else:
            target = os.path.abspath(os.path.join(self.data_path, url_path[1:]))
            if os.path.isfile(target):
                self.send_file(target, "application/x-binary")
            else:
                self.wfile.write('HTTP/1.1 404\r\nContent-Length: 0\r\n\r\n' )

def main(data_path="."):
    logging.info("listen http on 8880")
    httpd = HTTPServer(('', 8880), TestHttpServer, data_path)

    http_thread = threading.Thread(target=httpd.serve_forever)
    http_thread.setDaemon(True)
    http_thread.start()

    while True:
        time.sleep(10)

if __name__ == "__main__":
    if len(sys.argv) > 2:
        data_path = sys.argv[1]
    else:
        data_path = "."

    try:
        main(data_path=data_path)
    except Exception:
        import traceback
        traceback.print_exc(file=sys.stdout)
    except KeyboardInterrupt:
        sys.exit()
    <commit_msg>Fix bug https://github.com/XX-net/XX-Net/issues/1727<commit_after>import os
import urlparse
import datetime
import threading
import mimetools
import socket
import errno
import xlog
import sys
import select
import time

logging = xlog.Logger()

class HttpServerHandler():
    default_request_version = "HTTP/1.1"
    MessageClass = mimetools.Message
    rbufsize = -1
    wbufsize = 0

    def __init__(self, sock, client, args):
        self.connection = sock
        self.rfile = socket._fileobject(self.connection, "rb", self.rbufsize)
        self.wfile = socket._fileobject(self.connection, "wb", self.wbufsize)
        self.client_address = client
        self.args = args
        self.setup()

    def setup(self):
        pass

    def handle(self):
        #logging.info('Connected from %r', self.client_address)
        while True:
            try:
                self.close_connection = 1
                self.handle_one_request()
            except Exception as e:
                #logging.warn("handle err:%r close", e)
                self.close_connection = 1

            if self.close_connection:
                break
        self.connection.close()
        #logging.debug("closed from %s:%d", self.client_address[0], self.client_address[1])

    def address_string(self):
        return '%s:%s' % self.client_address[:2]

    def parse_request(self):
        self.command = None  # set in case of error on the first line
        self.request_version = version = self.default_request_version

        requestline = self.raw_requestline
        requestline = requestline.rstrip('\r\n')
        self.requestline = requestline
        words = requestline.split()
        if len(words) == 3:
            command, path, version = words
            if version[:5] != 'HTTP/':
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            try:
                base_version_number = version.split('/', 1)[1]
                version_number = base_version_number.split(".")
                # RFC 2145 section 3.1 says there can be only one "." and
                #   - major and minor numbers MUST be treated as
                #      separate integers;
                #   - HTTP/2.4 is a lower version than HTTP/2.13, which in
                #      turn is lower than HTTP/12.3;
                #   - Leading zeros MUST be ignored by recipients.
                if len(version_number) != 2:
                    raise ValueError
                version_number = int(version_number[0]), int(version_number[1])
            except (ValueError, IndexError):
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            if version_number >= (1, 1):
                self.close_connection = 0
            if version_number >= (2, 0):
                self.send_error(505,
                          "Invalid HTTP Version (%s)" % base_version_number)
                return False
        elif len(words) == 2:
            command, path = words
            self.close_connection = 1
            if command != 'GET':
                self.send_error(400,
                                "Bad HTTP/0.9 request type (%r)" % command)
                return False
        elif not words:
            return False
        else:
            self.send_error(400, "Bad request syntax (%r)" % requestline)
            return False
        self.command, self.path, self.request_version = command, path, version

        # Examine the headers and look for a Connection directive
        self.headers = self.MessageClass(self.rfile, 0)

        conntype = self.headers.get('Connection', "")
        if conntype.lower() == 'close':
            self.close_connection = 1
        elif conntype.lower() == 'keep-alive':
            self.close_connection = 0
        return True

    def handle_one_request(self):
        try:
            try:
                self.raw_requestline = self.rfile.readline(65537)
            except Exception as e:
                #logging.warn("simple server handle except %r", e)
                return

            if len(self.raw_requestline) > 65536:
                #logging.warn("recv command line too large")
                return
            if not self.raw_requestline:
                #logging.warn("closed")
                return

            self.parse_request()

            if self.command == "GET":
                self.do_GET()
            elif self.command == "POST":
                self.do_POST()
            elif self.command == "CONNECT":
                self.do_CONNECT()
            elif self.command == "HEAD":
                self.do_HEAD()
            elif self.command == "DELETE":
                self.do_DELETE()
            elif self.command == "OPTIONS":
                self.do_OPTIONS()
            elif self.command == "PUT":
                self.do_PUT()
            else:
                logging.warn("unhandler cmd:%s", self.command)
                return

            self.wfile.flush() #actually send the response if not already done.
            self.close_connection = 0
        except socket.error as e:
            #logging.warn("socket error:%r", e)
            pass
        except IOError as e:
            if e.errno == errno.EPIPE:
                logging.warn("PIPE error:%r", e)
                pass
            else:
                logging.warn("IOError:%r", e)
                pass
        #except OpenSSL.SSL.SysCallError as e:
        #    logging.warn("socket error:%r", e)
        except Exception as e:
            logging.exception("handler:%r", e)
            pass

    def do_GET(self):
        pass

    def do_POST(self):
        pass

    def do_PUT(self):
        pass

    def do_DELETE(self):
        pass

    def do_OPTIONS(self):
        pass

    def do_HEAD(self):
        pass

    def do_CONNECT(self):
        pass

    def send_not_found(self):
        self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found')

    def send_response(self, mimetype, data):
        no_cache = "Cache-Control: no-cache, no-store, must-revalidate\r\nPragma: no-cache\r\nExpires: 0\r\n"
        try:
            self.wfile.write(('HTTP/1.1 200\r\n%sAccess-Control-Allow-Origin: *\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n'
                 % (no_cache, mimetype, len(data))).encode())
            self.wfile.write(data)
        except:
            pass

    def send_error(self, code, message=None):
        self.wfile.write('HTTP/1.1 %d\r\n' % code)
        self.wfile.write('Connection: close\r\n\r\n')
        if message:
            self.wfile.write(message)

    def send_response_data(self, mimetype="", content="", heads="", status=200):
        data = []
        data.append('HTTP/1.1 %d\r\n' % status)
        if len(mimetype):
            data.append('Content-Type: %s\r\n' % mimetype)
        data.append('Content-Length: %s\r\n' % len(content))
        if len(heads):
            data.append(heads)
        data.append("\r\n")

        if len(content) + len(heads) < 1024:
            data.append(content)
            data_str = "".join(data)
            self.wfile.write(data_str)
        else:
            data_str = "".join(data)
            self.wfile.write(data_str)
            if len(content):
                self.wfile.write(content)

    def send_file(self, filename, mimetype):
        try:
            if not os.path.isfile(filename):
                self.send_not_found()
                return

            file_size = os.path.getsize(filename)
            tme = (datetime.datetime.today()+datetime.timedelta(minutes=330)).strftime('%a, %d %b %Y %H:%M:%S GMT')
            head = 'HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nCache-Control:public, max-age=31536000\r\n'
            head += 'Expires: %s\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (tme, mimetype, file_size)
            self.wfile.write(head.encode())

            with open(filename, 'rb') as fp:
                while True:
                    data = fp.read(65535)
                    if not data:
                        break
                    self.wfile.write(data)
        except:
            pass
            #logging.warn("download broken")

class HTTPServer():
    def __init__(self, address, handler, args=(), use_https=False, cert=""):
        self.sockets = None
        self.running = True
        if isinstance(address, tuple):
            self.server_address = [address]
        else:
            #server can listen multi-port
            self.server_address = address
        self.handler = handler
        self.args = args
        self.use_https = use_https
        self.cert = cert
        self.init_socket()
        #logging.info("server %s:%d started.", address[0], address[1])

    def init_socket(self):
        if self.sockets is not None:
            self.server_close()

        self.sockets = []
        for addr in self.server_address:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            try:
                sock.bind(addr)
            except Exception as e:
                logging.error("bind to %s:%d fail", addr[0], addr[1])
                raise e

            if self.use_https:
                import OpenSSL
                ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
                #server.pem's location (containing the server private key and the server certificate).
                fpem = self.cert
                ctx.use_privatekey_file(fpem)
                ctx.use_certificate_file(fpem)
                sock = OpenSSL.SSL.Connection(ctx, sock)
            sock.listen(200)
            self.sockets.append(sock)
            logging.info("server %s:%d started.", addr[0], addr[1])

    def serve_forever(self):
        while self.running:
            r, w, e = select.select(self.sockets, [], [], 1)
            for rsock in r:
                try:
                    (sock, address) = rsock.accept()
                except IOError as e:
                    logging.warn("socket accept fail(errno: %s).", e.args[0])
                    if e.args[0] == 10022:
                        logging.info("restart socket server.")
                        self.init_socket()
                    break
                self.process_connect(sock, address)

    def process_connect(self, sock, address):
        #logging.debug("connect from %s:%d", address[0], address[1])
        client_obj = self.handler(sock, address, self.args)
        client_thread = threading.Thread(target=client_obj.handle)
        client_thread.start()

    def shutdown(self):
        self.running = False

    def server_close(self):
        for sock in self.sockets:
            sock.close()

class TestHttpServer(HttpServerHandler):
    def __init__(self, sock, client, args):
        self.data_path = args
        HttpServerHandler.__init__(self, sock, client, args)

    def generate_random_lowercase(self, n):
        min_lc = ord(b'a')
        len_lc = 26
        ba = bytearray(os.urandom(n))
        for i, b in enumerate(ba):
            ba[i] = min_lc + b % len_lc # convert 0..255 to 97..122
        #sys.stdout.buffer.write(ba)
        return ba

    def do_GET(self):
        url_path = urlparse.urlparse(self.path).path
        req = urlparse.urlparse(self.path).query
        reqs = urlparse.parse_qs(req, keep_blank_values=True)

        #logging.debug("GET %s from %s:%d", self.path, self.client_address[0], self.client_address[1])

        if url_path == '/':
            data = "OK\r\n"
            self.wfile.write('HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nContent-Length: %d\r\n\r\n%s' %(len(data), data) )
        elif url_path == '/null':
            mimetype = "application/x-binary"
            if "size" in reqs:
                file_size = int(reqs['size'][0])
            else:
                file_size = 1024 * 1024 * 1024

            self.wfile.write('HTTP/1.1 200\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (mimetype, file_size))
            start = 0
            data = self.generate_random_lowercase(65535)
            while start < file_size:
                left = file_size - start
                send_batch = min(left, 65535)
                self.wfile.write(data[:send_batch])
                start += send_batch
        else:
            target = os.path.abspath(os.path.join(self.data_path, url_path[1:]))
            if os.path.isfile(target):
                self.send_file(target, "application/x-binary")
            else:
                self.wfile.write('HTTP/1.1 404\r\nContent-Length: 0\r\n\r\n' )

def main(data_path="."):
    logging.info("listen http on 8880")
    httpd = HTTPServer(('', 8880), TestHttpServer, data_path)

    http_thread = threading.Thread(target=httpd.serve_forever)
    http_thread.setDaemon(True)
    http_thread.start()

    while True:
        time.sleep(10)

if __name__ == "__main__":
    if len(sys.argv) > 2:
        data_path = sys.argv[1]
    else:
        data_path = "."

    try:
        main(data_path=data_path)
    except Exception:
        import traceback
        traceback.print_exc(file=sys.stdout)
    except KeyboardInterrupt:
        sys.exit()
    <|endoftext|>


<commit_before>import torch
import os
import re
import pdb
from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel

class ModelIO(object):

    def load_model_specific(self, checkpoint):
        """
        Do something with the checkpoint
        :param checkpoint:
        :return:
        """
        raise NotImplementedError

    def get_save_dict(self):
        """
        Return specific things for the model
        :return:
        """
        raise NotImplementedError

    # -------------------------
    # OPTIONAL HOOKS
    # -------------------------
    def on_hpc_save(self):
        """
        Hook to do whatever you need right before Slurm manager saves the model
        :return:
        """
        pass

    def on_hpc_load(self):
        """
        Hook to do whatever you need right before Slurm manager loads the model
        :return:
        """
        pass

class TrainerIO(object):

    def __get_model(self):
        print(type(self.model))
        is_dp_module = type(self.model) is LightningDistributedDataParallel or type(self.model) is LightningDataParallel
        model = self.model.module if is_dp_module else self.model
        return model

    # --------------------
    # MODEL SAVE CHECKPOINT
    # --------------------
    def save_checkpoint(self, filepath):
        checkpoint = self.dump_checkpoint()

        # do the actual save
        torch.save(checkpoint, filepath)

    def dump_checkpoint(self):

        checkpoint = {
            'epoch': self.current_epoch,
            'global_step': self.global_step
        }

        if self.checkpoint_callback is not None:
            checkpoint['checkpoint_callback_best'] = self.checkpoint_callback.best

        if self.early_stop_callback is not None:
            checkpoint['early_stop_callback_wait'] = self.early_stop_callback.wait
            checkpoint['early_stop_callback_patience'] = self.early_stop_callback.patience

        optimizer_states = []
        for i, optimizer in enumerate(self.optimizers):
            optimizer_states.append(optimizer.state_dict())

        checkpoint['optimizer_states'] = optimizer_states

        # request what to save from the model
        model = self.__get_model()
        checkpoint_dict = model.get_save_dict()

        # merge trainer and model saving items
        checkpoint.update(checkpoint_dict)
        return checkpoint

    # --------------------
    # HPC IO
    # --------------------
    def enable_auto_hpc_walltime_manager(self):  # pragma: no cover
        if self.cluster is None:
            return

        # allow test tube to handle model check pointing automatically
        self.cluster.set_checkpoint_save_function(
            self.hpc_save,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'experiment': self.experiment
            }
        )
        self.cluster.set_checkpoint_load_function(
            self.hpc_load,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'on_gpu': self.on_gpu
            }
        )

    def restore_training_state(self, checkpoint):
        """
        Restore trainer state.
        Model will get its change to update
        :param checkpoint:
        :return:
        """
        if self.checkpoint_callback is not None:
            self.checkpoint_callback.best = checkpoint['checkpoint_callback_best']

        if self.early_stop_callback is not None:
            self.early_stop_callback.wait = checkpoint['early_stop_callback_wait']
            self.early_stop_callback.patience = checkpoint['early_stop_callback_patience']

        self.global_step = checkpoint['global_step']
        self.current_epoch = checkpoint['epoch']

        # restore the optimizers
        optimizer_states = checkpoint['optimizer_states']
        for optimizer, opt_state in zip(self.optimizers, optimizer_states):
            optimizer.load_state_dict(opt_state)

    # ----------------------------------
    # PRIVATE OPS
    # ----------------------------------
    def hpc_save(self, folderpath, experiment):
        # make sure the checkpoint folder exists
        os.makedirs(folderpath, exist_ok=True)

        # save exp to make sure we get all the metrics
        experiment.save()

        # close experiment to avoid issues
        experiment.close()

        ckpt_number = self.max_ckpt_in_folder(folderpath) + 1

        if not os.path.exists(folderpath):
            os.makedirs(folderpath, exist_ok=True)
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, ckpt_number)

        # give model a chance to do something on hpc_save
        model = self.__get_model()
        model.on_hpc_save()

        # request what to save from the model
        checkpoint_dict = self.dump_checkpoint()

        # do the actual save
        torch.save(checkpoint_dict, filepath)

    def hpc_load(self, folderpath, on_gpu):
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, self.max_ckpt_in_folder(folderpath))

        if on_gpu:
            checkpoint = torch.load(filepath)
        else:
            checkpoint = torch.load(filepath, map_location=lambda storage, loc: storage)

        # load training state
        self.restore_training_state(checkpoint)

        # load model state
        model = self.__get_model()
        model.load_model_specific(checkpoint)

        # call model hook
        model.on_hpc_load()

    def max_ckpt_in_folder(self, path):
        files = os.listdir(path)
        files = [x for x in files if 'ckpt_' in x]
        if len(files) == 0:
            return 0

        ckpt_vs = []
        for name in files:
            name = name.split('ckpt_')[-1]
            name = re.sub('[^0-9]', '', name)
            ckpt_vs.append(int(name))

        return max(ckpt_vs)

def load_hparams_from_tags_csv(tags_csv):
    from argparse import Namespace
    import pandas as pd

    tags_df = pd.read_csv(tags_csv)
    dic = tags_df.to_dict(orient='records')

    ns_dict = {row['key']: convert(row['value']) for row in dic}

    ns = Namespace(**ns_dict)
    return ns

def convert(val):
    constructors = [int, float, str]

    if type(val) is str:
        if val.lower() == 'true':
            return True
        if val.lower() == 'false':
            return False

    for c in constructors:
        try:
            return c(val)
        except ValueError:
            pass
    return val
<commit_msg>removed save model logging<commit_after>import torch
import os
import re
import pdb
from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel

class ModelIO(object):

    def load_model_specific(self, checkpoint):
        """
        Do something with the checkpoint
        :param checkpoint:
        :return:
        """
        raise NotImplementedError

    def get_save_dict(self):
        """
        Return specific things for the model
        :return:
        """
        raise NotImplementedError

    # -------------------------
    # OPTIONAL HOOKS
    # -------------------------
    def on_hpc_save(self):
        """
        Hook to do whatever you need right before Slurm manager saves the model
        :return:
        """
        pass

    def on_hpc_load(self):
        """
        Hook to do whatever you need right before Slurm manager loads the model
        :return:
        """
        pass

class TrainerIO(object):

    def __get_model(self):
        is_dp_module = type(self.model) is LightningDistributedDataParallel or type(self.model) is LightningDataParallel
        model = self.model.module if is_dp_module else self.model
        return model

    # --------------------
    # MODEL SAVE CHECKPOINT
    # --------------------
    def save_checkpoint(self, filepath):
        checkpoint = self.dump_checkpoint()

        # do the actual save
        torch.save(checkpoint, filepath)

    def dump_checkpoint(self):

        checkpoint = {
            'epoch': self.current_epoch,
            'global_step': self.global_step
        }

        if self.checkpoint_callback is not None:
            checkpoint['checkpoint_callback_best'] = self.checkpoint_callback.best

        if self.early_stop_callback is not None:
            checkpoint['early_stop_callback_wait'] = self.early_stop_callback.wait
            checkpoint['early_stop_callback_patience'] = self.early_stop_callback.patience

        optimizer_states = []
        for i, optimizer in enumerate(self.optimizers):
            optimizer_states.append(optimizer.state_dict())

        checkpoint['optimizer_states'] = optimizer_states

        # request what to save from the model
        model = self.__get_model()
        checkpoint_dict = model.get_save_dict()

        # merge trainer and model saving items
        checkpoint.update(checkpoint_dict)
        return checkpoint

    # --------------------
    # HPC IO
    # --------------------
    def enable_auto_hpc_walltime_manager(self):  # pragma: no cover
        if self.cluster is None:
            return

        # allow test tube to handle model check pointing automatically
        self.cluster.set_checkpoint_save_function(
            self.hpc_save,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'experiment': self.experiment
            }
        )
        self.cluster.set_checkpoint_load_function(
            self.hpc_load,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'on_gpu': self.on_gpu
            }
        )

    def restore_training_state(self, checkpoint):
        """
        Restore trainer state.
        Model will get its change to update
        :param checkpoint:
        :return:
        """
        if self.checkpoint_callback is not None:
            self.checkpoint_callback.best = checkpoint['checkpoint_callback_best']

        if self.early_stop_callback is not None:
            self.early_stop_callback.wait = checkpoint['early_stop_callback_wait']
            self.early_stop_callback.patience = checkpoint['early_stop_callback_patience']

        self.global_step = checkpoint['global_step']
        self.current_epoch = checkpoint['epoch']

        # restore the optimizers
        optimizer_states = checkpoint['optimizer_states']
        for optimizer, opt_state in zip(self.optimizers, optimizer_states):
            optimizer.load_state_dict(opt_state)

    # ----------------------------------
    # PRIVATE OPS
    # ----------------------------------
    def hpc_save(self, folderpath, experiment):
        # make sure the checkpoint folder exists
        os.makedirs(folderpath, exist_ok=True)

        # save exp to make sure we get all the metrics
        experiment.save()

        # close experiment to avoid issues
        experiment.close()

        ckpt_number = self.max_ckpt_in_folder(folderpath) + 1

        if not os.path.exists(folderpath):
            os.makedirs(folderpath, exist_ok=True)
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, ckpt_number)

        # give model a chance to do something on hpc_save
        model = self.__get_model()
        model.on_hpc_save()

        # request what to save from the model
        checkpoint_dict = self.dump_checkpoint()

        # do the actual save
        torch.save(checkpoint_dict, filepath)

    def hpc_load(self, folderpath, on_gpu):
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, self.max_ckpt_in_folder(folderpath))

        if on_gpu:
            checkpoint = torch.load(filepath)
        else:
            checkpoint = torch.load(filepath, map_location=lambda storage, loc: storage)

        # load training state
        self.restore_training_state(checkpoint)

        # load model state
        model = self.__get_model()
        model.load_model_specific(checkpoint)

        # call model hook
        model.on_hpc_load()

    def max_ckpt_in_folder(self, path):
        files = os.listdir(path)
        files = [x for x in files if 'ckpt_' in x]
        if len(files) == 0:
            return 0

        ckpt_vs = []
        for name in files:
            name = name.split('ckpt_')[-1]
            name = re.sub('[^0-9]', '', name)
            ckpt_vs.append(int(name))

        return max(ckpt_vs)

def load_hparams_from_tags_csv(tags_csv):
    from argparse import Namespace
    import pandas as pd

    tags_df = pd.read_csv(tags_csv)
    dic = tags_df.to_dict(orient='records')

    ns_dict = {row['key']: convert(row['value']) for row in dic}

    ns = Namespace(**ns_dict)
    return ns

def convert(val):
    constructors = [int, float, str]

    if type(val) is str:
        if val.lower() == 'true':
            return True
        if val.lower() == 'false':
            return False

    for c in constructors:
        try:
            return c(val)
        except ValueError:
            pass
    return val
<|endoftext|>


<commit_before>import torch
import os
import re
import pdb
from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel

class ModelIO(object):

    def on_load_checkpoint(self, checkpoint):
        """
        Do something with the checkpoint
        Gives model a chance to load something before state_dict is restored
        :param checkpoint:
        :return:
        """
        pass

    def on_save_checkpoint(self, checkpoint):
        """
        Give the model a chance to add something to the checkpoint.
        state_dict is already there
        """
        pass

    # -------------------------
    # OPTIONAL HOOKS
    # -------------------------
    def on_hpc_save(self, checkpoint):
        """
        Hook to do whatever you need right before Slurm manager saves the model
        :return:
        """
        pass

    def on_hpc_load(self, checkpoint):
        """
        Hook to do whatever you need right before Slurm manager loads the model
        :return:
        """
        pass

class TrainerIO(object):

    def __get_model(self):
        is_dp_module = type(self.model) is LightningDistributedDataParallel or type(self.model) is LightningDataParallel
        model = self.model.module if is_dp_module else self.model
        return model

    # --------------------
    # MODEL SAVE CHECKPOINT
    # --------------------
    def save_checkpoint(self, filepath):
        checkpoint = self.dump_checkpoint()

        # do the actual save
        torch.save(checkpoint, filepath)

    def dump_checkpoint(self):

        checkpoint = {
            'epoch': self.current_epoch,
            'global_step': self.global_step
        }

        if self.checkpoint_callback is not None:
            checkpoint['checkpoint_callback_best'] = self.checkpoint_callback.best

        if self.early_stop_callback is not None:
            checkpoint['early_stop_callback_wait'] = self.early_stop_callback.wait
            checkpoint['early_stop_callback_patience'] = self.early_stop_callback.patience

        optimizer_states = []
        for i, optimizer in enumerate(self.optimizers):
            optimizer_states.append(optimizer.state_dict())

        checkpoint['optimizer_states'] = optimizer_states

        # add the state_dict from the model
        model = self.__get_model()
        checkpoint['state_dict'] = model.state_dict()

        # give the model a chance to add a few things
        model.on_save_checkpoint(checkpoint)

        return checkpoint

    # --------------------
    # HPC IO
    # --------------------
    def enable_auto_hpc_walltime_manager(self):
        if self.cluster is None:
            return

        # allow test tube to handle model check pointing automatically
        self.cluster.set_checkpoint_save_function(
            self.hpc_save,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'experiment': self.experiment
            }
        )
        self.cluster.set_checkpoint_load_function(
            self.hpc_load,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'on_gpu': self.on_gpu
            }
        )

    def restore_training_state(self, checkpoint):
        """
        Restore trainer state.
        Model will get its change to update
        :param checkpoint:
        :return:
        """
        if self.checkpoint_callback is not None:
            self.checkpoint_callback.best = checkpoint['checkpoint_callback_best']

        if self.early_stop_callback is not None:
            self.early_stop_callback.wait = checkpoint['early_stop_callback_wait']
            self.early_stop_callback.patience = checkpoint['early_stop_callback_patience']

        self.global_step = checkpoint['global_step']
        self.current_epoch = checkpoint['epoch']

        # restore the optimizers
        optimizer_states = checkpoint['optimizer_states']
        for optimizer, opt_state in zip(self.optimizers, optimizer_states):
            optimizer.load_state_dict(opt_state)

    # ----------------------------------
    # PRIVATE OPS
    # ----------------------------------
    def hpc_save(self, folderpath, experiment):
        # make sure the checkpoint folder exists
        os.makedirs(folderpath, exist_ok=True)

        # save exp to make sure we get all the metrics
        experiment.save()

        # close experiment to avoid issues
        experiment.close()

        ckpt_number = self.max_ckpt_in_folder(folderpath) + 1

        if not os.path.exists(folderpath):
            os.makedirs(folderpath, exist_ok=True)
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, ckpt_number)

        # give model a chance to do something on hpc_save
        model = self.__get_model()
        checkpoint = self.dump_checkpoint()

        model.on_hpc_save(checkpoint)

        # do the actual save
        torch.save(checkpoint, filepath)

        return filepath

    def hpc_load(self, folderpath, on_gpu):
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, self.max_ckpt_in_folder(folderpath))

        if on_gpu:
            checkpoint = torch.load(filepath)
        else:
            checkpoint = torch.load(filepath, map_location=lambda storage, loc: storage)

        # load training state (affects trainer only)
        self.restore_training_state(checkpoint)

        # load model state
        model = self.__get_model()

        # load the state_dict on the model automatically
        model.load_state_dict(checkpoint['state_dict'])

        # call model hook
        model.on_hpc_load(checkpoint)

    def max_ckpt_in_folder(self, path):
        files = os.listdir(path)
        files = [x for x in files if 'ckpt_' in x]
        if len(files) == 0:
            return 0

        ckpt_vs = []
        for name in files:
            name = name.split('ckpt_')[-1]
            name = re.sub('[^0-9]', '', name)
            ckpt_vs.append(int(name))

        return max(ckpt_vs)

def load_hparams_from_tags_csv(tags_csv):
    from argparse import Namespace
    import pandas as pd

    tags_df = pd.read_csv(tags_csv)
    dic = tags_df.to_dict(orient='records')

    ns_dict = {row['key']: convert(row['value']) for row in dic}

    ns = Namespace(**ns_dict)
    return ns

def convert(val):
    constructors = [int, float, str]

    if type(val) is str:
        if val.lower() == 'true':
            return True
        if val.lower() == 'false':
            return False

    for c in constructors:
        try:
            return c(val)
        except ValueError:
            pass
    return val
<commit_msg>running tests<commit_after>import torch
import os
import re
import pdb
from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel

class ModelIO(object):

    def on_load_checkpoint(self, checkpoint):
        """
        Do something with the checkpoint
        Gives model a chance to load something before state_dict is restored
        :param checkpoint:
        :return:
        """
        pass

    def on_save_checkpoint(self, checkpoint):
        """
        Give the model a chance to add something to the checkpoint.
        state_dict is already there
        """
        pass

    # -------------------------
    # OPTIONAL HOOKS
    # -------------------------
    def on_hpc_save(self, checkpoint):
        """
        Hook to do whatever you need right before Slurm manager saves the model
        :return:
        """
        pass

    def on_hpc_load(self, checkpoint):
        """
        Hook to do whatever you need right before Slurm manager loads the model
        :return:
        """
        pass

class TrainerIO(object):

    def __get_model(self):
        is_dp_module = type(self.model) is LightningDistributedDataParallel or type(self.model) is LightningDataParallel
        model = self.model.module if is_dp_module else self.model
        return model

    # --------------------
    # MODEL SAVE CHECKPOINT
    # --------------------
    def save_checkpoint(self, filepath):
        checkpoint = self.dump_checkpoint()

        # do the actual save
        torch.save(checkpoint, filepath)

    def dump_checkpoint(self):

        checkpoint = {
            'epoch': self.current_epoch,
            'global_step': self.global_step
        }

        if self.checkpoint_callback is not None:
            checkpoint['checkpoint_callback_best'] = self.checkpoint_callback.best

        if self.early_stop_callback is not None:
            checkpoint['early_stop_callback_wait'] = self.early_stop_callback.wait
            checkpoint['early_stop_callback_patience'] = self.early_stop_callback.patience

        # save optimizers
        optimizer_states = []
        for i, optimizer in enumerate(self.optimizers):
            optimizer_states.append(optimizer.state_dict())

        checkpoint['optimizer_states'] = optimizer_states

        # save lr schedulers
        lr_schedulers = []
        for i, scheduler in enumerate(self.lr_schedulers):
            lr_schedulers.append(scheduler.state_dict())

        checkpoint['lr_schedulers'] = lr_schedulers

        # add the state_dict from the model
        model = self.__get_model()
        checkpoint['state_dict'] = model.state_dict()

        # give the model a chance to add a few things
        model.on_save_checkpoint(checkpoint)

        return checkpoint

    # --------------------
    # HPC IO
    # --------------------
    def enable_auto_hpc_walltime_manager(self):
        if self.cluster is None:
            return

        # allow test tube to handle model check pointing automatically
        self.cluster.set_checkpoint_save_function(
            self.hpc_save,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'experiment': self.experiment
            }
        )
        self.cluster.set_checkpoint_load_function(
            self.hpc_load,
            kwargs={
                'folderpath': self.checkpoint_callback.filepath,
                'on_gpu': self.on_gpu
            }
        )

    def restore_training_state(self, checkpoint):
        """
        Restore trainer state.
        Model will get its change to update
        :param checkpoint:
        :return:
        """
        if self.checkpoint_callback is not None:
            self.checkpoint_callback.best = checkpoint['checkpoint_callback_best']

        if self.early_stop_callback is not None:
            self.early_stop_callback.wait = checkpoint['early_stop_callback_wait']
            self.early_stop_callback.patience = checkpoint['early_stop_callback_patience']

        self.global_step = checkpoint['global_step']
        self.current_epoch = checkpoint['epoch']

        # restore the optimizers
        optimizer_states = checkpoint['optimizer_states']
        for optimizer, opt_state in zip(self.optimizers, optimizer_states):
            optimizer.load_state_dict(opt_state)

        # restore the lr schedulers
        lr_schedulers = checkpoint['lr_schedulers']
        for scheduler, lrs_state in zip(self.lr_schedulers, lr_schedulers):
            scheduler.load_state_dict(lrs_state)

    # ----------------------------------
    # PRIVATE OPS
    # ----------------------------------
    def hpc_save(self, folderpath, experiment):
        # make sure the checkpoint folder exists
        os.makedirs(folderpath, exist_ok=True)

        # save exp to make sure we get all the metrics
        experiment.save()

        # close experiment to avoid issues
        experiment.close()

        ckpt_number = self.max_ckpt_in_folder(folderpath) + 1

        if not os.path.exists(folderpath):
            os.makedirs(folderpath, exist_ok=True)
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, ckpt_number)

        # give model a chance to do something on hpc_save
        model = self.__get_model()
        checkpoint = self.dump_checkpoint()

        model.on_hpc_save(checkpoint)

        # do the actual save
        torch.save(checkpoint, filepath)

        return filepath

    def hpc_load(self, folderpath, on_gpu):
        filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, self.max_ckpt_in_folder(folderpath))

        if on_gpu:
            checkpoint = torch.load(filepath)
        else:
            checkpoint = torch.load(filepath, map_location=lambda storage, loc: storage)

        # load training state (affects trainer only)
        self.restore_training_state(checkpoint)

        # load model state
        model = self.__get_model()

        # load the state_dict on the model automatically
        model.load_state_dict(checkpoint['state_dict'])

        # call model hook
        model.on_hpc_load(checkpoint)

    def max_ckpt_in_folder(self, path):
        files = os.listdir(path)
        files = [x for x in files if 'ckpt_' in x]
        if len(files) == 0:
            return 0

        ckpt_vs = []
        for name in files:
            name = name.split('ckpt_')[-1]
            name = re.sub('[^0-9]', '', name)
            ckpt_vs.append(int(name))

        return max(ckpt_vs)

def load_hparams_from_tags_csv(tags_csv):
    from argparse import Namespace
    import pandas as pd

    tags_df = pd.read_csv(tags_csv)
    dic = tags_df.to_dict(orient='records')

    ns_dict = {row['key']: convert(row['value']) for row in dic}

    ns = Namespace(**ns_dict)
    return ns

def convert(val):
    constructors = [int, float, str]

    if type(val) is str:
        if val.lower() == 'true':
            return True
        if val.lower() == 'false':
            return False

    for c in constructors:
        try:
            return c(val)
        except ValueError:
            pass
    return val
<|endoftext|>


<commit_before># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox
import cybox.bindings.cybox_common as common_binding
from cybox.common import HashList, Integer

class ByteRun(cybox.Entity):
    _binding = common_binding
    _binding_class = common_binding.ByteRunType
    _namespace = 'http://cybox.mitre.org/common-2'

    offset = cybox.TypedField("Offset", Integer)
    file_system_offset = cybox.TypedField("File_System_Offset", Integer)
    image_offset = cybox.TypedField("Image_Offset", Integer)
    length = cybox.TypedField("Length", Integer)
    hashes = cybox.TypedField("Hashes", HashList)
    byte_run_data = cybox.TypedField("Byte_Run_Data")

class ByteRuns(cybox.EntityList):
    _binding_class = common_binding.ByteRunsType
    _binding_var = "Byte_Run"
    _contained_type = ByteRun
    _namespace = 'http://cybox.mitre.org/common-2'
<commit_msg>Added byte_order property to ByteRun class<commit_after># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox
import cybox.bindings.cybox_common as common_binding
from cybox.common import HashList, Integer, String

class ByteRun(cybox.Entity):
    _binding = common_binding
    _binding_class = common_binding.ByteRunType
    _namespace = 'http://cybox.mitre.org/common-2'

    offset = cybox.TypedField("Offset", Integer)
    byte_order = cybox.TypedField("Byte_Order", String)
    file_system_offset = cybox.TypedField("File_System_Offset", Integer)
    image_offset = cybox.TypedField("Image_Offset", Integer)
    length = cybox.TypedField("Length", Integer)
    hashes = cybox.TypedField("Hashes", HashList)
    byte_run_data = cybox.TypedField("Byte_Run_Data")

class ByteRuns(cybox.EntityList):
    _binding_class = common_binding.ByteRunsType
    _binding_var = "Byte_Run"
    _contained_type = ByteRun
    _namespace = 'http://cybox.mitre.org/common-2'
<|endoftext|>


<commit_before>
def observable(target, observatory):
    """Merit function to evaluate if a target is observable.
    Args:
        target (Target): Target object to evaluate.
        observatory (Observatory): The observatory object for which to evaluate
        the target.
    Returns:
        (1, observable): Returns 1 as the merit (a merit value of 1 indicates
        that all elevations are equally meritorious).  Will return True for
        observable if the target is observable or return Fale if not (which
        vetoes the target.
    """
    # Return 1 as merit if none of the time steps returned False (unobservable)
    return (1, True)
<commit_msg>* Basic observable check<commit_after>from astropy.time import Time

def observable(target, observatory):
    """Merit function to evaluate if a target is observable.
    Args:
        target (Target): Target object to evaluate.
        observatory (Observatory): The observatory object for which to evaluate
        the target.
    Returns:
        (1, observable): Returns 1 as the merit (a merit value of 1 indicates
        that all elevations are equally meritorious).  Will return True for
        observable if the target is observable or return Fale if not (which
        vetoes the target.
    """
    if observatory.target_is_up(Time.now(), target, observatory.horizon):
        return (1, True)

    return (0, False)
<|endoftext|>


<commit_before>"""This module provides I/O classes and routines for Cyclopts-related functionality.

:author: Matthew Gidden <matthew.gidden _at_ gmail.com>
"""
import numpy as np
import tables as t
import math
import datetime
from collections import defaultdict

import cyclopts
import cyclopts.tools as tools
import cyclopts.analysis as analysis

def value_mapping(x, y, tbl=None, uuids=True):
    """Returns a mapping from x to a list of ys in a table. A table can be
    supplied, or the underlying table will be used by default. If uuids is
    true, the cyclopts.tools.str_to_uuid function is used for both x and
    y."""
    ret = defaultdict(list)
    if uuids:
        for row in tbl.iterrows():
            ret[tools.str_to_uuid(row[x])].append(tools.str_to_uuid(row[y]))
    else:
        for row in tbl.iterrows():
            ret[row[x]].append(row[y])
    return ret

class Table(object):
    """A thin wrapper for a PyTables Table to be used by Cyclopts.
    """

    def __init__(self, h5file=None, path=None, dt=None, chunksize=None,
                 cachesize=None):
        """Parameters
        ----------
        h5file : PyTables File
            the hdf5 file
        path : string
            the absolute path to the table
        dt : np.dtype, optional
            the dtype for the table
        chunksize : int, optional
            the table chunksize, Cyclopts will optimize for a 32Kb L1 cache by
            default
        cachesize : int, optional
            the size of data to cache before writing, defaults to 100 times the
            chunksize
        """
        self.h5file = h5file
        self.path = path if path is not None else '/'
        self.dt = dt if dt is not None else np.dtype(None)
        # l1 cache size / row size / 2
        # factor of 2 is ideal for reading/writing speed (per @scopatz's advice)
        chunksize = chunksize if chunksize is not None \
            else math.floor(32 * 1024 / float(dt.itemsize) / 2)
        self.chunksize = int(chunksize)
        # 100 seems right, eh?
        factor = int(1e2)
        self.cachesize = factor * self.chunksize if cachesize is None else cachesize
        self.prefix = '/'.join(self.path.split('/')[:-1])
        if not self.prefix.startswith('/'):
            self.prefix = '/{0}'.format(self.prefix)
        self.name = self.path.split('/')[-1]
        self._data = np.empty(shape=(self.cachesize), dtype=self.dt)
        self._idx = 0
        self.n_writes = 0

        if self.h5file is not None and self.path in self.h5file:
            self._tbl = self.h5file.get_node(self.path)
        else:
            self._tbl = None

    def __del__(self):
        del self._data

    def create(self):
        """Creates a table in the h5file. This must be called before writing."""
        groups = [x for x in self.prefix.split('/') if x]
        prefix = ''
        for name in groups:
            path = '/'.join([prefix, name])
            prefix = '/' if not prefix else prefix
            if not path in self.h5file:
                self.h5file.create_group(prefix, name, title=name,
                                         filters=tools.FILTERS)
                self.h5file.flush()
            prefix = path

        self.h5file.create_table(self.prefix,
                                 self.name,
                                 description=self.dt,
                                 filters=tools.FILTERS,
                                 chunkshape=(self.chunksize,))

        self._tbl = self.h5file.get_node(self.path)

    def table(self):
        return self._tbl

    def value_mapping(self, x, y, tbl=None, uuids=True):
        """Returns the result of value_mapping() using the underlying table."""
        return value_mapping(c, y, tbl=self._tbl, uuids=uuids)

    def rows_where(self, cond, condvars=None):
        return self._tbl.read_where(cond, condvars=condvars)

    def uuid_rows(self, uuid, colname='instid'):
        condvars = {'uuid': uuid.bytes}
        return self.rows_where("""{0} == uuid""".format(colname),
                               condvars=condvars)

    def append_data(self, data):
        """Appends data to the Table. If the cachesize limit is reached, data is
        written to disc.

        Parameters
        ----------
        data : array-like
            data to append to the table
        """
        ndata = len(data)
        idx = self._idx
        arylen = self.cachesize
        # just add data, no writing
        if ndata + idx < arylen:
            self._idx += ndata
            self._data[idx:self._idx] = data
            return

        # writing
        space = arylen - idx
        n_writes = 1 + int(math.floor(float(ndata - space) / arylen))
        self._data[idx:arylen] = data[:space]
        self._idx = arylen
        self.flush()
        for i in range(n_writes - 1):
            start = i * arylen + space
            stop = (i + 1) * arylen + space
            self.flush(data[start:stop])
        self._idx = ndata - (n_writes - 1) * arylen - space
        if self._idx > 0:
            self._data[:self._idx] = data[-self._idx:]

    def writeable(self):
        return self._tbl is not None and self._tbl._v_file._iswritable()

    def flush(self, data=None):
        """Writes cached data to the table."""
        if not self.writeable() and data is not None and self._idx != 0:
            # not writeable but there was data to write
            raise IOError(("Cannot write data to the table {0} in unwriteable"
                           " file {1}").format(self._tbl._v_name,
                                               self._tbl._v_file.filename))
        if not self.writeable():
            # not writeable, don't do anything
            return

        if data is None:
            self._tbl.append(self._data[:self._idx])
            self._idx = 0
        else:
            self._tbl.append(data)
        self._tbl.flush()
        self.n_writes += 1

_result_dt = np.dtype([
                ("solnid", ('str', 16)), # 16 bytes for uuid
                ("instid", ('str', 16)), # 16 bytes for uuid
                ("solver", ('str', 30)), # 30 seems long enough, right?
                ("problem", ('str', 30)), # 30 seems long enough, right?
                ("time", np.float64),
                ("objective", np.float64),
                ("cyclopts_version", ('str', 12)),
                # len(dtime.datetime.now().isoformat(' ')) == 26
                ("timestamp", ('str', 26)),
                ])

class ResultTable(Table):
    """A Cyclopts Table for generic results.
    """

    def __init__(self, h5file, path='/Results', chunksize=None):
        """Parameters
        ----------
        h5file : PyTables File
            the hdf5 file
        path : string
            the absolute path to the table
        chunksize : int, optional
            the table chunksize, Cyclopts will optimize for a 32Kb L1 cache by
            default
        """
        super(ResultTable, self).__init__(h5file, path, _result_dt, chunksize)

    def record_soln(self, soln, soln_uuid, inst_uuid, solver):
        self.append_data([(
                    soln_uuid.bytes,
                    inst_uuid.bytes,
                    solver.type,
                    soln.type,
                    soln.time,
                    soln.objective,
                    cyclopts.__version__,
                    datetime.datetime.now().isoformat(' '),
                    )])

class PathMap(analysis.PathMap):
    """A simple container class for mapping columns to Hdf5 paths
    for the Results table"""

    def __init__(self, col=None):
        super(PathMap, self).__init__(col)

    @property
    def path(self):
        return '/Results'

class TableManager(object):
    """A managing class that performs RAII for its tables by creating them if
    needed upon acquisition and flushing them upon deletion. Tables can be
    accessed through the manager by its tables member, which is a dictionary
    from table names to Table objects."""

    def __init__(self, h5file, tables):
        """Parameters
        ----------
        h5file : PyTables File
            the hdf5 file
        tables : list of Tables
            the list of tables to manage
        """
        self.tables = {tbl.path.split('/')[-1]: tbl for tbl in tables}
        self.h5file = h5file
        for tbl in self.tables.values():
            if tbl.path not in self.h5file and self.h5file.mode is not 'r':
                tbl.create()

    def __del__(self):
        if self.h5file.isopen and self.h5file.mode is not 'r':
            self.flush_tables()

    def flush_tables(self):
        for tbl in self.tables.values():
            tbl.flush()

    def total_writes(self):
        return sum([tbl.n_writes for tbl in self.tables.values()])
<commit_msg>updated value_mapping api to be more consistent<commit_after>"""This module provides I/O classes and routines for Cyclopts-related functionality.

:author: Matthew Gidden <matthew.gidden _at_ gmail.com>
"""
import numpy as np
import tables as t
import math
import datetime
from collections import defaultdict

import cyclopts
import cyclopts.tools as tools
import cyclopts.analysis as analysis

def value_mapping(tbl, x, y, uuids=True):
    """Returns a mapping from x to a list of ys in a table. A table can be
    supplied, or the underlying table will be used by default. If uuids is
    true, the cyclopts.tools.str_to_uuid function is used for both x and
    y."""
    ret = defaultdict(list)
    if uuids:
        for row in tbl.iterrows():
            ret[tools.str_to_uuid(row[x])].append(tools.str_to_uuid(row[y]))
    else:
        for row in tbl.iterrows():
            ret[row[x]].append(row[y])
    return ret

class Table(object):
    """A thin wrapper for a PyTables Table to be used by Cyclopts.
    """

    def __init__(self, h5file=None, path=None, dt=None, chunksize=None,
                 cachesize=None):
        """Parameters
        ----------
        h5file : PyTables File
            the hdf5 file
        path : string
            the absolute path to the table
        dt : np.dtype, optional
            the dtype for the table
        chunksize : int, optional
            the table chunksize, Cyclopts will optimize for a 32Kb L1 cache by
            default
        cachesize : int, optional
            the size of data to cache before writing, defaults to 100 times the
            chunksize
        """
        self.h5file = h5file
        self.path = path if path is not None else '/'
        self.dt = dt if dt is not None else np.dtype(None)
        # l1 cache size / row size / 2
        # factor of 2 is ideal for reading/writing speed (per @scopatz's advice)
        chunksize = chunksize if chunksize is not None \
            else math.floor(32 * 1024 / float(dt.itemsize) / 2)
        self.chunksize = int(chunksize)
        # 100 seems right, eh?
        factor = int(1e2)
        self.cachesize = factor * self.chunksize if cachesize is None else cachesize
        self.prefix = '/'.join(self.path.split('/')[:-1])
        if not self.prefix.startswith('/'):
            self.prefix = '/{0}'.format(self.prefix)
        self.name = self.path.split('/')[-1]
        self._data = np.empty(shape=(self.cachesize), dtype=self.dt)
        self._idx = 0
        self.n_writes = 0

        if self.h5file is not None and self.path in self.h5file:
            self._tbl = self.h5file.get_node(self.path)
        else:
            self._tbl = None

    def __del__(self):
        del self._data

    def create(self):
        """Creates a table in the h5file. This must be called before writing."""
        groups = [x for x in self.prefix.split('/') if x]
        prefix = ''
        for name in groups:
            path = '/'.join([prefix, name])
            prefix = '/' if not prefix else prefix
            if not path in self.h5file:
                self.h5file.create_group(prefix, name, title=name,
                                         filters=tools.FILTERS)
                self.h5file.flush()
            prefix = path

        self.h5file.create_table(self.prefix,
                                 self.name,
                                 description=self.dt,
                                 filters=tools.FILTERS,
                                 chunkshape=(self.chunksize,))

        self._tbl = self.h5file.get_node(self.path)

    def table(self):
        return self._tbl

    def value_mapping(self, x, y, uuids=True):
        """Returns the result of value_mapping() using the underlying table."""
        return value_mapping(self._tbl, x, y, uuids=uuids)

    def rows_where(self, cond, condvars=None):
        return self._tbl.read_where(cond, condvars=condvars)

    def uuid_rows(self, uuid, colname='instid'):
        condvars = {'uuid': uuid.bytes}
        return self.rows_where("""{0} == uuid""".format(colname),
                               condvars=condvars)

    def append_data(self, data):
        """Appends data to the Table. If the cachesize limit is reached, data is
        written to disc.

        Parameters
        ----------
        data : array-like
            data to append to the table
        """
        ndata = len(data)
        idx = self._idx
        arylen = self.cachesize
        # just add data, no writing
        if ndata + idx < arylen:
            self._idx += ndata
            self._data[idx:self._idx] = data
            return

        # writing
        space = arylen - idx
        n_writes = 1 + int(math.floor(float(ndata - space) / arylen))
        self._data[idx:arylen] = data[:space]
        self._idx = arylen
        self.flush()
        for i in range(n_writes - 1):
            start = i * arylen + space
            stop = (i + 1) * arylen + space
            self.flush(data[start:stop])
        self._idx = ndata - (n_writes - 1) * arylen - space
        if self._idx > 0:
            self._data[:self._idx] = data[-self._idx:]

    def writeable(self):
        return self._tbl is not None and self._tbl._v_file._iswritable()

    def flush(self, data=None):
        """Writes cached data to the table."""
        if not self.writeable() and data is not None and self._idx != 0:
            # not writeable but there was data to write
            raise IOError(("Cannot write data to the table {0} in unwriteable"
                           " file {1}").format(self._tbl._v_name,
                                               self._tbl._v_file.filename))
        if not self.writeable():
            # not writeable, don't do anything
            return

        if data is None:
            self._tbl.append(self._data[:self._idx])
            self._idx = 0
        else:
            self._tbl.append(data)
        self._tbl.flush()
        self.n_writes += 1

_result_dt = np.dtype([
                ("solnid", ('str', 16)), # 16 bytes for uuid
                ("instid", ('str', 16)), # 16 bytes for uuid
                ("solver", ('str', 30)), # 30 seems long enough, right?
                ("problem", ('str', 30)), # 30 seems long enough, right?
                ("time", np.float64),
                ("objective", np.float64),
                ("cyclopts_version", ('str', 12)),
                # len(dtime.datetime.now().isoformat(' ')) == 26
                ("timestamp", ('str', 26)),
                ])

class ResultTable(Table):
    """A Cyclopts Table for generic results.
    """

    def __init__(self, h5file, path='/Results', chunksize=None):
        """Parameters
        ----------
        h5file : PyTables File
            the hdf5 file
        path : string
            the absolute path to the table
        chunksize : int, optional
            the table chunksize, Cyclopts will optimize for a 32Kb L1 cache by
            default
        """
        super(ResultTable, self).__init__(h5file, path, _result_dt, chunksize)

    def record_soln(self, soln, soln_uuid, inst_uuid, solver):
        self.append_data([(
                    soln_uuid.bytes,
                    inst_uuid.bytes,
                    solver.type,
                    soln.type,
                    soln.time,
                    soln.objective,
                    cyclopts.__version__,
                    datetime.datetime.now().isoformat(' '),
                    )])

class PathMap(analysis.PathMap):
    """A simple container class for mapping columns to Hdf5 paths
    for the Results table"""

    def __init__(self, col=None):
        super(PathMap, self).__init__(col)

    @property
    def path(self):
        return '/Results'

class TableManager(object):
    """A managing class that performs RAII for its tables by creating them if
    needed upon acquisition and flushing them upon deletion. Tables can be
    accessed through the manager by its tables member, which is a dictionary
    from table names to Table objects."""

    def __init__(self, h5file, tables):
        """Parameters
        ----------
        h5file : PyTables File
            the hdf5 file
        tables : list of Tables
            the list of tables to manage
        """
        self.tables = {tbl.path.split('/')[-1]: tbl for tbl in tables}
        self.h5file = h5file
        for tbl in self.tables.values():
            if tbl.path not in self.h5file and self.h5file.mode is not 'r':
                tbl.create()

    def __del__(self):
        if self.h5file.isopen and self.h5file.mode is not 'r':
            self.flush_tables()

    def flush_tables(self):
        for tbl in self.tables.values():
            tbl.flush()

    def total_writes(self):
        return sum([tbl.n_writes for tbl in self.tables.values()])
<|endoftext|>


<commit_before>from urlparse import urlsplit

# use test database settings from non-django existdb test
#from test_existdb import settings as exist_settings
from test_existdb.test_db import EXISTDB_SERVER_URL, EXISTDB_ROOT_COLLECTION, EXISTDB_TEST_COLLECTION

# Django settings for eulcore project.
DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

#We will not be using a RDB but this will allow tests to run
DATABASE_ENGINE = 'sqlite3'
DATABASE_NAME = 'no_db'

# eXist DB settings imported above

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = '%p=r8r%&a7$e2y8)w2fh+&@)duz!3=ps82t^x^p9w)-u2#@h8#'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
#     'django.template.loaders.eggs.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
)

ROOT_URLCONF = ''

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    # needed for test dependencies:
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',

    # our apps to test:
    'eulcore.django.emory_ldap',
    'eulcore.django.existdb',
    'eulcore.django.ldap',
    'eulcore.django.testsetup',
    'eulcore.django.ldap.emory',
    'south',
)

AUTH_PROFILE_MODULE = 'emory_ldap.EmoryLDAPUserProfile'

try:
    # use xmlrunner if it's installed; default runner otherwise. download
    # it from http://github.com/danielfm/unittest-xml-reporting/ to output
    # test results in JUnit-compatible XML.
    import xmlrunner
    TEST_RUNNER='xmlrunner.extra.djangotestrunner.run_tests'
    TEST_OUTPUT_DIR='test-results'
except ImportError:
    pass
<commit_msg>and revert test settings changes which shouldnt have been included in last commit<commit_after>from urlparse import urlsplit

# use test database settings from non-django existdb test
#from test_existdb import settings as exist_settings
from test_existdb.test_db import EXISTDB_SERVER_URL, EXISTDB_ROOT_COLLECTION, EXISTDB_TEST_COLLECTION

# Django settings for eulcore project.
DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

#We will not be using a RDB but this will allow tests to run
DATABASE_ENGINE = 'sqlite3'
DATABASE_NAME = 'no_db'

# eXist DB settings imported above

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = '%p=r8r%&a7$e2y8)w2fh+&@)duz!3=ps82t^x^p9w)-u2#@h8#'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
#     'django.template.loaders.eggs.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
)

ROOT_URLCONF = ''

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    # needed for test dependencies:
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',

    # our apps to test:
    'eulcore.django.emory_ldap',
    'eulcore.django.existdb',
    'eulcore.django.ldap',
    'eulcore.django.testsetup',
)

AUTH_PROFILE_MODULE = 'emory_ldap.EmoryLDAPUserProfile'

try:
    # use xmlrunner if it's installed; default runner otherwise. download
    # it from http://github.com/danielfm/unittest-xml-reporting/ to output
    # test results in JUnit-compatible XML.
    import xmlrunner
    TEST_RUNNER='xmlrunner.extra.djangotestrunner.run_tests'
    TEST_OUTPUT_DIR='test-results'
except ImportError:
    pass
<|endoftext|>


<commit_before>from django import template
from approver.models import Project, Person, User
from django.utils import timezone

import approver.constants as constants

register = template.Library()

@register.inclusion_tag('templatetags/footer.html')
def footer():
    """
    Renders a footer with the version number found in constants
    """
    input_dict={
        'version_number': constants.VERSION_NUMBER,
        'count_registered_projects': 0,
        'count_monthly_projects': 0,
        'count_active_users': 0,
        'count_approved_projects': 0,
    }
    input_dict['count_registered_projects'] = Project.objects.count()
    input_dict['count_monthly_projects'] = Project.objects.filter(created__month=timezone.now().month).count()
    input_dict['count_active_users'] = User.objects.exclude(last_login=None).count()
    input_dict['count_approved_projects'] = Project.objects.exclude(approval_date=None).count()
    return input_dict
<commit_msg>Fix bug with footer having the wrong pending project count<commit_after>from django import template
from approver.models import Project, Person, User
from django.utils import timezone

import approver.constants as constants

register = template.Library()

@register.inclusion_tag('templatetags/footer.html')
def footer():
    """
    Renders a footer with the version number found in constants
    """
    input_dict={
        'version_number': constants.VERSION_NUMBER,
        'count_registered_projects': 0,
        'count_monthly_projects': 0,
        'count_active_users': 0,
        'count_approved_projects': 0,
    }
    input_dict['count_registered_projects'] = Project.objects.exclude(approval_date__isnull=False).count()
    input_dict['count_monthly_projects'] = Project.objects.filter(created__month=timezone.now().month).count()
    input_dict['count_active_users'] = User.objects.exclude(last_login=None).count()
    input_dict['count_approved_projects'] = Project.objects.exclude(approval_date=None).count()
    return input_dict
<|endoftext|>


<commit_before>from panorama.wrappers import Video

import gdata.youtube.service

class Youtube(Video):

    def __init__(self, url, options=None):
        self.url = url
        self.id = self.get_video_id()

    def get_video_entry(self):
        if not hasattr(self, "entry"):
            try:
                self.entry = (gdata.youtube.service.YouTubeService().
                              GetYouTubeVideoEntry(video_id=self.id))
            except:
                print "Youtube video with id %s doesn't exists" % self.id
                return
        return self.entry

    def get_video_id(self):
        """
        Returns the video ID from the video URL.

        Returns:
            A string with the youtube video id.
        """
        if not hasattr(self, "id"):
            self.id = self.get_url_parameters('v')
        return self.id

    def get_title(self):
        if not hasattr(self, "title"):
            self.title = self.get_video_entry().media.title.text
        return self.title

    def get_description(self):
        if not hasattr(self, "description"):
            self.description = self.get_video_entry().media.description.text
        return self.description

    def get_thumbnails(self):
        if not hasattr(self, "thumbnail"):
            self.thumbnails = []
            for thumbnail in self.get_video_entry().media.thumbnail:
                self.thumbnails.append(thumbnail.url)
        return self.thumbnails

    def get_thumbnail(self):
        """
        From the list of all thumbnails returns only the first.

        Returns:
            A string with a url for the thumbnail.
        """
        if not hasattr(self, "thumbnail"):
            self.thumbnail = self.get_thumbnails()[0]
        return self.thumbnail

    def get_duration(self):
        if not hasattr(self, "duration"):
            self.duration = self.get_video_entry().media.duration.seconds
        return self.duration

    def get_flv(self):
        """
        With this handler, this function do exactly the same that the
        function get_embed_url()
        """
        if not hasattr(self, "embed_url"):
            self.embed_url = self.get_video_entry().GetSwfUrl()
        return self.embed_url

    def get_embed_url(self):
        if not hasattr(self, "embed_url"):
            self.embed_url = self.get_video_entry().GetSwfUrl()
        return self.embed_url

    def get_embed_html(self, options=None):
        if not hasattr(self, "embed_html"):
            self.embed_html = (
                    '<object width="%(width)s" height="%(height)s">'
                    '<param name="movie" value="%(movie)s">'
                    '<param name="allowFullScreen" value="true">'
                    '<param name="allowscriptaccess" value="always">'
                    '<param name="wmode" value="transparent">'
                    '<embed '
                    'src="%(movie)s" type="application/x-shockwave-flash"'
                    'allowscriptaccess="always" allowfullscreen="true"'
                    'width="%(width)s" height="%(height)s">'
                    '</object>'
                ) % ({
                    'width': options.get('width') or 560 if options else 560,
                    'height': options.get('height') or 349 if options else 349,
                    'movie': self.get_embed_url()
                })
        return self.embed_html

    def get_download_url(self):
        if not hasattr(self, "download_url"):
            self.download_url = self.get_video_entry().GetSwfUrl()
        return self.download_url

    def get_tags(self):
        if not hasattr(self, "tags"):
            self.tags = self.get_video_entry().media.keywords.text.split(',')
        return self.tags
<commit_msg>Added Vimeo wrapper<commit_after>import gdata.youtube.service

from panorama.wrappers import Video

class Youtube(Video):

    def __init__(self, url, options=None):
        self.url = url
        self.id = self.get_video_id()

    def get_video_entry(self):
        if not hasattr(self, "entry"):
            try:
                self.entry = (gdata.youtube.service.YouTubeService().
                              GetYouTubeVideoEntry(video_id=self.id))
            except:
                print "Youtube video with id %s doesn't exists" % self.id
                return
        return self.entry

    def get_video_id(self):
        """
        Returns the video ID from the video URL.

        Returns:
            A string with the youtube video id.
        """
        if not hasattr(self, "id"):
            self.id = self.get_url_parameters('v')
        return self.id

    def get_title(self):
        if not hasattr(self, "title"):
            self.title = self.get_video_entry().media.title.text
        return self.title

    def get_description(self):
        if not hasattr(self, "description"):
            self.description = self.get_video_entry().media.description.text
        return self.description

    def get_thumbnails(self):
        if not hasattr(self, "thumbnails"):
            self.thumbnails = []
            for thumbnail in self.get_video_entry().media.thumbnail:
                self.thumbnails.append(thumbnail.url)
        return self.thumbnails

    def get_thumbnail(self):
        """
        From the list of all thumbnails returns only the first.

        Returns:
            A string with a url for the thumbnail.
        """
        if not hasattr(self, "thumbnail"):
            self.thumbnail = self.get_thumbnails()[0]
        return self.thumbnail

    def get_duration(self):
        if not hasattr(self, "duration"):
            self.duration = self.get_video_entry().media.duration.seconds
        return self.duration

    def get_flv(self):
        """
        With this handler, this function do exactly the same that the
        function get_embed_url()
        """
        if not hasattr(self, "embed_url"):
            self.embed_url = self.get_embed_url()
        return self.embed_url

    def get_embed_url(self):
        if not hasattr(self, "embed_url"):
            self.embed_url = self.get_video_entry().GetSwfUrl()
        return self.embed_url

    def get_embed_html(self, options=None):
        if not hasattr(self, "embed_html"):
            self.embed_html = (
                    '<object width="%(width)s" height="%(height)s">'
                    '<param name="movie" value="%(movie)s">'
                    '<param name="allowFullScreen" value="true">'
                    '<param name="allowscriptaccess" value="always">'
                    '<param name="wmode" value="transparent">'
                    '<embed '
                    'src="%(movie)s" type="application/x-shockwave-flash"'
                    'allowscriptaccess="always" allowfullscreen="true"'
                    'width="%(width)s" height="%(height)s">'
                    '</object>'
                ) % ({
                    'width': options.get('width') or 560 if options else 560,
                    'height': options.get('height') or 349 if options else 349,
                    'movie': self.get_embed_url()
                })
        return self.embed_html

    def get_download_url(self):
        if not hasattr(self, "download_url"):
            self.download_url = self.get_video_entry().GetSwfUrl()
        return self.download_url

    def get_tags(self):
        if not hasattr(self, "tags"):
            self.tags = self.get_video_entry().media.keywords.text.split(',')
        return self.tags
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *

from cyder.core.ctnr.views import *
from cyder.core.urls import core_urls

urlpatterns = core_urls('ctnr') + patterns('',
    url(r'(?P<pk>[\w-]+)/add_user/$', 'add_user', name='ctnr-add-user'),
    url(r'(?P<pk>[\w-]+)?/?change/$', 'change_ctnr', name='ctnr-change'),
)
<commit_msg>fix ctnr urls<commit_after>from django.conf.urls.defaults import *

from cyder.core.ctnr.views import *
from cyder.core.urls import core_urls

urlpatterns = core_urls('ctnr') + patterns('',
    url(r'(?P<pk>[\w-]+)/add_user/$', add_user, name='ctnr-add-user'),
    url(r'(?P<pk>[\w-]+)?/?change/$', change_ctnr, name='ctnr-change'),
    url(r'(?P<pk>[\w-]+)/$', CtnrDetailView.as_view(), name='ctnr-detail'),
)
<|endoftext|>