<commit_before>import re
import os
import sys
import stat
import time
import errno
import struct
import select
import socket
import logging
import tempfile
import threading
from ctypes import *
from ctypes.util import find_library
from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP

from gconf import gconf
import repce
from repce import RepceServer, RepceClient
from master import GMaster

UrlRX  = re.compile('\A(\w+)://(.*)')
HostRX = re.compile('[a-z\d](?:[a-z\d.-]*[a-z\d])?', re.I)
UserRX = re.compile("[\w!\#$%&'*+-\/=?^_`{|}~]+")

def sup(x, *a, **kw):
    return getattr(super(type(x), x), sys._getframe(1).f_code.co_name)(*a, **kw)

def desugar(ustr):
    m = re.match('([^:]*):(.*)', ustr)
    if m:
        if not m.groups()[0]:
            return "gluster://localhost" + ustr
        elif '@' in m.groups()[0] or re.search('[:/]', m.groups()[1]):
            return "ssh://" + ustr
        else:
            return "gluster://#{str}"
    else:
        return "file://" + os.path.abspath(ustr)

def parse_url(ustr):
    m = UrlRX.match(ustr)
    if not m:
        ustr = desugar(ustr)
    m = UrlRX.match(ustr)
    if not m:
        raise RuntimeError("malformed url")
    sch, path = m.groups()
    this = sys.modules[__name__]
    if not hasattr(this, sch.upper()):
        raise RuntimeError("unknown url scheme " + sch)
    return getattr(this, sch.upper())(path)

class Xattr(object):

    libc = CDLL(find_library("libc"))

    @classmethod
    def geterrno(cls):
        return c_int.in_dll(cls.libc, 'errno').value

    @classmethod
    def raise_oserr(cls):
        errn = cls.geterrno()
        raise OSError(errn, os.strerror(errn))

    @classmethod
    def lgetxattr(cls, path, attr, siz=0):
        if siz:
            buf = create_string_buffer('\0' * siz)
        else:
            buf = None
        ret = cls.libc.lgetxattr(path, attr, buf, siz)
        if ret == -1:
            cls.raise_oserr()
        if siz:
            return buf.raw[:ret]
        else:
            return ret

    @classmethod
    def lsetxattr(cls, path, attr, val):
        ret = cls.libc.lsetxattr(path, attr, val, len(val), 0)
        if ret == -1:
            cls.raise_oserr()

class Server(object):

    GX_NSPACE = "trusted.glusterfs"

    @staticmethod
    def entries(path):
        try:
            return os.listdir(path)
        except OSError:
            ex = sys.exc_info()[1]
            if ex.errno == ENOTDIR:
                return []
            else:
                raise

    @classmethod
    def purge(cls, path, entries=None):
        me_also = entries == None
        if not entries:
            try:
                entries = os.listdir(path)
            except OSError:
                ex = sys.exc_info()[1]
                if ex.errno in (ENOTDIR, ENOENT, ELOOP):
                    try:
                        os.unlink(path)
                        return
                    except OSError:
                        ex = sys.exc_info()[1]
                        if ex.errno != ENOENT:
                            raise
                else:
                    raise
        for e in entries:
            cls.purge(os.path.join(path, e))
        if me_also:
            os.rmdir(path)

    @classmethod
    def _create(cls, path, ctor):
        try:
            ctor(path)
        except OSError:
            ex = sys.exc_info()[1]
            if ex.errno == EEXIST:
                cls.purge(path)
                return ctor(path)
            raise

    @classmethod
    def mkdir(cls, path):
        cls._create(path, os.mkdir)

    @classmethod
    def symlink(cls, lnk, path):
        cls._create(path, lambda p: os.symlink(lnk, p))

    @classmethod
    def xtime(cls, path, uuid):
        try:
            return struct.unpack('!II', Xattr.lgetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), 8))
        except OSError:
            ex = sys.exc_info()[1]
            if ex.errno in (ENOENT, ENODATA):
                return ex.errno
            else:
                raise

    @classmethod
    def set_xtime(cls, path, uuid, mark):
	try:
	    Xattr.lsetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), struct.pack('!II', *mark))
	except OSError:
	    ex = sys.exc_info()[1]
	    if ex.errno == ENOENT:
		logging.error ("File for which the setxattr to be done is not present")

    @staticmethod
    def setattr(path, adct):
        own = adct.get('own')
        if own:
            os.lchown(path, *own)
        mode = adct.get('mode')
        if mode:
            os.chmod(path, stat.S_IMODE(mode))
        times = adct.get('times')
        if times:
            os.utime(path, times)

    @staticmethod
    def pid():
        return os.getpid()

    lastping = 0
    @classmethod
    def ping(cls):
        cls.lastping += 1
        return cls.lastping

    @staticmethod
    def version():
        return 1.0

class SlaveLocal(object):

    def can_connect_to(self, remote):
        return not remote

    def service_loop(self):
        repce = RepceServer(self.server, sys.stdin, sys.stdout, int(gconf.sync_jobs))
        t = threading.Thread(target=repce.service_loop)
        t.setDaemon(True)
        t.start()
        logging.info("slave listening")
        if gconf.timeout and int(gconf.timeout) > 0:
            while True:
                lp = self.server.lastping
                time.sleep(int(gconf.timeout))
                if lp == self.server.lastping:
                    logging.info("connection inactive for %d seconds, stopping" % int(gconf.timeout))
                    break
        else:
            select.select((), (), ())

class SlaveRemote(object):

    def connect_remote(self, rargs=[], **opts):
        slave = opts.get('slave', self.url)
        ix, ox = os.pipe()
        iy, oy = os.pipe()
        pid = os.fork()
        if not pid:
            os.close(ox)
            os.dup2(ix, sys.stdin.fileno())
            os.close(iy)
            os.dup2(oy, sys.stdout.fileno())
            argv = rargs + gconf.remote_gsyncd.split() + ['-N', '--listen', '--timeout', str(gconf.timeout), slave]
            os.execvp(argv[0], argv)
        os.close(ix)
        os.close(oy)
        return self.start_fd_client(iy, ox, **opts)

    def start_fd_client(self, i, o, **opts):
        self.server = RepceClient(i, o)
        rv = self.server.__version__()
        exrv = {'proto': repce.repce_version, 'object': Server.version()}
        da0 = (rv, exrv)
        da1 = ({}, {})
        for i in range(2):
            for k, v in da0[i].iteritems():
                da1[i][k] = int(v)
        if da1[0] != da1[1]:
            raise RuntimeError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
        if gconf.timeout and int(gconf.timeout) > 0:
            def pinger():
                while True:
                    self.server.ping()
                    time.sleep(int(gconf.timeout) * 0.5)
            t = threading.Thread(target=pinger)
            t.setDaemon(True)
            t.start()

    def rsync(self, files, *args):
        if not files:
            raise RuntimeError("no files to sync")
        logging.debug("files: " + ", ".join(files))
        argv = gconf.rsync_command.split() + gconf.rsync_extra.split() + ['-aR'] + files + list(args)
        return os.spawnvp(os.P_WAIT, argv[0], argv) == 0

class AbstractUrl(object):

    def __init__(self, path, pattern):
        m = re.search(pattern, path)
        if not m:
            raise RuntimeError("malformed path")
        self.path = path
        return m.groups()

    def scheme(self):
        return type(self).__name__.lower()

    def canonical_path(self):
        return self.path

    def get_url(self, canonical=False):
        if canonical:
            pa = self.canonical_path()
        else:
            pa = self.path
        return "://".join((self.scheme(), pa))

    @property
    def url(self):
        return self.get_url()

  ### Concrete resource classes ###

class FILE(AbstractUrl, SlaveLocal, SlaveRemote):

    class FILEServer(Server):
        pass

    server = FILEServer

    def __init__(self, path):
        sup(self, path, '^/')

    def connect(self):
        os.chdir(self.path)

    def rsync(self, files):
        return sup(self, files, self.path)

class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):

    class GLUSTERServer(Server):

        @classmethod
        def volume_info(cls):
            vm = struct.unpack('!' + 'B'*19 + 'II',
                               Xattr.lgetxattr('.', '.'.join([cls.GX_NSPACE, 'volume-mark']), 27))
            m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(['%02x' % x for x in vm[2:18]]))
            uuid = '-'.join(m.groups())
            return { 'version': vm[0:2],
                     'uuid'   : uuid,
                     'retval' : vm[18],
                     'volume_mark': vm[-2:] }

    server = GLUSTERServer

    def __init__(self, path):
        self.host, self.volume = sup(self, path, '^(%s):(.+)' % HostRX.pattern)

    def canonical_path(self):
        return ':'.join([socket.gethostbyname(self.host), self.volume])

    def can_connect_to(self, remote):
        return True

    def connect(self):
        d = tempfile.mkdtemp()
        try:
            argv = [gconf.gluster_command] + \
                    (gconf.gluster_log_level and ['-L', gConf.gluster_log_level] or []) + \
                    ['-l', gconf.gluster_log_file, '-s', self.host,
                     '--volfile-id', self.volume, '--client-pid=-1', d]
            if os.spawnvp(os.P_WAIT, argv[0], argv):
                raise RuntimeError("command failed: " + " ".join(argv))
            logging.debug('auxiliary glusterfs mount in place')
            os.chdir(d)
            argv = ['umount', '-l', d]
            if os.spawnvp(os.P_WAIT, argv[0], argv):
                raise RuntimeError("command failed: " + " ".join(argv))
        finally:
            try:
                os.rmdir(d)
            except:
                logging.warn('stale mount possibly left behind on ' + d)
        logging.debug('auxiliary glusterfs mount prepared')

    def connect_remote(self, *a, **kw):
        sup(self, *a, **kw)
        self.slavedir = "/proc/%d/cwd" % self.server.pid()

    def service_loop(self, *args):
        if args:
            GMaster(self, args[0]).crawl()
        else:
            sup(self, *args)

    def rsync(self, files):
        return sup(self, files, self.slavedir)

class SSH(AbstractUrl, SlaveRemote):

    def __init__(self, path):
        self.remote_addr, inner_url = sup(self, path,
                                          '^((?:%s@)?%s):(.+)' % tuple([ r.pattern for r in (UserRX, HostRX) ]))
        self.inner_rsc = parse_url(inner_url)

    def canonical_path(self):
        m = re.match('([^@]+)@(.+)', self.remote_addr)
        if m:
            u, h = m.groups()
        else:
            u, h = os.getlogin(), self.remote_addr
        remote_addr = '@'.join([u, socket.gethostbyname(h)])
        return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)])

    def can_connect_to(self, remote):
        return False

    def start_fd_client(self, *a, **opts):
        if opts['deferred']:
            return a
        sup(self, *a)
        ityp = type(self.inner_rsc)
        if ityp == FILE:
            slavepath = self.inner_rsc.path
        elif ityp == GLUSTER:
            slavepath = "/proc/%d/cwd" % self.server.pid()
        else:
            raise NotImplementedError
        self.slaveurl = ':'.join([self.remote_addr, slavepath])

    def connect_remote(self, go_daemon=None):
        if go_daemon == 'done':
            return self.start_fd_client(*self.fd_pair)
        gconf.setup_ssh_ctl(tempfile.mkdtemp())
        deferred = go_daemon == 'postconn'
        ret = sup(self, gconf.ssh_command.split() + gconf.ssh_ctl_args + [self.remote_addr], slave=self.inner_rsc.url, deferred=deferred)
        if deferred:
            # send a message to peer so that we can wait for
            # the answer from which we know connection is
            # established and we can proceed with daemonization
            # (doing that too early robs the ssh passwd prompt...)
            # However, we'd better not start the RepceClient
            # before daemonization (that's not preserved properly
            # in daemon), we just do a an ad-hoc linear put/get.
            i, o = ret
            inf = os.fdopen(i)
            repce.send(o, None, '__version__')
            select.select((inf,), (), ())
            repce.recv(inf)
            # hack hack hack: store a global reference to the file
            # to save it from getting GC'd which implies closing it
            gconf._in_fd_reference = inf
            self.fd_pair = (i, o)
            return 'should'

    def rsync(self, files):
        return sup(self, files, '-ze', " ".join(gconf.ssh_command.split() + gconf.ssh_ctl_args), self.slaveurl)
<commit_msg>syncdaemon: Handling of deleted symlinks (of directories). When a symlink of a directory of master is deleted. The corresponding delete in slave is handled as rmdir() of the symlink assuming the file to be directory<commit_after>import re
import os
import sys
import stat
import time
import errno
import struct
import select
import socket
import logging
import tempfile
import threading
from ctypes import *
from ctypes.util import find_library
from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP

from gconf import gconf
import repce
from repce import RepceServer, RepceClient
from master import GMaster

UrlRX  = re.compile('\A(\w+)://(.*)')
HostRX = re.compile('[a-z\d](?:[a-z\d.-]*[a-z\d])?', re.I)
UserRX = re.compile("[\w!\#$%&'*+-\/=?^_`{|}~]+")

def sup(x, *a, **kw):
    return getattr(super(type(x), x), sys._getframe(1).f_code.co_name)(*a, **kw)

def desugar(ustr):
    m = re.match('([^:]*):(.*)', ustr)
    if m:
        if not m.groups()[0]:
            return "gluster://localhost" + ustr
        elif '@' in m.groups()[0] or re.search('[:/]', m.groups()[1]):
            return "ssh://" + ustr
        else:
            return "gluster://#{str}"
    else:
        return "file://" + os.path.abspath(ustr)

def parse_url(ustr):
    m = UrlRX.match(ustr)
    if not m:
        ustr = desugar(ustr)
    m = UrlRX.match(ustr)
    if not m:
        raise RuntimeError("malformed url")
    sch, path = m.groups()
    this = sys.modules[__name__]
    if not hasattr(this, sch.upper()):
        raise RuntimeError("unknown url scheme " + sch)
    return getattr(this, sch.upper())(path)

class Xattr(object):

    libc = CDLL(find_library("libc"))

    @classmethod
    def geterrno(cls):
        return c_int.in_dll(cls.libc, 'errno').value

    @classmethod
    def raise_oserr(cls):
        errn = cls.geterrno()
        raise OSError(errn, os.strerror(errn))

    @classmethod
    def lgetxattr(cls, path, attr, siz=0):
        if siz:
            buf = create_string_buffer('\0' * siz)
        else:
            buf = None
        ret = cls.libc.lgetxattr(path, attr, buf, siz)
        if ret == -1:
            cls.raise_oserr()
        if siz:
            return buf.raw[:ret]
        else:
            return ret

    @classmethod
    def lsetxattr(cls, path, attr, val):
        ret = cls.libc.lsetxattr(path, attr, val, len(val), 0)
        if ret == -1:
            cls.raise_oserr()

class Server(object):

    GX_NSPACE = "trusted.glusterfs"

    @staticmethod
    def entries(path):
        try:
            return os.listdir(path)
        except OSError:
            ex = sys.exc_info()[1]
            if ex.errno == ENOTDIR:
                return []
            else:
                raise

    @classmethod
    def purge(cls, path, entries=None):
        me_also = entries == None
        if not entries:
            try:
                entries = os.listdir(path)
            except OSError:
                ex = sys.exc_info()[1]
                if ex.errno in (ENOTDIR, ENOENT, ELOOP):
                    try:
                        os.unlink(path)
                        return
                    except OSError:
                        ex = sys.exc_info()[1]
                        if ex.errno != ENOENT:
                            raise
                else:
                    raise
        for e in entries:
            cls.purge(os.path.join(path, e))
        if me_also:
            try:
                os.rmdir(path)
            except OSError:
                ex = sys.exc_info()[1]
                if ex.errno == ENOTDIR:
                    try:
                        os.unlink(path)
                        return
                    except OSError:
                        ex = sys.exc_info()[1]
                        if ex.errno != ENOENT:
                            raise
                elif ex.errno == ENOENT:
                    logging.debug ("Trying to delete a file which is not present")
                else:
                    raise

    @classmethod
    def _create(cls, path, ctor):
        try:
            ctor(path)
        except OSError:
            ex = sys.exc_info()[1]
            if ex.errno == EEXIST:
                cls.purge(path)
                return ctor(path)
            raise

    @classmethod
    def mkdir(cls, path):
        cls._create(path, os.mkdir)

    @classmethod
    def symlink(cls, lnk, path):
        cls._create(path, lambda p: os.symlink(lnk, p))

    @classmethod
    def xtime(cls, path, uuid):
        try:
            return struct.unpack('!II', Xattr.lgetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), 8))
        except OSError:
            ex = sys.exc_info()[1]
            if ex.errno in (ENOENT, ENODATA):
                return ex.errno
            else:
                raise

    @classmethod
    def set_xtime(cls, path, uuid, mark):
	try:
	    Xattr.lsetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), struct.pack('!II', *mark))
	except OSError:
	    ex = sys.exc_info()[1]
	    if ex.errno == ENOENT:
		logging.error ("File for which the setxattr to be done is not present")

    @staticmethod
    def setattr(path, adct):
        own = adct.get('own')
        if own:
            os.lchown(path, *own)
        mode = adct.get('mode')
        if mode:
            os.chmod(path, stat.S_IMODE(mode))
        times = adct.get('times')
        if times:
            os.utime(path, times)

    @staticmethod
    def pid():
        return os.getpid()

    lastping = 0
    @classmethod
    def ping(cls):
        cls.lastping += 1
        return cls.lastping

    @staticmethod
    def version():
        return 1.0

class SlaveLocal(object):

    def can_connect_to(self, remote):
        return not remote

    def service_loop(self):
        repce = RepceServer(self.server, sys.stdin, sys.stdout, int(gconf.sync_jobs))
        t = threading.Thread(target=repce.service_loop)
        t.setDaemon(True)
        t.start()
        logging.info("slave listening")
        if gconf.timeout and int(gconf.timeout) > 0:
            while True:
                lp = self.server.lastping
                time.sleep(int(gconf.timeout))
                if lp == self.server.lastping:
                    logging.info("connection inactive for %d seconds, stopping" % int(gconf.timeout))
                    break
        else:
            select.select((), (), ())

class SlaveRemote(object):

    def connect_remote(self, rargs=[], **opts):
        slave = opts.get('slave', self.url)
        ix, ox = os.pipe()
        iy, oy = os.pipe()
        pid = os.fork()
        if not pid:
            os.close(ox)
            os.dup2(ix, sys.stdin.fileno())
            os.close(iy)
            os.dup2(oy, sys.stdout.fileno())
            argv = rargs + gconf.remote_gsyncd.split() + ['-N', '--listen', '--timeout', str(gconf.timeout), slave]
            os.execvp(argv[0], argv)
        os.close(ix)
        os.close(oy)
        return self.start_fd_client(iy, ox, **opts)

    def start_fd_client(self, i, o, **opts):
        self.server = RepceClient(i, o)
        rv = self.server.__version__()
        exrv = {'proto': repce.repce_version, 'object': Server.version()}
        da0 = (rv, exrv)
        da1 = ({}, {})
        for i in range(2):
            for k, v in da0[i].iteritems():
                da1[i][k] = int(v)
        if da1[0] != da1[1]:
            raise RuntimeError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
        if gconf.timeout and int(gconf.timeout) > 0:
            def pinger():
                while True:
                    self.server.ping()
                    time.sleep(int(gconf.timeout) * 0.5)
            t = threading.Thread(target=pinger)
            t.setDaemon(True)
            t.start()

    def rsync(self, files, *args):
        if not files:
            raise RuntimeError("no files to sync")
        logging.debug("files: " + ", ".join(files))
        argv = gconf.rsync_command.split() + gconf.rsync_extra.split() + ['-aR'] + files + list(args)
        return os.spawnvp(os.P_WAIT, argv[0], argv) == 0

class AbstractUrl(object):

    def __init__(self, path, pattern):
        m = re.search(pattern, path)
        if not m:
            raise RuntimeError("malformed path")
        self.path = path
        return m.groups()

    def scheme(self):
        return type(self).__name__.lower()

    def canonical_path(self):
        return self.path

    def get_url(self, canonical=False):
        if canonical:
            pa = self.canonical_path()
        else:
            pa = self.path
        return "://".join((self.scheme(), pa))

    @property
    def url(self):
        return self.get_url()

  ### Concrete resource classes ###

class FILE(AbstractUrl, SlaveLocal, SlaveRemote):

    class FILEServer(Server):
        pass

    server = FILEServer

    def __init__(self, path):
        sup(self, path, '^/')

    def connect(self):
        os.chdir(self.path)

    def rsync(self, files):
        return sup(self, files, self.path)

class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):

    class GLUSTERServer(Server):

        @classmethod
        def volume_info(cls):
            vm = struct.unpack('!' + 'B'*19 + 'II',
                               Xattr.lgetxattr('.', '.'.join([cls.GX_NSPACE, 'volume-mark']), 27))
            m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(['%02x' % x for x in vm[2:18]]))
            uuid = '-'.join(m.groups())
            return { 'version': vm[0:2],
                     'uuid'   : uuid,
                     'retval' : vm[18],
                     'volume_mark': vm[-2:] }

    server = GLUSTERServer

    def __init__(self, path):
        self.host, self.volume = sup(self, path, '^(%s):(.+)' % HostRX.pattern)

    def canonical_path(self):
        return ':'.join([socket.gethostbyname(self.host), self.volume])

    def can_connect_to(self, remote):
        return True

    def connect(self):
        d = tempfile.mkdtemp()
        try:
            argv = [gconf.gluster_command] + \
                    (gconf.gluster_log_level and ['-L', gConf.gluster_log_level] or []) + \
                    ['-l', gconf.gluster_log_file, '-s', self.host,
                     '--volfile-id', self.volume, '--client-pid=-1', d]
            if os.spawnvp(os.P_WAIT, argv[0], argv):
                raise RuntimeError("command failed: " + " ".join(argv))
            logging.debug('auxiliary glusterfs mount in place')
            os.chdir(d)
            argv = ['umount', '-l', d]
            if os.spawnvp(os.P_WAIT, argv[0], argv):
                raise RuntimeError("command failed: " + " ".join(argv))
        finally:
            try:
                os.rmdir(d)
            except:
                logging.warn('stale mount possibly left behind on ' + d)
        logging.debug('auxiliary glusterfs mount prepared')

    def connect_remote(self, *a, **kw):
        sup(self, *a, **kw)
        self.slavedir = "/proc/%d/cwd" % self.server.pid()

    def service_loop(self, *args):
        if args:
            GMaster(self, args[0]).crawl()
        else:
            sup(self, *args)

    def rsync(self, files):
        return sup(self, files, self.slavedir)

class SSH(AbstractUrl, SlaveRemote):

    def __init__(self, path):
        self.remote_addr, inner_url = sup(self, path,
                                          '^((?:%s@)?%s):(.+)' % tuple([ r.pattern for r in (UserRX, HostRX) ]))
        self.inner_rsc = parse_url(inner_url)

    def canonical_path(self):
        m = re.match('([^@]+)@(.+)', self.remote_addr)
        if m:
            u, h = m.groups()
        else:
            u, h = os.getlogin(), self.remote_addr
        remote_addr = '@'.join([u, socket.gethostbyname(h)])
        return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)])

    def can_connect_to(self, remote):
        return False

    def start_fd_client(self, *a, **opts):
        if opts['deferred']:
            return a
        sup(self, *a)
        ityp = type(self.inner_rsc)
        if ityp == FILE:
            slavepath = self.inner_rsc.path
        elif ityp == GLUSTER:
            slavepath = "/proc/%d/cwd" % self.server.pid()
        else:
            raise NotImplementedError
        self.slaveurl = ':'.join([self.remote_addr, slavepath])

    def connect_remote(self, go_daemon=None):
        if go_daemon == 'done':
            return self.start_fd_client(*self.fd_pair)
        gconf.setup_ssh_ctl(tempfile.mkdtemp())
        deferred = go_daemon == 'postconn'
        ret = sup(self, gconf.ssh_command.split() + gconf.ssh_ctl_args + [self.remote_addr], slave=self.inner_rsc.url, deferred=deferred)
        if deferred:
            # send a message to peer so that we can wait for
            # the answer from which we know connection is
            # established and we can proceed with daemonization
            # (doing that too early robs the ssh passwd prompt...)
            # However, we'd better not start the RepceClient
            # before daemonization (that's not preserved properly
            # in daemon), we just do a an ad-hoc linear put/get.
            i, o = ret
            inf = os.fdopen(i)
            repce.send(o, None, '__version__')
            select.select((inf,), (), ())
            repce.recv(inf)
            # hack hack hack: store a global reference to the file
            # to save it from getting GC'd which implies closing it
            gconf._in_fd_reference = inf
            self.fd_pair = (i, o)
            return 'should'

    def rsync(self, files):
        return sup(self, files, '-ze', " ".join(gconf.ssh_command.split() + gconf.ssh_ctl_args), self.slaveurl)
<|endoftext|>


<commit_before>import warnings

import pandas as pd
import numpy as np
from scipy import interpolate
from typing import Union, List, Tuple

class Interpolation:
    """A callable that returns the result of an interpolation function over input data.

        Attributes
        ----------
        data :
            The data from which to build the interpolation. Contains
            cateogrical_parameters and continuous_parameters.
        categorical_parameters :
            Column names to be used as categorical parameters in Interpolation
            to select between interpolation functions.
        continuous_parameters :
            Column names to be used as continuous parameters in Interpolation. If
            bin edges, should be of the form (column name used in call, column name
            for left bin edge, column name for right bin edge). If order is 0 and
            any continuous parameter is given without bin edges, continuous bins
            are created using the given parameter as midpoints.
        order :
            Order of interpolation.
        """

    def __init__(self, data: pd.DataFrame, categorical_parameters: Union[List[str], Tuple[str]],
                 continuous_parameters: Union[List[str], Tuple[str], List[List[str]], Tuple[Tuple[str, str, str]],
                                              List[Tuple[str, str, str]], Tuple[List[str]]], order: int):

        self.key_columns = categorical_parameters
        self.parameter_columns, self._data, self.value_columns = validate_parameters(data, categorical_parameters,
                                                                                continuous_parameters, order)
        self.order = order

        if self.key_columns:
            # Since there are key_columns we need to group the table by those
            # columns to get the sub-tables to fit
            sub_tables = self._data.groupby(list(self.key_columns))
        else:
            # There are no key columns so we will fit the whole table
            sub_tables = {None: self._data}.items()

        self.interpolations = {}

        for key, base_table in sub_tables:
            if base_table.empty:    # if one of the key columns is a category and not all values are present in data
                continue
            # if order 0, we can interpolate all values at once
            if order == 0:
                self.interpolations[key] = Order0Interp(base_table, self.parameter_columns, self.value_columns)
            else:
                # For each permutation of the key columns build interpolations
                self.interpolations[key] = {}

                for value_column in self.value_columns:
                    # For each value in the table build an interpolation function
                    if len(self.parameter_columns) == 2:
                        # 2 variable interpolation
                        index, column = self.parameter_columns
                        table = base_table.pivot(index=index, columns=column, values=value_column)
                        x = table.index.values
                        y = table.columns.values
                        z = table.values
                        func = interpolate.RectBivariateSpline(x=x, y=y, z=z, ky=order, kx=order).ev
                    else:
                        # 1 variable interpolation
                        base_table = base_table.sort_values(by=self.parameter_columns[0])
                        x = base_table[self.parameter_columns[0]]
                        y = base_table[value_column]
                        func = interpolate.InterpolatedUnivariateSpline(x, y, k=order)
                    self.interpolations[key][value_column] = func

    def __call__(self, interpolants: pd.DataFrame) -> pd.DataFrame:
        """Get the interpolated results for the parameters in interpolants.

        Parameters
         ----------
        interpolants :
            Data frame containing the parameters to interpolate..

        Returns
        -------
        pd.DataFrame
            A table with the interpolated values for the given interpolants.
        """

        validate_call_data(interpolants, self.key_columns, self.parameter_columns)

        if self.key_columns:
            sub_tables = interpolants.groupby(list(self.key_columns))
        else:
            sub_tables = [(None, interpolants)]
        result = pd.DataFrame(index=interpolants.index, columns=self.value_columns, dtype=np.float64)
        for key, sub_table in sub_tables:
            if sub_table.empty:
                continue
            if self.order == 0: # we can interpolate all value columns at once
                df = self.interpolations[key](sub_table)
                #for v in self.value_columns:
                #    result.loc[sub_table.index, v] = df.loc[sub_table.index, v]
                result.loc[sub_table.index, self.value_columns] = df.loc[sub_table.index, self.value_columns]
            else:
                funcs = self.interpolations[key]
                parameters = tuple(sub_table[k] for k in self.parameter_columns)
                for value_column, func in funcs.items():
                    out = func(*parameters)
                    # This reshape is necessary because RectBivariateSpline and InterpolatedUnivariateSpline return results
                    # in slightly different shapes and we need them to be consistent
                    if out.shape:
                        result.loc[sub_table.index, value_column] = out.reshape((out.shape[0],))
                    else:
                        result.loc[sub_table.index, value_column] = out

        return result

    def __repr__(self):
        return "Interpolation()"

def validate_parameters(data, categorical_parameters, continuous_parameters, order):
    if order not in [0, 1]:
        raise ValueError('Only order 0 and order 1 interpolations are supported. '
                         f'You specified {order}')

    if len(continuous_parameters) not in [1, 2] and order != 0:
        raise ValueError("Interpolation over more than two parameters is only supported"
                         "for order 0. For all other orders, only interpolation over 1 or "
                         "2 variables is supported")

    if len(continuous_parameters) < 1:
        raise ValueError("You must supply at least one continuous parameter over which to interpolate.")

    out = []
    for p in continuous_parameters:

        if isinstance(p, (List, Tuple)) and len(p) != 3:
            raise ValueError(f'If you specify bin edges for a parameter, they must be in the form '
                             f'(column name when called, column name of left bin edge, column name of right bin edge). '
                             f'You specified {p}.')

        param_col = p[1] if isinstance(p, (List, Tuple)) else p

        if len(data[param_col].unique()) > order:
            out.append(p)
        else:
            warnings.warn(f"You requested an order {order} interpolation over the parameter {param_col}, "
                          f"however there are only {len(data[param_col].unique())} unique values for {param_col}"
                          f"which is insufficient to support the requested interpolation order."
                          f"The parameter will be dropped from the interpolation.")
            data = data.drop(param_col, axis='columns')

    # break out the individual columns from binned column name lists
    param_cols = ([col for p in continuous_parameters if isinstance(p, (List, Tuple)) for col in p]
                  + [p for p in continuous_parameters if isinstance(p, str)])
    # These are the columns which the interpolation function will approximate
    value_columns = sorted(data.columns.difference(set(categorical_parameters) | set(param_cols)))
    if not value_columns:
        raise ValueError(f"No non-parameter data. Available columns: {data.columns}, "
                         f"Parameter columns: {set(categorical_parameters)|set(continuous_parameters)}")
    return out, data, value_columns

def validate_call_data(data, key_columns, parameter_columns):
    if not isinstance(data, pd.DataFrame):
        raise TypeError(f'Interpolations can only be called on pandas.DataFrames. You'
                        f'passed {type(data)}.')
    callable_param_cols = ([p[0] for p in parameter_columns if isinstance(p, (List, Tuple))]
                           + [p for p in parameter_columns if isinstance(p, str)])
    if not set(callable_param_cols) <= set(data.columns.values.tolist()):
        raise ValueError(f'The continuous parameter columns with which you built the Interpolation must all '
                         f'be present in the data you call it on. The Interpolation has key '
                         f'columns: {callable_param_cols} and your data has columns: '
                         f'{data.columns.values.tolist()}')

    if key_columns and not set(key_columns) <= set(data.columns.values.tolist()):
        raise ValueError(f'The key (categorical) columns with which you built the Interpolation must all'
                         f'be present in the data you call it on. The Interpolation has key'
                         f'columns: {key_columns} and your data has columns: '
                         f'{data.columns.values.tolist()}')

def check_data_complete(data, parameter_columns):
    """ For any parameters specified with edges, make sure edges
    don't overlap and don't have any gaps. Assumes that edges are
    specified with ends and starts overlapping (but one exclusive and
    the other inclusive) so can check that end of previous == start
    of current.

    If multiple parameters, make sure all combinations of parameters
    are present in data."""

    param_edges = [p[1:] for p in parameter_columns if isinstance(p, (Tuple, List))] # strip out call column name
    param_points = [p for p in parameter_columns if isinstance(p, str)]

    # check no overlaps/gaps
    for p in param_edges:
        other_params = param_points + [p_ed[0] for p_ed in param_edges if p_ed != p]
        if other_params:
            sub_tables = data.groupby(list(other_params))
        else:
            sub_tables = {None: data}.items()

        n_p_total = len(set(data[p[0]]))

        for _, table in sub_tables:

            param_data = table[[p[0], p[1]]].copy().sort_values(by=p[0])
            start, end = param_data[p[0]].reset_index(drop=True), param_data[p[1]].reset_index(drop=True)

            if len(set(start)) < n_p_total:
                raise ValueError(f'You must provide a value for every combination of {parameter_columns}.')

            if len(start) <= 1:
                continue
            for i in range(1, len(start)):
                e = end[i-1]
                s = start[i]
                if e > s or s == start[i-1]:
                    raise ValueError(f'Parameter data must not contain overlaps. Parameter {p} '
                                     f'contains overlapping data.')
                if e < s:
                    raise NotImplementedError(f'Interpolation only supported for parameter columns '
                                              f'with continuous bins. Parameter {p} contains '
                                              f'non-continuous bins.')

class Order0Interp:
    """A callable that returns the result of order 0 interpolation over input data.

    Attributes
    ----------
    data :
        The data from which to build the interpolation. Contains
        cateogrical_parameters and continuous_parameters.
    parameter_columns :
        Column names to be used as parameters in Interpolation.
    """
    def __init__(self, data, parameter_columns, value_columns):
        """

        Parameters
        ----------
        data :
            Dataframe used to build interpolation.
        parameter_columns :
            Parameter columns. Should be of form (column name used in call,
            column name for left bin edge, column name for right bin edge)
            or column name. If given as single column name, assumed to be
            midpoint of bin and continuous bins created

        """
        check_data_complete(data, parameter_columns)
        self.data = data.copy()
        self.value_columns = value_columns

        # (column name used in call, column name for left edge): [ordered left edges of bins]
        self.parameter_bins = {}

        for p in parameter_columns:
            if isinstance(p, (List, Tuple)):
                param = (p[0], p[1]) # no need for right edge because already confirmed continuous
                left_edge = self.data[param[1]].drop_duplicates().sort_values()
            else:  # make left edge assuming p is the midpoint and bins are continuous
                left_edge = make_left_edge(self.data, p)
                self.data[f'{p}_left'] = self.data[p].apply(lambda x: left_edge[x])
                param = (p, f'{p}_left')  # no need for right edge because constructed continuous
            self.parameter_bins[param] = left_edge.tolist()

    def __call__(self, interpolants: pd.DataFrame) -> pd.DataFrame:
        """Find the bins for each parameter for each interpolant in interpolants
        and return the values from data there.

        Parameters
        ----------
        interpolants:
            Data frame containing the parameters to interpolate..

        Returns
        -------
        pd.DataFrame
            A table with the interpolated values for the given interpolants.

        """
        # build a dataframe where we have the start of each parameter bin for each interpolant
        interpolant_bins = pd.DataFrame(index=interpolants.index)

        merge_cols = []
        for cols, bins in self.parameter_bins.items():
            merge_cols.append(cols[1])
            interpolant_col = interpolants[cols[0]]
            bin_indices = np.digitize(interpolant_col, bins)
            # digitize uses 0 to indicate <min and len(bins) for >max so adjust to actual indices into bin_indices
            bin_indices = [x-1 if x > 0 else x for x in bin_indices]

            interpolant_bins[cols[1]] = [bins[i] for i in bin_indices]

        index = interpolant_bins.index

        interp_vals = interpolant_bins.merge(self.data, how='left', on=merge_cols).set_index(index)
        return interp_vals[self.value_columns]

def make_left_edge(data: pd.DataFrame, col: str) -> pd.Series:
    """

    Parameters
    ----------
    data :
        A dataframe containing column col.
    col :
        String name of column containing midpoints for which bins should be constructed.

    Returns
    -------
        pd.Series
            A series indexed the same as col with left edge values for a bin constructed
            from each value in col in data as the midpoint.
    """
    mid_pts = data[[col]].drop_duplicates().sort_values(by=col).reset_index(drop=True)
    mid_pts['shift'] = mid_pts[col].shift()

    mid_pts['left'] = mid_pts.apply(lambda row: (row[col] if pd.isna(row['shift'])
                                                   else 0.5 * (row[col] + row['shift'])), axis=1)

    return mid_pts.set_index(col)['left']

<commit_msg>cleaning up doc<commit_after>import warnings

import pandas as pd
import numpy as np
from scipy import interpolate
from typing import Union, List, Tuple

class Interpolation:
    """A callable that returns the result of an interpolation function over input data.

        Attributes
        ----------
        data :
            The data from which to build the interpolation. Contains
            cateogrical_parameters and continuous_parameters.
        categorical_parameters :
            Column names to be used as categorical parameters in Interpolation
            to select between interpolation functions.
        continuous_parameters :
            Column names to be used as continuous parameters in Interpolation. If
            bin edges, should be of the form (column name used in call, column name
            for left bin edge, column name for right bin edge). If order is 0 and
            any continuous parameter is given without bin edges, continuous bins
            are created using the given parameter as midpoints.
        order :
            Order of interpolation.
        """

    def __init__(self, data: pd.DataFrame, categorical_parameters: Union[List[str], Tuple[str]],
                 continuous_parameters: Union[List[str], Tuple[str], List[List[str]], Tuple[Tuple[str, str, str]],
                                              List[Tuple[str, str, str]], Tuple[List[str]]], order: int):

        self.key_columns = categorical_parameters
        self.parameter_columns, self._data, self.value_columns = validate_parameters(data, categorical_parameters,
                                                                                continuous_parameters, order)
        self.order = order

        if self.key_columns:
            # Since there are key_columns we need to group the table by those
            # columns to get the sub-tables to fit
            sub_tables = self._data.groupby(list(self.key_columns))
        else:
            # There are no key columns so we will fit the whole table
            sub_tables = {None: self._data}.items()

        self.interpolations = {}

        for key, base_table in sub_tables:
            if base_table.empty:    # if one of the key columns is a category and not all values are present in data
                continue
            # if order 0, we can interpolate all values at once
            if order == 0:
                self.interpolations[key] = Order0Interp(base_table, self.parameter_columns, self.value_columns)
            else:
                # For each permutation of the key columns build interpolations
                self.interpolations[key] = {}

                for value_column in self.value_columns:
                    # For each value in the table build an interpolation function
                    if len(self.parameter_columns) == 2:
                        # 2 variable interpolation
                        index, column = self.parameter_columns
                        table = base_table.pivot(index=index, columns=column, values=value_column)
                        x = table.index.values
                        y = table.columns.values
                        z = table.values
                        func = interpolate.RectBivariateSpline(x=x, y=y, z=z, ky=order, kx=order).ev
                    else:
                        # 1 variable interpolation
                        base_table = base_table.sort_values(by=self.parameter_columns[0])
                        x = base_table[self.parameter_columns[0]]
                        y = base_table[value_column]
                        func = interpolate.InterpolatedUnivariateSpline(x, y, k=order)
                    self.interpolations[key][value_column] = func

    def __call__(self, interpolants: pd.DataFrame) -> pd.DataFrame:
        """Get the interpolated results for the parameters in interpolants.

        Parameters
         ----------
        interpolants :
            Data frame containing the parameters to interpolate..

        Returns
        -------
        pd.DataFrame
            A table with the interpolated values for the given interpolants.
        """

        validate_call_data(interpolants, self.key_columns, self.parameter_columns)

        if self.key_columns:
            sub_tables = interpolants.groupby(list(self.key_columns))
        else:
            sub_tables = [(None, interpolants)]
        result = pd.DataFrame(index=interpolants.index, columns=self.value_columns, dtype=np.float64)
        for key, sub_table in sub_tables:
            if sub_table.empty:
                continue
            if self.order == 0: # we can interpolate all value columns at once
                df = self.interpolations[key](sub_table)
                #for v in self.value_columns:
                #    result.loc[sub_table.index, v] = df.loc[sub_table.index, v]
                result.loc[sub_table.index, self.value_columns] = df.loc[sub_table.index, self.value_columns]
            else:
                funcs = self.interpolations[key]
                parameters = tuple(sub_table[k] for k in self.parameter_columns)
                for value_column, func in funcs.items():
                    out = func(*parameters)
                    # This reshape is necessary because RectBivariateSpline and InterpolatedUnivariateSpline return results
                    # in slightly different shapes and we need them to be consistent
                    if out.shape:
                        result.loc[sub_table.index, value_column] = out.reshape((out.shape[0],))
                    else:
                        result.loc[sub_table.index, value_column] = out

        return result

    def __repr__(self):
        return "Interpolation()"

def validate_parameters(data, categorical_parameters, continuous_parameters, order):
    if order not in [0, 1]:
        raise ValueError('Only order 0 and order 1 interpolations are supported. '
                         f'You specified {order}')

    if len(continuous_parameters) not in [1, 2] and order != 0:
        raise ValueError("Interpolation over more than two parameters is only supported"
                         "for order 0. For all other orders, only interpolation over 1 or "
                         "2 variables is supported")

    if len(continuous_parameters) < 1:
        raise ValueError("You must supply at least one continuous parameter over which to interpolate.")

    out = []
    for p in continuous_parameters:

        if isinstance(p, (List, Tuple)) and len(p) != 3:
            raise ValueError(f'If you specify bin edges for a parameter, they must be in the form '
                             f'(column name when called, column name of left bin edge, column name of right bin edge). '
                             f'You specified {p}.')

        param_col = p[1] if isinstance(p, (List, Tuple)) else p

        if len(data[param_col].unique()) > order:
            out.append(p)
        else:
            warnings.warn(f"You requested an order {order} interpolation over the parameter {param_col}, "
                          f"however there are only {len(data[param_col].unique())} unique values for {param_col}"
                          f"which is insufficient to support the requested interpolation order."
                          f"The parameter will be dropped from the interpolation.")
            data = data.drop(param_col, axis='columns')

    # break out the individual columns from binned column name lists
    param_cols = ([col for p in continuous_parameters if isinstance(p, (List, Tuple)) for col in p]
                  + [p for p in continuous_parameters if isinstance(p, str)])
    # These are the columns which the interpolation function will approximate
    value_columns = sorted(data.columns.difference(set(categorical_parameters) | set(param_cols)))
    if not value_columns:
        raise ValueError(f"No non-parameter data. Available columns: {data.columns}, "
                         f"Parameter columns: {set(categorical_parameters)|set(continuous_parameters)}")
    return out, data, value_columns

def validate_call_data(data, key_columns, parameter_columns):
    if not isinstance(data, pd.DataFrame):
        raise TypeError(f'Interpolations can only be called on pandas.DataFrames. You'
                        f'passed {type(data)}.')
    callable_param_cols = ([p[0] for p in parameter_columns if isinstance(p, (List, Tuple))]
                           + [p for p in parameter_columns if isinstance(p, str)])
    if not set(callable_param_cols) <= set(data.columns.values.tolist()):
        raise ValueError(f'The continuous parameter columns with which you built the Interpolation must all '
                         f'be present in the data you call it on. The Interpolation has key '
                         f'columns: {callable_param_cols} and your data has columns: '
                         f'{data.columns.values.tolist()}')

    if key_columns and not set(key_columns) <= set(data.columns.values.tolist()):
        raise ValueError(f'The key (categorical) columns with which you built the Interpolation must all'
                         f'be present in the data you call it on. The Interpolation has key'
                         f'columns: {key_columns} and your data has columns: '
                         f'{data.columns.values.tolist()}')

def check_data_complete(data, parameter_columns):
    """ For any parameters specified with edges, make sure edges
    don't overlap and don't have any gaps. Assumes that edges are
    specified with ends and starts overlapping (but one exclusive and
    the other inclusive) so can check that end of previous == start
    of current.

    If multiple parameters, make sure all combinations of parameters
    are present in data."""

    param_edges = [p[1:] for p in parameter_columns if isinstance(p, (Tuple, List))] # strip out call column name
    param_points = [p for p in parameter_columns if isinstance(p, str)]

    # check no overlaps/gaps
    for p in param_edges:
        other_params = param_points + [p_ed[0] for p_ed in param_edges if p_ed != p]
        if other_params:
            sub_tables = data.groupby(list(other_params))
        else:
            sub_tables = {None: data}.items()

        n_p_total = len(set(data[p[0]]))

        for _, table in sub_tables:

            param_data = table[[p[0], p[1]]].copy().sort_values(by=p[0])
            start, end = param_data[p[0]].reset_index(drop=True), param_data[p[1]].reset_index(drop=True)

            if len(set(start)) < n_p_total:
                raise ValueError(f'You must provide a value for every combination of {parameter_columns}.')

            if len(start) <= 1:
                continue
            for i in range(1, len(start)):
                e = end[i-1]
                s = start[i]
                if e > s or s == start[i-1]:
                    raise ValueError(f'Parameter data must not contain overlaps. Parameter {p} '
                                     f'contains overlapping data.')
                if e < s:
                    raise NotImplementedError(f'Interpolation only supported for parameter columns '
                                              f'with continuous bins. Parameter {p} contains '
                                              f'non-continuous bins.')

class Order0Interp:
    """A callable that returns the result of order 0 interpolation over input data.

    Attributes
    ----------
    data :
        The data from which to build the interpolation. Contains
        categorical_parameters and continuous_parameters.
    parameter_columns :
        Column names to be used as parameters in Interpolation.
    """
    def __init__(self, data, parameter_columns, value_columns):
        """

        Parameters
        ----------
        data :
            Dataframe used to build interpolation.
        parameter_columns :
            Parameter columns. Should be of form (column name used in call,
            column name for left bin edge, column name for right bin edge)
            or column name. If given as single column name, assumed to be
            midpoint of bin and continuous bins created. Assumes left bin
            edges are inclusive and right exclusive.

        """
        check_data_complete(data, parameter_columns)
        self.data = data.copy()
        self.value_columns = value_columns

        # (column name used in call, column name for left edge): [ordered left edges of bins]
        self.parameter_bins = {}

        for p in parameter_columns:
            if isinstance(p, (List, Tuple)):
                param = (p[0], p[1]) # no need for right edge because already confirmed continuous
                left_edge = self.data[param[1]].drop_duplicates().sort_values()
            else:  # make left edge assuming p is the midpoint and bins are continuous
                left_edge = make_left_edge(self.data, p)
                self.data[f'{p}_left'] = self.data[p].apply(lambda x: left_edge[x])
                param = (p, f'{p}_left')  # no need for right edge because constructed continuous
            self.parameter_bins[param] = left_edge.tolist()

    def __call__(self, interpolants: pd.DataFrame) -> pd.DataFrame:
        """Find the bins for each parameter for each interpolant in interpolants
        and return the values from data there.

        Parameters
        ----------
        interpolants:
            Data frame containing the parameters to interpolate..

        Returns
        -------
        pd.DataFrame
            A table with the interpolated values for the given interpolants.

        """
        # build a dataframe where we have the start of each parameter bin for each interpolant
        interpolant_bins = pd.DataFrame(index=interpolants.index)

        merge_cols = []
        for cols, bins in self.parameter_bins.items():
            merge_cols.append(cols[1])
            interpolant_col = interpolants[cols[0]]
            bin_indices = np.digitize(interpolant_col, bins)
            # digitize uses 0 to indicate <min and len(bins) for >max so adjust to actual indices into bin_indices
            bin_indices = [x-1 if x > 0 else x for x in bin_indices]

            interpolant_bins[cols[1]] = [bins[i] for i in bin_indices]

        index = interpolant_bins.index

        interp_vals = interpolant_bins.merge(self.data, how='left', on=merge_cols).set_index(index)
        return interp_vals[self.value_columns]

def make_left_edge(data: pd.DataFrame, col: str) -> pd.Series:
    """

    Parameters
    ----------
    data :
        A dataframe containing column col.
    col :
        String name of column containing midpoints for which bins should be constructed.

    Returns
    -------
        pd.Series
            A series indexed the same as col with left edge values for a bin constructed
            from each value in col in data as the midpoint.
    """
    mid_pts = data[[col]].drop_duplicates().sort_values(by=col).reset_index(drop=True)
    mid_pts['shift'] = mid_pts[col].shift()

    mid_pts['left'] = mid_pts.apply(lambda row: (row[col] if pd.isna(row['shift'])
                                                   else 0.5 * (row[col] + row['shift'])), axis=1)

    return mid_pts.set_index(col)['left']

<|endoftext|>


<commit_before>import os
import time
import signal
import logging
from subprocess import list2cmdline, Popen, CalledProcessError

LOG = logging.getLogger(__name__)

class TarArchiveAction(object):
    def __init__(self, snap_datadir, archive_stream, config):
        self.snap_datadir = snap_datadir
        self.archive_stream = archive_stream
        self.config = config

    def __call__(self, event, snapshot_fsm, snapshot_vol):
        argv = [
            'tar',
            '--create',
            '--file', '-',
            '--verbose',
            '--totals',
            '--directory', self.snap_datadir,
            '.'
        ]

        for param in self.config['exclude']:
            argv.insert(-1, "--exclude")
            argv.insert(-1, os.path.join('.', param))

        LOG.info("Running: %s > %s", list2cmdline(argv), self.archive_stream.name)

        archive_log = os.path.dirname(self.archive_stream.name)
        archive_log = os.path.join(archive_log, 'archive.log')
        process = Popen(argv,
                        preexec_fn=os.setsid,
                        stdout=self.archive_stream,
                        stderr=open(archive_log, 'w'),
                        close_fds=True)
        while process.poll() is None:
            if signal.SIGINT in snapshot_fsm.sigmgr.pending:
                os.kill(process.pid, signal.SIGKILL)
            time.sleep(0.5)

        if signal.SIGINT in snapshot_fsm.sigmgr.pending:
            raise KeyboardInterrupt("Interrupted")

        if process.returncode != 0:
            raise CalledProcessError(process.returncode, "tar")
<commit_msg>close the archive stream as part of tarring -- ensures we don't unmount before all the data is written<commit_after>import os
import time
import signal
import logging
from subprocess import list2cmdline, Popen, CalledProcessError

LOG = logging.getLogger(__name__)

class TarArchiveAction(object):
    def __init__(self, snap_datadir, archive_stream, config):
        self.snap_datadir = snap_datadir
        self.archive_stream = archive_stream
        self.config = config

    def __call__(self, event, snapshot_fsm, snapshot_vol):
        argv = [
            'tar',
            '--create',
            '--file', '-',
            '--verbose',
            '--totals',
            '--directory', self.snap_datadir,
            '.'
        ]

        for param in self.config['exclude']:
            argv.insert(-1, "--exclude")
            argv.insert(-1, os.path.join('.', param))

        LOG.info("Running: %s > %s", list2cmdline(argv), self.archive_stream.name)

        archive_log = os.path.dirname(self.archive_stream.name)
        archive_log = os.path.join(archive_log, 'archive.log')
        process = Popen(argv,
                        preexec_fn=os.setsid,
                        stdout=self.archive_stream,
                        stderr=open(archive_log, 'w'),
                        close_fds=True)
        while process.poll() is None:
            if signal.SIGINT in snapshot_fsm.sigmgr.pending:
                os.kill(process.pid, signal.SIGKILL)
            time.sleep(0.5)

        self.archive_stream.close()

        if signal.SIGINT in snapshot_fsm.sigmgr.pending:
            raise KeyboardInterrupt("Interrupted")

        if process.returncode != 0:
            raise CalledProcessError(process.returncode, "tar")
<|endoftext|>


<commit_before>"""The Fast Gradient Method attack."""

import numpy as np
import tensorflow as tf

def fast_gradient_method(model_fn, x, eps, norm, loss_fn=None, clip_min=None, clip_max=None, y=None,
                         targeted=False, sanity_checks=False):
  """
  Tensorflow 2.0 implementation of the Fast Gradient Method.
  :param model_fn: a callable that takes an input tensor and returns the model logits.
  :param x: input tensor.
  :param eps: epsilon (input variation parameter); see https://arxiv.org/abs/1412.6572.
  :param norm: Order of the norm (mimics NumPy). Possible values: np.inf, 1 or 2.
  :param loss_fn: loss function that takes (labels, logits) as arguments and returns loss.
                  default function is 'tf.nn.sparse_softmax_cross_entropy_with_logits'
  :param clip_min: (optional) float. Minimum float value for adversarial example components.
  :param clip_max: (optional) float. Maximum float value for adversarial example components.
  :param y: (optional) Tensor with true labels. If targeted is true, then provide the
            target label. Otherwise, only provide this parameter if you'd like to use true
            labels when crafting adversarial samples. Otherwise, model predictions are used
            as labels to avoid the "label leaking" effect (explained in this paper:
            https://arxiv.org/abs/1611.01236). Default is None.
  :param targeted: (optional) bool. Is the attack targeted or untargeted?
            Untargeted, the default, will try to make the label incorrect.
            Targeted will instead try to move in the direction of being more like y.
  :param sanity_checks: bool, if True, include asserts (Turn them off to use less runtime /
            memory or for unit tests that intentionally pass strange input)
  :return: a tensor for the adversarial example
  """
  if norm not in [np.inf, 1, 2]:
    raise ValueError("Norm order must be either np.inf, 1, or 2.")

  if loss_fn is None:
    loss_fn = tf.nn.sparse_softmax_cross_entropy_with_logits

  asserts = []

  # If a data range was specified, check that the input was in that range
  if clip_min is not None:
    asserts.append(tf.math.greater_equal(x, clip_min))

  if clip_max is not None:
    asserts.append(tf.math.less_equal(x, clip_max))

  if y is None:
    # Using model predictions as ground truth to avoid label leaking
    y = tf.argmax(model_fn(x), 1)

  grad = compute_gradient(model_fn, loss_fn, x, y, targeted)

  optimal_perturbation = optimize_linear(grad, eps, norm)
  # Add perturbation to original example to obtain adversarial example
  adv_x = x + optimal_perturbation

  # If clipping is needed, reset all values outside of [clip_min, clip_max]
  if (clip_min is not None) or (clip_max is not None):
    # We don't currently support one-sided clipping
    assert clip_min is not None and clip_max is not None
    adv_x = tf.clip_by_value(adv_x, clip_min, clip_max)

  if sanity_checks:
    assert np.all(asserts)
  return adv_x

# Due to performance reasons, this function is wrapped inside of tf.function decorator.
# Not using the decorator here, or letting the user wrap the attack in tf.function is way
# slower on Tensorflow 2.0.0-alpha0.
@tf.function
def compute_gradient(model_fn, loss_fn, x, y, targeted):
  """
  Computes the gradient of the loss with respect to the input tensor.
  :param model_fn: a callable that takes an input tensor and returns the model logits.
  :param loss_fn: loss function that takes (labels, logits) as arguments and returns loss.
  :param x: input tensor
  :param y: Tensor with true labels. If targeted is true, then provide the target label.
  :param targeted:  bool. Is the attack targeted or untargeted? Untargeted, the default, will
                    try to make the label incorrect. Targeted will instead try to move in the
                    direction of being more like y.
  :return: A tensor containing the gradient of the loss with respect to the input tensor.
  """

  with tf.GradientTape() as g:
    g.watch(x)
    # Compute loss
    loss = loss_fn(labels=y, logits=model_fn(x))
    if targeted:  # attack is targeted, minimize loss of target label rather than maximize loss of correct label
      loss = -loss

  # Define gradient of loss wrt input
  grad = g.gradient(loss, x)
  return grad

def optimize_linear(grad, eps, norm=np.inf):
  """
  Solves for the optimal input to a linear function under a norm constraint.

  Optimal_perturbation = argmax_{eta, ||eta||_{norm} < eps} dot(eta, grad)

  :param grad: tf tensor containing a batch of gradients
  :param eps: float scalar specifying size of constraint region
  :param norm: int specifying order of norm
  :returns:
    tf tensor containing optimal perturbation
  """

  # Convert the iterator returned by `range` into a list.
  axis = list(range(1, len(grad.get_shape())))
  avoid_zero_div = 1e-12
  if norm == np.inf:
    optimal_perturbation = tf.sign(grad)
  elif norm == 1:
    abs_grad = tf.abs(grad)
    sign = tf.sign(grad)
    max_abs_grad = tf.reduce_max(abs_grad, axis, keepdims=True)
    tied_for_max = tf.dtypes.cast(tf.equal(abs_grad, max_abs_grad), dtype=tf.float32)
    num_ties = tf.reduce_sum(tied_for_max, axis, keepdims=True)
    optimal_perturbation = sign * tied_for_max / num_ties
  elif norm == 2:
    square = tf.maximum(avoid_zero_div, tf.reduce_sum(tf.square(grad), axis, keepdims=True))
    optimal_perturbation = grad / tf.sqrt(square)
  else:
    raise NotImplementedError("Only L-inf, L1 and L2 norms are currently implemented.")

  # Scale perturbation to be the solution for the norm=eps rather than norm=1 problem
  scaled_perturbation = tf.multiply(eps, optimal_perturbation)
  return scaled_perturbation
<commit_msg>Update fast_gradient_method.py<commit_after>"""The Fast Gradient Method attack."""

import numpy as np
import tensorflow as tf

def fast_gradient_method(model_fn, x, eps, norm, loss_fn=None, clip_min=None, clip_max=None, y=None,
                         targeted=False, sanity_checks=False):
  """
  Tensorflow 2.0 implementation of the Fast Gradient Method.
  :param model_fn: a callable that takes an input tensor and returns the model logits.
  :param x: input tensor.
  :param eps: epsilon (input variation parameter); see https://arxiv.org/abs/1412.6572.
  :param norm: Order of the norm (mimics NumPy). Possible values: np.inf, 1 or 2.
  :param loss_fn: (optional) loss function that takes (labels, logits) as arguments and returns loss.
                  default function is 'tf.nn.sparse_softmax_cross_entropy_with_logits'
  :param clip_min: (optional) float. Minimum float value for adversarial example components.
  :param clip_max: (optional) float. Maximum float value for adversarial example components.
  :param y: (optional) Tensor with true labels. If targeted is true, then provide the
            target label. Otherwise, only provide this parameter if you'd like to use true
            labels when crafting adversarial samples. Otherwise, model predictions are used
            as labels to avoid the "label leaking" effect (explained in this paper:
            https://arxiv.org/abs/1611.01236). Default is None.
  :param targeted: (optional) bool. Is the attack targeted or untargeted?
            Untargeted, the default, will try to make the label incorrect.
            Targeted will instead try to move in the direction of being more like y.
  :param sanity_checks: bool, if True, include asserts (Turn them off to use less runtime /
            memory or for unit tests that intentionally pass strange input)
  :return: a tensor for the adversarial example
  """
  if norm not in [np.inf, 1, 2]:
    raise ValueError("Norm order must be either np.inf, 1, or 2.")

  if loss_fn is None:
    loss_fn = tf.nn.sparse_softmax_cross_entropy_with_logits

  asserts = []

  # If a data range was specified, check that the input was in that range
  if clip_min is not None:
    asserts.append(tf.math.greater_equal(x, clip_min))

  if clip_max is not None:
    asserts.append(tf.math.less_equal(x, clip_max))

  if y is None:
    # Using model predictions as ground truth to avoid label leaking
    y = tf.argmax(model_fn(x), 1)

  grad = compute_gradient(model_fn, loss_fn, x, y, targeted)

  optimal_perturbation = optimize_linear(grad, eps, norm)
  # Add perturbation to original example to obtain adversarial example
  adv_x = x + optimal_perturbation

  # If clipping is needed, reset all values outside of [clip_min, clip_max]
  if (clip_min is not None) or (clip_max is not None):
    # We don't currently support one-sided clipping
    assert clip_min is not None and clip_max is not None
    adv_x = tf.clip_by_value(adv_x, clip_min, clip_max)

  if sanity_checks:
    assert np.all(asserts)
  return adv_x

# Due to performance reasons, this function is wrapped inside of tf.function decorator.
# Not using the decorator here, or letting the user wrap the attack in tf.function is way
# slower on Tensorflow 2.0.0-alpha0.
@tf.function
def compute_gradient(model_fn, loss_fn, x, y, targeted):
  """
  Computes the gradient of the loss with respect to the input tensor.
  :param model_fn: a callable that takes an input tensor and returns the model logits.
  :param loss_fn: loss function that takes (labels, logits) as arguments and returns loss.
  :param x: input tensor
  :param y: Tensor with true labels. If targeted is true, then provide the target label.
  :param targeted:  bool. Is the attack targeted or untargeted? Untargeted, the default, will
                    try to make the label incorrect. Targeted will instead try to move in the
                    direction of being more like y.
  :return: A tensor containing the gradient of the loss with respect to the input tensor.
  """

  with tf.GradientTape() as g:
    g.watch(x)
    # Compute loss
    loss = loss_fn(labels=y, logits=model_fn(x))
    if targeted:  # attack is targeted, minimize loss of target label rather than maximize loss of correct label
      loss = -loss

  # Define gradient of loss wrt input
  grad = g.gradient(loss, x)
  return grad

def optimize_linear(grad, eps, norm=np.inf):
  """
  Solves for the optimal input to a linear function under a norm constraint.

  Optimal_perturbation = argmax_{eta, ||eta||_{norm} < eps} dot(eta, grad)

  :param grad: tf tensor containing a batch of gradients
  :param eps: float scalar specifying size of constraint region
  :param norm: int specifying order of norm
  :returns:
    tf tensor containing optimal perturbation
  """

  # Convert the iterator returned by `range` into a list.
  axis = list(range(1, len(grad.get_shape())))
  avoid_zero_div = 1e-12
  if norm == np.inf:
    optimal_perturbation = tf.sign(grad)
  elif norm == 1:
    abs_grad = tf.abs(grad)
    sign = tf.sign(grad)
    max_abs_grad = tf.reduce_max(abs_grad, axis, keepdims=True)
    tied_for_max = tf.dtypes.cast(tf.equal(abs_grad, max_abs_grad), dtype=tf.float32)
    num_ties = tf.reduce_sum(tied_for_max, axis, keepdims=True)
    optimal_perturbation = sign * tied_for_max / num_ties
  elif norm == 2:
    square = tf.maximum(avoid_zero_div, tf.reduce_sum(tf.square(grad), axis, keepdims=True))
    optimal_perturbation = grad / tf.sqrt(square)
  else:
    raise NotImplementedError("Only L-inf, L1 and L2 norms are currently implemented.")

  # Scale perturbation to be the solution for the norm=eps rather than norm=1 problem
  scaled_perturbation = tf.multiply(eps, optimal_perturbation)
  return scaled_perturbation
<|endoftext|>


<commit_before>#!/usr/bin/env python

from csv import DictReader
from urllib import urlopen, urlencode
from bs4 import BeautifulSoup

def find_examples(soup):
    ''' Generate a list of examples from a starting point.
    '''
    el = soup.find(id='middle-column').find(class_='inside').find(text='Illustrative Examples:')

    if not el:
        # Nothing.
        return

    for i in range(2):
        el = el.nextSibling

        if el.name != 'br':
            # There are supposed to be two breaks after "Illustrative Examples"
            return

    while True:
        el = el.nextSibling

        if el.name is not None:
            # Expected plain text here.
            return

        yield unicode(el)

        el = el.nextSibling

        if el.name != 'br':
            # Expected a break here
            return

for row in DictReader(urlopen('http://forever.codeforamerica.org.s3.amazonaws.com/NAICS/6-digit_2012_Codes.csv')):

    q = dict(code=row['2012 NAICS Code'], search='2012 NAICS Search')
    url = 'http://www.census.gov/cgi-bin/sssd/naics/naicsrch?' + urlencode(q)
    html = urlopen(url).read()
    soup = BeautifulSoup(html)

    print row['2012 NAICS Title']

    for example in find_examples(soup):
        print ' ', example
<commit_msg>Added HTML scraper for cross references<commit_after>#!/usr/bin/env python

from re import compile
from csv import DictReader
from urllib import urlopen, urlencode
from bs4 import BeautifulSoup

def find_examples(soup):
    ''' Generate a list of examples from a starting point.
    '''
    el = soup.find(text='Illustrative Examples:')

    if not el:
        # Nothing.
        return

    for i in range(2):
        el = el.nextSibling

        if el.name != 'br':
            # There are supposed to be two breaks after "Illustrative Examples"
            return

    while True:
        el = el.nextSibling

        if el.name is not None:
            # Expected plain text here.
            return

        yield unicode(el)

        el = el.nextSibling

        if el.name != 'br':
            # Expected a break here
            return

def find_crossreferences(soup):
    '''
    '''
    el = soup.find(text=compile(r'^Cross-References.'))

    if not el:
        # Nothing.
        return

    el = el.nextSibling.nextSibling.nextSibling

    if el.name != 'ul':
        # Expected an unordered list after some blank lines.
        return

    for el in el.find_all('li'):
        code, text = None, ''

        for el in el.contents:
            if el.name == 'a':
                text += el.string
                code = el.string
            else:
                text += unicode(el)

        yield code, text

rows = DictReader(urlopen('http://forever.codeforamerica.org.s3.amazonaws.com/NAICS/6-digit_2012_Codes.csv'))

rows = [
    {'2012 NAICS Code': 423390, '2012 NAICS Title': 'Other Construction Material Merchant Wholesalers'},
    {'2012 NAICS Code': 111110, '2012 NAICS Title': 'Soybean Farming'},
    {'2012 NAICS Code': 111310, '2012 NAICS Title': 'Orange Groves'},
    ]

for row in rows:

    q = dict(code=row['2012 NAICS Code'], search='2012 NAICS Search')
    url = 'http://www.census.gov/cgi-bin/sssd/naics/naicsrch?' + urlencode(q)
    html = urlopen(url).read()
    soup = BeautifulSoup(html)
    soup = soup.find(id='middle-column').find(class_='inside')

    print row['2012 NAICS Title']

    for example in find_examples(soup):
        print ' ', example

    for (code, text) in find_crossreferences(soup):
        print ' ', code, text
<|endoftext|>


<commit_before>from django.core.management.base import NoArgsCommand

class Command(NoArgsCommand):
    help = "Runs the command-line client for the current DATABASE_ENGINE."

    requires_model_validation = False

    def handle_noargs(self, **options):
        from django.db import connection
        connection.client.runshell()
<commit_msg>Fixed #8978 -- We now print a helpful error message for 'manage.py dbshell' if the client executable is not found. Previously we were displaying a traceback<commit_after>from django.core.management.base import NoArgsCommand, CommandError

class Command(NoArgsCommand):
    help = "Runs the command-line client for the current DATABASE_ENGINE."

    requires_model_validation = False

    def handle_noargs(self, **options):
        from django.db import connection
        try:
            connection.client.runshell()
        except OSError:
            # Note that we're assuming OSError means that the client program
            # isn't installed. There's a possibility OSError would be raised
            # for some other reason, in which case this error message would be
            # inaccurate. Still, this message catches the common case.
            raise CommandError('You appear not to have the %r program installed or on your path.' % \
                connection.client.executable_name)
<|endoftext|>


<commit_before>from django.db.backends import BaseDatabaseIntrospection
import cx_Oracle
import re

foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")

class DatabaseIntrospection(BaseDatabaseIntrospection):
    # Maps type objects to Django Field types.
    data_types_reverse = {
        cx_Oracle.CLOB: 'TextField',
        cx_Oracle.DATETIME: 'DateField',
        cx_Oracle.FIXED_CHAR: 'CharField',
        cx_Oracle.NCLOB: 'TextField',
        cx_Oracle.NUMBER: 'DecimalField',
        cx_Oracle.STRING: 'CharField',
        cx_Oracle.TIMESTAMP: 'DateTimeField',
    }

    try:
        data_types_reverse[cx_Oracle.NATIVE_FLOAT] = 'FloatField'
    except AttributeError:
        pass

    try:
        data_types_reverse[cx_Oracle.UNICODE] = 'CharField'
    except AttributeError:
        pass

    def get_field_type(self, data_type, description):
        # If it's a NUMBER with scale == 0, consider it an IntegerField
        if data_type == cx_Oracle.NUMBER and description[5] == 0:
            if description[4] > 11:
                return 'BigIntegerField'
            else:
                return 'IntegerField'
        else:
            return super(DatabaseIntrospection, self).get_field_type(
                data_type, description)

    def get_table_list(self, cursor):
        "Returns a list of table names in the current database."
        cursor.execute("SELECT TABLE_NAME FROM USER_TABLES")
        return [row[0].lower() for row in cursor.fetchall()]

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        cursor.execute("SELECT * FROM %s WHERE ROWNUM < 2" % self.connection.ops.quote_name(table_name))
        description = []
        for desc in cursor.description:
            description.append((desc[0].lower(),) + desc[1:])
        return description

    def table_name_converter(self, name):
        "Table name comparison is case insensitive under Oracle"
        return name.lower()

    def _name_to_index(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: field_index} for the given table.
        Indexes are 0-based.
        """
        return dict([(d[0], i) for i, d in enumerate(self.get_table_description(cursor, table_name))])

    def get_relations(self, cursor, table_name):
        """
        Returns a dictionary of {field_index: (field_index_other_table, other_table)}
        representing all relationships to the given table. Indexes are 0-based.
        """
        table_name = table_name.upper()
        cursor.execute("""
    SELECT ta.column_id - 1, tb.table_name, tb.column_id - 1
    FROM   user_constraints, USER_CONS_COLUMNS ca, USER_CONS_COLUMNS cb,
           user_tab_cols ta, user_tab_cols tb
    WHERE  user_constraints.table_name = %s AND
           ta.table_name = %s AND
           ta.column_name = ca.column_name AND
           ca.table_name = %s AND
           user_constraints.constraint_name = ca.constraint_name AND
           user_constraints.r_constraint_name = cb.constraint_name AND
           cb.table_name = tb.table_name AND
           cb.column_name = tb.column_name AND
           ca.position = cb.position""", [table_name, table_name, table_name])

        relations = {}
        for row in cursor.fetchall():
            relations[row[0]] = (row[2], row[1])
        return relations

    def get_indexes(self, cursor, table_name):
        """
        Returns a dictionary of fieldname -> infodict for the given table,
        where each infodict is in the format:
            {'primary_key': boolean representing whether it's the primary key,
             'unique': boolean representing whether it's a unique index}
        """
        # This query retrieves each index on the given table, including the
        # first associated field name
        # "We were in the nick of time; you were in great peril!"
        sql = """\
SELECT LOWER(all_tab_cols.column_name) AS column_name,
       CASE user_constraints.constraint_type
           WHEN 'P' THEN 1 ELSE 0
       END AS is_primary_key,
       CASE user_indexes.uniqueness
           WHEN 'UNIQUE' THEN 1 ELSE 0
       END AS is_unique
FROM   all_tab_cols, user_cons_columns, user_constraints, user_ind_columns, user_indexes
WHERE  all_tab_cols.column_name = user_cons_columns.column_name (+)
  AND  all_tab_cols.table_name = user_cons_columns.table_name (+)
  AND  user_cons_columns.constraint_name = user_constraints.constraint_name (+)
  AND  user_constraints.constraint_type (+) = 'P'
  AND  user_ind_columns.column_name (+) = all_tab_cols.column_name
  AND  user_ind_columns.table_name (+) = all_tab_cols.table_name
  AND  user_indexes.uniqueness (+) = 'UNIQUE'
  AND  user_indexes.index_name (+) = user_ind_columns.index_name
  AND  all_tab_cols.table_name = UPPER(%s)
"""
        cursor.execute(sql, [table_name])
        indexes = {}
        for row in cursor.fetchall():
            indexes[row[0]] = {'primary_key': row[1], 'unique': row[2]}
        return indexes
<commit_msg>Fixed another case-sensitivity bug in Oracle introspection.  This one was harmless, but it was causing an irritating test failure.<commit_after>from django.db.backends import BaseDatabaseIntrospection
import cx_Oracle
import re

foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")

class DatabaseIntrospection(BaseDatabaseIntrospection):
    # Maps type objects to Django Field types.
    data_types_reverse = {
        cx_Oracle.CLOB: 'TextField',
        cx_Oracle.DATETIME: 'DateField',
        cx_Oracle.FIXED_CHAR: 'CharField',
        cx_Oracle.NCLOB: 'TextField',
        cx_Oracle.NUMBER: 'DecimalField',
        cx_Oracle.STRING: 'CharField',
        cx_Oracle.TIMESTAMP: 'DateTimeField',
    }

    try:
        data_types_reverse[cx_Oracle.NATIVE_FLOAT] = 'FloatField'
    except AttributeError:
        pass

    try:
        data_types_reverse[cx_Oracle.UNICODE] = 'CharField'
    except AttributeError:
        pass

    def get_field_type(self, data_type, description):
        # If it's a NUMBER with scale == 0, consider it an IntegerField
        if data_type == cx_Oracle.NUMBER and description[5] == 0:
            if description[4] > 11:
                return 'BigIntegerField'
            else:
                return 'IntegerField'
        else:
            return super(DatabaseIntrospection, self).get_field_type(
                data_type, description)

    def get_table_list(self, cursor):
        "Returns a list of table names in the current database."
        cursor.execute("SELECT TABLE_NAME FROM USER_TABLES")
        return [row[0].lower() for row in cursor.fetchall()]

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        cursor.execute("SELECT * FROM %s WHERE ROWNUM < 2" % self.connection.ops.quote_name(table_name))
        description = []
        for desc in cursor.description:
            description.append((desc[0].lower(),) + desc[1:])
        return description

    def table_name_converter(self, name):
        "Table name comparison is case insensitive under Oracle"
        return name.lower()

    def _name_to_index(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: field_index} for the given table.
        Indexes are 0-based.
        """
        return dict([(d[0], i) for i, d in enumerate(self.get_table_description(cursor, table_name))])

    def get_relations(self, cursor, table_name):
        """
        Returns a dictionary of {field_index: (field_index_other_table, other_table)}
        representing all relationships to the given table. Indexes are 0-based.
        """
        table_name = table_name.upper()
        cursor.execute("""
    SELECT ta.column_id - 1, tb.table_name, tb.column_id - 1
    FROM   user_constraints, USER_CONS_COLUMNS ca, USER_CONS_COLUMNS cb,
           user_tab_cols ta, user_tab_cols tb
    WHERE  user_constraints.table_name = %s AND
           ta.table_name = %s AND
           ta.column_name = ca.column_name AND
           ca.table_name = %s AND
           user_constraints.constraint_name = ca.constraint_name AND
           user_constraints.r_constraint_name = cb.constraint_name AND
           cb.table_name = tb.table_name AND
           cb.column_name = tb.column_name AND
           ca.position = cb.position""", [table_name, table_name, table_name])

        relations = {}
        for row in cursor.fetchall():
            relations[row[0]] = (row[2], row[1].lower())
        return relations

    def get_indexes(self, cursor, table_name):
        """
        Returns a dictionary of fieldname -> infodict for the given table,
        where each infodict is in the format:
            {'primary_key': boolean representing whether it's the primary key,
             'unique': boolean representing whether it's a unique index}
        """
        # This query retrieves each index on the given table, including the
        # first associated field name
        # "We were in the nick of time; you were in great peril!"
        sql = """\
SELECT LOWER(all_tab_cols.column_name) AS column_name,
       CASE user_constraints.constraint_type
           WHEN 'P' THEN 1 ELSE 0
       END AS is_primary_key,
       CASE user_indexes.uniqueness
           WHEN 'UNIQUE' THEN 1 ELSE 0
       END AS is_unique
FROM   all_tab_cols, user_cons_columns, user_constraints, user_ind_columns, user_indexes
WHERE  all_tab_cols.column_name = user_cons_columns.column_name (+)
  AND  all_tab_cols.table_name = user_cons_columns.table_name (+)
  AND  user_cons_columns.constraint_name = user_constraints.constraint_name (+)
  AND  user_constraints.constraint_type (+) = 'P'
  AND  user_ind_columns.column_name (+) = all_tab_cols.column_name
  AND  user_ind_columns.table_name (+) = all_tab_cols.table_name
  AND  user_indexes.uniqueness (+) = 'UNIQUE'
  AND  user_indexes.index_name (+) = user_ind_columns.index_name
  AND  all_tab_cols.table_name = UPPER(%s)
"""
        cursor.execute(sql, [table_name])
        indexes = {}
        for row in cursor.fetchall():
            indexes[row[0]] = {'primary_key': row[1], 'unique': row[2]}
        return indexes
<|endoftext|>


<commit_before>'''Automatic active(current/up-level) url highlighting(adding css class)
with django template tag
'''
from hashlib import md5
from lxml.html import fromstring, tostring
from django import template
from django.core.cache import cache
from classytags.core import Tag, Options
from classytags.arguments import MultiKeywordArgument
from django_activeurl import settings

register = template.Library()

class ActiveUrl(Tag):
    '''By default addition css_class is "active"
        for parent tag of all  which parent is
.
        Quick example
        {% activeurl %}

                    /page/

        {% endactiveurl %}
        will be rendered to

                    page

        if your current request.get_full_path() starts with /page/
        starts with logic decided for applying "active" status
        for up-level  in your menus/submenus
    '''

    name = 'activeurl'

    # template tag arguments
    options = Options(
        MultiKeywordArgument('kwargs', required=False),
        blocks=[('endactiveurl', 'nodelist')],
    )

    def check_active(self, url, el, full_path, css_class, parent_tag):
        '''check url "active" status, apply css_class to html element'''
        # check non empty href parameter
        if url.attrib.get('href'):
            # skip "root" url
            if url.attrib['href'] != '/':
                # compare href parameter with full path
                if full_path.startswith(url.attrib['href']):
                    # check parent tag has "class" attribute
                    if el.attrib.get('class'):
                        # prevent multiple "class" adding
                        if not css_class in el.attrib['class']:
                            # check for empty "class" attribute
                            if el.attrib['class']:
                                # append "active" class
                                el.attrib['class'] += ' ' + css_class
                            else:
                                # set "class" attribute
                                el.attrib['class'] = css_class
                    else:
                        # create "class" attribute
                        el.attrib['class'] = css_class
                    return True
        return False

    def render_tag(self, context, kwargs, nodelist):
        '''renders html with "active" urls inside template tag'''
        # set attributes from kwargs
        default_kwargs = settings.DEFAULT_KWARGS
        default_kwargs.update(kwargs)
        kwargs = default_kwargs
        css_class = kwargs['css_class']
        parent_tag = kwargs['parent_tag']

        # accept from template parent_tag values such as False, None, ''
        if not parent_tag:
            parent_tag = ''

        # flag for prevent html rebuilding, when no "active" urls found
        processed = False

        # get request from context
        request = context['request']

        # get full path from request
        full_path = request.get_full_path()

        # render content inside template tag
        context.push()
        content = nodelist.render(context)
        context.pop()

        # try to take rendered html with "active" urls from cache
        if settings.CACHE_ACTIVE_URL:
            data = content + css_class + parent_tag + full_path
            data = data.encode('utf-8', 'ignore')
            cache_key = settings.CACHE_ACTIVE_URL_PREFIX \
                + md5(data).hexdigest()

            from_cache = cache.get(cache_key)
            if from_cache:
                return from_cache

        # build html tree from content inside template tag
        tree = fromstring(content)

        # if parent_tag is False\None\empty, so "active" status will be applied
        # directly to
        if not parent_tag:
            # xpath query to get all
            urls = tree.xpath('//a')

            # check "active" status for all urls
            for url in urls:
                if self.check_active(url, url, full_path, css_class,
                                     parent_tag):
                    processed = True

            # prevent html rebuild if no one of urls is "active"
            if processed:
                # build html from tree
                content = tostring(tree)
        # otherwise css_class must be applied to parent_tag
        else:
            # xpath query to get all parents tags
            els = tree.xpath('//%s' % parent_tag)
            for el in els:
                # xpath query to get all  inside current tag
                urls = el.xpath('a')
                # check "active" status for all urls
                for url in urls:
                    if self.check_active(url, el, full_path, css_class,
                                         parent_tag):
                        # flag for rebuild html tree
                        processed = True
                        # stop check other  inside current parent_tag
                        break

            # do not rebuild html if no "active" urls inside parent_tag
            if processed:
                # build html from tree
                content = tostring(tree)

        # write rendered html to cache, if caching is enabled
        if settings.CACHE_ACTIVE_URL:
            cache.set(cache_key, content,
                      settings.CACHE_ACTIVE_URL_TIMEOUT)

        return content

# register new template tag
register.tag(ActiveUrl)
<commit_msg>Small logic fix<commit_after>'''Automatic active(current/up-level) url highlighting(adding css class)
with django template tag
'''
from hashlib import md5
from lxml.html import fromstring, tostring
from django import template
from django.core.cache import cache
from classytags.core import Tag, Options
from classytags.arguments import MultiKeywordArgument
from django_activeurl import settings

register = template.Library()

class ActiveUrl(Tag):
    '''By default addition css_class is "active"
        for parent tag of all  which parent is
.
        Quick example
        {% activeurl %}

                    /page/

        {% endactiveurl %}
        will be rendered to

                    page

        if your current request.get_full_path() starts with /page/
        starts with logic decided for applying "active" status
        for up-level  in your menus/submenus
    '''

    name = 'activeurl'

    # template tag arguments
    options = Options(
        MultiKeywordArgument('kwargs', required=False),
        blocks=[('endactiveurl', 'nodelist')],
    )

    def check_active(self, url, el, full_path, css_class, parent_tag):
        '''check url "active" status, apply css_class to html element'''
        # check non empty href parameter
        if url.attrib.get('href'):
            # skip "root" url
            if url.attrib['href'] != '/':
                # compare href parameter with full path
                if full_path.startswith(url.attrib['href']):
                    # check parent tag has "class" attribute or it is empty
                    if el.attrib.get('class'):
                        # prevent multiple "class" adding
                        if not css_class in el.attrib['class']:
                            # append "active" class
                            el.attrib['class'] += ' ' + css_class
                    else:
                        # create or set "class" attribute
                        el.attrib['class'] = css_class
                    return True
        return False

    def render_tag(self, context, kwargs, nodelist):
        '''renders html with "active" urls inside template tag'''
        # set attributes from kwargs
        default_kwargs = settings.DEFAULT_KWARGS
        default_kwargs.update(kwargs)
        kwargs = default_kwargs
        css_class = kwargs['css_class']
        parent_tag = kwargs['parent_tag']

        # accept from template parent_tag values such as False, None, ''
        if not parent_tag:
            parent_tag = ''

        # flag for prevent html rebuilding, when no "active" urls found
        processed = False

        # get request from context
        request = context['request']

        # get full path from request
        full_path = request.get_full_path()

        # render content inside template tag
        context.push()
        content = nodelist.render(context)
        context.pop()

        # try to take rendered html with "active" urls from cache
        if settings.CACHE_ACTIVE_URL:
            data = content + css_class + parent_tag + full_path
            data = data.encode('utf-8', 'ignore')
            cache_key = settings.CACHE_ACTIVE_URL_PREFIX \
                + md5(data).hexdigest()

            from_cache = cache.get(cache_key)
            if from_cache:
                return from_cache

        # build html tree from content inside template tag
        tree = fromstring(content)

        # if parent_tag is False\None\empty, so "active" status will be applied
        # directly to
        if not parent_tag:
            # xpath query to get all
            urls = tree.xpath('//a')

            # check "active" status for all urls
            for url in urls:
                if self.check_active(url, url, full_path, css_class,
                                     parent_tag):
                    processed = True

            # prevent html rebuild if no one of urls is "active"
            if processed:
                # build html from tree
                content = tostring(tree)
        # otherwise css_class must be applied to parent_tag
        else:
            # xpath query to get all parents tags
            els = tree.xpath('//%s' % parent_tag)
            for el in els:
                # xpath query to get all  inside current tag
                urls = el.xpath('a')
                # check "active" status for all urls
                for url in urls:
                    if self.check_active(url, el, full_path, css_class,
                                         parent_tag):
                        # flag for rebuild html tree
                        processed = True
                        # stop check other  inside current parent_tag
                        break

            # do not rebuild html if no "active" urls inside parent_tag
            if processed:
                # build html from tree
                content = tostring(tree)

        # write rendered html to cache, if caching is enabled
        if settings.CACHE_ACTIVE_URL:
            cache.set(cache_key, content,
                      settings.CACHE_ACTIVE_URL_TIMEOUT)

        return content

# register new template tag
register.tag(ActiveUrl)
<|endoftext|>