<commit_before>import re
from django.http import HttpResponse
from django.contrib import messages
from django.views.decorators.http import require_GET, require_POST
from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, render, redirect
from django.utils.translation import ugettext_lazy as _
from models import Device, Group, Product, Session, Result

@require_GET
@login_required
def devices(request):
    context = {}
    context['title'] = _('Devices')
    context['devices'] = Device.objects.all()
    return render(request, 'cygapp/devices.html', context)

@require_GET
@login_required
def device(request, deviceID):
    try:
        device = Device.objects.get(pk=deviceID)
    except Device.DoesNotExist:
        device = None
        messages.error(request, _('Device not found!'))

    context = {}
    context['devices'] = Device.objects.all().order_by('description')
    context['title'] = _('Devices')

    if device:
        context['device'] = device
        members = device.groups.all().order_by('name')
        context['members'] = members
        context['products'] = Product.objects.all().order_by('name')

        groups = Group.objects.exclude(id__in = members.values_list('id',
            flat=True))
        context['groups'] = groups
        context['title'] = _('Device ') + device.description

    return render(request, 'cygapp/devices.html', context)

@require_GET
@login_required
def add(request):
    context = {}
    context['title'] = _('New device')
    context['groups'] = Group.objects.all().order_by('name')
    context['products'] = Product.objects.all().order_by('name')
    context['devices'] = Device.objects.all()
    context['device'] = Device()
    return render(request, 'cygapp/devices.html', context)

@require_POST
@login_required
def save(request):
    deviceID = request.POST['deviceId']
    if not (deviceID == 'None' or re.match(r'^\d+$', deviceID)):
        return HttpResponse(status=400)

    members = []
    if request.POST['memberlist'] != '':
        members = request.POST['memberlist'].split(',')

    for member in members:
        if not re.match(r'^\d+$', member):
            return HttpResponse(status=400)

    value = request.POST['value'].lower()
    if not re.match(r'^[a-f0-9]+$', value):
        return HttpResponse(status=400)

    description = request.POST['description']
    if not re.match(r'^[\S ]{0,50}$', description):
        return HttpResponse(status=400)

    productID = request.POST['product']
    if not re.match(r'^\d+$', productID):
        return HttpResponse(status=400)

    try:
        product = Product.objects.get(pk=productID)
    except Product.DoesNotExist:
        return HttpResponse(status=400)

    if deviceID == 'None':
        device = Device.objects.create(value=value, description=description,
                product=product)
    else:
        device = get_object_or_404(Device, pk=deviceID)
        device.value = value
        device.description = description
        device.product = product
        device.save()

    if members:
        device.groups.clear()
        members = Group.objects.filter(id__in=members)
        for member in members:
            device.groups.add(member)

        device.save()

    messages.success(request, _('Device saved!'))
    return redirect('/devices/%d' % device.id)

@require_POST
@login_required
def delete(request, deviceID):
    device = get_object_or_404(Device, pk=deviceID)
    device.delete()

    messages.success(request, _('Device deleted!'))
    return redirect('/devices')

@require_GET
@login_required
def simulate(request, deviceID):
    device = get_object_or_404(Device, pk=deviceID)

    context = {}
    context['device'] = device
    context['title'] = _('Simulation report for ') + str(device)

    sessions = Session.objects.filter(device=device)
    context['session_count'] = len(sessions)
    context['group_set'] = device.get_group_set()

    if context['session_count'] > 0:
        session = sessions.latest('time')
        context['last_session'] = session.time
        context['last_user'] = session.identity.data
        context['last_result'] = session.recommendation
    else:
        context['last_session'] = _('Never')
        context['last_user'] = _('No one')
        context['last_result'] = _('N/A')

    enforcements = []
    for group in context['group_set']:
        for e in group.enforcements.all():
            try:
                result =  Result.objects.filter(
                        session__device=device,policy=e.policy).latest()
                enforcements.append((e, result.recommendation,
                    device.is_due_for(e)))
            except Result.DoesNotExist:
                enforcements.append((e, _('N/A'), device.is_due_for(e)))

    context['enforcements'] = enforcements

    return render(request, 'cygapp/device_report.html', context)
<commit_msg>Close tannerli/cygnet-doc#60<commit_after>import re
from datetime import datetime
from django.http import HttpResponse
from django.contrib import messages
from django.views.decorators.http import require_GET, require_POST
from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, render, redirect
from django.utils.translation import ugettext_lazy as _
from models import Device, Group, Product, Session, Result

@require_GET
@login_required
def devices(request):
    context = {}
    context['title'] = _('Devices')
    context['devices'] = Device.objects.all()
    return render(request, 'cygapp/devices.html', context)

@require_GET
@login_required
def device(request, deviceID):
    try:
        device = Device.objects.get(pk=deviceID)
    except Device.DoesNotExist:
        device = None
        messages.error(request, _('Device not found!'))

    context = {}
    context['devices'] = Device.objects.all().order_by('description')
    context['title'] = _('Devices')

    if device:
        context['device'] = device
        members = device.groups.all().order_by('name')
        context['members'] = members
        context['products'] = Product.objects.all().order_by('name')

        groups = Group.objects.exclude(id__in = members.values_list('id',
            flat=True))
        context['groups'] = groups
        context['title'] = _('Device ') + device.description

    return render(request, 'cygapp/devices.html', context)

@require_GET
@login_required
def add(request):
    context = {}
    context['title'] = _('New device')
    context['groups'] = Group.objects.all().order_by('name')
    context['products'] = Product.objects.all().order_by('name')
    context['devices'] = Device.objects.all()
    context['device'] = Device()
    return render(request, 'cygapp/devices.html', context)

@require_POST
@login_required
def save(request):
    deviceID = request.POST['deviceId']
    if not (deviceID == 'None' or re.match(r'^\d+$', deviceID)):
        return HttpResponse(status=400)

    members = []
    if request.POST['memberlist'] != '':
        members = request.POST['memberlist'].split(',')

    for member in members:
        if not re.match(r'^\d+$', member):
            return HttpResponse(status=400)

    value = request.POST['value'].lower()
    if not re.match(r'^[a-f0-9]+$', value):
        return HttpResponse(status=400)

    description = request.POST['description']
    if not re.match(r'^[\S ]{0,50}$', description):
        return HttpResponse(status=400)

    productID = request.POST['product']
    if not re.match(r'^\d+$', productID):
        return HttpResponse(status=400)

    try:
        product = Product.objects.get(pk=productID)
    except Product.DoesNotExist:
        return HttpResponse(status=400)

    if deviceID == 'None':
        device = Device.objects.create(value=value, description=description,
                product=product, created=datetime.today())
    else:
        device = get_object_or_404(Device, pk=deviceID)
        device.value = value
        device.description = description
        device.product = product
        device.save()

    if members:
        device.groups.clear()
        members = Group.objects.filter(id__in=members)
        for member in members:
            device.groups.add(member)

        device.save()

    messages.success(request, _('Device saved!'))
    return redirect('/devices/%d' % device.id)

@require_POST
@login_required
def delete(request, deviceID):
    device = get_object_or_404(Device, pk=deviceID)
    device.delete()

    messages.success(request, _('Device deleted!'))
    return redirect('/devices')

@require_GET
@login_required
def simulate(request, deviceID):
    device = get_object_or_404(Device, pk=deviceID)

    context = {}
    context['device'] = device
    context['title'] = _('Simulation report for ') + str(device)

    sessions = Session.objects.filter(device=device)
    context['session_count'] = len(sessions)
    context['group_set'] = device.get_group_set()

    if context['session_count'] > 0:
        session = sessions.latest('time')
        context['last_session'] = session.time
        context['last_user'] = session.identity.data
        context['last_result'] = session.recommendation
    else:
        context['last_session'] = _('Never')
        context['last_user'] = _('No one')
        context['last_result'] = _('N/A')

    enforcements = []
    for group in context['group_set']:
        for e in group.enforcements.all():
            try:
                result =  Result.objects.filter(
                        session__device=device,policy=e.policy).latest()
                enforcements.append((e, result.recommendation,
                    device.is_due_for(e)))
            except Result.DoesNotExist:
                enforcements.append((e, _('N/A'), device.is_due_for(e)))

    context['enforcements'] = enforcements

    return render(request, 'cygapp/device_report.html', context)
<|endoftext|>


<commit_before>import argparse

from .client import ArchivesSpaceClient
from .reporter import CSVReporter
from .validator import Validator

class DACSspace:
    """Base DACSspace class. Fetches data from AS, validates and reports results."""

    def run(self, published_only, invalid_only):
        client = ArchivesSpaceClient()
        validator = Validator()
        reporter = CSVReporter()
        data = client.get_resources(published_only)
        results = []
        for obj in data:
            result = validator.validate(obj)
            results.append(result)
        reporter.write_report(results, invalid_only)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Fetches data from AS, validates and reports results")
    parser.add_argument(
        '--published_only',
        help='Fetches only published records from AS',
        action='store_true')
    parser.add_argument(
        '--invalid_only',
        help='Reports only invalid data',
        action='store_true')
    args = parser.parse_args()

    DACSspace().run(args.published_only, args.invalid_only)
<commit_msg>update imports, add schema arguments<commit_after>import argparse

from client import ArchivesSpaceClient
from reporter import CSVReporter
from validator import Validator

class DACSspace:
    """Base DACSspace class. Fetches data from AS, validates and reports results."""

    def run(self, published_only, invalid_only,
            schema_identifier, schema_filepath):
        client = ArchivesSpaceClient()
        validator = Validator(schema_identifier, schema_filepath)
        reporter = CSVReporter()
        data = client.get_resources(published_only)
        results = []
        for obj in data:
            result = validator.validate(obj)
            results.append(result)
        reporter.write_report(results, invalid_only)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Fetches data from AS, validates and reports results")
    parser.add_argument(
        '--published_only',
        help='Fetches only published records from AS',
        action='store_true')
    parser.add_argument(
        '--invalid_only',
        help='Reports only invalid data',
        action='store_true')
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
        '--schema_identifier',
        help='Choose schema_identifier or schema_filepath. schema_identifier default is single_level_required.json',
        type=str, default='single_level_required.json')
    group.add_argument(
        '--schema_filepath',
        help='Choose schema_identifier or schema_filepath. Schema_filepath default is None, only one of schema_identifier',
        type=str, default=None)
    args = parser.parse_args()

DACSspace().run(
    args.published_only,
    args.invalid_only,
    args.schema_identifier,
    args.schema_filepath)
<|endoftext|>


<commit_before>import errno
import fcntl
import logging
import os
import signal
import sys
import time
import traceback

import fs
import sockinherit

logger = logging.getLogger(__name__)

class Daemon:
    """
    A generic daemon class.

    Usage: subclass the Daemon class and override the run() method
    """

    def __init__(self, pidfile, run=lambda: None,
                 stdin='/dev/null', stdout='/dev/null', stderr='/dev/null', foreground=False):

        self._run = run
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = pidfile
        # NOTE: We need to open another separate file to avoid the file
        #       reopened # again. Which case, process lose file lock.
        #
        # From "man fcntl":
        # As well as being removed by an explicit F_UNLCK, record locks are
        # automatically released when the process terminates  or  if  it
        # closes  any  file descriptor  referring  to  a  file  on which locks
        # are held.  This is bad: it means that a process can lose the locks
        # on a file like /etc/passwd or /etc/mtab when for some reason a
        # library function decides to open, read and close it.
        self.lockfile = pidfile + ".lock"
        self.lockfp = None
        self.foreground = foreground

    def daemonize(self):
        """
        do the UNIX double-fork magic, see Stevens' "Advanced
        Programming in the UNIX Environment" for details (ISBN 0201563177)
        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
        """

        if self.foreground:
            return

        try:

            pid = os.fork()
            if pid > 0:
                # exit first parent
                sys.exit(0)

        except OSError as e:

            sys.stderr.write("fork #1 failed: %d (%s)\n"
                             % (e.errno, e.strerror))
            sys.exit(1)

        # decouple from parent environment
        # os.chdir("/")
        os.setsid()
        os.umask(0)

        # do second fork
        try:

            pid = os.fork()
            if pid > 0:
                # exit from second parent
                sys.exit(0)

        except OSError as e:

            sys.stderr.write("fork #2 failed: %d (%s)\n"
                             % (e.errno, e.strerror))
            sys.exit(1)

        # redirect standard file descriptors
        sys.stdout.flush()
        sys.stderr.flush()
        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        se = file(self.stderr, 'a+', 0)
        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        logger.info("OK daemonized")

    def trylock_or_exit(self, timeout=10):

        interval = 0.1
        n = int(timeout / interval) + 1
        flag = fcntl.LOCK_EX | fcntl.LOCK_NB

        for ii in range(n):

            fd = os.open(self.lockfile, os.O_RDWR | os.O_CREAT)

            fcntl.fcntl(fd, fcntl.F_SETFD,
                        fcntl.fcntl(fd, fcntl.F_GETFD, 0)
                        | fcntl.FD_CLOEXEC)

            try:
                fcntl.lockf(fd, flag)

                self.lockfp = os.fdopen(fd, 'w+r')
                break

            except IOError as e:
                os.close(fd)
                if e[0] == errno.EAGAIN:
                    time.sleep(interval)
                else:
                    raise

        else:
            logger.info("Failure acquiring lock %s" % (self.lockfile, ))
            sys.exit(1)

        logger.info("OK acquired lock %s" % (self.lockfile))

    def unlock(self):

        if self.lockfp is None:
            return

        fd = self.lockfp.fileno()
        fcntl.lockf(fd, fcntl.LOCK_UN)
        self.lockfp.close()
        self.lockfp = None

    def start(self):

        self.daemonize()
        self.init_proc()
        self.run()

    def init_proc(self):
        self.trylock_or_exit()
        self.write_pid_or_exit()

    def write_pid_or_exit(self):

        self.pf = open(self.pidfile, 'w+r')
        pf = self.pf

        fd = pf.fileno()
        fcntl.fcntl(fd, fcntl.F_SETFD,
                    fcntl.fcntl(fd, fcntl.F_GETFD, 0)
                    | fcntl.FD_CLOEXEC)

        try:
            pid = os.getpid()
            logger.debug('write pid:' + str(pid))

            pf.truncate(0)
            pf.write(str(pid))
            pf.flush()
        except Exception as e:
            logger.error('write pid failed.' + repr(e))
            sys.exit(0)

    def stop_pid(self):
        return self.stop()

    def stop(self):

        pid = None
        if not os.path.exists(self.pidfile):

            logger.debug('pidfile not exist:' + self.pidfile)
            return

        while 1:

            try:
                pid = fs.read_file(self.pidfile)
                pid = int(pid)
                os.kill(pid, signal.SIGTERM)
                return

            except Exception as e:
                logger.debug('get pid failed.' + str(e) + str(pid))
                # file been deleted?
                break

            time.sleep(0.1)

    def restart(self):
        """
        Restart the daemon
        """

        self.stop()
        self.start()

    def run(self):
        """
        You should override this method when you subclass Daemon.
        It will be called after the process has been
        daemonized by start() or restart().
        """

        self._run()

        return

def standard_daemonize(run_func, pidfn,
                       inheritable=None,
                       upgrade_arg=None,
                       cleanup=None):

    d = Daemon(pidfn, None)

    logger.info("sys.argv: " + repr(sys.argv))

    try:
        if len(sys.argv) == 1:
            d.init_proc()
            run_func()

        elif len(sys.argv) == 2:

            if 'start' == sys.argv[1]:
                d.daemonize()
                if inheritable is not None:
                    sockinherit.init_inheritable(inheritable)
                    install_upgrade_sig(d, inheritable, upgrade_arg, cleanup)

                d.init_proc()
                run_func()

            elif 'stop' == sys.argv[1]:
                d.stop_pid()

            elif 'restart' == sys.argv[1]:
                if inheritable is None:
                    d.stop_pid()
                    d.daemonize()
                    d.init_proc()
                else:
                    signal_to_upgrade(pidfn)
                    # if it returns, upgrade failed. fall back to normal
                    # restart.

                    d.stop_pid()

                    d.daemonize()
                    sockinherit.init_inheritable(inheritable)
                    install_upgrade_sig(d, inheritable, upgrade_arg, cleanup)
                    d.init_proc()

                run_func()

            else:
                logger.error("Unknown command: %s" % (sys.argv[1]))
                print "Unknown command"
                sys.exit(2)

            sys.exit(0)
        else:
            print "usage: %s start|stop|restart" % sys.argv[0]
            sys.exit(2)

    except Exception as e:
        logger.error(traceback.format_exc())
        logger.error(repr(e))

def signal_to_upgrade(pidFile):

    pid = _get_pid(pidFile)
    logger.info("old pid: " + repr(pid))

    if pid is not None:

        try:
            os.kill(pid, signal.SIGUSR2)
        except OSError:
            # no such process
            return

        logger.info("SIGUSR2 sent to: " + repr(pid))

        for ii in range(30):
            time.sleep(0.1)
            pid2 = _get_pid(pidFile)
            if pid2 is not None and pid != pid2:
                logger.info("new pid generated, upgrade succeed: "
                            + repr(pid2))
                sys.exit(0)

def _get_pid(fn):

    try:
        with open(fn, 'r') as f:
            cont = f.read()
    except OSError:
        return None

    try:
        return int(cont)
    except ValueError:
        return None

def install_upgrade_sig(d, inheritable, upgrade_arg, cleanup):

    heir = {}
    for name, val in inheritable.items():
        heir[name] = val['sock'].fileno()

    def sig_up(*args, **argkv):
        if callable(upgrade_arg):
            exe, args, cwd = upgrade_arg()
        else:
            exe, args, cwd = upgrade_arg

        sockinherit.upgrade(exe, args, cwd, heir)

        logger.info("parent spwaned new process")
        logger.info("parent sleep 1 second")
        time.sleep(1)

        d.unlock()
        logger.info("parent released lock, about to exit")

        (cleanup or sys.exit)()

    signal.signal(signal.SIGUSR2, sig_up)
<commit_msg>simplify<commit_after>import errno
import fcntl
import logging
import os
import signal
import sys
import time

logger = logging.getLogger(__name__)

class Daemon(object):

    def __init__(self,
                 pidfile,
                 stdin='/dev/null',
                 stdout='/dev/null',
                 stderr='/dev/null'):

        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = pidfile
        # NOTE: We need to open another separate file to avoid the file
        #       reopened # again. Which case, process lose file lock.
        #
        # From "man fcntl":
        # As well as being removed by an explicit F_UNLCK, record locks are
        # automatically released when the process terminates  or  if  it
        # closes  any  file descriptor  referring  to  a  file  on which locks
        # are held.  This is bad: it means that a process can lose the locks
        # on a file like /etc/passwd or /etc/mtab when for some reason a
        # library function decides to open, read and close it.
        self.lockfile = pidfile + ".lock"
        self.lockfp = None

    def daemonize(self):
        """
        do the UNIX double-fork magic, see Stevens' "Advanced
        Programming in the UNIX Environment" for details (ISBN 0201563177)
        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
        """

        try:

            pid = os.fork()
            if pid > 0:
                # exit first parent
                sys.exit(0)

        except OSError as e:
            logger.error("fork #1 failed: " + repr(e))
            sys.exit(1)

        # decouple from parent environment
        os.setsid()
        os.umask(0)

        # do second fork
        try:

            pid = os.fork()
            if pid > 0:
                # exit from second parent
                sys.exit(0)

        except OSError as e:
            logger.error("fork #2 failed: " + repr(e))
            sys.exit(1)

        # redirect standard file descriptors
        sys.stdout.flush()
        sys.stderr.flush()
        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        se = file(self.stderr, 'a+', 0)
        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        logger.info("OK daemonized")

    def trylock_or_exit(self, timeout=10):

        interval = 0.1
        n = int(timeout / interval) + 1
        flag = fcntl.LOCK_EX | fcntl.LOCK_NB

        for ii in range(n):

            fd = os.open(self.lockfile, os.O_RDWR | os.O_CREAT)

            fcntl.fcntl(fd, fcntl.F_SETFD,
                        fcntl.fcntl(fd, fcntl.F_GETFD, 0)
                        | fcntl.FD_CLOEXEC)

            try:
                fcntl.lockf(fd, flag)

                self.lockfp = os.fdopen(fd, 'w+r')
                break

            except IOError as e:
                os.close(fd)
                if e[0] == errno.EAGAIN:
                    time.sleep(interval)
                else:
                    raise

        else:
            logger.info("Failure acquiring lock %s" % (self.lockfile, ))
            sys.exit(1)

        logger.info("OK acquired lock %s" % (self.lockfile))

    def unlock(self):

        if self.lockfp is None:
            return

        fd = self.lockfp.fileno()
        fcntl.lockf(fd, fcntl.LOCK_UN)
        self.lockfp.close()
        self.lockfp = None

    def start(self):

        self.daemonize()
        self.init_proc()

    def init_proc(self):
        self.trylock_or_exit()
        self.write_pid_or_exit()

    def write_pid_or_exit(self):

        self.pf = open(self.pidfile, 'w+r')
        pf = self.pf

        fd = pf.fileno()
        fcntl.fcntl(fd, fcntl.F_SETFD,
                    fcntl.fcntl(fd, fcntl.F_GETFD, 0)
                    | fcntl.FD_CLOEXEC)

        try:
            pid = os.getpid()
            logger.debug('write pid:' + str(pid))

            pf.truncate(0)
            pf.write(str(pid))
            pf.flush()
        except Exception as e:
            logger.exception('write pid failed.' + repr(e))
            sys.exit(0)

    def stop(self):

        pid = None

        if not os.path.exists(self.pidfile):

            logger.debug('pidfile not exist:' + self.pidfile)
            return

        try:
            pid = _read_file(self.pidfile)
            pid = int(pid)
            os.kill(pid, signal.SIGTERM)
            return

        except Exception as e:
            logger.warn('{e} while get and kill pid={pid}'.format(
                e=repr(e), pid=pid))

def _read_file(fn):
    with open(fn, 'r') as f:
        return f.read()

def standard_daemonize(run_func, pidfn):

    logging.basicConfig(stream=sys.stderr)
    logging.getLogger(__name__).setLevel(logging.DEBUG)

    d = Daemon(pidfn, None)

    logger.info("sys.argv: " + repr(sys.argv))

    try:
        if len(sys.argv) == 1:
            d.init_proc()
            run_func()

        elif len(sys.argv) == 2:

            if 'start' == sys.argv[1]:
                d.start()
                run_func()

            elif 'stop' == sys.argv[1]:
                d.stop()

            elif 'restart' == sys.argv[1]:
                d.stop()
                d.start()
                run_func()

            else:
                logger.error("Unknown command: %s" % (sys.argv[1]))
                print "Unknown command"
                sys.exit(2)

            sys.exit(0)
        else:
            print "usage: %s start|stop|restart" % sys.argv[0]
            sys.exit(2)

    except Exception as e:
        logger.exception(repr(e))
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""An abstract base class for all Dakota experiments."""

from abc import ABCMeta, abstractmethod
import types
import yaml

class DakotaBase(object):

    """Describe features common to all Dakota experiments."""

    __metaclass__ = ABCMeta

    @abstractmethod
    def __init__(self, component=None, template_file=None,
                 method=None, configuration_file='config.yaml',
                 run_directory='.', input_files=None,
                 data_file='dakota.dat',
                 variable_type='continuous_design',
                 variable_descriptors=None, interface='direct',
                 analysis_driver='rosenbrock',
                 is_objective_function=False, response_descriptors=None,
                 response_files=None, response_statistics=None, **kwargs):
        """Create a set of default experiment parameters."""
        self.component = component
        self.configuration_file = configuration_file
        self.run_directory = run_directory
        self.template_file = template_file
        self.input_files = input_files
        self.data_file = data_file
        self.method = method
        self.variable_type = variable_type
        self.variable_descriptors = [] if variable_descriptors is None \
                                    else variable_descriptors
        self.interface = interface
        self.analysis_driver = analysis_driver
        self.parameters_file = 'params.in'
        self.results_file = 'results.out'
        self.is_objective_function = is_objective_function
        self.response_descriptors = [] if response_descriptors is None \
                                    else response_descriptors
        self.response_files = response_files
        self.response_statistics = response_statistics

    @classmethod
    def from_file_like(cls, file_like):
        """Create a DakotaBase instance from a file-like object.

        Parameters
        ----------
        file_like : file_like
            Input parameter file.

        Returns
        -------
        DakotaBase
            A new DakotaBase instance.

        """
        config = {}
        if isinstance(file_like, types.StringTypes):
            with open(file_like, 'r') as fp:
                config = yaml.load(fp.read())
        else:
            config = yaml.load(file_like)
        return cls(**config)

    def environment_block(self):
        """Define the environment block of a Dakota input file."""
        s = '# Dakota input file\n' \
            + 'environment\n' \
            + '  tabular_data\n' \
            + '    tabular_data_file = {!r}\n\n'.format(self.data_file)
        return(s)

    @abstractmethod
    def method_block(self):
        """Define the method block of a Dakota input file."""
        s = 'method\n' \
            + '  {}\n\n'.format(self.method)
        return(s)

    def variables_block(self):
        """Define the variables block of a Dakota input file."""
        s = 'variables\n' \
            + ' {0} = {1}\n'.format(self.variable_type,
                                    len(self.variable_descriptors))
        s += '    descriptors ='
        for vd in self.variable_descriptors:
            s += ' {!r}'.format(vd)
        s += '\n\n'
        return(s)

    def interface_block(self):
        """Define the interface block of a Dakota input file."""
        s = 'interface\n' \
            + '  {}\n'.format(self.interface) \
            + '  analysis_driver = {!r}\n'.format(self.analysis_driver)
        if self.component is not None:
            s += '  analysis_components = {!r}\n'.format(self.configuration_file)
        if self.interface is not 'direct':
            s += '  parameters_file = {!r}\n'.format(self.parameters_file) \
                 + '  results_file = {!r}\n'.format(self.results_file) \
                 + '  work_directory\n' \
                 + '    named \'run\'\n' \
                 + '    directory_tag\n' \
                 + '    directory_save\n' \
                 + '  file_save\n'
        s += '\n'
        return(s)

    def responses_block(self):
        """Define the responses block of a Dakota input file."""
        n_responses = len(self.response_descriptors)
        s = 'responses\n'
        if self.is_objective_function:
            s += '  objective_functions = {}\n'.format(n_responses)
        else:
            s += '  response_functions = {}\n'.format(n_responses)
        s += '    response_descriptors ='
        for rd in self.response_descriptors:
            s += ' {!r}'.format(rd)
        s += '\n' \
             + '  no_gradients\n' \
             + '  no_hessians\n'
        return(s)
<commit_msg>Make response_descriptors a property<commit_after>#! /usr/bin/env python
"""An abstract base class for all Dakota experiments."""

from abc import ABCMeta, abstractmethod
import types
import yaml

class DakotaBase(object):

    """Describe features common to all Dakota experiments."""

    __metaclass__ = ABCMeta

    @abstractmethod
    def __init__(self,
                 component=None,
                 template_file=None,
                 method=None,
                 configuration_file='config.yaml',
                 run_directory='.',
                 input_files=None,
                 data_file='dakota.dat',
                 variable_type='continuous_design',
                 variable_descriptors=None,
                 interface='direct',
                 analysis_driver='rosenbrock',
                 is_objective_function=False,
                 response_descriptors=(),
                 response_files=None,
                 response_statistics=None,
                 **kwargs):
        """Create a set of default experiment parameters."""
        self.component = component
        self.configuration_file = configuration_file
        self.run_directory = run_directory
        self.template_file = template_file
        self.input_files = input_files
        self.data_file = data_file
        self.method = method
        self.variable_type = variable_type
        self.variable_descriptors = [] if variable_descriptors is None \
                                    else variable_descriptors
        self.interface = interface
        self.analysis_driver = analysis_driver
        self.parameters_file = 'params.in'
        self.results_file = 'results.out'
        self.is_objective_function = is_objective_function
        self._response_descriptors = response_descriptors
        self.response_files = response_files
        self.response_statistics = response_statistics

    @property
    def response_descriptors(self):
        """Labels attached to Dakota responses."""
        return self._response_descriptors

    @response_descriptors.setter
    def response_descriptors(self, value):
        """Set labels for Dakota responses.

        Parameters
        ----------
        value : list or tuple of str
          The new response labels.

        """
        if not isinstance(value, (tuple, list)):
            raise TypeError("Descriptor must be a tuple or a list")
        self._response_descriptors = value

    @classmethod
    def from_file_like(cls, file_like):
        """Create a DakotaBase instance from a file-like object.

        Parameters
        ----------
        file_like : file_like
            Input parameter file.

        Returns
        -------
        DakotaBase
            A new DakotaBase instance.

        """
        config = {}
        if isinstance(file_like, types.StringTypes):
            with open(file_like, 'r') as fp:
                config = yaml.load(fp.read())
        else:
            config = yaml.load(file_like)
        return cls(**config)

    def environment_block(self):
        """Define the environment block of a Dakota input file."""
        s = '# Dakota input file\n' \
            + 'environment\n' \
            + '  tabular_data\n' \
            + '    tabular_data_file = {!r}\n\n'.format(self.data_file)
        return(s)

    @abstractmethod
    def method_block(self):
        """Define the method block of a Dakota input file."""
        s = 'method\n' \
            + '  {}\n\n'.format(self.method)
        return(s)

    def variables_block(self):
        """Define the variables block of a Dakota input file."""
        s = 'variables\n' \
            + ' {0} = {1}\n'.format(self.variable_type,
                                    len(self.variable_descriptors))
        s += '    descriptors ='
        for vd in self.variable_descriptors:
            s += ' {!r}'.format(vd)
        s += '\n\n'
        return(s)

    def interface_block(self):
        """Define the interface block of a Dakota input file."""
        s = 'interface\n' \
            + '  {}\n'.format(self.interface) \
            + '  analysis_driver = {!r}\n'.format(self.analysis_driver)
        if self.component is not None:
            s += '  analysis_components = {!r}\n'.format(self.configuration_file)
        if self.interface is not 'direct':
            s += '  parameters_file = {!r}\n'.format(self.parameters_file) \
                 + '  results_file = {!r}\n'.format(self.results_file) \
                 + '  work_directory\n' \
                 + '    named \'run\'\n' \
                 + '    directory_tag\n' \
                 + '    directory_save\n' \
                 + '  file_save\n'
        s += '\n'
        return(s)

    def responses_block(self):
        """Define the responses block of a Dakota input file."""
        n_responses = len(self.response_descriptors)
        s = 'responses\n'
        if self.is_objective_function:
            s += '  objective_functions = {}\n'.format(n_responses)
        else:
            s += '  response_functions = {}\n'.format(n_responses)
        s += '    response_descriptors ='
        for rd in self.response_descriptors:
            s += ' {!r}'.format(rd)
        s += '\n' \
             + '  no_gradients\n' \
             + '  no_hessians\n'
        return(s)
<|endoftext|>


<commit_before>from celery.execute import send_task
from django.db import models
from ckeditor.fields import HTMLField
from gallery.managers import GalleryManager

class Gallery(models.Model):
    name = models.CharField(max_length=100)
    slug = models.SlugField(unique=True)
    description = HTMLField(null=True, blank=True)
    parent_gallery = models.ForeignKey('self', null=True, blank=True)
    published = models.BooleanField(default=False)
    order = models.PositiveSmallIntegerField(blank=True, null=True)
    date_created = models.DateTimeField(auto_now_add=True)
    date_modified = models.DateTimeField(auto_now=True)

    objects = GalleryManager()

    class Meta:
        verbose_name_plural = "galleries"
        ordering = ('order',)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ("gallery_detail", (), {"gallery": self.slug})

    def picture_count(self):
        """Returns the number of pictures in the gallery."""
        return Picture.objects.filter(gallery=self).count()
    picture_count.short_description = 'Number of Pictures'

class Picture(models.Model):
    name = models.CharField(max_length=100)
    slug = models.SlugField(unique=True)
    original = models.ImageField(upload_to='original')
    viewable = models.ImageField(upload_to='viewable')
    thumbnail = models.ImageField(upload_to='thumbnail')
    description = HTMLField(null=True, blank=True)
    gallery = models.ForeignKey(Gallery)
    order = models.PositiveSmallIntegerField(null=True, blank=True)
    date_created = models.DateTimeField(auto_now_add=True)
    date_modified = models.DateTimeField(auto_now=True)

    class Meta:
        ordering = ('order',)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ("picture_detail", (), {
            "picture": self.slug,
            "gallery": self.gallery.slug,
        })

    def save(self, **kwargs):
        """
        Saves a Picture instance. Return True if a thumbnail and viewable is
        generated, False otherwise.
        """
        # Determine if new file, if not, get path of old original image
        if self.id is not None:
            old_path = Picture.objects.get(pk=self.id).original.path
        else:
            old_path = None

        #Save the image before kicking off the celery task (we need the id)
        super(Picture, self).save(**kwargs)

        # If new Picture or change original file
        # create a celery task to generate the viewable and thumbnail.
        if old_path is None or self.original.path != old_path:
            send_task("tasks.GenerateImages", self.id)

<commit_msg>Fixed bug with creating job for a celery task.<commit_after>from celery.execute import send_task
from django.db import models
from ckeditor.fields import HTMLField
from gallery.managers import GalleryManager

class Gallery(models.Model):
    name = models.CharField(max_length=100)
    slug = models.SlugField(unique=True)
    description = HTMLField(null=True, blank=True)
    parent_gallery = models.ForeignKey('self', null=True, blank=True)
    published = models.BooleanField(default=False)
    order = models.PositiveSmallIntegerField(blank=True, null=True)
    date_created = models.DateTimeField(auto_now_add=True)
    date_modified = models.DateTimeField(auto_now=True)

    objects = GalleryManager()

    class Meta:
        verbose_name_plural = "galleries"
        ordering = ('order',)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ("gallery_detail", (), {"gallery": self.slug})

    def picture_count(self):
        """Returns the number of pictures in the gallery."""
        return Picture.objects.filter(gallery=self).count()
    picture_count.short_description = 'Number of Pictures'

class Picture(models.Model):
    name = models.CharField(max_length=100)
    slug = models.SlugField(unique=True)
    original = models.ImageField(upload_to='original')
    viewable = models.ImageField(upload_to='viewable')
    thumbnail = models.ImageField(upload_to='thumbnail')
    description = HTMLField(null=True, blank=True)
    gallery = models.ForeignKey(Gallery)
    order = models.PositiveSmallIntegerField(null=True, blank=True)
    date_created = models.DateTimeField(auto_now_add=True)
    date_modified = models.DateTimeField(auto_now=True)

    class Meta:
        ordering = ('order',)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ("picture_detail", (), {
            "picture": self.slug,
            "gallery": self.gallery.slug,
        })

    def save(self, **kwargs):
        """
        Saves a Picture instance. Return True if a thumbnail and viewable is
        generated, False otherwise.
        """
        # Determine if new file, if not, get path of old original image
        if self.id is not None:
            old_path = Picture.objects.get(pk=self.id).original.path
        else:
            old_path = None

        #Save the image before kicking off the celery task (we need the id)
        super(Picture, self).save(**kwargs)

        # If new Picture or change original file
        # create a celery task to generate the viewable and thumbnail.
        if old_path is None or self.original.path != old_path:
            send_task("gallery.tasks.GenerateImages", [self.id])

<|endoftext|>


<commit_before>__version__ = "1.13.0"

_js_dist_dependencies = [
    {
        "external_url": {
            "prod": [
                "https://unpkg.com/@babel/polyfill@7.12.1/dist/polyfill.min.js",
                "https://unpkg.com/react@16.14.0/umd/react.production.min.js",
                "https://unpkg.com/react-dom@16.14.0/umd/react-dom.production.min.js",
                "https://unpkg.com/prop-types@15.8.1/prop-types.min.js",
            ],
            "dev": [
                "https://unpkg.com/@babel/polyfill@7.12.1/dist/polyfill.min.js",
                "https://unpkg.com/react@16.14.0/umd/react.development.js",
                "https://unpkg.com/react-dom@16.14.0/umd/react-dom.development.js",
                "https://unpkg.com/prop-types@15.8.1/prop-types.js",
            ],
        },
        "relative_package_path": {
            "prod": [
                "deps/polyfill@7.12.1.min.js",
                "deps/react@16.14.0.min.js",
                "deps/react-dom@16.14.0.min.js",
                "deps/prop-types@15.8.1.min.js",
            ],
            "dev": [
                "deps/polyfill@7.12.1.min.js",
                "deps/react@16.14.0.js",
                "deps/react-dom@16.14.0.js",
                "deps/prop-types@15.8.1.js",
            ],
        },
        "namespace": "dash",
    }
]

_js_dist = [
    {
        "relative_package_path": "dash-renderer/build/dash_renderer.min.js",
        "dev_package_path": "dash-renderer/build/dash_renderer.dev.js",
        "external_url": "https://unpkg.com/dash-renderer@1.13.0"
        "/build/dash_renderer.min.js",
        "namespace": "dash",
    },
    {
        "relative_package_path": "dash-renderer/build/dash_renderer.min.js.map",
        "dev_package_path": "dash-renderer/build/dash_renderer.dev.js.map",
        "namespace": "dash",
        "dynamic": True,
    },
]
<commit_msg>Bump dash renderer version.<commit_after>__version__ = "1.14.0"

_js_dist_dependencies = [
    {
        "external_url": {
            "prod": [
                "https://unpkg.com/@babel/polyfill@7.12.1/dist/polyfill.min.js",
                "https://unpkg.com/react@16.14.0/umd/react.production.min.js",
                "https://unpkg.com/react-dom@16.14.0/umd/react-dom.production.min.js",
                "https://unpkg.com/prop-types@15.8.1/prop-types.min.js",
            ],
            "dev": [
                "https://unpkg.com/@babel/polyfill@7.12.1/dist/polyfill.min.js",
                "https://unpkg.com/react@16.14.0/umd/react.development.js",
                "https://unpkg.com/react-dom@16.14.0/umd/react-dom.development.js",
                "https://unpkg.com/prop-types@15.8.1/prop-types.js",
            ],
        },
        "relative_package_path": {
            "prod": [
                "deps/polyfill@7.12.1.min.js",
                "deps/react@16.14.0.min.js",
                "deps/react-dom@16.14.0.min.js",
                "deps/prop-types@15.8.1.min.js",
            ],
            "dev": [
                "deps/polyfill@7.12.1.min.js",
                "deps/react@16.14.0.js",
                "deps/react-dom@16.14.0.js",
                "deps/prop-types@15.8.1.js",
            ],
        },
        "namespace": "dash",
    }
]

_js_dist = [
    {
        "relative_package_path": "dash-renderer/build/dash_renderer.min.js",
        "dev_package_path": "dash-renderer/build/dash_renderer.dev.js",
        "external_url": "https://unpkg.com/dash-renderer@1.14.0"
        "/build/dash_renderer.min.js",
        "namespace": "dash",
    },
    {
        "relative_package_path": "dash-renderer/build/dash_renderer.min.js.map",
        "dev_package_path": "dash-renderer/build/dash_renderer.dev.js.map",
        "namespace": "dash",
        "dynamic": True,
    },
]
<|endoftext|>


<commit_before>#!/usr/bin/env python

DESC = """EPP script to copy 'concentration' and 'concentration unit' for each
sample sample in the 'Qubit Result File' to the 'Concentration' and 'Conc. Units'
fields of the output analytes of the process.

Warnings are generated to the user and stored in regular log file wich allso
contains regular execution information in the folowing cases:

1) missing row names (samples) in file
2) duplicated row names (samples)
3) missing value (concentrations)
4) values found but for some reason are not successfully copied:

Can be executed in the background or triggered by a user pressing a "blue button".

Written by Maya Brandi
"""

import os
import sys
import logging
import numpy as np

from argparse import ArgumentParser
from requests import HTTPError
from genologics.lims import Lims
from genologics.config import BASEURI,USERNAME,PASSWORD
from genologics.entities import Process
from genologics.epp import EppLogger
from genologics.epp import ReadResultFiles
from genologics.epp import set_field

def main(lims, pid, epp_logger):
    process = Process(lims,id = pid)
    sample_names = map(lambda a: a.name, process.analytes()[0])
    target_files = process.result_files()
    file_handler = ReadResultFiles(process)
    files = file_handler.shared_files['Qubit Result File']
    qubit_result_file = file_handler.format_file(files,
                                                 name = 'Qubit Result File',
                                                 first_header = 'Sample',
                                                 find_keys = sample_names)
    missing_samples = 0
    low_conc = 0
    bad_formated = 0
    abstract = []
    udfs = dict(process.udf.items())
    if udfs.has_key("Minimum required concentration (ng/ul)"):
        min_conc = udfs["Minimum required concentration (ng/ul)"]
    else:
        min_conc = None
        abstract.append("Set 'Minimum required concentration (ng/ul)' to get qc-flaggs based on this treshold!")
    for target_file in target_files:
        print '*****'
        print dir(target_file)
#        print target_file.container
#        print target_file.files
#        print target_file.location
#        print target_file.root
#        print dir(target_file)
        sample = target_file.samples[0].name
        print target_file.samples
        if qubit_result_file.has_key(sample):
            sample_mesurements = qubit_result_file[sample]
            if "Sample Concentration" in sample_mesurements.keys():
                conc, unit = sample_mesurements["Sample Concentration"]
                if conc == 'Out Of Range':
                    target_file.qc_flag = "FAILED"
                elif conc.replace('.','').isdigit():
                    conc = float(conc)
                    if unit == 'ng/mL':
                        conc = np.true_divide(conc, 1000)
                    if min_conc:
                        if conc < min_conc:
                            target_file.qc_flag = "FAILED"
                            low_conc +=1
                        else:
                            target_file.qc_flag = "PASSED"
                    target_file.udf['Concentration'] = conc
                    target_file.udf['Conc. Units'] = 'ng/ul'
                else:
                    bad_formated += 1
                set_field(target_file)
        else:
            missing_samples += 1

    if low_conc:
        abstract.append('{0}/{1} samples have low concentration.'.format(low_conc, len(target_files)))
    if missing_samples:
        abstract.append('{0}/{1} samples are missing in Qubit Result File.'.format(missing_samples, len(target_files)))
    if bad_formated:
        abstract.append('There are {0} badly formated samples in Qubit Result File. Please fix these to get proper results.'.format(bad_formated))

    print >> sys.stderr, ' '.join(abstract)

if __name__ == "__main__":
    parser = ArgumentParser(description=DESC)
    parser.add_argument('--pid', default = '24-38458', dest = 'pid',
                        help='Lims id for current Process')
    parser.add_argument('--log', dest = 'log',
                        help=('File name for standard log file, '
                              'for runtime information and problems.'))

    args = parser.parse_args()

    lims = Lims(BASEURI,USERNAME,PASSWORD)
    lims.check_version()

    with EppLogger(log_file=args.log, lims=lims, prepend=True) as epp_logger:
        main(lims, args.pid, epp_logger)

<commit_msg>debugging missing pool names<commit_after>#!/usr/bin/env python

DESC = """EPP script to copy 'concentration' and 'concentration unit' for each
sample sample in the 'Qubit Result File' to the 'Concentration' and 'Conc. Units'
fields of the output analytes of the process.

Warnings are generated to the user and stored in regular log file wich allso
contains regular execution information in the folowing cases:

1) missing row names (samples) in file
2) duplicated row names (samples)
3) missing value (concentrations)
4) values found but for some reason are not successfully copied:

Can be executed in the background or triggered by a user pressing a "blue button".

Written by Maya Brandi
"""

import os
import sys
import logging
import numpy as np

from argparse import ArgumentParser
from requests import HTTPError
from genologics.lims import Lims
from genologics.config import BASEURI,USERNAME,PASSWORD
from genologics.entities import Process
from genologics.epp import EppLogger
from genologics.epp import ReadResultFiles
from genologics.epp import set_field

def main(lims, pid, epp_logger):
    process = Process(lims,id = pid)
    sample_names = map(lambda a: a.name, process.analytes()[0])
    target_files = process.result_files()
    file_handler = ReadResultFiles(process)
    files = file_handler.shared_files['Qubit Result File']
    qubit_result_file = file_handler.format_file(files,
                                                 name = 'Qubit Result File',
                                                 first_header = 'Sample',
                                                 find_keys = sample_names)
    missing_samples = 0
    low_conc = 0
    bad_formated = 0
    abstract = []
    udfs = dict(process.udf.items())
    if udfs.has_key("Minimum required concentration (ng/ul)"):
        min_conc = udfs["Minimum required concentration (ng/ul)"]
    else:
        min_conc = None
        abstract.append("Set 'Minimum required concentration (ng/ul)' to get qc-flaggs based on this treshold!")
    for target_file in target_files:
        print '*****'
        print dir(target_file)
        try:
            print target_file.container
        except:
            print 'no  target_file.container'
        try:
            print target_file.files
        except:
            print 'no files'
        try:
            print target_file.location
        except:
            print 'no loc'
#        print target_file.root
#        print dir(target_file)
        sample = target_file.samples[0].name
        print target_file.samples
        if qubit_result_file.has_key(sample):
            sample_mesurements = qubit_result_file[sample]
            if "Sample Concentration" in sample_mesurements.keys():
                conc, unit = sample_mesurements["Sample Concentration"]
                if conc == 'Out Of Range':
                    target_file.qc_flag = "FAILED"
                elif conc.replace('.','').isdigit():
                    conc = float(conc)
                    if unit == 'ng/mL':
                        conc = np.true_divide(conc, 1000)
                    if min_conc:
                        if conc < min_conc:
                            target_file.qc_flag = "FAILED"
                            low_conc +=1
                        else:
                            target_file.qc_flag = "PASSED"
                    target_file.udf['Concentration'] = conc
                    target_file.udf['Conc. Units'] = 'ng/ul'
                else:
                    bad_formated += 1
                set_field(target_file)
        else:
            missing_samples += 1

    if low_conc:
        abstract.append('{0}/{1} samples have low concentration.'.format(low_conc, len(target_files)))
    if missing_samples:
        abstract.append('{0}/{1} samples are missing in Qubit Result File.'.format(missing_samples, len(target_files)))
    if bad_formated:
        abstract.append('There are {0} badly formated samples in Qubit Result File. Please fix these to get proper results.'.format(bad_formated))

    print >> sys.stderr, ' '.join(abstract)

if __name__ == "__main__":
    parser = ArgumentParser(description=DESC)
    parser.add_argument('--pid', default = '24-38458', dest = 'pid',
                        help='Lims id for current Process')
    parser.add_argument('--log', dest = 'log',
                        help=('File name for standard log file, '
                              'for runtime information and problems.'))

    args = parser.parse_args()

    lims = Lims(BASEURI,USERNAME,PASSWORD)
    lims.check_version()

    with EppLogger(log_file=args.log, lims=lims, prepend=True) as epp_logger:
        main(lims, args.pid, epp_logger)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import logging
import traceback
import click
from os import path

# Add top-level module path to sys.path before importing tubular code.
sys.path.append( path.dirname( path.dirname( path.abspath(__file__) ) ) )

from tubular import asgard

logging.basicConfig(stream=sys.stdout, level=logging.INFO)

@click.command()
@click.option('--asg_name', envvar='ASG_NAME', help='the name of the Autoscale Group to delete', required=True)
def delete_asg(asg_name):
    try:
        asgard.delete_asg(asg_name, True)
    except Exception, e:
        traceback.print_exc()
        click.secho("Error Deploying AMI: {0}.\nMessage: {1}".format(asg_name, e.message), fg='red')
        sys.exit(1)

    sys.exit(0)

if __name__ == "__main__":
    delete_asg()
<commit_msg>fixed typo in delete-asg.py script<commit_after>#!/usr/bin/env python
import sys
import logging
import traceback
import click
from os import path

# Add top-level module path to sys.path before importing tubular code.
sys.path.append( path.dirname( path.dirname( path.abspath(__file__) ) ) )

from tubular import asgard

logging.basicConfig(stream=sys.stdout, level=logging.INFO)

@click.command()
@click.option('--asg_name', envvar='ASG_NAME', help='the name of the Autoscale Group to delete', required=True)
def delete_asg(asg_name):
    try:
        asgard.delete_asg(asg_name, True)
    except Exception, e:
        traceback.print_exc()
        click.secho("Error Deleting ASG: {0}.\nMessage: {1}".format(asg_name, e.message), fg='red')
        sys.exit(1)

    sys.exit(0)

if __name__ == "__main__":
    delete_asg()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    Lexing error finder
    ~~~~~~~~~~~~~~~~~~~

    For the source files given on the command line, display
    the text where Error tokens are being generated, along
    with some context.

    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys, os

try:
    import pygments
except ImportError:
    # try parent path
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from pygments import highlight
from pygments.lexer import RegexLexer
from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
from pygments.token import Error, _TokenType

class DebuggingRegexLexer(RegexLexer):
    """Make the state stack, position and current match instance attributes."""

    def get_tokens_unprocessed(self, text, stack=('root',)):
        """
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        self.pos = 0
        tokendefs = self._tokens
        self.statestack = list(stack)
        statetokens = tokendefs[self.statestack[-1]]
        while 1:
            for rexmatch, action, new_state in statetokens:
                self.m = rexmatch(text, self.pos)
                if self.m:
                    if type(action) is _TokenType:
                        yield self.pos, action, self.m.group()
                    else:
                        for item in action(self, self.m):
                            yield item
                    self.pos = self.m.end()
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            for state in new_state:
                                if state == '#pop':
                                    self.statestack.pop()
                                elif state == '#push':
                                    self.statestack.append(self.statestack[-1])
                                else:
                                    self.statestack.append(state)
                        elif isinstance(new_state, int):
                            # pop
                            del self.statestack[new_state:]
                        elif new_state == '#push':
                            self.statestack.append(self.statestack[-1])
                        else:
                            assert False, 'wrong state def: %r' % new_state
                        statetokens = tokendefs[self.statestack[-1]]
                    break
            else:
                try:
                    if text[self.pos] == '\n':
                        # at EOL, reset state to 'root'
                        self.pos += 1
                        self.statestack = ['root']
                        statetokens = tokendefs['root']
                        yield self.pos, Text, u'\n'
                        continue
                    yield self.pos, Error, text[self.pos]
                    self.pos += 1
                except IndexError:
                    break

def main(fn):
    try:
        lx = get_lexer_for_filename(os.path.basename(fn))
    except ValueError:
        try:
            name, rest = fn.split('_', 1)
            lx = get_lexer_by_name(name)
        except ValueError:
            raise AssertionError('no lexer found for file %r' % fn)
    debug_lexer = False
    # does not work for e.g. ExtendedRegexLexers
    if lx.__class__.__bases__ == (RegexLexer,):
        lx.__class__.__bases__ = (DebuggingRegexLexer,)
        debug_lexer = True
    lno = 1
    text = file(fn, 'U').read()
    text = text.strip('\n') + '\n'
    text = text.decode('latin1')
    ntext = []
    states = []
    for type, val in lx.get_tokens(text):
        lno += val.count('\n')
        if type == Error:
            print 'Error parsing', fn, 'on line', lno
            print 'Previous tokens' + (debug_lexer and ' and states' or '') + ':'
            for i in range(len(ntext) - num, len(ntext)):
                reprs = map(repr, ntext[i])
                print '   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0],
                if debug_lexer:
                    print ' ' + ' ' * (29-len(reprs[0])) + repr(states[i]),
                print
            print 'Error token:'
            l = len(repr(val))
            print '   ' + repr(val),
            if debug_lexer:
                print ' ' * (60-l) + repr(lx.statestack),
            print
            print
            return 1
        ntext.append((type,val))
        if debug_lexer:
            states.append(lx.statestack[:])
    return 0

num = 10

if __name__ == '__main__':
    if sys.argv[1][:2] == '-n':
        num = int(sys.argv[1][2:])
        del sys.argv[1]
    ret = 0
    for f in sys.argv[1:]:
        ret += main(f)
    sys.exit(bool(ret))
<commit_msg>Add an option to show all tokens.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    Lexing error finder
    ~~~~~~~~~~~~~~~~~~~

    For the source files given on the command line, display
    the text where Error tokens are being generated, along
    with some context.

    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys, os

try:
    import pygments
except ImportError:
    # try parent path
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from pygments.lexer import RegexLexer
from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
from pygments.token import Error, Text, _TokenType

class DebuggingRegexLexer(RegexLexer):
    """Make the state stack, position and current match instance attributes."""

    def get_tokens_unprocessed(self, text, stack=('root',)):
        """
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        self.pos = 0
        tokendefs = self._tokens
        self.statestack = list(stack)
        statetokens = tokendefs[self.statestack[-1]]
        while 1:
            for rexmatch, action, new_state in statetokens:
                self.m = rexmatch(text, self.pos)
                if self.m:
                    if type(action) is _TokenType:
                        yield self.pos, action, self.m.group()
                    else:
                        for item in action(self, self.m):
                            yield item
                    self.pos = self.m.end()
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            for state in new_state:
                                if state == '#pop':
                                    self.statestack.pop()
                                elif state == '#push':
                                    self.statestack.append(self.statestack[-1])
                                else:
                                    self.statestack.append(state)
                        elif isinstance(new_state, int):
                            # pop
                            del self.statestack[new_state:]
                        elif new_state == '#push':
                            self.statestack.append(self.statestack[-1])
                        else:
                            assert False, 'wrong state def: %r' % new_state
                        statetokens = tokendefs[self.statestack[-1]]
                    break
            else:
                try:
                    if text[self.pos] == '\n':
                        # at EOL, reset state to 'root'
                        self.pos += 1
                        self.statestack = ['root']
                        statetokens = tokendefs['root']
                        yield self.pos, Text, u'\n'
                        continue
                    yield self.pos, Error, text[self.pos]
                    self.pos += 1
                except IndexError:
                    break

def main(fn):
    try:
        lx = get_lexer_for_filename(os.path.basename(fn))
    except ValueError:
        try:
            name, rest = fn.split('_', 1)
            lx = get_lexer_by_name(name)
        except ValueError:
            raise AssertionError('no lexer found for file %r' % fn)
    debug_lexer = False
    # does not work for e.g. ExtendedRegexLexers
    if lx.__class__.__bases__ == (RegexLexer,):
        lx.__class__.__bases__ = (DebuggingRegexLexer,)
        debug_lexer = True
    lno = 1
    text = file(fn, 'U').read()
    text = text.strip('\n') + '\n'
    text = text.decode('latin1')
    ntext = []
    states = []

    def show_token(tok):
        reprs = map(repr, tok)
        print '   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0],
        if debug_lexer:
            print ' ' + ' ' * (29-len(reprs[0])) + repr(states[i]),
        print

    for type, val in lx.get_tokens(text):
        lno += val.count('\n')
        if type == Error:
            print 'Error parsing', fn, 'on line', lno
            print 'Previous tokens' + (debug_lexer and ' and states' or '') + ':'
            for i in range(len(ntext) - num, len(ntext)):
                show_token(ntext[i])
            print 'Error token:'
            l = len(repr(val))
            print '   ' + repr(val),
            if debug_lexer:
                print ' ' * (60-l) + repr(lx.statestack),
            print
            print
            return 1
        ntext.append((type,val))
        if debug_lexer:
            states.append(lx.statestack[:])
    if showall:
        for tok in ntext:
            show_token(tok)
    return 0

num = 10
showall = False

if __name__ == '__main__':
    import getopt
    opts, args = getopt.getopt(sys.argv[1:], 'n:a')
    for opt, val in opts:
        if opt == '-n':
            num = int(val)
        elif opt == '-a':
            showall = True
    ret = 0
    for f in args:
        ret += main(f)
    sys.exit(bool(ret))
<|endoftext|>


<commit_before>"""TensorFlow workspace initialization. Consult the WORKSPACE on how to use it."""

load("//third_party/googleapis:repository_rules.bzl", "config_googleapis")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_toolchains//repositories:repositories.bzl", bazel_toolchains_repositories = "repositories")
load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies")
load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
load("@local_config_android//:android.bzl", "android_workspace")
load("@rules_cc//cc:repositories.bzl", "rules_cc_toolchains")

def _tf_bind():
    """Bind targets for some external repositories"""
    ##############################################################################
    # BIND DEFINITIONS
    #
    # Please do not add bind() definitions unless we have no other choice.
    # If that ends up being the case, please leave a comment explaining
    # why we can't depend on the canonical build target.

    # Needed by Protobuf
    native.bind(
        name = "grpc_cpp_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin",
    )
    native.bind(
        name = "grpc_python_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_python_plugin",
    )

    native.bind(
        name = "grpc_lib",
        actual = "@com_github_grpc_grpc//:grpc++",
    )

    native.bind(
        name = "grpc_lib_unsecure",
        actual = "@com_github_grpc_grpc//:grpc++_unsecure",
    )

    # Needed by Protobuf
    native.bind(
        name = "python_headers",
        actual = str(Label("//third_party/python_runtime:headers")),
    )

    # Needed by Protobuf
    native.bind(
        name = "six",
        actual = "@six_archive//:six",
    )

def workspace():
    http_archive(
        name = "inception_v1",
        build_file = "//:models.BUILD",
        sha256 = "7efe12a8363f09bc24d7b7a450304a15655a57a7751929b2c1593a71183bb105",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/inception_v1.zip",
        ],
    )

    http_archive(
        name = "mobile_ssd",
        build_file = "//:models.BUILD",
        sha256 = "bddd81ea5c80a97adfac1c9f770e6f55cbafd7cce4d3bbe15fbeb041e6b8f3e8",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_android_export.zip",
        ],
    )

    http_archive(
        name = "mobile_multibox",
        build_file = "//:models.BUILD",
        sha256 = "859edcddf84dddb974c36c36cfc1f74555148e9c9213dedacf1d6b613ad52b96",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/mobile_multibox_v1a.zip",
        ],
    )

    http_archive(
        name = "stylize",
        build_file = "//:models.BUILD",
        sha256 = "3d374a730aef330424a356a8d4f04d8a54277c425e274ecb7d9c83aa912c6bfa",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/stylize_v1.zip",
        ],
    )

    http_archive(
        name = "speech_commands",
        build_file = "//:models.BUILD",
        sha256 = "c3ec4fea3158eb111f1d932336351edfe8bd515bb6e87aad4f25dbad0a600d0c",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/speech_commands_v0.01.zip",
        ],
    )

    http_archive(
        name = "person_detect_data",
        sha256 = "170542270da256994ce24d1e357f6e84a54fdaf7d28ff2b74725a40b70b082cf",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_grayscale_2020_05_24.zip",
        ],
    )

    rules_cc_toolchains()

    bazel_toolchains_repositories()

    # Use `swift_rules_dependencies` to fetch the toolchains. With the
    # `git_repository` rules above, the following call will skip redefining them.
    swift_rules_dependencies()

    android_workspace()

    # If a target is bound twice, the later one wins, so we have to do tf bindings
    # at the end of the WORKSPACE file.
    _tf_bind()

    grpc_extra_deps()
    config_googleapis()

# Alias so it can be loaded without assigning to a different symbol to prevent
# shadowing previous loads and trigger a buildifier warning.
tf_workspace0 = workspace
<commit_msg>Updates Apple rules for Bazel<commit_after>"""TensorFlow workspace initialization. Consult the WORKSPACE on how to use it."""

load("//third_party/googleapis:repository_rules.bzl", "config_googleapis")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_toolchains//repositories:repositories.bzl", bazel_toolchains_repositories = "repositories")
load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies")
load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies")
load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies")
load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
load("@local_config_android//:android.bzl", "android_workspace")
load("@rules_cc//cc:repositories.bzl", "rules_cc_toolchains")

def _tf_bind():
    """Bind targets for some external repositories"""
    ##############################################################################
    # BIND DEFINITIONS
    #
    # Please do not add bind() definitions unless we have no other choice.
    # If that ends up being the case, please leave a comment explaining
    # why we can't depend on the canonical build target.

    # Needed by Protobuf
    native.bind(
        name = "grpc_cpp_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin",
    )
    native.bind(
        name = "grpc_python_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_python_plugin",
    )

    native.bind(
        name = "grpc_lib",
        actual = "@com_github_grpc_grpc//:grpc++",
    )

    native.bind(
        name = "grpc_lib_unsecure",
        actual = "@com_github_grpc_grpc//:grpc++_unsecure",
    )

    # Needed by Protobuf
    native.bind(
        name = "python_headers",
        actual = str(Label("//third_party/python_runtime:headers")),
    )

    # Needed by Protobuf
    native.bind(
        name = "six",
        actual = "@six_archive//:six",
    )

def workspace():
    http_archive(
        name = "inception_v1",
        build_file = "//:models.BUILD",
        sha256 = "7efe12a8363f09bc24d7b7a450304a15655a57a7751929b2c1593a71183bb105",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/inception_v1.zip",
        ],
    )

    http_archive(
        name = "mobile_ssd",
        build_file = "//:models.BUILD",
        sha256 = "bddd81ea5c80a97adfac1c9f770e6f55cbafd7cce4d3bbe15fbeb041e6b8f3e8",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_android_export.zip",
        ],
    )

    http_archive(
        name = "mobile_multibox",
        build_file = "//:models.BUILD",
        sha256 = "859edcddf84dddb974c36c36cfc1f74555148e9c9213dedacf1d6b613ad52b96",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/mobile_multibox_v1a.zip",
        ],
    )

    http_archive(
        name = "stylize",
        build_file = "//:models.BUILD",
        sha256 = "3d374a730aef330424a356a8d4f04d8a54277c425e274ecb7d9c83aa912c6bfa",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/stylize_v1.zip",
        ],
    )

    http_archive(
        name = "speech_commands",
        build_file = "//:models.BUILD",
        sha256 = "c3ec4fea3158eb111f1d932336351edfe8bd515bb6e87aad4f25dbad0a600d0c",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/models/speech_commands_v0.01.zip",
        ],
    )

    http_archive(
        name = "person_detect_data",
        sha256 = "170542270da256994ce24d1e357f6e84a54fdaf7d28ff2b74725a40b70b082cf",
        urls = [
            "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_grayscale_2020_05_24.zip",
        ],
    )

    rules_cc_toolchains()

    bazel_toolchains_repositories()

    # Apple rules for Bazel. https://github.com/bazelbuild/rules_apple.
    apple_rules_dependencies()
    swift_rules_dependencies()
    apple_support_dependencies()

    android_workspace()

    # If a target is bound twice, the later one wins, so we have to do tf bindings
    # at the end of the WORKSPACE file.
    _tf_bind()

    grpc_extra_deps()
    config_googleapis()

# Alias so it can be loaded without assigning to a different symbol to prevent
# shadowing previous loads and trigger a buildifier warning.
tf_workspace0 = workspace
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

import tensorflow as tf

def identity(x, name=None):
    """The identity activation function, Shortcut is ``linear``.

    Parameters
    ----------
    x : a tensor input
        input(s)

    Returns
    --------
    A `Tensor` with the same type as `x`.
    """
    return x

# Shortcut
linear = identity

def ramp(x=None, v_min=0, v_max=1, name=None):
    """The ramp activation function.

    Parameters
    ----------
    x : a tensor input
        input(s)
    v_min : float
        if input(s) smaller than v_min, change inputs to v_min
    v_max : float
        if input(s) greater than v_max, change inputs to v_max
    name : a string or None
        An optional name to attach to this activation function.

    Returns
    --------
    A `Tensor` with the same type as `x`.
    """
    return tf.clip_by_value(x, clip_value_min=v_min, clip_value_max=v_max, name=name)

def leaky_relu(x=None, alpha=0.1, name="LeakyReLU"):
    """The LeakyReLU, Shortcut is ``lrelu``.

    Modified version of ReLU, introducing a nonzero gradient for negative
    input.

    Parameters
    ----------
    x : A `Tensor` with type `float`, `double`, `int32`, `int64`, `uint8`,
        `int16`, or `int8`.
    alpha : `float`. slope.
    name : a string or None
        An optional name to attach to this activation function.

    Examples
    ---------
    >>> network = tl.layers.DenseLayer(network, n_units=100, name = 'dense_lrelu',
    ...                 act= lambda x : tl.act.lrelu(x, 0.2))

    References
    ------------
    - `Rectifier Nonlinearities Improve Neural Network Acoustic Models, Maas et al. (2013) <http://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf>`_
    """
    with tf.name_scope(name) as scope:
        # x = tf.nn.relu(x)
        # m_x = tf.nn.relu(-x)
        # x -= alpha * m_x
        x = tf.maximum(x, alpha * x)
    return x

#Shortcut
lrelu = leaky_relu

def swish(x, name='swish'):
    """The Swish function, see `Swish: a Self-Gated Activation Function <https://arxiv.org/abs/1710.05941>`_.

    Parameters
    ----------
    x : a tensor input
        input(s)

    Returns
    --------
    A `Tensor` with the same type as `x`.
    """
    with tf.name_scope(name) as scope:
        x =  tf.nn.sigmoid(x) * x
    return x

def pixel_wise_softmax(output, name='pixel_wise_softmax'):
    """Return the softmax outputs of images, every pixels have multiple label, the sum of a pixel is 1.
    Usually be used for image segmentation.

    Parameters
    ------------
    output : tensor
        - For 2d image, 4D tensor [batch_size, height, weight, channel], channel >= 2.
        - For 3d image, 5D tensor [batch_size, depth, height, weight, channel], channel >= 2.

    Examples
    ---------
    >>> outputs = pixel_wise_softmax(network.outputs)
    >>> dice_loss = 1 - dice_coe(outputs, y_, epsilon=1e-5)

    References
    -----------
    - `tf.reverse <https://www.tensorflow.org/versions/master/api_docs/python/array_ops.html#reverse>`_
    """
    with tf.name_scope(name) as scope:
        return tf.nn.softmax(output)
        ## old implementation
        # exp_map = tf.exp(output)
        # if output.get_shape().ndims == 4:   # 2d image
        #     evidence = tf.add(exp_map, tf.reverse(exp_map, [False, False, False, True]))
        # elif output.get_shape().ndims == 5: # 3d image
        #     evidence = tf.add(exp_map, tf.reverse(exp_map, [False, False, False, False, True]))
        # else:
        #     raise Exception("output parameters should be 2d or 3d image, not %s" % str(output._shape))
        # return tf.div(exp_map, evidence)
<commit_msg>scope name - lrelu<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

import tensorflow as tf

def identity(x, name=None):
    """The identity activation function, Shortcut is ``linear``.

    Parameters
    ----------
    x : a tensor input
        input(s)

    Returns
    --------
    A `Tensor` with the same type as `x`.
    """
    return x

# Shortcut
linear = identity

def ramp(x=None, v_min=0, v_max=1, name=None):
    """The ramp activation function.

    Parameters
    ----------
    x : a tensor input
        input(s)
    v_min : float
        if input(s) smaller than v_min, change inputs to v_min
    v_max : float
        if input(s) greater than v_max, change inputs to v_max
    name : a string or None
        An optional name to attach to this activation function.

    Returns
    --------
    A `Tensor` with the same type as `x`.
    """
    return tf.clip_by_value(x, clip_value_min=v_min, clip_value_max=v_max, name=name)

def leaky_relu(x=None, alpha=0.1, name="lrelu"):
    """The LeakyReLU, Shortcut is ``lrelu``.

    Modified version of ReLU, introducing a nonzero gradient for negative
    input.

    Parameters
    ----------
    x : A `Tensor` with type `float`, `double`, `int32`, `int64`, `uint8`,
        `int16`, or `int8`.
    alpha : `float`. slope.
    name : a string or None
        An optional name to attach to this activation function.

    Examples
    ---------
    >>> network = tl.layers.DenseLayer(network, n_units=100, name = 'dense_lrelu',
    ...                 act= lambda x : tl.act.lrelu(x, 0.2))

    References
    ------------
    - `Rectifier Nonlinearities Improve Neural Network Acoustic Models, Maas et al. (2013) <http://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf>`_
    """
    # with tf.name_scope(name) as scope:
        # x = tf.nn.relu(x)
        # m_x = tf.nn.relu(-x)
        # x -= alpha * m_x
    x = tf.maximum(x, alpha * x, name=name)
    return x

#Shortcut
lrelu = leaky_relu

def swish(x, name='swish'):
    """The Swish function, see `Swish: a Self-Gated Activation Function <https://arxiv.org/abs/1710.05941>`_.

    Parameters
    ----------
    x : a tensor input
        input(s)

    Returns
    --------
    A `Tensor` with the same type as `x`.
    """
    with tf.name_scope(name) as scope:
        x =  tf.nn.sigmoid(x) * x
    return x

def pixel_wise_softmax(output, name='pixel_wise_softmax'):
    """Return the softmax outputs of images, every pixels have multiple label, the sum of a pixel is 1.
    Usually be used for image segmentation.

    Parameters
    ------------
    output : tensor
        - For 2d image, 4D tensor [batch_size, height, weight, channel], channel >= 2.
        - For 3d image, 5D tensor [batch_size, depth, height, weight, channel], channel >= 2.

    Examples
    ---------
    >>> outputs = pixel_wise_softmax(network.outputs)
    >>> dice_loss = 1 - dice_coe(outputs, y_, epsilon=1e-5)

    References
    -----------
    - `tf.reverse <https://www.tensorflow.org/versions/master/api_docs/python/array_ops.html#reverse>`_
    """
    with tf.name_scope(name) as scope:
        return tf.nn.softmax(output)
        ## old implementation
        # exp_map = tf.exp(output)
        # if output.get_shape().ndims == 4:   # 2d image
        #     evidence = tf.add(exp_map, tf.reverse(exp_map, [False, False, False, True]))
        # elif output.get_shape().ndims == 5: # 3d image
        #     evidence = tf.add(exp_map, tf.reverse(exp_map, [False, False, False, False, True]))
        # else:
        #     raise Exception("output parameters should be 2d or 3d image, not %s" % str(output._shape))
        # return tf.div(exp_map, evidence)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# File: pool.py
# Author: Yuxin Wu <ppwwyyxx@gmail.com>
import tensorflow as tf
import numpy as np

from .common import layer_register
from ..utils.argtools import shape2d, shape4d
from ._test import TestModel

__all__ = ['MaxPooling', 'FixedUnPooling', 'AvgPooling', 'GlobalAvgPooling',
           'BilinearUpSample']

def _Pooling(func, x, shape, stride, padding, data_format):
    padding = padding.upper()
    shape = shape4d(shape, data_format=data_format)
    if stride is None:
        stride = shape
    else:
        stride = shape4d(stride, data_format=data_format)

    return func(x, ksize=shape,
                strides=stride, padding=padding,
                data_format=data_format,
                name='output')

@layer_register()
def MaxPooling(x, shape, stride=None, padding='VALID', data_format='NHWC'):
    """
    Max Pooling on 4D tensors.

    Args:
        x (tf.Tensor): a 4D tensor.
        shape: int or (h, w) tuple
        stride: int or (h, w) tuple. Defaults to be the same as shape.
        padding (str): 'valid' or 'same'.

    Returns:
        tf.Tensor named ``output``.
    """
    return _Pooling(tf.nn.max_pool, x, shape, stride, padding,
                    data_format=data_format)

@layer_register()
def AvgPooling(x, shape, stride=None, padding='VALID', data_format='NHWC'):
    """
    Average Pooling on 4D tensors.

    Args:
        x (tf.Tensor): a 4D tensor.
        shape: int or (h, w) tuple
        stride: int or (h, w) tuple. Defaults to be the same as shape.
        padding (str): 'valid' or 'same'.

    Returns:
        tf.Tensor named ``output``.
    """
    return _Pooling(tf.nn.avg_pool, x, shape, stride, padding,
                    data_format=data_format)

@layer_register()
def GlobalAvgPooling(x, data_format='NHWC'):
    """
    Global average pooling as in the paper `Network In Network
    <http://arxiv.org/abs/1312.4400>`_.

    Args:
        x (tf.Tensor): a NHWC tensor.
    Returns:
        tf.Tensor: a NC tensor named ``output``.
    """
    assert x.get_shape().ndims == 4
    assert data_format in ['NHWC', 'NCHW']
    axis = [1, 2] if data_format == 'NHWC' else [2, 3]
    return tf.reduce_mean(x, axis, name='output')

def UnPooling2x2ZeroFilled(x):
    # https://github.com/tensorflow/tensorflow/issues/2169
    out = tf.concat([x, tf.zeros_like(x)], 3)
    out = tf.concat([out, tf.zeros_like(out)], 2)

    sh = x.get_shape().as_list()
    if None not in sh[1:]:
        out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]]
        return tf.reshape(out, out_size)
    else:
        shv = tf.shape(x)
        ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]]))
        ret.set_shape([None, None, None, sh[3]])
        return ret

@layer_register()
def FixedUnPooling(x, shape, unpool_mat=None):
    """
    Unpool the input with a fixed matrix to perform kronecker product with.

    Args:
        x (tf.Tensor): a NHWC tensor
        shape: int or (h, w) tuple
        unpool_mat: a tf.Tensor or np.ndarray 2D matrix with size=shape.
            If is None, will use a matrix with 1 at top-left corner.

    Returns:
        tf.Tensor: a NHWC tensor.
    """
    shape = shape2d(shape)

    # a faster implementation for this special case
    if shape[0] == 2 and shape[1] == 2 and unpool_mat is None:
        return UnPooling2x2ZeroFilled(x)

    input_shape = tf.shape(x)
    if unpool_mat is None:
        mat = np.zeros(shape, dtype='float32')
        mat[0][0] = 1
        unpool_mat = tf.constant(mat, name='unpool_mat')
    elif isinstance(unpool_mat, np.ndarray):
        unpool_mat = tf.constant(unpool_mat, name='unpool_mat')
    assert unpool_mat.get_shape().as_list() == list(shape)

    # perform a tensor-matrix kronecker product
    fx = tf.reshape(tf.transpose(x, [0, 3, 1, 2]), [-1])
    fx = tf.expand_dims(fx, -1)       # (bchw)x1
    mat = tf.expand_dims(tf.reshape(unpool_mat, [-1]), 0)  # 1x(shxsw)
    prod = tf.matmul(fx, mat)  # (bchw) x(shxsw)
    prod = tf.reshape(prod, tf.stack(
        [-1, input_shape[3], input_shape[1], input_shape[2], shape[0], shape[1]]))
    prod = tf.transpose(prod, [0, 2, 4, 3, 5, 1])
    prod = tf.reshape(prod, tf.stack(
        [-1, input_shape[1] * shape[0], input_shape[2] * shape[1], input_shape[3]]))
    return prod

@layer_register()
def BilinearUpSample(x, shape):
    """
    Deterministic bilinearly-upsample the input images.

    Args:
        x (tf.Tensor): a NHWC tensor
        shape (int): the upsample factor

    Returns:
        tf.Tensor: a NHWC tensor.
    """
    inp_shape = x.get_shape().as_list()
    ch = inp_shape[3]
    assert ch is not None

    shape = int(shape)
    filter_shape = 2 * shape

    def bilinear_conv_filler(s):
        """
        s: width, height of the conv filter
        See https://github.com/BVLC/caffe/blob/master/include%2Fcaffe%2Ffiller.hpp#L244
        """
        f = np.ceil(float(s) / 2)
        c = float(2 * f - 1 - f % 2) / (2 * f)
        ret = np.zeros((s, s), dtype='float32')
        for x in range(s):
            for y in range(s):
                ret[x, y] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
        return ret
    w = bilinear_conv_filler(filter_shape)
    w = np.repeat(w, ch * ch).reshape((filter_shape, filter_shape, ch, ch))

    weight_var = tf.constant(w, tf.float32,
                             shape=(filter_shape, filter_shape, ch, ch),
                             name='bilinear_upsample_filter')
    x = tf.pad(x, [[0, 0], [shape - 1, shape - 1], [shape - 1, shape - 1], [0, 0]], mode='SYMMETRIC')
    out_shape = tf.shape(x) * tf.constant([1, shape, shape, 1], tf.int32)
    deconv = tf.nn.conv2d_transpose(x, weight_var, out_shape,
                                    [1, shape, shape, 1], 'SAME')
    edge = shape * (shape - 1)
    deconv = deconv[:, edge:-edge, edge:-edge, :]

    if inp_shape[1]:
        inp_shape[1] *= shape
    if inp_shape[2]:
        inp_shape[2] *= shape
    deconv.set_shape(inp_shape)
    return deconv

class TestPool(TestModel):
    def test_FixedUnPooling(self):
        h, w = 3, 4
        mat = np.random.rand(h, w, 3).astype('float32')
        inp = self.make_variable(mat)
        inp = tf.reshape(inp, [1, h, w, 3])
        output = FixedUnPooling('unpool', inp, 2)
        res = self.run_variable(output)
        self.assertEqual(res.shape, (1, 2 * h, 2 * w, 3))

        # mat is on cornser
        ele = res[0, ::2, ::2, 0]
        self.assertTrue((ele == mat[:, :, 0]).all())
        # the rest are zeros
        res[0, ::2, ::2, :] = 0
        self.assertTrue((res == 0).all())

    def test_BilinearUpSample(self):
        h, w = 12, 12
        scale = 2

        mat = np.random.rand(h, w).astype('float32')
        inp = self.make_variable(mat)
        inp = tf.reshape(inp, [1, h, w, 1])

        output = BilinearUpSample('upsample', inp, scale)
        res = self.run_variable(output)[0, :, :, 0]

        from skimage.transform import rescale
        res2 = rescale(mat, scale, mode='edge')

        diff = np.abs(res2 - res)

        # if not diff.max() < 1e-4:
        #     import IPython
        #     IPython.embed(config=IPython.terminal.ipapp.load_default_config())
        self.assertTrue(diff.max() < 1e-4, diff.max())
<commit_msg>simplify FixedUnpooling with tensordot, add NCHW support (but not for the special 2x2 case) (fix #314)<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# File: pool.py
# Author: Yuxin Wu <ppwwyyxx@gmail.com>
import tensorflow as tf
import numpy as np

from .common import layer_register
from ..utils.argtools import shape2d, shape4d
from ._test import TestModel

__all__ = ['MaxPooling', 'FixedUnPooling', 'AvgPooling', 'GlobalAvgPooling',
           'BilinearUpSample']

def _Pooling(func, x, shape, stride, padding, data_format):
    padding = padding.upper()
    shape = shape4d(shape, data_format=data_format)
    if stride is None:
        stride = shape
    else:
        stride = shape4d(stride, data_format=data_format)

    return func(x, ksize=shape,
                strides=stride, padding=padding,
                data_format=data_format,
                name='output')

@layer_register()
def MaxPooling(x, shape, stride=None, padding='VALID', data_format='NHWC'):
    """
    Max Pooling on 4D tensors.

    Args:
        x (tf.Tensor): a 4D tensor.
        shape: int or (h, w) tuple
        stride: int or (h, w) tuple. Defaults to be the same as shape.
        padding (str): 'valid' or 'same'.

    Returns:
        tf.Tensor named ``output``.
    """
    return _Pooling(tf.nn.max_pool, x, shape, stride, padding,
                    data_format=data_format)

@layer_register()
def AvgPooling(x, shape, stride=None, padding='VALID', data_format='NHWC'):
    """
    Average Pooling on 4D tensors.

    Args:
        x (tf.Tensor): a 4D tensor.
        shape: int or (h, w) tuple
        stride: int or (h, w) tuple. Defaults to be the same as shape.
        padding (str): 'valid' or 'same'.

    Returns:
        tf.Tensor named ``output``.
    """
    return _Pooling(tf.nn.avg_pool, x, shape, stride, padding,
                    data_format=data_format)

@layer_register()
def GlobalAvgPooling(x, data_format='NHWC'):
    """
    Global average pooling as in the paper `Network In Network
    <http://arxiv.org/abs/1312.4400>`_.

    Args:
        x (tf.Tensor): a NHWC tensor.
    Returns:
        tf.Tensor: a NC tensor named ``output``.
    """
    assert x.get_shape().ndims == 4
    assert data_format in ['NHWC', 'NCHW']
    axis = [1, 2] if data_format == 'NHWC' else [2, 3]
    return tf.reduce_mean(x, axis, name='output')

def UnPooling2x2ZeroFilled(x):
    # https://github.com/tensorflow/tensorflow/issues/2169
    out = tf.concat([x, tf.zeros_like(x)], 3)
    out = tf.concat([out, tf.zeros_like(out)], 2)

    sh = x.get_shape().as_list()
    if None not in sh[1:]:
        out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]]
        return tf.reshape(out, out_size)
    else:
        shv = tf.shape(x)
        ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]]))
        ret.set_shape([None, None, None, sh[3]])
        return ret

@layer_register()
def FixedUnPooling(x, shape, unpool_mat=None, data_format='NHWC'):
    """
    Unpool the input with a fixed matrix to perform kronecker product with.

    Args:
        x (tf.Tensor): a 4D image tensor
        shape: int or (h, w) tuple
        unpool_mat: a tf.Tensor or np.ndarray 2D matrix with size=shape.
            If is None, will use a matrix with 1 at top-left corner.

    Returns:
        tf.Tensor: a 4D image tensor.
    """
    shape = shape2d(shape)

    # a faster implementation for this special case
    if shape[0] == 2 and shape[1] == 2 and unpool_mat is None and data_format == 'NHWC':
        return UnPooling2x2ZeroFilled(x)

    input_shape = tf.shape(x)
    if unpool_mat is None:
        mat = np.zeros(shape, dtype='float32')
        mat[0][0] = 1
        unpool_mat = tf.constant(mat, name='unpool_mat')
    elif isinstance(unpool_mat, np.ndarray):
        unpool_mat = tf.constant(unpool_mat, name='unpool_mat')
    assert unpool_mat.get_shape().as_list() == list(shape)

    if data_format == 'NHWC':
        x = tf.transpose(x, [0, 3, 1, 2])
    # perform a tensor-matrix kronecker product
    x = tf.expand_dims(x, -1)       # bchwx1
    mat = tf.expand_dims(unpool_mat, 0)  # 1xshxsw
    prod = tf.tensordot(x, mat, axes=1)  # bxcxhxwxshxsw
    if data_format == 'NHWC':
        prod = tf.transpose(prod, [0, 2, 4, 3, 5, 1])
        prod = tf.reshape(prod, tf.stack(
            [-1, input_shape[1] * shape[0], input_shape[2] * shape[1], input_shape[3]]))
    else:
        prod = tf.transpose(prod, [0, 1, 2, 4, 3, 5])
        prod = tf.reshape(prod, tf.stack(
            [-1, input_shape[3], input_shape[1] * shape[0], input_shape[2] * shape[1]]))
    # TODO static shape inference
    return prod

@layer_register()
def BilinearUpSample(x, shape):
    """
    Deterministic bilinearly-upsample the input images.

    Args:
        x (tf.Tensor): a NHWC tensor
        shape (int): the upsample factor

    Returns:
        tf.Tensor: a NHWC tensor.
    """
    inp_shape = x.get_shape().as_list()
    ch = inp_shape[3]
    assert ch is not None

    shape = int(shape)
    filter_shape = 2 * shape

    def bilinear_conv_filler(s):
        """
        s: width, height of the conv filter
        See https://github.com/BVLC/caffe/blob/master/include%2Fcaffe%2Ffiller.hpp#L244
        """
        f = np.ceil(float(s) / 2)
        c = float(2 * f - 1 - f % 2) / (2 * f)
        ret = np.zeros((s, s), dtype='float32')
        for x in range(s):
            for y in range(s):
                ret[x, y] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
        return ret
    w = bilinear_conv_filler(filter_shape)
    w = np.repeat(w, ch * ch).reshape((filter_shape, filter_shape, ch, ch))

    weight_var = tf.constant(w, tf.float32,
                             shape=(filter_shape, filter_shape, ch, ch),
                             name='bilinear_upsample_filter')
    x = tf.pad(x, [[0, 0], [shape - 1, shape - 1], [shape - 1, shape - 1], [0, 0]], mode='SYMMETRIC')
    out_shape = tf.shape(x) * tf.constant([1, shape, shape, 1], tf.int32)
    deconv = tf.nn.conv2d_transpose(x, weight_var, out_shape,
                                    [1, shape, shape, 1], 'SAME')
    edge = shape * (shape - 1)
    deconv = deconv[:, edge:-edge, edge:-edge, :]

    if inp_shape[1]:
        inp_shape[1] *= shape
    if inp_shape[2]:
        inp_shape[2] *= shape
    deconv.set_shape(inp_shape)
    return deconv

class TestPool(TestModel):
    def test_FixedUnPooling(self):
        h, w = 3, 4
        mat = np.random.rand(h, w, 3).astype('float32')
        inp = self.make_variable(mat)
        inp = tf.reshape(inp, [1, h, w, 3])
        output = FixedUnPooling('unpool', inp, 2)
        res = self.run_variable(output)
        self.assertEqual(res.shape, (1, 2 * h, 2 * w, 3))

        # mat is on cornser
        ele = res[0, ::2, ::2, 0]
        self.assertTrue((ele == mat[:, :, 0]).all())
        # the rest are zeros
        res[0, ::2, ::2, :] = 0
        self.assertTrue((res == 0).all())

    def test_BilinearUpSample(self):
        h, w = 12, 12
        scale = 2

        mat = np.random.rand(h, w).astype('float32')
        inp = self.make_variable(mat)
        inp = tf.reshape(inp, [1, h, w, 1])

        output = BilinearUpSample('upsample', inp, scale)
        res = self.run_variable(output)[0, :, :, 0]

        from skimage.transform import rescale
        res2 = rescale(mat, scale, mode='edge')

        diff = np.abs(res2 - res)

        # if not diff.max() < 1e-4:
        #     import IPython
        #     IPython.embed(config=IPython.terminal.ipapp.load_default_config())
        self.assertTrue(diff.max() < 1e-4, diff.max())
<|endoftext|>