<commit_before>"""
Internal shared-state variables such as config settings and host lists.
"""

import os
import re
import socket
import sys
from optparse import make_option

from fabric.utils import abort
from fabric.network import HostConnectionCache
from fabric.version import get_version

#
# Paramiko
#

try:
    import paramiko as ssh
except ImportError:
    abort("paramiko is a required module. Please install it:\n\t$ sudo easy_install paramiko")

#
# Win32 flag
#

# Impacts a handful of platform specific behaviors.
win32 = sys.platform in ['win32', 'cygwin']

#
# Environment dictionary - support structures
#

class _AttributeDict(dict):
    """
    Dictionary subclass enabling attribute lookup/assignment of keys/values.

    For example::

        >>> m = _AttributeDict({'foo': 'bar'})
        >>> m.foo
        bar
        >>> m.foo = 'not bar'
        >>> m['foo']
        not bar

    ``_AttributeDict`` objects also provide ``.first()`` which acts like
    ``.get()`` but accepts multiple keys as arguments, and returns the value of
    the first hit, e.g.::

        >>> m = _AttributeDict({'foo': 'bar', 'biz': 'baz'})
        >>> m.first('wrong', 'incorrect', 'foo', 'biz')
        bar

    """
    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            # to conform with __getattr__ spec
            raise AttributeError(key)

    def __setattr__(self, key, value):
        self[key] = value

    def first(self, *names):
        for name in names:
            value = self.get(name)
            if value:
                return value

# By default, if the user (including code using Fabric as a library) doesn't
# set the username, we obtain the currently running username and use that.
def _get_system_username():
    """
    Obtain name of current system user, which will be default connection user.
    """
    if not win32:
        import pwd
        return pwd.getpwuid(os.getuid())[0]
    else:
        import win32api
        import win32security
        import win32profile
        return win32api.GetUserName()

def _rc_path():
    """
    Return platform-specific default file path for $HOME/.fabricrc.
    """
    rc_file = '.fabricrc'
    if not win32:
        return os.path.expanduser("~/" + rc_file)
    else:
        from win32com.shell.shell import SHGetSpecialFolderPath
        from win32com.shell.shellcon import CSIDL_PROFILE
        return "%s/%s" % (
            SHGetSpecialFolderPath(0,CSIDL_PROFILE),
            rc_file
        )

# Options/settings which exist both as environment keys and which can be set
# on the command line, are defined here. When used via `fab` they will be added
# to the optparse parser, and either way they are added to `env` below (i.e.
# the 'dest' value becomes the environment key and the value, the env value).
#
# Keep in mind that optparse changes hyphens to underscores when automatically
# deriving the `dest` name, e.g. `--reject-unknown-hosts` becomes
# `reject_unknown_hosts`.
#
# Furthermore, *always* specify some sort of default to avoid ending up with
# optparse.NO_DEFAULT (currently a two-tuple)! None is better than ''.
env_options = [

    # By default, we accept unknown hosts' keys. This option allows users to
    # disable that behavior (which means Fabric will raise an exception and
    # terminate when an unknown host key is received from a server).
    make_option('-r', '--reject-unknown-hosts',
        action='store_true',
        default=False,
        help="reject unknown hosts"
    ),

    # By default, we load the user's ~/.ssh/known_hosts file. In some cases
    # users may not want this to occur.
    make_option('-D', '--disable-known-hosts',
        action='store_true',
        default=False,
        help="do not load user known_hosts file"
    ),

    # Username
    make_option('-u', '--user',
        default=_get_system_username(),
        help="username to use when connecting to remote hosts"
    ),

    # Password
    make_option('-p', '--password',
        default=None,
        help="password for use with authentication and/or sudo"
    ),

    # Global host list
    make_option('-H', '--hosts',
        default=None,
        help="comma-separated list of hosts to operate on"
    ),

    # Global role list
    make_option('-R', '--roles',
        default=None,
        help="comma-separated list of roles to operate on"
    ),

    # Private key file
    make_option('-i',
        action='append',
        dest='key_filename',
        default=None,
        help="path to SSH private key file. May be repeated."
    ),

    # Fabfile name to look for
    make_option('-f', '--fabfile',
        default='fabfile.py',
        help="name of fabfile to load, e.g. 'fabfile.py' or '../other.py'"
    ),

    # Default error-handling behavior
    make_option('-w', '--warn-only',
        action='store_true',
        default=False,
        help="warn, instead of abort, when commands fail"
    ),

    # Shell used when running remote commands
    make_option('-s', '--shell',
        default='/bin/bash -l -c',
        help="specify a new shell, defaults to '/bin/bash -l -c'"
    ),

    # Config file location
    make_option('-c', '--config',
        dest='rcfile',
        default=_rc_path(),
        help="specify location of config file to use"
    ),

    # Verbosity controls, analogous to context_managers.(hide|show)
    make_option('--hide',
        metavar='LEVELS',
        help="comma-separated list of output levels to hide"
    ),
    make_option('--show',
        metavar='LEVELS',
        help="comma-separated list of output levels to show"
    )

]

#
# Environment dictionary - actual dictionary object
#

# Global environment dict. Currently a catchall for everything: config settings
# such as global deep/broad mode, host lists, username etc.
# Most default values are specified in `env_options` above, in the interests of
# preserving DRY.
env = _AttributeDict({
    # Version number for --version
    'version': get_version(),
    'sudo_prompt': 'sudo password:',
    'use_shell': True,
    'roledefs': {},
    'cwd': ''
})

# Add in option defaults
for option in env_options:
    env[option.dest] = option.default

#
# Command dictionary
#

# Keys are the command/function names, values are the callables themselves.
# This is filled in when main() runs.
commands = {}

#
# Host connection dict/cache
#

connections = HostConnectionCache()

#
# Output controls
#

class _AliasDict(_AttributeDict):
    """
    `_AttributeDict` subclass that allows for "aliasing" of keys to other keys.

    Upon creation, takes an ``aliases`` mapping, which should map alias names
    to lists of key names. Aliases do not store their own value, but instead
    set (override) all mapped keys' values. For example, in the following
    `_AliasDict`, calling ``mydict['foo'] = True`` will set the values of
    ``mydict['bar']``, ``mydict['biz']`` and ``mydict['baz']`` all to True::

        mydict = _AliasDict(
            {'biz': True, 'baz': False},
            aliases={'foo': ['bar', 'biz', 'baz']}
        )

    Because it is possible for the aliased values to be in a heterogenous
    state, reading aliases is not supported -- only writing to them is allowed.
    This also means they will not show up in e.g. ``dict.keys()``.

    ..note::

        Aliases are recursive, so you may refer to an alias within the key list
        of another alias. Naturally, this means that you can end up with
        infinite loops if you're not careful.

    `_AliasDict` provides a special function, `expand_aliases`, which will take
    a list of keys as an argument and will return that list of keys with any
    aliases expanded. This function will **not** dedupe, so any aliases which
    overlap will result in duplicate keys in the resulting list.
    """
    def __init__(self, arg=None, aliases=None):
        init = super(_AliasDict, self).__init__
        if arg is not None:
            init(arg)
        else:
            init()
        # Can't use super() here because of _AttributeDict's setattr override
        dict.__setattr__(self, 'aliases', aliases)

    def __setitem__(self, key, value):
        if key in self.aliases:
            for aliased in self.aliases[key]:
                self[aliased] = value
        else:
            return super(_AliasDict, self).__setitem__(key, value)

    def expand_aliases(self, keys):
        ret = []
        for key in keys:
            if key in self.aliases:
                ret.extend(self.expand_aliases(self.aliases[key]))
            else:
                ret.append(key)
        return ret

# Keys are "levels" or "groups" of output, values are always boolean,
# determining whether output falling into the given group is printed or not
# printed.
#
# By default, everything except 'debug' is printed, as this is what the average
# user, and new users, are most likely to expect.
#
# See docs/usage.rst for details on what these levels mean.
output = _AliasDict({
    'status': True,
    'aborts': True,
    'warnings': True,
    'running': True,
    'stdout': True,
    'stderr': True,
    'debug': False

}, aliases={
    'everything': ['warnings', 'running', 'output'],
    'output': ['stdout', 'stderr']
})
<commit_msg>Some trivial changes so that doctests pass.<commit_after>"""
Internal shared-state variables such as config settings and host lists.
"""

import os
import re
import socket
import sys
from optparse import make_option

from fabric.utils import abort
from fabric.network import HostConnectionCache
from fabric.version import get_version

#
# Paramiko
#

try:
    import paramiko as ssh
except ImportError:
    abort("paramiko is a required module. Please install it:\n\t$ sudo easy_install paramiko")

#
# Win32 flag
#

# Impacts a handful of platform specific behaviors.
win32 = sys.platform in ['win32', 'cygwin']

#
# Environment dictionary - support structures
#

class _AttributeDict(dict):
    """
    Dictionary subclass enabling attribute lookup/assignment of keys/values.

    For example::

        >>> m = _AttributeDict({'foo': 'bar'})
        >>> m.foo
        'bar'
        >>> m.foo = 'not bar'
        >>> m['foo']
        'not bar'

    ``_AttributeDict`` objects also provide ``.first()`` which acts like
    ``.get()`` but accepts multiple keys as arguments, and returns the value of
    the first hit, e.g.::

        >>> m = _AttributeDict({'foo': 'bar', 'biz': 'baz'})
        >>> m.first('wrong', 'incorrect', 'foo', 'biz')
        'bar'

    """
    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            # to conform with __getattr__ spec
            raise AttributeError(key)

    def __setattr__(self, key, value):
        self[key] = value

    def first(self, *names):
        for name in names:
            value = self.get(name)
            if value:
                return value

# By default, if the user (including code using Fabric as a library) doesn't
# set the username, we obtain the currently running username and use that.
def _get_system_username():
    """
    Obtain name of current system user, which will be default connection user.
    """
    if not win32:
        import pwd
        return pwd.getpwuid(os.getuid())[0]
    else:
        import win32api
        import win32security
        import win32profile
        return win32api.GetUserName()

def _rc_path():
    """
    Return platform-specific default file path for $HOME/.fabricrc.
    """
    rc_file = '.fabricrc'
    if not win32:
        return os.path.expanduser("~/" + rc_file)
    else:
        from win32com.shell.shell import SHGetSpecialFolderPath
        from win32com.shell.shellcon import CSIDL_PROFILE
        return "%s/%s" % (
            SHGetSpecialFolderPath(0,CSIDL_PROFILE),
            rc_file
        )

# Options/settings which exist both as environment keys and which can be set
# on the command line, are defined here. When used via `fab` they will be added
# to the optparse parser, and either way they are added to `env` below (i.e.
# the 'dest' value becomes the environment key and the value, the env value).
#
# Keep in mind that optparse changes hyphens to underscores when automatically
# deriving the `dest` name, e.g. `--reject-unknown-hosts` becomes
# `reject_unknown_hosts`.
#
# Furthermore, *always* specify some sort of default to avoid ending up with
# optparse.NO_DEFAULT (currently a two-tuple)! None is better than ''.
env_options = [

    # By default, we accept unknown hosts' keys. This option allows users to
    # disable that behavior (which means Fabric will raise an exception and
    # terminate when an unknown host key is received from a server).
    make_option('-r', '--reject-unknown-hosts',
        action='store_true',
        default=False,
        help="reject unknown hosts"
    ),

    # By default, we load the user's ~/.ssh/known_hosts file. In some cases
    # users may not want this to occur.
    make_option('-D', '--disable-known-hosts',
        action='store_true',
        default=False,
        help="do not load user known_hosts file"
    ),

    # Username
    make_option('-u', '--user',
        default=_get_system_username(),
        help="username to use when connecting to remote hosts"
    ),

    # Password
    make_option('-p', '--password',
        default=None,
        help="password for use with authentication and/or sudo"
    ),

    # Global host list
    make_option('-H', '--hosts',
        default=None,
        help="comma-separated list of hosts to operate on"
    ),

    # Global role list
    make_option('-R', '--roles',
        default=None,
        help="comma-separated list of roles to operate on"
    ),

    # Private key file
    make_option('-i',
        action='append',
        dest='key_filename',
        default=None,
        help="path to SSH private key file. May be repeated."
    ),

    # Fabfile name to look for
    make_option('-f', '--fabfile',
        default='fabfile.py',
        help="name of fabfile to load, e.g. 'fabfile.py' or '../other.py'"
    ),

    # Default error-handling behavior
    make_option('-w', '--warn-only',
        action='store_true',
        default=False,
        help="warn, instead of abort, when commands fail"
    ),

    # Shell used when running remote commands
    make_option('-s', '--shell',
        default='/bin/bash -l -c',
        help="specify a new shell, defaults to '/bin/bash -l -c'"
    ),

    # Config file location
    make_option('-c', '--config',
        dest='rcfile',
        default=_rc_path(),
        help="specify location of config file to use"
    ),

    # Verbosity controls, analogous to context_managers.(hide|show)
    make_option('--hide',
        metavar='LEVELS',
        help="comma-separated list of output levels to hide"
    ),
    make_option('--show',
        metavar='LEVELS',
        help="comma-separated list of output levels to show"
    )

]

#
# Environment dictionary - actual dictionary object
#

# Global environment dict. Currently a catchall for everything: config settings
# such as global deep/broad mode, host lists, username etc.
# Most default values are specified in `env_options` above, in the interests of
# preserving DRY.
env = _AttributeDict({
    # Version number for --version
    'version': get_version(),
    'sudo_prompt': 'sudo password:',
    'use_shell': True,
    'roledefs': {},
    'cwd': ''
})

# Add in option defaults
for option in env_options:
    env[option.dest] = option.default

#
# Command dictionary
#

# Keys are the command/function names, values are the callables themselves.
# This is filled in when main() runs.
commands = {}

#
# Host connection dict/cache
#

connections = HostConnectionCache()

#
# Output controls
#

class _AliasDict(_AttributeDict):
    """
    `_AttributeDict` subclass that allows for "aliasing" of keys to other keys.

    Upon creation, takes an ``aliases`` mapping, which should map alias names
    to lists of key names. Aliases do not store their own value, but instead
    set (override) all mapped keys' values. For example, in the following
    `_AliasDict`, calling ``mydict['foo'] = True`` will set the values of
    ``mydict['bar']``, ``mydict['biz']`` and ``mydict['baz']`` all to True::

        mydict = _AliasDict(
            {'biz': True, 'baz': False},
            aliases={'foo': ['bar', 'biz', 'baz']}
        )

    Because it is possible for the aliased values to be in a heterogenous
    state, reading aliases is not supported -- only writing to them is allowed.
    This also means they will not show up in e.g. ``dict.keys()``.

    ..note::

        Aliases are recursive, so you may refer to an alias within the key list
        of another alias. Naturally, this means that you can end up with
        infinite loops if you're not careful.

    `_AliasDict` provides a special function, `expand_aliases`, which will take
    a list of keys as an argument and will return that list of keys with any
    aliases expanded. This function will **not** dedupe, so any aliases which
    overlap will result in duplicate keys in the resulting list.
    """
    def __init__(self, arg=None, aliases=None):
        init = super(_AliasDict, self).__init__
        if arg is not None:
            init(arg)
        else:
            init()
        # Can't use super() here because of _AttributeDict's setattr override
        dict.__setattr__(self, 'aliases', aliases)

    def __setitem__(self, key, value):
        if key in self.aliases:
            for aliased in self.aliases[key]:
                self[aliased] = value
        else:
            return super(_AliasDict, self).__setitem__(key, value)

    def expand_aliases(self, keys):
        ret = []
        for key in keys:
            if key in self.aliases:
                ret.extend(self.expand_aliases(self.aliases[key]))
            else:
                ret.append(key)
        return ret

# Keys are "levels" or "groups" of output, values are always boolean,
# determining whether output falling into the given group is printed or not
# printed.
#
# By default, everything except 'debug' is printed, as this is what the average
# user, and new users, are most likely to expect.
#
# See docs/usage.rst for details on what these levels mean.
output = _AliasDict({
    'status': True,
    'aborts': True,
    'warnings': True,
    'running': True,
    'stdout': True,
    'stderr': True,
    'debug': False

}, aliases={
    'everything': ['warnings', 'running', 'output'],
    'output': ['stdout', 'stderr']
})
<|endoftext|>


<commit_before>from django.contrib import admin
from django.conf.urls import url
from django.contrib import messages

from . import models
from .forms import UploadFileForm, UploadAnnualFileForm
from django_q.tasks import async_task

@admin.register(models.FinancialYear)
class FinancialYearAdmin(admin.ModelAdmin):
    list_display = ("budget_year", "active")

@admin.register(models.BudgetPhase)
class BudgetPhaseAdmin(admin.ModelAdmin):
    list_display = ("name",)

@admin.register(models.Project)
class ProjectAdmin(admin.ModelAdmin):
    list_display = (
        "project_number",
        "project_type",
        "function",
        "ward_location",
        "latitude",
        "longitude",
    )

@admin.register(models.Expenditure)
class ExpenditureAdmin(admin.ModelAdmin):
    list_display = (
        "project",
        "budget_phase",
        "financial_year",
        "amount",
    )
    list_filter = ("budget_phase", "financial_year")

@admin.register(models.ProjectQuarterlySpend)
class QuarterlySpendAdmin(admin.ModelAdmin):
    list_display = ("project", "financial_year", "q1", "q2", "q3", "q4")
    list_filter = ["financial_year"]

#@admin.register(models.QuarterlySpendFile)
class SpendFileAdmin(admin.ModelAdmin):
    list_display = ("financial_year", "document", "status")
    form = UploadFileForm

    def save_model(self, request, obj, form, change):
        messages.add_message(
            request, messages.INFO, "Dataset is currently being processed."
        )
        super().save_model(request, obj, form, change)
        task_id = async_task("infrastructure.upload.process_document", obj.id)

@admin.register(models.AnnualSpendFile)
class AnnualSpendFileAdmin(admin.ModelAdmin):
    list_display = ("financial_year", "document", "status")
    form = UploadAnnualFileForm

    def save_model(self, request, obj, form, change):
        messages.add_message(
            request, messages.INFO, "Dataset is currently being processed."
        )
        super().save_model(request, obj, form, change)
        task_id = async_task("infrastructure.upload.process_document", obj.id)
<commit_msg>Delete quarterly spend file admin<commit_after>from django.contrib import admin
from django.conf.urls import url
from django.contrib import messages

from . import models
from .forms import UploadFileForm, UploadAnnualFileForm
from django_q.tasks import async_task

@admin.register(models.FinancialYear)
class FinancialYearAdmin(admin.ModelAdmin):
    list_display = ("budget_year", "active")

@admin.register(models.BudgetPhase)
class BudgetPhaseAdmin(admin.ModelAdmin):
    list_display = ("name",)

@admin.register(models.Project)
class ProjectAdmin(admin.ModelAdmin):
    list_display = (
        "project_number",
        "project_type",
        "function",
        "ward_location",
        "latitude",
        "longitude",
    )

@admin.register(models.Expenditure)
class ExpenditureAdmin(admin.ModelAdmin):
    list_display = (
        "project",
        "budget_phase",
        "financial_year",
        "amount",
    )
    list_filter = ("budget_phase", "financial_year")

@admin.register(models.ProjectQuarterlySpend)
class QuarterlySpendAdmin(admin.ModelAdmin):
    list_display = ("project", "financial_year", "q1", "q2", "q3", "q4")
    list_filter = ["financial_year"]

@admin.register(models.AnnualSpendFile)
class AnnualSpendFileAdmin(admin.ModelAdmin):
    list_display = ("financial_year", "document", "status")
    form = UploadAnnualFileForm

    def save_model(self, request, obj, form, change):
        messages.add_message(
            request, messages.INFO, "Dataset is currently being processed."
        )
        super().save_model(request, obj, form, change)
        task_id = async_task("infrastructure.upload.process_document", obj.id)
<|endoftext|>


<commit_before>"""Virtual Adversarial Ladder"""
import tensorflow as tf
from src.utils import count_trainable_params, preprocess, get_batch_ops
from src.vat import Adversary
from src.ladder import Ladder, Encoder, Decoder

class LadderWithVAN(Ladder):
    def get_corrupted_encoder(self, inputs, bn, train_flag, params,
                              start_layer=0, update_batch_stats=False,
                              scope='enc', reuse=True):
        return VANEncoder(
            inputs, bn, train_flag, params, self.clean.logits,
            this_encoder_noise=params.corrupt_sd,
            start_layer=start_layer, update_batch_stats=update_batch_stats,
            scope=scope, reuse=reuse)

class VANEncoder(Encoder):
    def __init__(
            self, inputs, bn, is_training, params, clean_logits,
            this_encoder_noise=0.0, start_layer=0, update_batch_stats=True,
            scope='enc', reuse=None):

        self.params = params
        self.clean_logits = clean_logits

        super(VANEncoder, self).__init__(
            inputs, bn, is_training, params,
            this_encoder_noise=this_encoder_noise,
            start_layer=start_layer,
            update_batch_stats=update_batch_stats,
            scope=scope, reuse=reuse
        )

    def get_vadv_noise(self, inputs, l):
        join, split_lu, labeled, unlabeled = get_batch_ops(self.batch_size)

        adv = Adversary(
            bn=self.bn,
            params=self.params,
            layer_eps=self.params.epsilon[l],
            start_layer=l
        )

        x = unlabeled(inputs)
        logit = unlabeled(self.clean_logits)

        ul_noise = adv.generate_virtual_adversarial_perturbation(
            x=x, logit=logit, is_training=self.is_training)

        return join(tf.zeros(tf.shape(labeled(inputs))), ul_noise)

    def print_progress(self, l_out):
        el = self.encoder_layers
        print("Layer {}: {} -> {}, epsilon {}".format(l_out, el[l_out - 1], el[l_out],
              self.params.epsilon.get(l_out - 1)))

    def generate_noise(self, inputs, l):

        if self.noise_sd > 0.0:
            noise = tf.random_normal(tf.shape(inputs)) * self.noise_sd

            if self.params.model == "n" and l==0:
                noise += self.get_vadv_noise(inputs, l)

            elif self.params.model == "nlw" and (l + 1 < self.num_layers):
                # don't add adversarial noise to logits
                noise += self.get_vadv_noise(inputs, l)

        else:
            noise = tf.zeros(tf.shape(inputs))

        return inputs + noise

def get_vat_cost(ladder, train_flag, params):
    unlabeled = lambda x: x[params.batch_size:] if x is not None else x

    def get_layer_vat_cost(l):

        adv = Adversary(bn=ladder.bn,
                        params=params,
                        layer_eps=params.epsilon[l],
                        start_layer=l)

        # VAT on unlabeled only
        return (
            adv.virtual_adversarial_loss(
                x=ladder.corr.unlabeled.z[l],
                logit=unlabeled(ladder.corr.logits),  # should this be clean?
                is_training=train_flag)
        )

    if params.model == "clw":
        vat_costs = []
        for l in range(ladder.num_layers):
            vat_costs.append(get_layer_vat_cost(l))
        vat_cost = tf.add_n(vat_costs)

    elif params.model == "c":
        vat_cost = get_layer_vat_cost(0)

    else:
        vat_cost = 0.0

    return vat_cost

def build_graph(params):
    model = params.model
    # -----------------------------
    # Placeholder setup
    inputs_placeholder = tf.placeholder(
        tf.float32, shape=(None, params.encoder_layers[0]))
    inputs = preprocess(inputs_placeholder, params)
    outputs = tf.placeholder(tf.float32)
    train_flag = tf.placeholder(tf.bool)

    if model == "c" or model == "clw":
        ladder = Ladder(inputs, outputs, train_flag, params)
        vat_cost = get_vat_cost(ladder, train_flag, params)

    elif model == "n" or model == "nlw":
        ladder = LadderWithVAN(inputs, outputs, train_flag, params)
        vat_cost = 0.0

    else:
        ladder = Ladder(inputs, outputs, train_flag, params)
        vat_cost = 0.0

    # -----------------------------
    # Loss, accuracy and training steps
    loss = ladder.cost + ladder.u_cost + vat_cost
    accuracy = tf.reduce_mean(
        tf.cast(
            tf.equal(ladder.predict, tf.argmax(outputs, 1)),
            "float")) * tf.constant(100.0)

    learning_rate = tf.Variable(params.initial_learning_rate, trainable=False)
    beta1 = tf.Variable(params.beta1, trainable=False)
    train_step = tf.train.AdamOptimizer(learning_rate,
                                        beta1=beta1).minimize(loss)

    # add the updates of batch normalization statistics to train_step
    bn_updates = tf.group(*ladder.bn.bn_assigns)
    with tf.control_dependencies([train_step]):
        train_step = tf.group(bn_updates)

    saver = tf.train.Saver(keep_checkpoint_every_n_hours=0.5,
                            max_to_keep=5)

    # Graph
    g = dict()
    g['images'] = inputs_placeholder
    g['labels'] = outputs
    g['train_flag'] = train_flag
    g['ladder'] = ladder
    g['saver'] = saver
    g['train_step'] = train_step
    g['lr'] = learning_rate
    g['beta1'] = beta1

    # Metrics
    m = dict()
    m['loss'] = loss
    m['cost'] = ladder.cost
    m['uc'] = ladder.u_cost
    m['vc'] = vat_cost
    m['acc'] = accuracy

    trainable_params = count_trainable_params()

    return g, m, trainable_params
<commit_msg>Make vat_cost where not needed a tensor<commit_after>"""Virtual Adversarial Ladder"""
import tensorflow as tf
from src.utils import count_trainable_params, preprocess, get_batch_ops
from src.vat import Adversary
from src.ladder import Ladder, Encoder, Decoder

class LadderWithVAN(Ladder):
    def get_corrupted_encoder(self, inputs, bn, train_flag, params,
                              start_layer=0, update_batch_stats=False,
                              scope='enc', reuse=True):
        return VANEncoder(
            inputs, bn, train_flag, params, self.clean.logits,
            this_encoder_noise=params.corrupt_sd,
            start_layer=start_layer, update_batch_stats=update_batch_stats,
            scope=scope, reuse=reuse)

class VANEncoder(Encoder):
    def __init__(
            self, inputs, bn, is_training, params, clean_logits,
            this_encoder_noise=0.0, start_layer=0, update_batch_stats=True,
            scope='enc', reuse=None):

        self.params = params
        self.clean_logits = clean_logits

        super(VANEncoder, self).__init__(
            inputs, bn, is_training, params,
            this_encoder_noise=this_encoder_noise,
            start_layer=start_layer,
            update_batch_stats=update_batch_stats,
            scope=scope, reuse=reuse
        )

    def get_vadv_noise(self, inputs, l):
        join, split_lu, labeled, unlabeled = get_batch_ops(self.batch_size)

        adv = Adversary(
            bn=self.bn,
            params=self.params,
            layer_eps=self.params.epsilon[l],
            start_layer=l
        )

        x = unlabeled(inputs)
        logit = unlabeled(self.clean_logits)

        ul_noise = adv.generate_virtual_adversarial_perturbation(
            x=x, logit=logit, is_training=self.is_training)

        return join(tf.zeros(tf.shape(labeled(inputs))), ul_noise)

    def print_progress(self, l_out):
        el = self.encoder_layers
        print("Layer {}: {} -> {}, epsilon {}".format(l_out, el[l_out - 1], el[l_out],
              self.params.epsilon.get(l_out - 1)))

    def generate_noise(self, inputs, l):

        if self.noise_sd > 0.0:
            noise = tf.random_normal(tf.shape(inputs)) * self.noise_sd

            if self.params.model == "n" and l==0:
                noise += self.get_vadv_noise(inputs, l)

            elif self.params.model == "nlw" and (l + 1 < self.num_layers):
                # don't add adversarial noise to logits
                noise += self.get_vadv_noise(inputs, l)

        else:
            noise = tf.zeros(tf.shape(inputs))

        return inputs + noise

def get_vat_cost(ladder, train_flag, params):
    unlabeled = lambda x: x[params.batch_size:] if x is not None else x

    def get_layer_vat_cost(l):

        adv = Adversary(bn=ladder.bn,
                        params=params,
                        layer_eps=params.epsilon[l],
                        start_layer=l)

        # VAT on unlabeled only
        return (
            adv.virtual_adversarial_loss(
                x=ladder.corr.unlabeled.z[l],
                logit=unlabeled(ladder.corr.logits),  # should this be clean?
                is_training=train_flag)
        )

    if params.model == "clw":
        vat_costs = []
        for l in range(ladder.num_layers):
            vat_costs.append(get_layer_vat_cost(l))
        vat_cost = tf.add_n(vat_costs)

    elif params.model == "c":
        vat_cost = get_layer_vat_cost(0)

    else:
        vat_cost = 0.0

    return vat_cost

def build_graph(params):
    model = params.model
    # -----------------------------
    # Placeholder setup
    inputs_placeholder = tf.placeholder(
        tf.float32, shape=(None, params.encoder_layers[0]))
    inputs = preprocess(inputs_placeholder, params)
    outputs = tf.placeholder(tf.float32)
    train_flag = tf.placeholder(tf.bool)

    if model == "c" or model == "clw":
        ladder = Ladder(inputs, outputs, train_flag, params)
        vat_cost = get_vat_cost(ladder, train_flag, params)

    elif model == "n" or model == "nlw":
        ladder = LadderWithVAN(inputs, outputs, train_flag, params)
        vat_cost = tf.zeros(1)

    else:
        ladder = Ladder(inputs, outputs, train_flag, params)
        vat_cost = tf.zeros(1)

    # -----------------------------
    # Loss, accuracy and training steps
    loss = ladder.cost + ladder.u_cost + vat_cost
    accuracy = tf.reduce_mean(
        tf.cast(
            tf.equal(ladder.predict, tf.argmax(outputs, 1)),
            "float")) * tf.constant(100.0)

    learning_rate = tf.Variable(params.initial_learning_rate, trainable=False)
    beta1 = tf.Variable(params.beta1, trainable=False)
    train_step = tf.train.AdamOptimizer(learning_rate,
                                        beta1=beta1).minimize(loss)

    # add the updates of batch normalization statistics to train_step
    bn_updates = tf.group(*ladder.bn.bn_assigns)
    with tf.control_dependencies([train_step]):
        train_step = tf.group(bn_updates)

    saver = tf.train.Saver(keep_checkpoint_every_n_hours=0.5,
                            max_to_keep=5)

    # Graph
    g = dict()
    g['images'] = inputs_placeholder
    g['labels'] = outputs
    g['train_flag'] = train_flag
    g['ladder'] = ladder
    g['saver'] = saver
    g['train_step'] = train_step
    g['lr'] = learning_rate
    g['beta1'] = beta1

    # Metrics
    m = dict()
    m['loss'] = loss
    m['cost'] = ladder.cost
    m['uc'] = ladder.u_cost
    m['vc'] = vat_cost
    m['acc'] = accuracy

    trainable_params = count_trainable_params()

    return g, m, trainable_params
<|endoftext|>


<commit_before>"""
Internal subroutines for e.g. aborting execution with an error message,
or performing indenting on multiline output.
"""
import os
import sys
import textwrap
from traceback import format_exc

def abort(msg):
    """
    Abort execution, print ``msg`` to stderr and exit with error status (1.)

    This function currently makes use of `sys.exit`_, which raises
    `SystemExit`_. Therefore, it's possible to detect and recover from inner
    calls to `abort` by using ``except SystemExit`` or similar.

    .. _sys.exit: http://docs.python.org/library/sys.html#sys.exit
    .. _SystemExit: http://docs.python.org/library/exceptions.html#exceptions.SystemExit
    """
    from fabric.state import output, env
    if not env.colorize_errors:
        red  = lambda x: x
    else:
        from colors import red

    if output.aborts:
        if isinstance(msg, unicode):
            msg_changed = msg.encode(sys.stderr.encoding)
        else:
            msg_changed = str(msg)
        sys.stderr.write(red("\nFatal error: %s\n" % msg_changed))
        sys.stderr.write(red("\nAborting.\n"))
    sys.exit(1)

def warn(msg):
    """
    Print warning message, but do not abort execution.

    This function honors Fabric's :doc:`output controls
    <../../usage/output_controls>` and will print the given ``msg`` to stderr,
    provided that the ``warnings`` output level (which is active by default) is
    turned on.
    """
    from fabric.state import output, env

    if not env.colorize_errors:
        magenta = lambda x: x
    else:
        from colors import magenta

    if output.warnings:
        if isinstance(msg, unicode):
            msg_changed = msg.encode(sys.stderr.encoding)
        else:
            msg_changed = str(msg)
        sys.stderr.write(magenta("\nWarning: %s\n\n" % msg_changed))

def indent(text, spaces=4, strip=False):
    """
    Return ``text`` indented by the given number of spaces.

    If text is not a string, it is assumed to be a list of lines and will be
    joined by ``\\n`` prior to indenting.

    When ``strip`` is ``True``, a minimum amount of whitespace is removed from
    the left-hand side of the given string (so that relative indents are
    preserved, but otherwise things are left-stripped). This allows you to
    effectively "normalize" any previous indentation for some inputs.
    """
    # Normalize list of strings into a string for dedenting. "list" here means
    # "not a string" meaning "doesn't have splitlines". Meh.
    if not hasattr(text, 'splitlines'):
        text = '\n'.join(text)
    # Dedent if requested
    if strip:
        text = textwrap.dedent(text)
    prefix = ' ' * spaces
    output = '\n'.join(prefix + line for line in text.splitlines())
    # Strip out empty lines before/aft
    output = output.strip()
    # Reintroduce first indent (which just got stripped out)
    output = prefix + output
    return output

def puts(text, show_prefix=None, end="\n", flush=False):
    """
    An alias for ``print`` whose output is managed by Fabric's output controls.

    In other words, this function simply prints to ``sys.stdout``, but will
    hide its output if the ``user`` :doc:`output level
    </usage/output_controls>` is set to ``False``.

    If ``show_prefix=False``, `puts` will omit the leading ``[hostname]``
    which it tacks on by default. (It will also omit this prefix if
    ``env.host_string`` is empty.)

    Newlines may be disabled by setting ``end`` to the empty string (``''``).
    (This intentionally mirrors Python 3's ``print`` syntax.)

    You may force output flushing (e.g. to bypass output buffering) by setting
    ``flush=True``.

    .. versionadded:: 0.9.2
    .. seealso:: `~fabric.utils.fastprint`
    """
    from fabric.state import output, env
    if show_prefix is None:
        show_prefix = env.output_prefix
    if output.user:
        prefix = ""
        if env.host_string and show_prefix:
            prefix = "[%s] " % env.host_string
        if isinstance(text, unicode):
            text_changed = text.encode(sys.stdout.encoding)
        else:
            text_changed = str(text)
        sys.stdout.write(prefix + text_changed + end)
        if flush:
            sys.stdout.flush()

def fastprint(text, show_prefix=False, end="", flush=True):
    """
    Print ``text`` immediately, without any prefix or line ending.

    This function is simply an alias of `~fabric.utils.puts` with different
    default argument values, such that the ``text`` is printed without any
    embellishment and immediately flushed.

    It is useful for any situation where you wish to print text which might
    otherwise get buffered by Python's output buffering (such as within a
    processor intensive ``for`` loop). Since such use cases typically also
    require a lack of line endings (such as printing a series of dots to
    signify progress) it also omits the traditional newline by default.

    .. note::

        Since `~fabric.utils.fastprint` calls `~fabric.utils.puts`, it is
        likewise subject to the ``user`` :doc:`output level
        </usage/output_controls>`.

    .. versionadded:: 0.9.2
    .. seealso:: `~fabric.utils.puts`
    """
    return puts(text=text, show_prefix=show_prefix, end=end, flush=flush)

def handle_prompt_abort(prompt_for):
    import fabric.state
    reason = "Needed to prompt for %s (host: %s), but %%s" % (
        prompt_for, fabric.state.env.host_string
    )
    # Explicit "don't prompt me bro"
    if fabric.state.env.abort_on_prompts:
        abort(reason % "abort-on-prompts was set to True")
    # Implicit "parallel == stdin/prompts have ambiguous target"
    if fabric.state.env.parallel:
        abort(reason % "input would be ambiguous in parallel mode")

class _AttributeDict(dict):
    """
    Dictionary subclass enabling attribute lookup/assignment of keys/values.

    For example::

        >>> m = _AttributeDict({'foo': 'bar'})
        >>> m.foo
        'bar'
        >>> m.foo = 'not bar'
        >>> m['foo']
        'not bar'

    ``_AttributeDict`` objects also provide ``.first()`` which acts like
    ``.get()`` but accepts multiple keys as arguments, and returns the value of
    the first hit, e.g.::

        >>> m = _AttributeDict({'foo': 'bar', 'biz': 'baz'})
        >>> m.first('wrong', 'incorrect', 'foo', 'biz')
        'bar'

    """
    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            # to conform with __getattr__ spec
            raise AttributeError(key)

    def __setattr__(self, key, value):
        self[key] = value

    def first(self, *names):
        for name in names:
            value = self.get(name)
            if value:
                return value

class _AliasDict(_AttributeDict):
    """
    `_AttributeDict` subclass that allows for "aliasing" of keys to other keys.

    Upon creation, takes an ``aliases`` mapping, which should map alias names
    to lists of key names. Aliases do not store their own value, but instead
    set (override) all mapped keys' values. For example, in the following
    `_AliasDict`, calling ``mydict['foo'] = True`` will set the values of
    ``mydict['bar']``, ``mydict['biz']`` and ``mydict['baz']`` all to True::

        mydict = _AliasDict(
            {'biz': True, 'baz': False},
            aliases={'foo': ['bar', 'biz', 'baz']}
        )

    Because it is possible for the aliased values to be in a heterogenous
    state, reading aliases is not supported -- only writing to them is allowed.
    This also means they will not show up in e.g. ``dict.keys()``.

    ..note::

        Aliases are recursive, so you may refer to an alias within the key list
        of another alias. Naturally, this means that you can end up with
        infinite loops if you're not careful.

    `_AliasDict` provides a special function, `expand_aliases`, which will take
    a list of keys as an argument and will return that list of keys with any
    aliases expanded. This function will **not** dedupe, so any aliases which
    overlap will result in duplicate keys in the resulting list.
    """
    def __init__(self, arg=None, aliases=None):
        init = super(_AliasDict, self).__init__
        if arg is not None:
            init(arg)
        else:
            init()
        # Can't use super() here because of _AttributeDict's setattr override
        dict.__setattr__(self, 'aliases', aliases)

    def __setitem__(self, key, value):
        # Attr test required to not blow up when deepcopy'd
        if hasattr(self, 'aliases') and key in self.aliases:
            for aliased in self.aliases[key]:
                self[aliased] = value
        else:
            return super(_AliasDict, self).__setitem__(key, value)

    def expand_aliases(self, keys):
        ret = []
        for key in keys:
            if key in self.aliases:
                ret.extend(self.expand_aliases(self.aliases[key]))
            else:
                ret.append(key)
        return ret

def _pty_size():
    """
    Obtain (rows, cols) tuple for sizing a pty on the remote end.

    Defaults to 80x24 (which is also the 'ssh' lib's default) but will detect
    local (stdout-based) terminal window size on non-Windows platforms.
    """
    from fabric.state import win32
    if not win32:
        import fcntl
        import termios
        import struct

    default_rows, default_cols = 24, 80
    rows, cols = default_rows, default_cols
    if not win32 and sys.stdout.isatty():
        # We want two short unsigned integers (rows, cols)
        fmt = 'HH'
        # Create an empty (zeroed) buffer for ioctl to map onto. Yay for C!
        buffer = struct.pack(fmt, 0, 0)
        # Call TIOCGWINSZ to get window size of stdout, returns our filled
        # buffer
        try:
            result = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ,
                buffer)
            # Unpack buffer back into Python data types
            rows, cols = struct.unpack(fmt, result)
            # Fall back to defaults if TIOCGWINSZ returns unreasonable values
            if rows == 0:
                rows = default_rows
            if cols == 0:
                cols = default_cols
        # Deal with e.g. sys.stdout being monkeypatched, such as in testing.
        # Or termios not having a TIOCGWINSZ.
        except AttributeError:
            pass
    return rows, cols

def error(message, func=None, exception=None, stdout=None, stderr=None):
    """
    Call ``func`` with given error ``message``.

    If ``func`` is None (the default), the value of ``env.warn_only``
    determines whether to call ``abort`` or ``warn``.

    If ``exception`` is given, it is inspected to get a string message, which
    is printed alongside the user-generated ``message``.

    If ``stdout`` and/or ``stderr`` are given, they are assumed to be strings
    to be printed.
    """
    import fabric.state
    if func is None:
        func = fabric.state.env.warn_only and warn or abort
    # If debug printing is on, append a traceback to the message
    if fabric.state.output.debug:
        message += "\n\n" + format_exc()
    # Otherwise, if we were given an exception, append its contents.
    elif exception is not None:
        # Figure out how to get a string out of the exception; EnvironmentError
        # subclasses, for example, "are" integers and .strerror is the string.
        # Others "are" strings themselves. May have to expand this further for
        # other error types.
        if hasattr(exception, 'strerror') and exception.strerror is not None:
            underlying = exception.strerror
        else:
            underlying = exception
        message += "\n\nUnderlying exception:\n" + indent(str(underlying))
    if func is abort:
        if stdout and not fabric.state.output.stdout:
            message += _format_error_output("Standard output", stdout)
        if stderr and not fabric.state.output.stderr:
            message += _format_error_output("Standard error", stderr)
    return func(message)

def _format_error_output(header, body):
    term_width = _pty_size()[1]
    header_side_length = (term_width - (len(header) + 2)) / 2
    mark = "="
    side = mark * header_side_length
    return "\n\n%s %s %s\n\n%s\n\n%s" % (
        side, header, side, body, mark * term_width
    )

# TODO: replace with collections.deque(maxlen=xxx) in Python 2.6
class RingBuffer(list):
    def __init__(self, value, maxlen):
        # Heh.
        self._super = super(RingBuffer, self)
        self._maxlen = maxlen
        return self._super.__init__(value)

    def _free(self):
        return self._maxlen - len(self)

    def append(self, value):
        if self._free() == 0:
            del self[0]
        return self._super.append(value)

    def extend(self, values):
        overage = len(values) - self._free()
        if overage > 0:
            del self[0:overage]
        return self._super.extend(values)

    # Paranoia from here on out.
    def insert(self, index, value):
        raise ValueError("Can't insert into the middle of a ring buffer!")

    def __setslice__(self, i, j, sequence):
        raise ValueError("Can't set a slice of a ring buffer!")

    def __setitem__(self, key, value):
        if isinstance(key, slice):
            raise ValueError("Can't set a slice of a ring buffer!")
        else:
            return self._super.__setitem__(key, value)

def apply_lcwd(path, env):
    # Apply CWD if a relative path
    if not os.path.isabs(path) and env.lcwd:
        path = os.path.join(env.lcwd, path)
    return path
<commit_msg>Refactor encoding logic<commit_after>"""
Internal subroutines for e.g. aborting execution with an error message,
or performing indenting on multiline output.
"""
import os
import sys
import textwrap
from traceback import format_exc

def _encode(msg, stream):
    if isinstance(msg, unicode):
        return msg.encode(stream.encoding)
    else:
        return str(msg)

def abort(msg):
    """
    Abort execution, print ``msg`` to stderr and exit with error status (1.)

    This function currently makes use of `sys.exit`_, which raises
    `SystemExit`_. Therefore, it's possible to detect and recover from inner
    calls to `abort` by using ``except SystemExit`` or similar.

    .. _sys.exit: http://docs.python.org/library/sys.html#sys.exit
    .. _SystemExit: http://docs.python.org/library/exceptions.html#exceptions.SystemExit
    """
    from fabric.state import output, env
    if not env.colorize_errors:
        red  = lambda x: x
    else:
        from colors import red

    if output.aborts:
        sys.stderr.write(red("\nFatal error: %s\n" % _encode(msg, sys.stderr)))
        sys.stderr.write(red("\nAborting.\n"))
    sys.exit(1)

def warn(msg):
    """
    Print warning message, but do not abort execution.

    This function honors Fabric's :doc:`output controls
    <../../usage/output_controls>` and will print the given ``msg`` to stderr,
    provided that the ``warnings`` output level (which is active by default) is
    turned on.
    """
    from fabric.state import output, env

    if not env.colorize_errors:
        magenta = lambda x: x
    else:
        from colors import magenta

    if output.warnings:
        msg = _encode(msg, sys.stderr)
        sys.stderr.write(magenta("\nWarning: %s\n\n" % msg))

def indent(text, spaces=4, strip=False):
    """
    Return ``text`` indented by the given number of spaces.

    If text is not a string, it is assumed to be a list of lines and will be
    joined by ``\\n`` prior to indenting.

    When ``strip`` is ``True``, a minimum amount of whitespace is removed from
    the left-hand side of the given string (so that relative indents are
    preserved, but otherwise things are left-stripped). This allows you to
    effectively "normalize" any previous indentation for some inputs.
    """
    # Normalize list of strings into a string for dedenting. "list" here means
    # "not a string" meaning "doesn't have splitlines". Meh.
    if not hasattr(text, 'splitlines'):
        text = '\n'.join(text)
    # Dedent if requested
    if strip:
        text = textwrap.dedent(text)
    prefix = ' ' * spaces
    output = '\n'.join(prefix + line for line in text.splitlines())
    # Strip out empty lines before/aft
    output = output.strip()
    # Reintroduce first indent (which just got stripped out)
    output = prefix + output
    return output

def puts(text, show_prefix=None, end="\n", flush=False):
    """
    An alias for ``print`` whose output is managed by Fabric's output controls.

    In other words, this function simply prints to ``sys.stdout``, but will
    hide its output if the ``user`` :doc:`output level
    </usage/output_controls>` is set to ``False``.

    If ``show_prefix=False``, `puts` will omit the leading ``[hostname]``
    which it tacks on by default. (It will also omit this prefix if
    ``env.host_string`` is empty.)

    Newlines may be disabled by setting ``end`` to the empty string (``''``).
    (This intentionally mirrors Python 3's ``print`` syntax.)

    You may force output flushing (e.g. to bypass output buffering) by setting
    ``flush=True``.

    .. versionadded:: 0.9.2
    .. seealso:: `~fabric.utils.fastprint`
    """
    from fabric.state import output, env
    if show_prefix is None:
        show_prefix = env.output_prefix
    if output.user:
        prefix = ""
        if env.host_string and show_prefix:
            prefix = "[%s] " % env.host_string
        sys.stdout.write(prefix + _encode(text, sys.stdout) + end)
        if flush:
            sys.stdout.flush()

def fastprint(text, show_prefix=False, end="", flush=True):
    """
    Print ``text`` immediately, without any prefix or line ending.

    This function is simply an alias of `~fabric.utils.puts` with different
    default argument values, such that the ``text`` is printed without any
    embellishment and immediately flushed.

    It is useful for any situation where you wish to print text which might
    otherwise get buffered by Python's output buffering (such as within a
    processor intensive ``for`` loop). Since such use cases typically also
    require a lack of line endings (such as printing a series of dots to
    signify progress) it also omits the traditional newline by default.

    .. note::

        Since `~fabric.utils.fastprint` calls `~fabric.utils.puts`, it is
        likewise subject to the ``user`` :doc:`output level
        </usage/output_controls>`.

    .. versionadded:: 0.9.2
    .. seealso:: `~fabric.utils.puts`
    """
    return puts(text=text, show_prefix=show_prefix, end=end, flush=flush)

def handle_prompt_abort(prompt_for):
    import fabric.state
    reason = "Needed to prompt for %s (host: %s), but %%s" % (
        prompt_for, fabric.state.env.host_string
    )
    # Explicit "don't prompt me bro"
    if fabric.state.env.abort_on_prompts:
        abort(reason % "abort-on-prompts was set to True")
    # Implicit "parallel == stdin/prompts have ambiguous target"
    if fabric.state.env.parallel:
        abort(reason % "input would be ambiguous in parallel mode")

class _AttributeDict(dict):
    """
    Dictionary subclass enabling attribute lookup/assignment of keys/values.

    For example::

        >>> m = _AttributeDict({'foo': 'bar'})
        >>> m.foo
        'bar'
        >>> m.foo = 'not bar'
        >>> m['foo']
        'not bar'

    ``_AttributeDict`` objects also provide ``.first()`` which acts like
    ``.get()`` but accepts multiple keys as arguments, and returns the value of
    the first hit, e.g.::

        >>> m = _AttributeDict({'foo': 'bar', 'biz': 'baz'})
        >>> m.first('wrong', 'incorrect', 'foo', 'biz')
        'bar'

    """
    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            # to conform with __getattr__ spec
            raise AttributeError(key)

    def __setattr__(self, key, value):
        self[key] = value

    def first(self, *names):
        for name in names:
            value = self.get(name)
            if value:
                return value

class _AliasDict(_AttributeDict):
    """
    `_AttributeDict` subclass that allows for "aliasing" of keys to other keys.

    Upon creation, takes an ``aliases`` mapping, which should map alias names
    to lists of key names. Aliases do not store their own value, but instead
    set (override) all mapped keys' values. For example, in the following
    `_AliasDict`, calling ``mydict['foo'] = True`` will set the values of
    ``mydict['bar']``, ``mydict['biz']`` and ``mydict['baz']`` all to True::

        mydict = _AliasDict(
            {'biz': True, 'baz': False},
            aliases={'foo': ['bar', 'biz', 'baz']}
        )

    Because it is possible for the aliased values to be in a heterogenous
    state, reading aliases is not supported -- only writing to them is allowed.
    This also means they will not show up in e.g. ``dict.keys()``.

    ..note::

        Aliases are recursive, so you may refer to an alias within the key list
        of another alias. Naturally, this means that you can end up with
        infinite loops if you're not careful.

    `_AliasDict` provides a special function, `expand_aliases`, which will take
    a list of keys as an argument and will return that list of keys with any
    aliases expanded. This function will **not** dedupe, so any aliases which
    overlap will result in duplicate keys in the resulting list.
    """
    def __init__(self, arg=None, aliases=None):
        init = super(_AliasDict, self).__init__
        if arg is not None:
            init(arg)
        else:
            init()
        # Can't use super() here because of _AttributeDict's setattr override
        dict.__setattr__(self, 'aliases', aliases)

    def __setitem__(self, key, value):
        # Attr test required to not blow up when deepcopy'd
        if hasattr(self, 'aliases') and key in self.aliases:
            for aliased in self.aliases[key]:
                self[aliased] = value
        else:
            return super(_AliasDict, self).__setitem__(key, value)

    def expand_aliases(self, keys):
        ret = []
        for key in keys:
            if key in self.aliases:
                ret.extend(self.expand_aliases(self.aliases[key]))
            else:
                ret.append(key)
        return ret

def _pty_size():
    """
    Obtain (rows, cols) tuple for sizing a pty on the remote end.

    Defaults to 80x24 (which is also the 'ssh' lib's default) but will detect
    local (stdout-based) terminal window size on non-Windows platforms.
    """
    from fabric.state import win32
    if not win32:
        import fcntl
        import termios
        import struct

    default_rows, default_cols = 24, 80
    rows, cols = default_rows, default_cols
    if not win32 and sys.stdout.isatty():
        # We want two short unsigned integers (rows, cols)
        fmt = 'HH'
        # Create an empty (zeroed) buffer for ioctl to map onto. Yay for C!
        buffer = struct.pack(fmt, 0, 0)
        # Call TIOCGWINSZ to get window size of stdout, returns our filled
        # buffer
        try:
            result = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ,
                buffer)
            # Unpack buffer back into Python data types
            rows, cols = struct.unpack(fmt, result)
            # Fall back to defaults if TIOCGWINSZ returns unreasonable values
            if rows == 0:
                rows = default_rows
            if cols == 0:
                cols = default_cols
        # Deal with e.g. sys.stdout being monkeypatched, such as in testing.
        # Or termios not having a TIOCGWINSZ.
        except AttributeError:
            pass
    return rows, cols

def error(message, func=None, exception=None, stdout=None, stderr=None):
    """
    Call ``func`` with given error ``message``.

    If ``func`` is None (the default), the value of ``env.warn_only``
    determines whether to call ``abort`` or ``warn``.

    If ``exception`` is given, it is inspected to get a string message, which
    is printed alongside the user-generated ``message``.

    If ``stdout`` and/or ``stderr`` are given, they are assumed to be strings
    to be printed.
    """
    import fabric.state
    if func is None:
        func = fabric.state.env.warn_only and warn or abort
    # If debug printing is on, append a traceback to the message
    if fabric.state.output.debug:
        message += "\n\n" + format_exc()
    # Otherwise, if we were given an exception, append its contents.
    elif exception is not None:
        # Figure out how to get a string out of the exception; EnvironmentError
        # subclasses, for example, "are" integers and .strerror is the string.
        # Others "are" strings themselves. May have to expand this further for
        # other error types.
        if hasattr(exception, 'strerror') and exception.strerror is not None:
            underlying = exception.strerror
        else:
            underlying = exception
        message += "\n\nUnderlying exception:\n" + indent(str(underlying))
    if func is abort:
        if stdout and not fabric.state.output.stdout:
            message += _format_error_output("Standard output", stdout)
        if stderr and not fabric.state.output.stderr:
            message += _format_error_output("Standard error", stderr)
    return func(message)

def _format_error_output(header, body):
    term_width = _pty_size()[1]
    header_side_length = (term_width - (len(header) + 2)) / 2
    mark = "="
    side = mark * header_side_length
    return "\n\n%s %s %s\n\n%s\n\n%s" % (
        side, header, side, body, mark * term_width
    )

# TODO: replace with collections.deque(maxlen=xxx) in Python 2.6
class RingBuffer(list):
    def __init__(self, value, maxlen):
        # Heh.
        self._super = super(RingBuffer, self)
        self._maxlen = maxlen
        return self._super.__init__(value)

    def _free(self):
        return self._maxlen - len(self)

    def append(self, value):
        if self._free() == 0:
            del self[0]
        return self._super.append(value)

    def extend(self, values):
        overage = len(values) - self._free()
        if overage > 0:
            del self[0:overage]
        return self._super.extend(values)

    # Paranoia from here on out.
    def insert(self, index, value):
        raise ValueError("Can't insert into the middle of a ring buffer!")

    def __setslice__(self, i, j, sequence):
        raise ValueError("Can't set a slice of a ring buffer!")

    def __setitem__(self, key, value):
        if isinstance(key, slice):
            raise ValueError("Can't set a slice of a ring buffer!")
        else:
            return self._super.__setitem__(key, value)

def apply_lcwd(path, env):
    # Apply CWD if a relative path
    if not os.path.isabs(path) and env.lcwd:
        path = os.path.join(env.lcwd, path)
    return path
<|endoftext|>


<commit_before>import Tkinter as tk
import ttk

class DoubleListSelector(tk.Frame):
    def __init__(self, master, **options):
        tk.Frame.__init__(self, master)

        self.source = options.get("source")
        self.selected = options.get("selected")

        self.source_list_var = tk.StringVar()
        self.source_list = tk.Listbox(self, exportselection=0, listvar=self.source_list_var, selectmode=tk.MULTIPLE)
        self.source_list.bind('<ButtonRelease-1>', self.source_list_changed)

        for item in self.source:
            if not item in self.selected:
                self.source_list.insert(tk.END, str(item))

        self.selected_list_var = tk.StringVar()
        self.selected_list = tk.Listbox(self, exportselection=0, listvar=self.selected_list_var, selectmode=tk.MULTIPLE)
        self.selected_list.bind('<ButtonRelease-1>', self.selected_list_changed)

        for item in self.selected:
            self.selected_list.insert(tk.END, str(item))

        buttons = tk.Frame(self)
        self.add_btn = tk.Button(buttons, text="Add", command=self.select_item, state=tk.DISABLED)
        self.add_btn.pack()

        self.remove_btn = tk.Button(buttons, text="Remove", command=self.unselect_item, state=tk.DISABLED)
        self.remove_btn.pack()

        self.selected_list.pack(side=tk.LEFT)
        buttons.pack(side=tk.LEFT)
        self.source_list.pack(side=tk.LEFT)

    def source_list_changed(self, ev):
        if self.source_list.curselection():
            self.add_btn.configure(state=tk.NORMAL)
        else:
            self.add_btn.configure(state=tk.DISABLED)

    def selected_list_changed(self, ev):
        if self.selected_list.curselection():
            self.remove_btn.configure(state=tk.NORMAL)
        else:
            self.remove_btn.configure(state=tk.DISABLED)

    def select_item(self):
        selected_items = []
        for i in self.source_list.curselection():
            item = self.source[int(i)]
            selected_items.append(item)

        for item in selected_items:
            self.source.remove(item)

        self.selected.extend(selected_items)
        self.update_listboxes()
        self.add_btn.configure(state=tk.DISABLED)

    def update_listboxes(self):
        self.selected_list_var.set(' '.join(map(str, self.selected)))
        self.source_list_var.set(' '.join(map(str, self.source)))

    def unselect_item(self):
        selected_items = []
        for i in self.selected_list.curselection():
            item = self.selected[int(i)]
            selected_items.append(item)

        for item in selected_items:
            self.selected.remove(item)

        self.source.extend(selected_items)
        self.update_listboxes()
        self.remove_btn.configure(state=tk.DISABLED)

configuration_schemas = {}

def get_configuration_schema(name):
    global configuration_schemas
    return configuration_schemas[name]

def register_configuration_schema(schema):
    global configuration_schemas
    configuration_schemas[schema.name] = schema

def list_configuration_schemas():
    global configuration_schemas
    return configuration_schemas.values()

class ConfigurationSchema:
    def __init__(self, name, **args):
        self._name = name
        self._sections = {}
        self._documentation = args.get('documentation') or "Not documented"
        self._parents = args.get("parents") or []
        register_configuration_schema(self)

    def section(self, section):
        self._sections[section.name] = section
        return self

    def sections(self):
        return self._sections.values()

    def get_section(self, name):
        return self._sections[name]

    def parents(self):
        return self._parents

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value
        return self

    @property
    def documentation(self):
        return self._documentation

    @documentation.setter
    def documentation(self, value):
        self._documentation = value
        return self

    def __str__(self):
        return self.name

class ConfigurationSchemaSection:
    def __init__(self, name, **args):
        self._name = name
        self._subsections = {}
        self._options = {}
        self._documentation = args.get('documentation') or 'Not documented'

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value
        return self

    def subsections(self):
        return self._subsections.values()

    def add_section(self, section):
        self._subsections[section.name] = section
        return self

    def options(self):
        return self._options.values()

    def add_option(self, option):
        self._options[option.name] = option
        return self

    def get_option(self, name):
        return self._options[name]

    @property
    def documentation(self):
        return self._documentation

    @documentation.setter
    def documentation(self, value):
        self._documentation = value
        return self

class ConfigurationSchemaOption:
    def __init__(self, name, option_type, **args):
        self._name = name
        self._option_type = option_type
        self._documentation = args.get('documentation') or 'Not documented'

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value
        return self

    @property
    def option_type(self):
        return self._option_type

    @option_type.setter
    def option_type(self, value):
        self._option_type = value
        return self

    @property
    def documentation(self):
        return self._documentation

    @documentation.setter
    def documentation(self, value):
        self._documentation = value
        return self

class ConfigurationSchemaNavigator(tk.Frame):
    def __init__(self, master, schemas):
        tk.Frame.__init__(self, master)

        self.schemas = schemas

        # The pane
        self.pane = tk.Frame(self)

        # The tree
        self.tree = ttk.Treeview(self.pane)
        ysb = ttk.Scrollbar(self.pane, orient='vertical', command=self.tree.yview)
        xsb = ttk.Scrollbar(self.pane, orient='horizontal', command=self.tree.xview)
        self.tree.configure(yscroll=ysb.set, xscroll=xsb.set)
        self.tree.heading('#0', text='Configuration schemas', anchor='w')

        for schema in schemas:
            self.insert_schema(schema)

        self.tree.tag_bind('schema', '<ButtonRelease-1>', self.select_schema)
        self.tree.tag_bind('section', '<ButtonRelease-1>', self.select_section)
        self.tree.tag_bind('option', '<ButtonRelease-1>', self.select_option)

        self.tree.grid(column=0, row=0, sticky=tk.N+tk.S)

        # The editor

        self.editor = ConfigurationSchemaEditor(self.pane, schemas[0])
        self.editor.grid(column=1, row=0)
        self.pane.pack()

    def select_schema(self, ev):
        item_id = str(self.tree.focus())
        schema = self.find_schema(item_id)
        print 'Selected schema was %s' % item_id
        print schema
        self.editor.grid_forget()
        self.editor = ConfigurationSchemaEditor(self.pane, schema)
        self.editor.grid(column=1, row=0)

    def find_schema(self, id):
        return next((sc for sc in self.schemas if sc.name == id), None)

    def select_section(self, ev):
        item_id = str(self.tree.focus())
        print 'Selected section was %s' % item_id
        section = self.find_section(item_id)
        print section
        self.editor.grid_forget()
        self.editor = ConfigurationSchemaSectionEditor(self.pane, section)
        self.editor.grid(column=1, row=0)

    def find_section(self, id):
        path = id.split('.')
        # First element is the schema, find it first
        schema = self.find_schema(path[0])

        section = schema
        i = 1
        while i <= len(path) - 1:
            section = section.get_section(path[i])
            i = i + 1

        return section

    def select_option(self, ev):
        item_id = str(self.tree.focus())
        print 'Selected option was %s' % item_id
        print self.find_option(item_id)

    def find_option(self, id):
        path= id.split('.')
        schema = self.find_schema(path[0])
        section = schema
        i = 1
        while i <= len(path) - 2:
            section = section.get_section(path[i])
            i = i + 1
        return section.get_option(path[len(path) - 1])

    def insert_schema(self, schema):
        sc = self.tree.insert('', 'end', schema.name, text=schema.name, tags='schema')
        for section in schema.sections():
            self.insert_section(section, sc)

    def insert_section(self, section, parent):
        sc = self.tree.insert(parent, 'end', parent + '.' + section.name, text=section.name, tags='section')
        for subsection in section.subsections():
            self.insert_section(subsection, sc)

        for option in section.options():
            self.tree.insert(sc, 'end', sc + '.' + option.name, text=option.name, tags='option')

class ConfigurationSchemaEditor(tk.Frame):
    def __init__(self, master, schema):
        tk.Frame.__init__(self, master)

        self.schema = schema
        tk.Label(self, text=schema.name + " configuration schema").pack()
        props = tk.Frame(self)
        tk.Label(props, text="Name: ").grid(row=0, column=0)
        self.schema_name = tk.StringVar()
        self.schema_name.set(schema.name or "")

        tk.Entry(props, textvariable=self.schema_name).grid(row=0, column=1)
        tk.Label(props, text="Parents:").grid(row=1, column=0)
        parents = DoubleListSelector(props, source=list_configuration_schemas(),
                                        selected=schema.parents())
        parents.grid(row=1, column=1)

        self.schema_documentation = schema.documentation
        tk.Label(props, text="Documentation").grid(row=2, column=0)
        text = tk.Text(props)
        text.insert(tk.END, self.schema_documentation)
        text.grid(row=2, column=1)

        tk.Button(props, text="Save").grid(row=3, column=1, sticky=tk.SE)
        props.pack()

class ConfigurationSchemaSectionEditor(tk.Frame):
    def __init__(self, master, section):
        tk.Frame.__init__(self, master)

        self.section = section

        tk.Label(self, text=section.name + " section").pack()

        f = tk.Frame(self)

        tk.Label(f, text="Name: ").grid(row=0, column=0)
        self.section_name = tk.StringVar()
        self.section_name.set(section.name or "")
        tk.Entry(f, textvariable=self.section_name).grid(row=0, column=1)

        self.section_documentation = section.documentation
        tk.Label(f, text="Documentation").grid(row=1, column=0)
        text = tk.Text(f)
        text.insert(tk.END, self.section_documentation)
        text.grid(row=1, column=1)

        tk.Button(f, text="Save").grid(row=2, column=1, sticky=tk.SE)
        f.pack()

class ConfigurationNavigator(tk.Frame):
    def __init__(self, master, configs):
        tk.Frame.__init__(self, master)

        configs = tk.Frame(self)

        self.tree = ttk.Treeview(configs)
        ysb = ttk.Scrollbar(configs, orient='vertical', command=self.tree.yview)
        xsb = ttk.Scrollbar(configs, orient='horizontal', command=self.tree.xview)
        self.tree.configure(yscroll=ysb.set, xscroll=xsb.set)
        self.tree.heading('#0', text='Configuration', anchor='w')

        #abspath = os.path.abspath(path)
        #root_node = self.tree.insert('', 'end', text="Configuration schemas", open=True)
        #self.process_directory(root_node, abspath)

        self.tree.grid(row=0, column=0)
        ysb.grid(row=0, column=1, sticky='ns')
        xsb.grid(row=1, column=0, sticky='ew')

        self.tree.bind("<Button-1>", self.onDoubleClick)
        configs.pack(side=tk.LEFT)

        self.panel = tk.Frame(self)
        tk.Label(self.panel, text="Hello!!").pack()
        self.panel.pack()

        self.grid()

    def onDoubleClick(self, event):
        if isinstance(self.tree.selection(), tuple):
            item = self.tree.selection()[0]
            print "You clicked on ", self.tree.item(item, "text")
            self.panel.pack_forget()
            self.panel= tk.Frame(self)

            self.panel.pack()
            tk.Label(self.panel, text=("You clicked on ", self.tree.item(item, "text"))).pack()

    def process_directory(self, parent, path):
        for p in os.listdir(path):
            abspath = os.path.join(path, p)
            isdir = os.path.isdir(abspath)
            oid = self.tree.insert(parent, 'end', text=p, open=False)
            if isdir:
                self.process_directory(oid, abspath)

if __name__ == '__main__':
    schemas = []
    sch1 = ConfigurationSchema("Web")
    host = ConfigurationSchemaOption("host", 'uri', documentation="Server host")
    s1 = ConfigurationSchemaSection("Server").add_option(host)
    sch1.section(s1)
    schemas.append(sch1)

    db = ConfigurationSchema("Database")
    db_engine = ConfigurationSchemaOption("engine", "option", documentation="The database engine")
    db_server = ConfigurationSchemaSection("Server").add_option(db_engine)
    db.section(db_server)
    schemas.append(db)

    root = tk.Tk()
    navigator = ConfigurationSchemaNavigator(root, schemas)
    navigator.pack()
    root.mainloop()<commit_msg>Schema options and editor<commit_after>import Tkinter as tk
import ttk

class DoubleListSelector(tk.Frame):
    def __init__(self, master, **options):
        tk.Frame.__init__(self, master)

        self.source = options.get("source")
        self.selected = options.get("selected")

        self.source_list_var = tk.StringVar()
        self.source_list = tk.Listbox(self, exportselection=0, listvar=self.source_list_var, selectmode=tk.MULTIPLE)
        self.source_list.bind('<ButtonRelease-1>', self.source_list_changed)

        for item in self.source:
            if not item in self.selected:
                self.source_list.insert(tk.END, str(item))

        self.selected_list_var = tk.StringVar()
        self.selected_list = tk.Listbox(self, exportselection=0, listvar=self.selected_list_var, selectmode=tk.MULTIPLE)
        self.selected_list.bind('<ButtonRelease-1>', self.selected_list_changed)

        for item in self.selected:
            self.selected_list.insert(tk.END, str(item))

        buttons = tk.Frame(self)
        self.add_btn = tk.Button(buttons, text="Add", command=self.select_item, state=tk.DISABLED)
        self.add_btn.pack()

        self.remove_btn = tk.Button(buttons, text="Remove", command=self.unselect_item, state=tk.DISABLED)
        self.remove_btn.pack()

        self.selected_list.pack(side=tk.LEFT)
        buttons.pack(side=tk.LEFT)
        self.source_list.pack(side=tk.LEFT)

    def source_list_changed(self, ev):
        if self.source_list.curselection():
            self.add_btn.configure(state=tk.NORMAL)
        else:
            self.add_btn.configure(state=tk.DISABLED)

    def selected_list_changed(self, ev):
        if self.selected_list.curselection():
            self.remove_btn.configure(state=tk.NORMAL)
        else:
            self.remove_btn.configure(state=tk.DISABLED)

    def select_item(self):
        selected_items = []
        for i in self.source_list.curselection():
            item = self.source[int(i)]
            selected_items.append(item)

        for item in selected_items:
            self.source.remove(item)

        self.selected.extend(selected_items)
        self.update_listboxes()
        self.add_btn.configure(state=tk.DISABLED)

    def update_listboxes(self):
        self.selected_list_var.set(' '.join(map(str, self.selected)))
        self.source_list_var.set(' '.join(map(str, self.source)))

    def unselect_item(self):
        selected_items = []
        for i in self.selected_list.curselection():
            item = self.selected[int(i)]
            selected_items.append(item)

        for item in selected_items:
            self.selected.remove(item)

        self.source.extend(selected_items)
        self.update_listboxes()
        self.remove_btn.configure(state=tk.DISABLED)

configuration_schemas = {}

def get_configuration_schema(name):
    global configuration_schemas
    return configuration_schemas[name]

def register_configuration_schema(schema):
    global configuration_schemas
    configuration_schemas[schema.name] = schema

def list_configuration_schemas():
    global configuration_schemas
    return configuration_schemas.values()

class ConfigurationSchema:
    def __init__(self, name, **args):
        self._name = name
        self._sections = {}
        self._documentation = args.get('documentation') or "Not documented"
        self._parents = args.get("parents") or []
        register_configuration_schema(self)

    def section(self, section):
        self._sections[section.name] = section
        return self

    def sections(self):
        return self._sections.values()

    def get_section(self, name):
        return self._sections[name]

    def parents(self):
        return self._parents

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value
        return self

    @property
    def documentation(self):
        return self._documentation

    @documentation.setter
    def documentation(self, value):
        self._documentation = value
        return self

    def __str__(self):
        return self.name

class ConfigurationSchemaSection:
    def __init__(self, name, **args):
        self._name = name
        self._subsections = {}
        self._options = {}
        self._documentation = args.get('documentation') or 'Not documented'

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value
        return self

    def subsections(self):
        return self._subsections.values()

    def add_section(self, section):
        self._subsections[section.name] = section
        return self

    def options(self):
        return self._options.values()

    def add_option(self, option):
        self._options[option.name] = option
        return self

    def get_option(self, name):
        return self._options[name]

    @property
    def documentation(self):
        return self._documentation

    @documentation.setter
    def documentation(self, value):
        self._documentation = value
        return self

class ConfigurationSchemaOption:
    def __init__(self, name, option_type, **args):
        self._name = name
        self._option_type = option_type
        self._documentation = args.get('documentation') or 'Not documented'

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value
        return self

    @property
    def option_type(self):
        return self._option_type

    @option_type.setter
    def option_type(self, value):
        self._option_type = value
        return self

    @property
    def documentation(self):
        return self._documentation

    @documentation.setter
    def documentation(self, value):
        self._documentation = value
        return self

class OptionType(object):
    _name = "Option type"

    @classmethod
    def option_name(cls):
        return cls._name

    @property
    def name(self):
        return self.__class__.option_name()

class StringOptionType(OptionType):
    _name = "String"

class NumberOptionType(OptionType):
    _name = "Number"

class BooleanOptionType(OptionType):
    _name = "Boolean"

class ChoiceOptionType(OptionType):
    _name = "Choice"

    def __init__(self, options):
        self._options = options

class ListOptionType(OptionType):
    _name = "List"

    def __init__(self, options):
        self._options = options

class ConfigurationSchemaNavigator(tk.Frame):
    def __init__(self, master, schemas):
        tk.Frame.__init__(self, master)

        self.schemas = schemas

        # The pane
        self.pane = tk.Frame(self)

        # The tree
        self.tree = ttk.Treeview(self.pane)
        ysb = ttk.Scrollbar(self.pane, orient='vertical', command=self.tree.yview)
        xsb = ttk.Scrollbar(self.pane, orient='horizontal', command=self.tree.xview)
        self.tree.configure(yscroll=ysb.set, xscroll=xsb.set)
        self.tree.heading('#0', text='Configuration schemas', anchor='w')

        for schema in schemas:
            self.insert_schema(schema)

        self.tree.tag_bind('schema', '<ButtonRelease-1>', self.select_schema)
        self.tree.tag_bind('section', '<ButtonRelease-1>', self.select_section)
        self.tree.tag_bind('option', '<ButtonRelease-1>', self.select_option)

        self.tree.grid(column=0, row=0, sticky=tk.N+tk.S)

        # The editor

        self.editor = ConfigurationSchemaEditor(self.pane, schemas[0])
        self.editor.grid(column=1, row=0)
        self.pane.pack()

    def select_schema(self, ev):
        item_id = str(self.tree.focus())
        schema = self.find_schema(item_id)
        print 'Selected schema was %s' % item_id
        print schema
        self.editor.grid_forget()
        self.editor = ConfigurationSchemaEditor(self.pane, schema)
        self.editor.grid(column=1, row=0)

    def find_schema(self, id):
        return next((sc for sc in self.schemas if sc.name == id), None)

    def select_section(self, ev):
        item_id = str(self.tree.focus())
        print 'Selected section was %s' % item_id
        section = self.find_section(item_id)
        print section
        self.editor.grid_forget()
        self.editor = ConfigurationSchemaSectionEditor(self.pane, section)
        self.editor.grid(column=1, row=0)

    def find_section(self, id):
        path = id.split('.')
        # First element is the schema, find it first
        schema = self.find_schema(path[0])

        section = schema
        i = 1
        while i <= len(path) - 1:
            section = section.get_section(path[i])
            i = i + 1

        return section

    def select_option(self, ev):
        item_id = str(self.tree.focus())
        print 'Selected option was %s' % item_id
        option = self.find_option(item_id)
        print self.find_option(item_id)
        self.editor.grid_forget()
        self.editor = ConfigurationSchemaOptionEditor(self.pane, option)
        self.editor.grid(column=1, row=0)

    def find_option(self, id):
        path= id.split('.')
        schema = self.find_schema(path[0])
        section = schema
        i = 1
        while i <= len(path) - 2:
            section = section.get_section(path[i])
            i = i + 1
        return section.get_option(path[len(path) - 1])

    def insert_schema(self, schema):
        sc = self.tree.insert('', 'end', schema.name, text=schema.name, tags='schema')
        for section in schema.sections():
            self.insert_section(section, sc)

    def insert_section(self, section, parent):
        sc = self.tree.insert(parent, 'end', parent + '.' + section.name, text=section.name, tags='section')
        for subsection in section.subsections():
            self.insert_section(subsection, sc)

        for option in section.options():
            self.tree.insert(sc, 'end', sc + '.' + option.name, text=option.name, tags='option')

class ConfigurationSchemaEditor(tk.Frame):
    def __init__(self, master, schema):
        tk.Frame.__init__(self, master)

        self.schema = schema
        tk.Label(self, text=schema.name + " configuration schema").pack()
        props = tk.Frame(self)
        tk.Label(props, text="Name: ").grid(row=0, column=0)
        self.schema_name = tk.StringVar()
        self.schema_name.set(schema.name or "")

        tk.Entry(props, textvariable=self.schema_name).grid(row=0, column=1)
        tk.Label(props, text="Parents:").grid(row=1, column=0)
        parents = DoubleListSelector(props, source=list_configuration_schemas(),
                                        selected=schema.parents())
        parents.grid(row=1, column=1)

        self.schema_documentation = schema.documentation
        tk.Label(props, text="Documentation").grid(row=2, column=0)
        text = tk.Text(props)
        text.insert(tk.END, self.schema_documentation)
        text.grid(row=2, column=1)

        tk.Button(props, text="Save").grid(row=3, column=1, sticky=tk.SE)
        props.pack()

class ConfigurationSchemaSectionEditor(tk.Frame):
    def __init__(self, master, section):
        tk.Frame.__init__(self, master)

        self.section = section

        tk.Label(self, text=section.name + " section").pack()

        f = tk.Frame(self)

        tk.Label(f, text="Name: ").grid(row=0, column=0)
        self.section_name = tk.StringVar()
        self.section_name.set(section.name or "")
        tk.Entry(f, textvariable=self.section_name).grid(row=0, column=1)

        self.section_documentation = section.documentation
        tk.Label(f, text="Documentation").grid(row=1, column=0)
        text = tk.Text(f)
        text.insert(tk.END, self.section_documentation)
        text.grid(row=1, column=1)

        tk.Button(f, text="Save").grid(row=2, column=1, sticky=tk.SE)
        f.pack()

class ConfigurationSchemaOptionEditor(tk.Frame):
    def __init__(self, master, option):
        tk.Frame.__init__(self, master)

        self.option = option

        tk.Label(self, text=option.name + " option").pack()

        f = tk.Frame(self)

        tk.Label(f, text="Name: ").grid(row=0, column=0)
        self.option_name = tk.StringVar()
        self.option_name.set(option.name or "")
        tk.Entry(f, textvariable=self.option_name).grid(row=0, column=1)

        tk.Label(f, text="Type: ").grid(row=1, column=0)
        self.option_type = tk.StringVar()
        if option.option_type:
            self.option_type.set(option.option_type.name)
        option_types = map(lambda o: o.option_name(), OptionType.__subclasses__())
        tk.OptionMenu(f, self.option_type, *option_types).grid(row=1, column=1)

        self.option_documentation = option.documentation
        tk.Label(f, text="Documentation:").grid(row=2, column=0)
        text = tk.Text(f)
        text.insert(tk.END, self.option_documentation)
        text.grid(row=2, column=1)

        tk.Button(f, text="Save").grid(row=3, column=1, sticky=tk.SE)
        f.pack()

class ConfigurationNavigator(tk.Frame):
    def __init__(self, master, configs):
        tk.Frame.__init__(self, master)

        configs = tk.Frame(self)

        self.tree = ttk.Treeview(configs)
        ysb = ttk.Scrollbar(configs, orient='vertical', command=self.tree.yview)
        xsb = ttk.Scrollbar(configs, orient='horizontal', command=self.tree.xview)
        self.tree.configure(yscroll=ysb.set, xscroll=xsb.set)
        self.tree.heading('#0', text='Configuration', anchor='w')

        #abspath = os.path.abspath(path)
        #root_node = self.tree.insert('', 'end', text="Configuration schemas", open=True)
        #self.process_directory(root_node, abspath)

        self.tree.grid(row=0, column=0)
        ysb.grid(row=0, column=1, sticky='ns')
        xsb.grid(row=1, column=0, sticky='ew')

        self.tree.bind("<Button-1>", self.onDoubleClick)
        configs.pack(side=tk.LEFT)

        self.panel = tk.Frame(self)
        tk.Label(self.panel, text="Hello!!").pack()
        self.panel.pack()

        self.grid()

    def onDoubleClick(self, event):
        if isinstance(self.tree.selection(), tuple):
            item = self.tree.selection()[0]
            print "You clicked on ", self.tree.item(item, "text")
            self.panel.pack_forget()
            self.panel= tk.Frame(self)

            self.panel.pack()
            tk.Label(self.panel, text=("You clicked on ", self.tree.item(item, "text"))).pack()

    def process_directory(self, parent, path):
        for p in os.listdir(path):
            abspath = os.path.join(path, p)
            isdir = os.path.isdir(abspath)
            oid = self.tree.insert(parent, 'end', text=p, open=False)
            if isdir:
                self.process_directory(oid, abspath)

if __name__ == '__main__':
    schemas = []
    sch1 = ConfigurationSchema("Web")
    host = ConfigurationSchemaOption("host", StringOptionType(), documentation="Server host")
    s1 = ConfigurationSchemaSection("Server").add_option(host)
    sch1.section(s1)
    schemas.append(sch1)

    db = ConfigurationSchema("Database")
    db_engine = ConfigurationSchemaOption("engine", ChoiceOptionType(["Postgresql", "Mysql"]), documentation="The database engine")
    db_server = ConfigurationSchemaSection("Server").add_option(db_engine)
    db.section(db_server)
    schemas.append(db)

    root = tk.Tk()
    navigator = ConfigurationSchemaNavigator(root, schemas)
    navigator.pack()
    root.mainloop()<|endoftext|>