<commit_before>#!/usr/bin/env python3
# Copyright (c) 2013-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Generate seeds.txt from Pieter's DNS seeder
#

import collections
import dns.resolver
import re
import sys

NSEEDS = 512

MAX_SEEDS_PER_ASN = 2

MIN_BLOCKS = 700000

# These are hosts that have been observed to be behaving strangely (e.g.
# aggressively connecting to every node).
SUSPICIOUS_HOSTS = {
    "23.92.36.9", "72.36.89.11", "130.211.129.106", "178.63.107.226",
    "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6",
    "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211",
    "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214",
    "54.94.195.96", "54.94.200.247"
}

PATTERN_IPV4 = re.compile(
    r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
PATTERN_ONION = re.compile(
    r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")

# Used to only select nodes with a user agent string compatible with the
# eCash network.
PATTERN_AGENT = re.compile(r"^(/Bitcoin ABC:0.(26|25).(\d+)\(.+\)/)")

def parseline(line):
    sline = line.split()
    if len(sline) < 11:
        return None

    # The user agent is at the end of the line. It may contain space, so we
    # concatenate.
    for i in range(12, len(sline)):
        sline[11] += ' ' + sline[i]

    # Remove leftovers
    del sline[12:]

    m = PATTERN_IPV4.match(sline[0])
    sortkey = None
    ip = None
    if m is None:
        m = PATTERN_IPV6.match(sline[0])
        if m is None:
            m = PATTERN_ONION.match(sline[0])
            if m is None:
                return None
            else:
                net = 'onion'
                ipstr = sortkey = m.group(1)
                port = int(m.group(2))
        else:
            net = 'ipv6'
            # Not interested in localhost
            if m.group(1) in ['::']:
                return None
            ipstr = m.group(1)
            # XXX parse IPv6 into number, could use name_to_ipv6 from
            # generate-seeds
            sortkey = ipstr
            port = int(m.group(2))
    else:
        # Do IPv4 sanity check
        ip = 0
        for i in range(0, 4):
            if int(m.group(i + 2)) < 0 or int(m.group(i + 2)) > 255:
                return None
            ip = ip + (int(m.group(i + 2)) << (8 * (3 - i)))
        if ip == 0:
            return None
        net = 'ipv4'
        sortkey = ip
        ipstr = m.group(1)
        port = int(m.group(6))
    # Skip bad results.
    if sline[1] == 0:
        return None
    # Extract uptime %.
    uptime30 = float(sline[7][:-1])
    # Extract Unix timestamp of last success.
    lastsuccess = int(sline[2])
    # Extract protocol version.
    version = int(sline[10])
    # Extract user agent.
    agent = sline[11][1:-1]
    # Extract service flags.
    service = int(sline[9], 16)
    # Extract blocks.
    blocks = int(sline[8])
    # Construct result.
    return {
        'net': net,
        'ip': ipstr,
        'port': port,
        'ipnum': ip,
        'uptime': uptime30,
        'lastsuccess': lastsuccess,
        'version': version,
        'agent': agent,
        'service': service,
        'blocks': blocks,
        'sortkey': sortkey,
    }

def filtermultiport(ips):
    '''Filter out hosts with more nodes per IP'''
    hist = collections.defaultdict(list)
    for ip in ips:
        hist[ip['sortkey']].append(ip)
    return [value[0] for (key, value) in list(hist.items()) if len(value) == 1]

# Based on Greg Maxwell's seed_filter.py

def filterbyasn(ips, max_per_asn, max_total):
    # Sift out ips by type
    ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
    ips_onion = [ip for ip in ips if ip['net'] == 'onion']

    # Filter IPv46 by ASN
    result = []
    asn_count = {}
    for ip in ips_ipv46:
        if len(result) == max_total:
            break
        try:
            if ip['net'] == 'ipv4':
                ipaddr = ip['ip']
                prefix = '.origin'
            else:
                # http://www.team-cymru.com/IP-ASN-mapping.html
                # 2001:4860:b002:23::68
                res = str()
                # pick the first 4 nibbles
                for nb in ip['ip'].split(':')[:4]:
                    # right padded with '0'
                    for c in nb.zfill(4):
                        # 2001 4860 b002 0023
                        res += c + '.'
                # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
                ipaddr = res.rstrip('.')
                prefix = '.origin6'

            asn = int([x.to_text() for x in dns.resolver.query('.'.join(
                       reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
                'TXT').response.answer][0].split('\"')[1].split(' ')[0])
            if asn not in asn_count:
                asn_count[asn] = 0
            if asn_count[asn] == max_per_asn:
                continue
            asn_count[asn] += 1
            result.append(ip)
        except Exception:
            sys.stderr.write(
                'ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')

    # Add back Onions
    result.extend(ips_onion)
    return result

def main():
    lines = sys.stdin.readlines()
    ips = [parseline(line) for line in lines]

    # Skip entries with valid address.
    ips = [ip for ip in ips if ip is not None]
    # Skip entries from suspicious hosts.
    ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
    # Enforce minimal number of blocks.
    ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
    # Require service bit 1.
    ips = [ip for ip in ips if (ip['service'] & 1) == 1]
    # Require at least 50% 30-day uptime.
    ips = [ip for ip in ips if ip['uptime'] > 50]
    # Require a known and recent user agent.
    ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]

    # Sort by availability (and use last success as tie breaker)
    ips.sort(key=lambda x:
             (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
    # Filter out hosts with multiple bitcoin ports, these are likely abusive
    ips = filtermultiport(ips)
    # Look up ASNs and limit results, both per ASN and globally.
    ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
    # Sort the results by IP address (for deterministic output).
    ips.sort(key=lambda x: (x['net'], x['sortkey']))

    for ip in ips:
        if ip['net'] == 'ipv6':
            print('[{}]:{}'.format(ip['ip'], ip['port']))
        else:
            print('{}:{}'.format(ip['ip'], ip['port']))

if __name__ == '__main__':
    main()
<commit_msg>Refactor & improve makeseeds<commit_after>#!/usr/bin/env python3
# Copyright (c) 2013-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Generate seeds.txt from Pieter's DNS seeder
#

import collections
import dns.resolver
import re
import sys

NSEEDS = 512

MAX_SEEDS_PER_ASN = 2

MIN_BLOCKS = 700000

# These are hosts that have been observed to be behaving strangely (e.g.
# aggressively connecting to every node).
SUSPICIOUS_HOSTS = {
    "23.92.36.9", "72.36.89.11", "130.211.129.106", "178.63.107.226",
    "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6",
    "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211",
    "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214",
    "54.94.195.96", "54.94.200.247"
}

PATTERN_IPV4 = re.compile(
    r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
PATTERN_ONION = re.compile(
    r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")

# Used to only select nodes with a user agent string compatible with the
# eCash network.
PATTERN_AGENT = re.compile(r"^(/Bitcoin ABC:0.(26|25).(\d+)\(.+\)/)")

def parseline(line):
    sline = line.split()
    if len(sline) < 11:
        return None

    # The user agent is at the end of the line. It may contain space, so we
    # concatenate.
    for i in range(12, len(sline)):
        sline[11] += ' ' + sline[i]

    # Remove leftovers
    del sline[12:]

    m = PATTERN_IPV4.match(sline[0])
    sortkey = None
    ip = None
    if m is None:
        m = PATTERN_IPV6.match(sline[0])
        if m is None:
            m = PATTERN_ONION.match(sline[0])
            if m is None:
                return None
            else:
                net = 'onion'
                ipstr = sortkey = m.group(1)
                port = int(m.group(2))
        else:
            net = 'ipv6'
            # Not interested in localhost
            if m.group(1) in ['::']:
                return None
            ipstr = m.group(1)
            # XXX parse IPv6 into number, could use name_to_ipv6 from
            # generate-seeds
            sortkey = ipstr
            port = int(m.group(2))
    else:
        # Do IPv4 sanity check
        ip = 0
        for i in range(0, 4):
            if int(m.group(i + 2)) < 0 or int(m.group(i + 2)) > 255:
                return None
            ip = ip + (int(m.group(i + 2)) << (8 * (3 - i)))
        if ip == 0:
            return None
        net = 'ipv4'
        sortkey = ip
        ipstr = m.group(1)
        port = int(m.group(6))
    # Skip bad results.
    if sline[1] == 0:
        return None
    # Extract uptime %.
    uptime30 = float(sline[7][:-1])
    # Extract Unix timestamp of last success.
    lastsuccess = int(sline[2])
    # Extract protocol version.
    version = int(sline[10])
    # Extract user agent.
    agent = sline[11][1:-1]
    # Extract service flags.
    service = int(sline[9], 16)
    # Extract blocks.
    blocks = int(sline[8])
    # Construct result.
    return {
        'net': net,
        'ip': ipstr,
        'port': port,
        'ipnum': ip,
        'uptime': uptime30,
        'lastsuccess': lastsuccess,
        'version': version,
        'agent': agent,
        'service': service,
        'blocks': blocks,
        'sortkey': sortkey,
    }

def dedup(ips):
    '''deduplicate by address,port'''
    d = {}
    for ip in ips:
        d[ip['ip'], ip['port']] = ip
    return list(d.values())

def filtermultiport(ips):
    '''Filter out hosts with more nodes per IP'''
    hist = collections.defaultdict(list)
    for ip in ips:
        hist[ip['sortkey']].append(ip)
    return [value[0] for (key, value) in list(hist.items()) if len(value) == 1]

def lookup_asn(net, ip):
    '''
    Look up the asn for an IP (4 or 6) address by querying cymru.com, or None
    if it could not be found.
    '''
    try:
        if net == 'ipv4':
            ipaddr = ip
            prefix = '.origin'
        else:
            # http://www.team-cymru.com/IP-ASN-mapping.html
            # 2001:4860:b002:23::68
            res = str()
            # pick the first 4 nibbles
            for nb in ip.split(':')[:4]:
                # right padded with '0'
                for c in nb.zfill(4):
                    # 2001 4860 b002 0023
                    res += c + '.'
            # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
            ipaddr = res.rstrip('.')
            prefix = '.origin6'

        asn = int([x.to_text() for x in dns.resolver.query('.'.join(
                   reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
            'TXT').response.answer][0].split('\"')[1].split(' ')[0])
        return asn
    except Exception:
        sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n')
        return None

# Based on Greg Maxwell's seed_filter.py

def filterbyasn(ips, max_per_asn, max_per_net):
    # Sift out ips by type
    ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
    ips_onion = [ip for ip in ips if ip['net'] == 'onion']

    # Filter IPv46 by ASN, and limit to max_per_net per network
    result = []
    net_count = collections.defaultdict(int)
    asn_count = collections.defaultdict(int)
    for ip in ips_ipv46:
        if net_count[ip['net']] == max_per_net:
            continue
        asn = lookup_asn(ip['net'], ip['ip'])
        if asn is None or asn_count[asn] == max_per_asn:
            continue
        asn_count[asn] += 1
        net_count[ip['net']] += 1
        result.append(ip)

    # Add back Onions (up to max_per_net)
    result.extend(ips_onion[0:max_per_net])
    return result

def ip_stats(ips):
    hist = collections.defaultdict(int)
    for ip in ips:
        if ip is not None:
            hist[ip['net']] += 1

    return '{:6d} {:6d} {:6d}'.format(
        hist['ipv4'], hist['ipv6'], hist['onion'])

def main():
    lines = sys.stdin.readlines()
    ips = [parseline(line) for line in lines]

    print(
        '\x1b[7m  IPv4   IPv6  Onion Pass                                               \x1b[0m',
        file=sys.stderr)
    print('{} Initial'.format(ip_stats(ips)), file=sys.stderr)
    # Skip entries with invalid address.
    ips = [ip for ip in ips if ip is not None]
    print(
        '{} Skip entries with invalid address'.format(ip_stats(ips)), file=sys.stderr)
    # Skip duplicates (in case multiple seeds files were concatenated)
    ips = dedup(ips)
    print(
        '{} After removing duplicates'.format(
            ip_stats(ips)),
        file=sys.stderr)
    # Skip entries from suspicious hosts.
    ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
    print(
        '{} Skip entries from suspicious hosts'.format(ip_stats(ips)), file=sys.stderr)
    # Enforce minimal number of blocks.
    ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
    print(
        '{} Enforce minimal number of blocks'.format(
            ip_stats(ips)),
        file=sys.stderr)
    # Require service bit 1.
    ips = [ip for ip in ips if (ip['service'] & 1) == 1]
    print('{} Require service bit 1'.format(ip_stats(ips)), file=sys.stderr)
    # Require at least 50% 30-day uptime for clearnet, 10% for onion.
    req_uptime = {
        'ipv4': 50,
        'ipv6': 50,
        'onion': 10,
    }
    ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
    print('{} Require minimum uptime'.format(ip_stats(ips)), file=sys.stderr)
    # Require a known and recent user agent.
    ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
    print(
        '{} Require a known and recent user agent'.format(ip_stats(ips)), file=sys.stderr)
    # Sort by availability (and use last success as tie breaker)
    ips.sort(key=lambda x:
             (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
    # Filter out hosts with multiple bitcoin ports, these are likely abusive
    ips = filtermultiport(ips)
    print(
        '{} Filter out hosts with multiple bitcoin ports'.format(ip_stats(ips)), file=sys.stderr)
    # Look up ASNs and limit results, both per ASN and globally.
    ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
    print(
        '{} Look up ASNs and limit results per ASN and per net'.format(ip_stats(ips)), file=sys.stderr)
    # Sort the results by IP address (for deterministic output).
    ips.sort(key=lambda x: (x['net'], x['sortkey']))
    for ip in ips:
        if ip['net'] == 'ipv6':
            print('[{}]:{}'.format(ip['ip'], ip['port']))
        else:
            print('{}:{}'.format(ip['ip'], ip['port']))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2016, the GPyOpt Authors
# Licensed under the BSD 3-clause license (see LICENSE.txt)

from .base import BOModel
import numpy as np

class RFModel(BOModel):
    """
    General class for handling a Ramdom Forest in GPyOpt.

    .. Note:: The model has beed wrapper 'as it is' from  Scikit-learn. Check
    http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestRegressor.html
    for further details.
    """

    analytical_gradient_prediction = False

    def __init__(self, bootstrap=True, criterion='mse', max_depth=None,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=500, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False):

        self.bootstrap = bootstrap
        self.criterion = criterion
        self.max_depth = max_depth
        self.max_features = max_features
        self.max_leaf_nodes = max_leaf_nodes
        self.min_samples_leaf = min_samples_leaf
        self.min_samples_split = min_samples_split
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.n_estimators = n_estimators
        self.n_jobs = n_jobs
        self.oob_score = oob_score
        self.random_state = random_state
        self.verbose = verbose
        self.warm_start = warm_start

        self.model = None

    def _create_model(self, X, Y):
        """
        Creates the model given some input data X and Y.
        """
        from sklearn.ensemble import RandomForestRegressor
        self.X = X
        self.Y = Y
        self.model = RandomForestRegressor(bootstrap = self.bootstrap,
                                           criterion = self.criterion,
                                           max_depth = self.max_depth,
                                           max_features = self.max_features,
                                           max_leaf_nodes = self.max_leaf_nodes,
                                           min_samples_leaf = self.min_samples_leaf,
                                           min_samples_split = self.min_samples_split,
                                           min_weight_fraction_leaf = self.min_weight_fraction_leaf,
                                           n_estimators = self.n_estimators,
                                           n_jobs = self.n_jobs,
                                           oob_score = self.oob_score,
                                           random_state = self.random_state,
                                           verbose = self.verbose,
                                           warm_start = self.warm_start)

        #self.model = RandomForestRegressor()
        self.model.fit(X,Y.flatten())

    def updateModel(self, X_all, Y_all, X_new, Y_new):
        """
        Updates the model with new observations.
        """
        self.X = X_all
        self.Y = Y_all
        if self.model is None:
            self._create_model(X_all, Y_all)
        else:
            self.model.fit(X_all, Y_all.flatten())

    def predict(self, X):
        """
        Predictions with the model. Returns posterior means and standard deviations at X.
        """
        X = np.atleast_2d(X)
        m = np.empty(shape=(0,1))
        s = np.empty(shape=(0,1))

        for k in range(X.shape[0]):
            preds = []
            for pred in self.model.estimators_:
                preds.append(pred.predict(X[k,:])[0])
            m = np.vstack((m ,np.array(preds).mean()))
            s = np.vstack((s ,np.array(preds).std()))
        return m, s

    def get_fmin(self):
        return self.model.predict(self.X).min()
<commit_msg>Correct typo in RFModel docstring<commit_after># Copyright (c) 2016, the GPyOpt Authors
# Licensed under the BSD 3-clause license (see LICENSE.txt)

from .base import BOModel
import numpy as np

class RFModel(BOModel):
    """
    General class for handling a Random Forest in GPyOpt.

    .. Note:: The model has beed wrapper 'as it is' from  Scikit-learn. Check
    http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestRegressor.html
    for further details.
    """

    analytical_gradient_prediction = False

    def __init__(self, bootstrap=True, criterion='mse', max_depth=None,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=500, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False):

        self.bootstrap = bootstrap
        self.criterion = criterion
        self.max_depth = max_depth
        self.max_features = max_features
        self.max_leaf_nodes = max_leaf_nodes
        self.min_samples_leaf = min_samples_leaf
        self.min_samples_split = min_samples_split
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.n_estimators = n_estimators
        self.n_jobs = n_jobs
        self.oob_score = oob_score
        self.random_state = random_state
        self.verbose = verbose
        self.warm_start = warm_start

        self.model = None

    def _create_model(self, X, Y):
        """
        Creates the model given some input data X and Y.
        """
        from sklearn.ensemble import RandomForestRegressor
        self.X = X
        self.Y = Y
        self.model = RandomForestRegressor(bootstrap = self.bootstrap,
                                           criterion = self.criterion,
                                           max_depth = self.max_depth,
                                           max_features = self.max_features,
                                           max_leaf_nodes = self.max_leaf_nodes,
                                           min_samples_leaf = self.min_samples_leaf,
                                           min_samples_split = self.min_samples_split,
                                           min_weight_fraction_leaf = self.min_weight_fraction_leaf,
                                           n_estimators = self.n_estimators,
                                           n_jobs = self.n_jobs,
                                           oob_score = self.oob_score,
                                           random_state = self.random_state,
                                           verbose = self.verbose,
                                           warm_start = self.warm_start)

        #self.model = RandomForestRegressor()
        self.model.fit(X,Y.flatten())

    def updateModel(self, X_all, Y_all, X_new, Y_new):
        """
        Updates the model with new observations.
        """
        self.X = X_all
        self.Y = Y_all
        if self.model is None:
            self._create_model(X_all, Y_all)
        else:
            self.model.fit(X_all, Y_all.flatten())

    def predict(self, X):
        """
        Predictions with the model. Returns posterior means and standard deviations at X.
        """
        X = np.atleast_2d(X)
        m = np.empty(shape=(0,1))
        s = np.empty(shape=(0,1))

        for k in range(X.shape[0]):
            preds = []
            for pred in self.model.estimators_:
                preds.append(pred.predict(X[k,:])[0])
            m = np.vstack((m ,np.array(preds).mean()))
            s = np.vstack((s ,np.array(preds).std()))
        return m, s

    def get_fmin(self):
        return self.model.predict(self.X).min()
<|endoftext|>


<commit_before>import sys
sys.path.append("../")

from Parsers.miseqParser import completeParseSamples, parseMetadata, getPairFiles
from Model.SequencingRun import SequencingRun
from Validation.offlineValidation import validateSampleSheet, validatePairFiles, validateSampleList
from Exceptions.SampleSheetError import SampleSheetError
from Exceptions.SequenceFileError import SequenceFileError

from ConfigParser import RawConfigParser
from pprint import pprint
from os import path

import wx

pathToModule=path.dirname(__file__)
if len(pathToModule)==0:
	pathToModule='.'

class MainPanel(wx.Panel):

	def __init__(self, parent):
		self.parent=parent
		wx.Panel.__init__(self, parent)

		self.sampleSheetFile=""
		self.seqRun=None
		self.browsePath="../"#os.getcwd()
		self.fileDlg=None
		self.confParser=RawConfigParser()
		self.configFile=pathToModule+"/../config.conf"
		self.confParser.read(self.configFile)

		self.LONG_BOX_SIZE=(300,32) #url and directories
		self.SHORT_BOX_SIZE=(200,32) #user and pass
		self.LABEL_TEXT_WIDTH=70
		self.LABEL_TEXT_HEIGHT=32

		self.topSizer = wx.BoxSizer(wx.VERTICAL)
		self.urlSizer = wx.BoxSizer(wx.HORIZONTAL)
		self.directorySizer = wx.BoxSizer(wx.HORIZONTAL)
		self.userPassContainer= wx.BoxSizer(wx.VERTICAL)
		self.usernameSizer = wx.BoxSizer(wx.HORIZONTAL)
		self.passwordSizer = wx.BoxSizer(wx.HORIZONTAL)
		self.logPanelContainer= wx.BoxSizer(wx.HORIZONTAL)
		self.logPanelSizer = wx.BoxSizer(wx.HORIZONTAL)

		self.addSelectSampleSheetSection()
		self.addURLsection()
		self.addUsernameSection()
		self.addPasswordSection()
		self.addLogPanelSection()

		self.topSizer.Add(self.directorySizer, proportion=0, flag=wx.ALL, border=5)
		self.topSizer.Add(self.urlSizer, proportion=0, flag=wx.ALL, border=5)

		self.topSizer.AddSpacer(10)

		self.userPassContainer.Add(self.usernameSizer, proportion=0, flag=wx.ALL, border=5)
		self.userPassContainer.Add(self.passwordSizer, proportion=0, flag=wx.ALL, border=5)

		self.logPanelContainer.Add(self.userPassContainer, proportion=0, flag=wx.ALL, border=5)
		self.logPanelContainer.Add(self.logPanelSizer, proportion=0, flag=wx.ALL, border=5)

		self.topSizer.Add(self.logPanelContainer, proportion=0, flag=wx.ALL, border=5)

		self.addUploadButton()

		self.SetSizer(self.topSizer)
		self.Layout()

		self.parent.Bind(wx.EVT_CLOSE, self.closeHandler)
		#self.progPanel=ProgressPanel(self.parent)
		#self.progPanel.Hide()

	def addURLsection(self):
		"""
		Adds URL text label and text box in to panel
		Sets a tooltip when hovering over text label or text box describing the URL that needs to be entered in the text box

		no return value
		"""
		self.baseUrlLabel= wx.StaticText(parent=self, id=-1, size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT), label="Base URL")
		self.baseUrlBox= wx.TextCtrl(self, size=self.LONG_BOX_SIZE)
		self.prevUrl=self.confParser.get("iridaUploader","baseURL")
		self.baseUrlBox.SetValue(self.prevUrl)

		self.saveUrlCheckbox = wx.CheckBox(parent=self, id=-1, label="Save URL for future use")
		if len(self.prevUrl)>0:
			self.saveUrlCheckbox.SetValue(True)
		else:
			self.saveUrlCheckbox.SetValue(False)
		self.Bind(wx.EVT_CHECKBOX, self.urlCheckboxHandler, self.saveUrlCheckbox)

		self.urlSizer.Add(self.baseUrlLabel, 0, wx.ALL,5)
		self.urlSizer.Add(self.baseUrlBox, 0, wx.ALL, 5)
		self.urlSizer.Add(self.saveUrlCheckbox, 0, wx.ALL, 5)

		tip="Enter the URL for the IRIDA server"
		self.baseUrlBox.SetToolTipString(tip)
		self.baseUrlLabel.SetToolTipString(tip)

	def addSelectSampleSheetSection(self):
		"""
		Adds data directory text label, text box and button in to panel
		Sets a tooltip when hovering over text label, text box or button describing what file needs to be selected

		Clicking the browse button or the text box launches the file dialog so that user can select SampleSheet file

		no return value
		"""

		self.directoryLabel= wx.StaticText(parent=self, id=-1,size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT), label="File path")
		self.directoryBox= wx.TextCtrl(self, size=self.LONG_BOX_SIZE)
		self.browseButton =wx.Button(self, label="Choose samplesheet file")
		self.browseButton.SetFocus()

		self.directorySizer.Add(self.directoryLabel,0, wx.ALL, 5)
		self.directorySizer.Add(self.directoryBox, 0, wx.ALL,5)
		self.directorySizer.Add(self.browseButton, 0 ,wx.ALL, 5)

		tip="Select the SampleSheet.csv file for the sequence files that you want to upload"
		self.directoryBox.SetToolTipString(tip)
		self.directoryLabel.SetToolTipString(tip)
		self.browseButton.SetToolTipString(tip)

		self.Bind(wx.EVT_BUTTON, self.openDirDialog, self.browseButton )
		self.directoryBox.Bind(wx.EVT_LEFT_DOWN, self.openDirDialog) #clicking directoryBox
		self.directoryBox.Bind(wx.EVT_SET_FOCUS , self.openDirDialog) #tabbing in to directorybox

	def addUsernameSection(self):
		"""
		Adds username text label and text box in to panel

		no return value
		"""
		self.usernameLabel= wx.StaticText(parent=self, id=-1, size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT),label="Username")
		self.usernameBox= wx.TextCtrl(self, size=self.SHORT_BOX_SIZE)
		self.usernameSizer.Add(self.usernameLabel)
		self.usernameSizer.Add(self.usernameBox)

	def addPasswordSection(self):
		"""
		Adds password text label and text box in to panel

		no return value
		"""
		self.passwordLabel= wx.StaticText(self, id=-1, size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT), label="Password")
		self.passwordBox= wx.TextCtrl(self, size=self.SHORT_BOX_SIZE, style=wx.TE_PASSWORD)
		self.passwordSizer.Add(self.passwordLabel)
		self.passwordSizer.Add(self.passwordBox)

	def addLogPanelSection(self):
		"""
		Adds log panel text control for displaying progress and errors

		no return value
		"""
		self.logPanel=wx.TextCtrl(self, id=-1, value="Waiting for user to select SampleSheet file.\n", size=(300,200), style=wx.TE_MULTILINE | wx.TE_READONLY )
		self.logPanelSizer.Add(self.logPanel)

	def addUploadButton(self):
		"""
		Adds upload button to panel

		no return value
		"""
		self.uploadButton =wx.Button(self, label="Upload")
		self.uploadButton.Disable()

		self.topSizer.AddStretchSpacer()
		self.topSizer.Add(self.uploadButton, 0, wx.BOTTOM|wx.ALIGN_CENTER)
		self.Bind(wx.EVT_BUTTON, self.uploadToServer, self.uploadButton)

		tip="Upload sequence files to IRIDA server. Select a valid SampleSheet file to enable button."
		self.uploadButton.SetToolTipString(tip)

	def displayWarning(self,warnMsg):
		"""Displays warning message

		arguments:
			warnMsg -- message to display in warning dialog baseUrlBox

		no return value
		"""
		self.logPanel.AppendText(warnMsg+"\n")
		warnDlg = wx.MessageDialog(parent=self, message=warnMsg, caption="Warning!", style=wx.OK | wx.ICON_WARNING)
		warnDlg.ShowModal()
		warnDlg.Destroy()

	def closeHandler(self, event):
		"""
		Function bound to window/MainFrame being closed (close button/alt+f4)
		Check status of url checkbox to see whether a new url to save has been entered
		this handles the case of the checkbox starting already checked, a new base url is typed and the checbkox is not clicked so it still remains checked. since the checkbox is not clicked the handler bound to it is not invoked so we call it here when closing the window to save if a new url has been enterred"
		destroy parent(MainFrame) to continue with regular closing procedure

		no return value
		"""
		self.urlCheckboxHandler()
		self.parent.Destroy()

	def urlCheckboxHandler(self, event=""):
		"""
		Function bound to url checkbox being clicked.
		If the checkbox is checked then save the url currently in the baseUrlBox to the config file
		If the checkbox is unchecked then write back the original value of baseURL.
		"""
		if self.saveUrlCheckbox.IsChecked():
			self.confParser.set("iridaUploader","baseURL",self.baseUrlBox.GetValue())
			with open(self.configFile, 'wb') as configfile:
				self.confParser.write(configfile)

		else:
			self.confParser.set("iridaUploader","baseURL",self.prevUrl)
			with open(self.configFile, 'wb') as configfile:
				self.confParser.write(configfile)

	def uploadToServer(self, event):
		"""
		Function bound to uploadButton being clicked
		Currently just prints values entered in text boxes and pairFiles in seqRun

		arguments:
			event -- EVT_BUTTON when upload button is clicked

		no return value
		"""

		print ("Server URL: " + self.baseUrlBox.GetValue() + "\n" + "User: " +self.usernameBox.GetValue() + "\n" + "Password: " + self.passwordBox.GetValue()).strip()

		if self.seqRun!=None:
			print self.seqRun.getWorkflow()
			pprint([self.seqRun.getPairFiles(sample.getID()) for sample in self.seqRun.getSamplesList()])

	def openDirDialog(self,event):
		"""
		Function bound to browseButton being clicked and directoryBox being clicked or tabbbed/focused
		Opens file dialog for user to select SampleSheet.csv
		Validates the selected SampleSheet
		If it's valid then proceed to create the sequence run (createSeqRun)
		else displays warning messagebox containing the error(s)

		arguments:
			event -- EVT_BUTTON when browse button is clicked
				or EVT_SET_FOCUS when directoryBox is focused via tabbing
				or EVT_LEFT_DOWN when directoryBox is clicked

		no return value
		"""

		self.browseButton.SetFocus()

		self.fileDlg= wx.FileDialog(self, "Select SampleSheet.csv file",  defaultDir=self.browsePath, style=wx.DD_DEFAULT_STYLE)

		if self.fileDlg.ShowModal() == wx.ID_OK:

			try:
				self.browsePath=self.fileDlg.GetDirectory()
				vRes=validateSampleSheet(self.fileDlg.GetPath())

				if vRes.isValid()==True:
					self.sampleSheetFile=self.fileDlg.GetPath()

					try:
						self.createSeqRun()
						self.directoryBox.SetValue(self.sampleSheetFile)
						self.uploadButton.Enable()
						self.logPanel.AppendText("Selected SampleSheet is valid\n")

					except (SampleSheetError, SequenceFileError),e:
						self.displayWarning(str(e))
						self.uploadButton.Disable()
						self.directoryBox.SetValue("")
						self.seqRun=None

				else:
					self.displayWarning(vRes.getErrors())
					self.uploadButton.Disable()
					self.directoryBox.SetValue("")
					self.seqRun=None

			except SampleSheetError, e:
				self.displayWarning(str(e))
				self.seqRun=None

		self.fileDlg.Destroy()

	def createSeqRun(self):
		"""
		Try to create a SequencingRun object and store in to self.seqRun
		Parses out the metadata dictionary and sampleslist from selected self.sampleSheetFile
		raises errors:
			if parsing raises/throws Exceptions
			if the parsed out samplesList fails validation
			if a pair file for a sample in samplesList fails validation
		these errors are expected to be caught by the calling function openDirDialog and it will be the one sending them to displayWarning

		no return value
		"""

		try:
			mDict=parseMetadata( self.sampleSheetFile )
			sList=completeParseSamples( self.sampleSheetFile )

		except SequenceFileError, e:
			raise SequenceFileError( str(e) )

		except SampleSheetError, e:
			raise SampleSheetError( str(e) )

		vRes=validateSampleList(sList)
		if vRes.isValid()==True:

			self.seqRun=SequencingRun()
			self.seqRun.setMetadata(mDict)
			self.seqRun.setSamplesList(sList)

		else:
			raise SequenceFileError( vRes.getErrors() )

		for sample in self.seqRun.getSamplesList():
			pfList=self.seqRun.getPairFiles( sample.getID() )

			vRes= validatePairFiles(pfList)
			if vRes.isValid()==False:
				raise SequenceFileError( vRes.getErrors() )

class MainFrame(wx.Frame):
	def __init__(self):
		self.WindowSize=(600,400)
		wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title="IRIDA Uploader", size=self.WindowSize, style=wx.DEFAULT_FRAME_STYLE ^ wx.RESIZE_BORDER ^ wx.MAXIMIZE_BOX)#use default frame style but disable border resize and maximize

		self.mp=MainPanel(self)
		self.Show()

if __name__ == "__main__":
	app = wx.App(False)
	frame = MainFrame()
	frame.Show()
	app.MainLoop()
<commit_msg>increase space in between url+file and user+pass+log<commit_after>import sys
sys.path.append("../")

from Parsers.miseqParser import completeParseSamples, parseMetadata, getPairFiles
from Model.SequencingRun import SequencingRun
from Validation.offlineValidation import validateSampleSheet, validatePairFiles, validateSampleList
from Exceptions.SampleSheetError import SampleSheetError
from Exceptions.SequenceFileError import SequenceFileError

from ConfigParser import RawConfigParser
from pprint import pprint
from os import path

import wx

pathToModule=path.dirname(__file__)
if len(pathToModule)==0:
	pathToModule='.'

class MainPanel(wx.Panel):

	def __init__(self, parent):
		self.parent=parent
		wx.Panel.__init__(self, parent)

		self.sampleSheetFile=""
		self.seqRun=None
		self.browsePath="../"#os.getcwd()
		self.fileDlg=None
		self.confParser=RawConfigParser()
		self.configFile=pathToModule+"/../config.conf"
		self.confParser.read(self.configFile)

		self.LONG_BOX_SIZE=(300,32) #url and directories
		self.SHORT_BOX_SIZE=(200,32) #user and pass
		self.LABEL_TEXT_WIDTH=70
		self.LABEL_TEXT_HEIGHT=32

		self.topSizer = wx.BoxSizer(wx.VERTICAL)
		self.urlSizer = wx.BoxSizer(wx.HORIZONTAL)
		self.directorySizer = wx.BoxSizer(wx.HORIZONTAL)
		self.userPassContainer= wx.BoxSizer(wx.VERTICAL)
		self.usernameSizer = wx.BoxSizer(wx.HORIZONTAL)
		self.passwordSizer = wx.BoxSizer(wx.HORIZONTAL)
		self.logPanelContainer= wx.BoxSizer(wx.HORIZONTAL)
		self.logPanelSizer = wx.BoxSizer(wx.HORIZONTAL)

		self.addSelectSampleSheetSection()
		self.addURLsection()
		self.addUsernameSection()
		self.addPasswordSection()
		self.addLogPanelSection()

		self.topSizer.Add(self.directorySizer, proportion=0, flag=wx.ALL, border=5)
		self.topSizer.Add(self.urlSizer, proportion=0, flag=wx.ALL, border=5)

		self.topSizer.AddSpacer(30)

		self.userPassContainer.Add(self.usernameSizer, proportion=0, flag=wx.ALL, border=5)
		self.userPassContainer.Add(self.passwordSizer, proportion=0, flag=wx.ALL, border=5)

		self.logPanelContainer.Add(self.userPassContainer, proportion=0, flag=wx.ALL, border=5)
		self.logPanelContainer.Add(self.logPanelSizer, proportion=0, flag=wx.ALL, border=5)

		self.topSizer.Add(self.logPanelContainer, proportion=0, flag=wx.ALL, border=5)

		self.addUploadButton()

		self.SetSizer(self.topSizer)
		self.Layout()

		self.parent.Bind(wx.EVT_CLOSE, self.closeHandler)
		#self.progPanel=ProgressPanel(self.parent)
		#self.progPanel.Hide()

	def addURLsection(self):
		"""
		Adds URL text label and text box in to panel
		Sets a tooltip when hovering over text label or text box describing the URL that needs to be entered in the text box

		no return value
		"""
		self.baseUrlLabel= wx.StaticText(parent=self, id=-1, size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT), label="Base URL")
		self.baseUrlBox= wx.TextCtrl(self, size=self.LONG_BOX_SIZE)
		self.prevUrl=self.confParser.get("iridaUploader","baseURL")
		self.baseUrlBox.SetValue(self.prevUrl)

		self.saveUrlCheckbox = wx.CheckBox(parent=self, id=-1, label="Save URL for future use")
		if len(self.prevUrl)>0:
			self.saveUrlCheckbox.SetValue(True)
		else:
			self.saveUrlCheckbox.SetValue(False)
		self.Bind(wx.EVT_CHECKBOX, self.urlCheckboxHandler, self.saveUrlCheckbox)

		self.urlSizer.Add(self.baseUrlLabel, 0, wx.ALL,5)
		self.urlSizer.Add(self.baseUrlBox, 0, wx.ALL, 5)
		self.urlSizer.Add(self.saveUrlCheckbox, 0, wx.ALL, 5)

		tip="Enter the URL for the IRIDA server"
		self.baseUrlBox.SetToolTipString(tip)
		self.baseUrlLabel.SetToolTipString(tip)

	def addSelectSampleSheetSection(self):
		"""
		Adds data directory text label, text box and button in to panel
		Sets a tooltip when hovering over text label, text box or button describing what file needs to be selected

		Clicking the browse button or the text box launches the file dialog so that user can select SampleSheet file

		no return value
		"""

		self.directoryLabel= wx.StaticText(parent=self, id=-1,size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT), label="File path")
		self.directoryBox= wx.TextCtrl(self, size=self.LONG_BOX_SIZE)
		self.browseButton =wx.Button(self, label="Choose samplesheet file")
		self.browseButton.SetFocus()

		self.directorySizer.Add(self.directoryLabel,0, wx.ALL, 5)
		self.directorySizer.Add(self.directoryBox, 0, wx.ALL,5)
		self.directorySizer.Add(self.browseButton, 0 ,wx.ALL, 5)

		tip="Select the SampleSheet.csv file for the sequence files that you want to upload"
		self.directoryBox.SetToolTipString(tip)
		self.directoryLabel.SetToolTipString(tip)
		self.browseButton.SetToolTipString(tip)

		self.Bind(wx.EVT_BUTTON, self.openDirDialog, self.browseButton )
		self.directoryBox.Bind(wx.EVT_LEFT_DOWN, self.openDirDialog) #clicking directoryBox
		self.directoryBox.Bind(wx.EVT_SET_FOCUS , self.openDirDialog) #tabbing in to directorybox

	def addUsernameSection(self):
		"""
		Adds username text label and text box in to panel

		no return value
		"""
		self.usernameLabel= wx.StaticText(parent=self, id=-1, size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT),label="Username")
		self.usernameBox= wx.TextCtrl(self, size=self.SHORT_BOX_SIZE)
		self.usernameSizer.Add(self.usernameLabel)
		self.usernameSizer.Add(self.usernameBox)

	def addPasswordSection(self):
		"""
		Adds password text label and text box in to panel

		no return value
		"""
		self.passwordLabel= wx.StaticText(self, id=-1, size=(self.LABEL_TEXT_WIDTH,self.LABEL_TEXT_HEIGHT), label="Password")
		self.passwordBox= wx.TextCtrl(self, size=self.SHORT_BOX_SIZE, style=wx.TE_PASSWORD)
		self.passwordSizer.Add(self.passwordLabel)
		self.passwordSizer.Add(self.passwordBox)

	def addLogPanelSection(self):
		"""
		Adds log panel text control for displaying progress and errors

		no return value
		"""
		self.logPanel=wx.TextCtrl(self, id=-1, value="Waiting for user to select SampleSheet file.\n", size=(300,200), style=wx.TE_MULTILINE | wx.TE_READONLY )
		self.logPanelSizer.Add(self.logPanel)

	def addUploadButton(self):
		"""
		Adds upload button to panel

		no return value
		"""
		self.uploadButton =wx.Button(self, label="Upload")
		self.uploadButton.Disable()

		self.topSizer.AddStretchSpacer()
		self.topSizer.Add(self.uploadButton, 0, wx.BOTTOM|wx.ALIGN_CENTER)
		self.Bind(wx.EVT_BUTTON, self.uploadToServer, self.uploadButton)

		tip="Upload sequence files to IRIDA server. Select a valid SampleSheet file to enable button."
		self.uploadButton.SetToolTipString(tip)

	def displayWarning(self,warnMsg):
		"""Displays warning message

		arguments:
			warnMsg -- message to display in warning dialog baseUrlBox

		no return value
		"""
		self.logPanel.AppendText(warnMsg+"\n")
		warnDlg = wx.MessageDialog(parent=self, message=warnMsg, caption="Warning!", style=wx.OK | wx.ICON_WARNING)
		warnDlg.ShowModal()
		warnDlg.Destroy()

	def closeHandler(self, event):
		"""
		Function bound to window/MainFrame being closed (close button/alt+f4)
		Check status of url checkbox to see whether a new url to save has been entered
		this handles the case of the checkbox starting already checked, a new base url is typed and the checbkox is not clicked so it still remains checked. since the checkbox is not clicked the handler bound to it is not invoked so we call it here when closing the window to save if a new url has been enterred"
		destroy parent(MainFrame) to continue with regular closing procedure

		no return value
		"""
		self.urlCheckboxHandler()
		self.parent.Destroy()

	def urlCheckboxHandler(self, event=""):
		"""
		Function bound to url checkbox being clicked.
		If the checkbox is checked then save the url currently in the baseUrlBox to the config file
		If the checkbox is unchecked then write back the original value of baseURL.
		"""
		if self.saveUrlCheckbox.IsChecked():
			self.confParser.set("iridaUploader","baseURL",self.baseUrlBox.GetValue())
			with open(self.configFile, 'wb') as configfile:
				self.confParser.write(configfile)

		else:
			self.confParser.set("iridaUploader","baseURL",self.prevUrl)
			with open(self.configFile, 'wb') as configfile:
				self.confParser.write(configfile)

	def uploadToServer(self, event):
		"""
		Function bound to uploadButton being clicked
		Currently just prints values entered in text boxes and pairFiles in seqRun

		arguments:
			event -- EVT_BUTTON when upload button is clicked

		no return value
		"""

		print ("Server URL: " + self.baseUrlBox.GetValue() + "\n" + "User: " +self.usernameBox.GetValue() + "\n" + "Password: " + self.passwordBox.GetValue()).strip()

		if self.seqRun!=None:
			print self.seqRun.getWorkflow()
			pprint([self.seqRun.getPairFiles(sample.getID()) for sample in self.seqRun.getSamplesList()])

	def openDirDialog(self,event):
		"""
		Function bound to browseButton being clicked and directoryBox being clicked or tabbbed/focused
		Opens file dialog for user to select SampleSheet.csv
		Validates the selected SampleSheet
		If it's valid then proceed to create the sequence run (createSeqRun)
		else displays warning messagebox containing the error(s)

		arguments:
			event -- EVT_BUTTON when browse button is clicked
				or EVT_SET_FOCUS when directoryBox is focused via tabbing
				or EVT_LEFT_DOWN when directoryBox is clicked

		no return value
		"""

		self.browseButton.SetFocus()

		self.fileDlg= wx.FileDialog(self, "Select SampleSheet.csv file",  defaultDir=self.browsePath, style=wx.DD_DEFAULT_STYLE)

		if self.fileDlg.ShowModal() == wx.ID_OK:

			try:
				self.browsePath=self.fileDlg.GetDirectory()
				vRes=validateSampleSheet(self.fileDlg.GetPath())

				if vRes.isValid()==True:
					self.sampleSheetFile=self.fileDlg.GetPath()

					try:
						self.createSeqRun()
						self.directoryBox.SetValue(self.sampleSheetFile)
						self.uploadButton.Enable()
						self.logPanel.AppendText("Selected SampleSheet is valid\n")

					except (SampleSheetError, SequenceFileError),e:
						self.displayWarning(str(e))
						self.uploadButton.Disable()
						self.directoryBox.SetValue("")
						self.seqRun=None

				else:
					self.displayWarning(vRes.getErrors())
					self.uploadButton.Disable()
					self.directoryBox.SetValue("")
					self.seqRun=None

			except SampleSheetError, e:
				self.displayWarning(str(e))
				self.seqRun=None

		self.fileDlg.Destroy()

	def createSeqRun(self):
		"""
		Try to create a SequencingRun object and store in to self.seqRun
		Parses out the metadata dictionary and sampleslist from selected self.sampleSheetFile
		raises errors:
			if parsing raises/throws Exceptions
			if the parsed out samplesList fails validation
			if a pair file for a sample in samplesList fails validation
		these errors are expected to be caught by the calling function openDirDialog and it will be the one sending them to displayWarning

		no return value
		"""

		try:
			mDict=parseMetadata( self.sampleSheetFile )
			sList=completeParseSamples( self.sampleSheetFile )

		except SequenceFileError, e:
			raise SequenceFileError( str(e) )

		except SampleSheetError, e:
			raise SampleSheetError( str(e) )

		vRes=validateSampleList(sList)
		if vRes.isValid()==True:

			self.seqRun=SequencingRun()
			self.seqRun.setMetadata(mDict)
			self.seqRun.setSamplesList(sList)

		else:
			raise SequenceFileError( vRes.getErrors() )

		for sample in self.seqRun.getSamplesList():
			pfList=self.seqRun.getPairFiles( sample.getID() )

			vRes= validatePairFiles(pfList)
			if vRes.isValid()==False:
				raise SequenceFileError( vRes.getErrors() )

class MainFrame(wx.Frame):
	def __init__(self):
		self.WindowSize=(600,400)
		wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title="IRIDA Uploader", size=self.WindowSize, style=wx.DEFAULT_FRAME_STYLE ^ wx.RESIZE_BORDER ^ wx.MAXIMIZE_BOX)#use default frame style but disable border resize and maximize

		self.mp=MainPanel(self)
		self.Show()

if __name__ == "__main__":
	app = wx.App(False)
	frame = MainFrame()
	frame.Show()
	app.MainLoop()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
from collections import Iterable
from datetime import timedelta
import copy

class Serializable:
    def validate(self):
        for c in self.__class__.__mro__:
            if not hasattr(c, '_validate'):
                continue

            for name, allowed in c._validate().items():
                if not hasattr(self, name):
                    raise AttributeError('%s.%s is missing' % (c._serialized_name(), name))

                val = getattr(self, name)

                if allowed is None:
                    if not isinstance(val, Iterable) and self.__class__.__name__ != 'RideSegment':
                        raise ValueError('%s.%s has non-iterable value: %s' % (c._serialized_name(), name, repr(getattr(self, name))))
                    if not c._validate_custom(self, name, val):
                        raise ValueError('%s.%s has invalid complex value: %s' % (c._serialized_name(), name, repr(getattr(self, name))))
                    continue

                if type(allowed) != tuple:
                    allowed = (allowed, )

                for a in allowed:
                    if a is None:
                        if val is None:
                            break
                    elif isinstance(val, a):
                        break
                else:
                    raise ValueError('%s.%s has to be %s, not %s' % (c._serialized_name(), name, self._validate_or(allowed), repr(getattr(self, name))))
        return True

    def _validate_or(self, items):
        if type(items) != tuple:
            return items.__name__
        out = []
        for item in items:
            if item is None:
                out.append('None')
            elif type(item) is tuple:
                out.append('Iterable(%s)' % self._validate_or(item))
            else:
                out.append(item.__name__)

        out = [(', ' + o) for o in out]
        if len(out) > 1:
            out[-1] = ' or ' + out[-1][2:]
        return ''.join(out)[2:]

    def _unserialize_typed(self, data, types=None):
        type_, data = data
        for t in types:
            if t._serialized_name() == type_:
                return t.unserialize(data)
        raise TypeError('Wrong datatype for unserialization')

    @classmethod
    def _unfold_subclasses(cls, allowed):
        if type(allowed) is not tuple:
            allowed = (allowed, )

        l = 0
        while len(allowed) != l:
            l = len(allowed)
            for a in allowed[:]:
                if a is not None and issubclass(a, Serializable):
                    allowed = tuple(set(allowed + tuple(a.__subclasses__())))
        return allowed

    def serialize(self, typed=False):
        self.validate()
        data = {}
        if hasattr(self, '_serialize'):
            data = self._serialize()
        else:
            for c in self.__class__.__mro__:
                if not hasattr(c, '_validate'):
                    continue

                for name, allowed in c._validate().items():
                    value = getattr(self, name)

                    if allowed is None:
                        n, v = c._serialize_custom(self, name)
                        if v is not None:
                            data[n] = v
                        continue

                    allowed = c._unfold_subclasses(allowed)

                    if len(allowed) == 1:
                        if isinstance(value, Serializable):
                            value = value.serialize()
                    else:
                        if value is None:
                            continue

                        if isinstance(value, Serializable):
                            t = len([a for a in allowed if a is not None and issubclass(a, Serializable)]) > 1
                            value = value.serialize(typed=t)

                    if isinstance(value, timedelta):
                        value = value.total_seconds()
                    data[name] = value

        if typed:
            return self.__class__._serialized_name(), data
        else:
            return data

    @classmethod
    def _serialized_name(cls):
        if hasattr(cls, 'Model'):
            return cls.Model.__name__ + '.' + cls.__name__
        else:
            return cls.__name__

    @classmethod
    def unserialize(cls, data):
        obj = cls()
        if hasattr(obj, '_unserialize'):
            obj._unserialize(data)
        else:
            for c in cls.__mro__:
                validate = {}
                if hasattr(c, '_validate'):
                    validate = c._validate()

                custom = hasattr(c, '_unserialize_custom')

                if not validate and not custom:
                    continue

                for name, value in data.items():
                    if name in validate and validate[name] is not None:
                        allowed = validate[name]

                        allowed = cls._unfold_subclasses(allowed)

                        if len(allowed) == 1:
                            if issubclass(allowed[0], timedelta):
                                value = timedelta(seconds=value)
                            elif issubclass(allowed[0], Serializable):
                                value = allowed[0].unserialize(value)
                        elif value is not None:
                            serializables = [a for a in allowed if a is not None and issubclass(a, Serializable)]
                            typed = len(serializables) > 1
                            if typed:
                                value = obj._unserialize_typed(value, serializables)
                            elif serializables:
                                value = serializables[0].unserialize(value)
                        setattr(obj, name, value)

                for name, value in data.items():
                    if (name not in validate or validate[name] is None) and custom:
                        c._unserialize_custom(obj, name, value)

        return obj

class MetaSearchable(type):
    def __init__(cls, a, b, c):
        cls.Request.Model = cls
        cls.Results.Model = cls

class Searchable(Serializable, metaclass=MetaSearchable):
    def matches(self, request):
        if not isinstance(request, Searchable.Request):
            raise TypeError('not a request')
        return request.matches(self)

    class Request(Serializable):
        def matches(self, obj):
            if not isinstance(obj, self.Model):
                raise TypeError('%s.Request can only match %s' % (self.Model.__name__, self.Model.__name__))
            obj.validate()
            return self._matches(obj)

        def _matches(self, obj):
            pass

    class Results(Serializable):
        def __init__(self, results=[], scored=False):
            super().__init__()
            if scored:
                self.results = tuple(results)
            else:
                self.results = tuple((r, None) for r in results)

        @classmethod
        def _validate(self):
            return {
                'results': None
            }

        def _validate_custom(self, name, value):
            if name == 'results':
                type_ = self.Model
                for v in value:
                    if type(v) != tuple or len(v) != 2 or not isinstance(v[0], type_):
                        return False
                return True

        def _serialize_custom(self, name):
            if name == 'results':
                return 'results', [(r[0].serialize(), r[1]) for r in self.results]

        def _unserialize_custom(self, name, data):
            if name == 'results':
                self.results = [(self.Model.unserialize(d[0]), d[1]) for d in data]

        def filter(self, request):
            if not self.results:
                return

            if not isinstance(request, self.Model.Request):
                raise TypeError('%s.Results can be filtered with %s' % (self.Model.__name__, self.Model.__name__))

            self.results = tuple(r for r in self.results if request.matches(r))

        def filtered(self, request):
            obj = copy.copy(self)
            obj.filter(request)
            return obj

        def __iter__(self):
            yield from (r[0] for r in self.results)

        def scored(self):
            yield from self.results

        def __getitem__(self, key):
            return self.results[key]

class Collectable(Searchable):
    @classmethod
    def _validate(cls):
        return {
            '_ids': None
        }

    def __init__(self):
        self._ids = {}

    def _validate_custom(self, name, value):
        if name == '_ids':
            if not isinstance(value, dict):
                return False

            for name, data in value.items():
                if not isinstance(name, str):
                    return False

                if not isinstance(data, (int, str, tuple)):
                    return False
            return True

    def _serialize_custom(self, name):
        if name == '_ids':
            return 'ids', self._ids

    def _unserialize_custom(self, name, data):
        if name == 'ids':
            for name, value in data.items():
                self._ids[name] = tuple(value) if isinstance(value, list) else value

class TripPart(Searchable):
    pass
<commit_msg>serialize datetime<commit_after>#!/usr/bin/env python3
from collections import Iterable
from datetime import datetime, timedelta
import copy

class Serializable:
    def validate(self):
        for c in self.__class__.__mro__:
            if not hasattr(c, '_validate'):
                continue

            for name, allowed in c._validate().items():
                if not hasattr(self, name):
                    raise AttributeError('%s.%s is missing' % (c._serialized_name(), name))

                val = getattr(self, name)

                if allowed is None:
                    if not isinstance(val, Iterable) and self.__class__.__name__ != 'RideSegment':
                        raise ValueError('%s.%s has non-iterable value: %s' % (c._serialized_name(), name, repr(getattr(self, name))))
                    if not c._validate_custom(self, name, val):
                        raise ValueError('%s.%s has invalid complex value: %s' % (c._serialized_name(), name, repr(getattr(self, name))))
                    continue

                if type(allowed) != tuple:
                    allowed = (allowed, )

                for a in allowed:
                    if a is None:
                        if val is None:
                            break
                    elif isinstance(val, a):
                        break
                else:
                    raise ValueError('%s.%s has to be %s, not %s' % (c._serialized_name(), name, self._validate_or(allowed), repr(getattr(self, name))))
        return True

    def _validate_or(self, items):
        if type(items) != tuple:
            return items.__name__
        out = []
        for item in items:
            if item is None:
                out.append('None')
            elif type(item) is tuple:
                out.append('Iterable(%s)' % self._validate_or(item))
            else:
                out.append(item.__name__)

        out = [(', ' + o) for o in out]
        if len(out) > 1:
            out[-1] = ' or ' + out[-1][2:]
        return ''.join(out)[2:]

    def _unserialize_typed(self, data, types=None):
        type_, data = data
        for t in types:
            if t._serialized_name() == type_:
                return t.unserialize(data)
        raise TypeError('Wrong datatype for unserialization')

    @classmethod
    def _unfold_subclasses(cls, allowed):
        if type(allowed) is not tuple:
            allowed = (allowed, )

        l = 0
        while len(allowed) != l:
            l = len(allowed)
            for a in allowed[:]:
                if a is not None and issubclass(a, Serializable):
                    allowed = tuple(set(allowed + tuple(a.__subclasses__())))
        return allowed

    def serialize(self, typed=False):
        self.validate()
        data = {}
        if hasattr(self, '_serialize'):
            data = self._serialize()
        else:
            for c in self.__class__.__mro__:
                if not hasattr(c, '_validate'):
                    continue

                for name, allowed in c._validate().items():
                    value = getattr(self, name)

                    if allowed is None:
                        n, v = c._serialize_custom(self, name)
                        if v is not None:
                            data[n] = v
                        continue

                    allowed = c._unfold_subclasses(allowed)

                    if len(allowed) == 1:
                        if isinstance(value, Serializable):
                            value = value.serialize()
                        elif isinstance(value, datetime):
                            value = value.strftime('%Y-%m-%d %H:%M:%S')
                    else:
                        if value is None:
                            continue

                        if isinstance(value, Serializable):
                            t = len([a for a in allowed if a is not None and issubclass(a, Serializable)]) > 1
                            value = value.serialize(typed=t)
                        elif isinstance(value, datetime):
                            value = value.strftime('%Y-%m-%d %H:%M:%S')

                    if isinstance(value, timedelta):
                        value = value.total_seconds()
                    data[name] = value

        if typed:
            return self.__class__._serialized_name(), data
        else:
            return data

    @classmethod
    def _serialized_name(cls):
        if hasattr(cls, 'Model'):
            return cls.Model.__name__ + '.' + cls.__name__
        else:
            return cls.__name__

    @classmethod
    def unserialize(cls, data):
        obj = cls()
        if hasattr(obj, '_unserialize'):
            obj._unserialize(data)
        else:
            for c in cls.__mro__:
                validate = {}
                if hasattr(c, '_validate'):
                    validate = c._validate()

                custom = hasattr(c, '_unserialize_custom')

                if not validate and not custom:
                    continue

                for name, value in data.items():
                    if name in validate and validate[name] is not None:
                        allowed = validate[name]

                        allowed = cls._unfold_subclasses(allowed)

                        if len(allowed) == 1:
                            # todo: serializable should be checked first
                            if issubclass(allowed[0], timedelta):
                                value = timedelta(seconds=value)
                            elif issubclass(allowed[0], datetime):
                                value = datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
                            elif issubclass(allowed[0], Serializable):
                                value = allowed[0].unserialize(value)
                        elif value is not None:
                            serializables = [a for a in allowed if a is not None and issubclass(a, Serializable)]
                            typed = len(serializables) > 1
                            if typed:
                                value = obj._unserialize_typed(value, serializables)
                            elif serializables:
                                value = serializables[0].unserialize(value)
                            elif datetime in allowed:
                                value = datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
                        setattr(obj, name, value)

                for name, value in data.items():
                    if (name not in validate or validate[name] is None) and custom:
                        c._unserialize_custom(obj, name, value)

        return obj

class MetaSearchable(type):
    def __init__(cls, a, b, c):
        cls.Request.Model = cls
        cls.Results.Model = cls

class Searchable(Serializable, metaclass=MetaSearchable):
    def matches(self, request):
        if not isinstance(request, Searchable.Request):
            raise TypeError('not a request')
        return request.matches(self)

    class Request(Serializable):
        def matches(self, obj):
            if not isinstance(obj, self.Model):
                raise TypeError('%s.Request can only match %s' % (self.Model.__name__, self.Model.__name__))
            obj.validate()
            return self._matches(obj)

        def _matches(self, obj):
            pass

    class Results(Serializable):
        def __init__(self, results=[], scored=False):
            super().__init__()
            if scored:
                self.results = tuple(results)
            else:
                self.results = tuple((r, None) for r in results)

        @classmethod
        def _validate(self):
            return {
                'results': None
            }

        def _validate_custom(self, name, value):
            if name == 'results':
                type_ = self.Model
                for v in value:
                    if type(v) != tuple or len(v) != 2 or not isinstance(v[0], type_):
                        return False
                return True

        def _serialize_custom(self, name):
            if name == 'results':
                return 'results', [(r[0].serialize(), r[1]) for r in self.results]

        def _unserialize_custom(self, name, data):
            if name == 'results':
                self.results = [(self.Model.unserialize(d[0]), d[1]) for d in data]

        def filter(self, request):
            if not self.results:
                return

            if not isinstance(request, self.Model.Request):
                raise TypeError('%s.Results can be filtered with %s' % (self.Model.__name__, self.Model.__name__))

            self.results = tuple(r for r in self.results if request.matches(r))

        def filtered(self, request):
            obj = copy.copy(self)
            obj.filter(request)
            return obj

        def __iter__(self):
            yield from (r[0] for r in self.results)

        def scored(self):
            yield from self.results

        def __getitem__(self, key):
            return self.results[key]

class Collectable(Searchable):
    @classmethod
    def _validate(cls):
        return {
            '_ids': None
        }

    def __init__(self):
        self._ids = {}

    def _validate_custom(self, name, value):
        if name == '_ids':
            if not isinstance(value, dict):
                return False

            for name, data in value.items():
                if not isinstance(name, str):
                    return False

                if not isinstance(data, (int, str, tuple)):
                    return False
            return True

    def _serialize_custom(self, name):
        if name == '_ids':
            return 'ids', self._ids

    def _unserialize_custom(self, name, data):
        if name == 'ids':
            for name, value in data.items():
                self._ids[name] = tuple(value) if isinstance(value, list) else value

class TripPart(Searchable):
    pass
<|endoftext|>


<commit_before>#import AsciiDammit
import re

map_chars = {u'°': u' degree',
             u'é': u'e',
             u'⁄': u'bkslsh',
             u'‘': u"'",
             u'<': u" lt ",
             u'—': u"-",
             u'Ô': u"O",
             u'Õ': u"O",
             u'£': u"pounds ",
             u'•': u"",
             u'\xc2\x97': u'-',
             u'\xc2\x91': u"'",
             u'\xc2\x92': u"'",
             u'\xc2\x93': u'"',
             u'\xc2\x94': u'"',
             u'\xe2\x80\x98': u"'",
             u'\xe2\x80\x99': u"'",
             u'\xe2\x80\x9a': u"'",
             u'\xe2\x80\x9b': u"'",
             u'\xe2\x80\x9c': u'"',
             u'\xe2\x80\x9d': u'"',
             u'\xe2\x80\x9f': u'"',
             u'\xe2\x80\x9e': u'"',
             u'\x60\x60': u'"',
             u'/': u'',
             u'<': u'',
             #u'\u2028': u' ',
             #u'\u2026': u' ',
             #u'\u2014': u' ',
             #u'\u2122': u' trademark ',
             #u'\u2022': u'-',
             u'\xa0': u' ',
             #u'\xfc': u'u',
             #u'\xf3': u'o',
             #u'\xe1': u'a',
             #u'\xe0': u'a',
             #u'\xe9': u'e',
             #u'\xef': u'i',
             #u'\xb5': u' ',
             #u'\xb6': u' ',
             #u'\xb7': u' dividedby ',
             #u'\xa7': u' ',
             #u'\xa9': u' copyright ',
             #u'\u0cb5': u' copyright ',
             #u'\u2018': "'",
             #u'\u2010': "-",
             #u'\u2014': '-',
             #u'\u2013': '-',
             u'\u00A0': u' ',
             u'\u201c': u'"',
             u'\u201d': u'"',
             u'\u2019': u"'",
             u'\t': u' '}

def clean_text(text):
    for s, r in map_chars.iteritems():
        _text = text.replace(s, r)
    _text = re.sub(r'(&[a-z0-9#]+;)+|\s\s+', ' ', _text)
    _text = re.sub(r'(https?|ftp):\/\/[^\s].*?', ' ', _text)
    #return AsciiDammit.asciiDammit(text)
    return _text
<commit_msg>fixing url regex<commit_after>#import AsciiDammit
import re

map_chars = {u'°': u' degree',
             u'é': u'e',
             u'⁄': u'bkslsh',
             u'‘': u"'",
             u'<': u" lt ",
             u'—': u"-",
             u'Ô': u"O",
             u'Õ': u"O",
             u'£': u"pounds ",
             u'•': u"",
             u'\xc2\x97': u'-',
             u'\xc2\x91': u"'",
             u'\xc2\x92': u"'",
             u'\xc2\x93': u'"',
             u'\xc2\x94': u'"',
             u'\xe2\x80\x98': u"'",
             u'\xe2\x80\x99': u"'",
             u'\xe2\x80\x9a': u"'",
             u'\xe2\x80\x9b': u"'",
             u'\xe2\x80\x9c': u'"',
             u'\xe2\x80\x9d': u'"',
             u'\xe2\x80\x9f': u'"',
             u'\xe2\x80\x9e': u'"',
             u'\x60\x60': u'"',
             u'/': u'',
             u'<': u'',
             #u'\u2028': u' ',
             #u'\u2026': u' ',
             #u'\u2014': u' ',
             #u'\u2122': u' trademark ',
             #u'\u2022': u'-',
             u'\xa0': u' ',
             #u'\xfc': u'u',
             #u'\xf3': u'o',
             #u'\xe1': u'a',
             #u'\xe0': u'a',
             #u'\xe9': u'e',
             #u'\xef': u'i',
             #u'\xb5': u' ',
             #u'\xb6': u' ',
             #u'\xb7': u' dividedby ',
             #u'\xa7': u' ',
             #u'\xa9': u' copyright ',
             #u'\u0cb5': u' copyright ',
             #u'\u2018': "'",
             #u'\u2010': "-",
             #u'\u2014': '-',
             #u'\u2013': '-',
             u'\u00A0': u' ',
             u'\u201c': u'"',
             u'\u201d': u'"',
             u'\u2019': u"'",
             u'\t': u' '}

def clean_text(text):
    for s, r in map_chars.iteritems():
        _text = text.replace(s, r)
    _text = re.sub(r'(&[a-z0-9#]+;)+|\s\s+', ' ', _text)
    _text = re.sub(r'(https?|ftp):\/\/[^\s]+', ' ', _text)
    #return AsciiDammit.asciiDammit(text)
    return _text
<|endoftext|>


<commit_before>'''Simple GAN model

'''

import logging

import torch
from torch import autograd
from torch.autograd import Variable
import torch.nn.functional as fun

from .gan import apply_penalty, f_divergence
from .modules.densenet import DenseNet

logger = logging.getLogger('cortex.models' + __name__)

resnet_discriminator_args_ = dict(dim_h=64, batch_norm=True, f_size=3, n_steps=3)
resnet_generator_args_ = dict(dim_h=64, batch_norm=True, f_size=3, n_steps=3)

mnist_discriminator_args_ = dict(dim_h=64, batch_norm=True, f_size=5, pad=2, stride=2, min_dim=7,
                                 nonlinearity='LeakyReLU')
mnist_generator_args_ = dict(dim_h=64, batch_norm=True, f_size=4, pad=1, stride=2, n_steps=2)

dcgan_discriminator_args_ = dict(dim_h=64, batch_norm=True, n_steps=3, nonlinearity='ReLU')
dcgan_generator_args_ = dict(dim_h=64, batch_norm=True, n_steps=3)

DEFAULTS = dict(
    data=dict(batch_size=dict(train=64, test=640), skip_last_batch=True,
              noise_variables=dict(z=('normal', 128), r=('normal', 1), f=('normal', 1))),
    optimizer=dict(
        optimizer='Adam',
        learning_rate=1e-4,
        clipping=dict(real_discriminator=0.1, fake_discriminator=0.1),
        updates_per_model=dict(discriminator=1, generator=1, real_discriminator=1, fake_discriminator=1, topnet=1)
    ),
    model=dict(model_type='dcgan', dim_d=1, dim_e=1, discriminator_args=None, generator_args=None),
    procedures=dict(measure='proxy_gan', boundary_seek=False, penalty=1.0, meta_penalty=1.0),
    train=dict(
        epochs=200,
        summary_updates=100,
        archive_every=10
    )
)

def setup(model=None, data=None, procedure=None, **kwargs):
    data['noise_variables']['r'] = ('normal', model['dim_d'])
    data['noise_variables']['f'] = ('normal', model['dim_d'])

def vral(nets, data_handler, measure=None, boundary_seek=False, penalty=None,
         real_mu=1.0, fake_mu=0.0, lam=100., vral_type=None, meta_penalty=None):
    # Variables
    X = data_handler['images']
    Z = data_handler['z']
    Rr = data_handler['r']
    Fr = data_handler['f']

    # Nets
    discriminator = nets['discriminator']
    generator = nets['generator']
    topnet = nets['topnet']
    real_discriminator = nets['real_discriminator']
    fake_discriminator = nets['fake_discriminator']
    gen_out = generator(Z, nonlinearity=fun.tanh)

    Rfphi = discriminator(X)
    Ffphi = discriminator(gen_out)
    Rf = topnet(Rfphi)
    Ff = topnet(Ffphi)

    if vral_type == 'fisher':
        real_r = real_discriminator(Rr)
        real_f = real_discriminator(Rf)
        fake_r = fake_discriminator(Fr)
        fake_f = fake_discriminator(Ff - Ff.mean())
        real_g = real_discriminator(Ff - real_mu)
    else:
        real_r = real_discriminator(Rr)
        real_f = real_discriminator(Rf - real_mu)
        fake_r = fake_discriminator(Fr)
        fake_f = fake_discriminator(Ff - fake_mu)
        real_g = real_discriminator(Ff - real_mu)

    if measure == 'mmd':
        assert False, 'Doesnt work right now'
        B, D = real_r.size()
        eye_B = Variable(torch.eye(B).cuda())
        def mmd_mat(x, y, remove_diagonal=False):
            mat = x.unsqueeze(2).expand(B, D, D) - y.unsqueeze(2).expand(B, D, D)
            term = torch.exp(-(mat ** 2).sum(1).sum(1))
            if remove_diagonal:
                term = ((1. - eye_B) * term).sum()
                return term / (B * (B - 1.))
            else:
                return term.mean()

        rrr = mmd_mat(real_r, real_r, remove_diagonal=True)
        rff = mmd_mat(real_f, real_f, remove_diagonal=True)
        rrf = mmd_mat(real_r, real_f)

        frr = mmd_mat(fake_r, fake_r, remove_diagonal=True)
        fff = mmd_mat(fake_f, fake_f, remove_diagonal=True)
        frf = mmd_mat(fake_r, fake_f)

        rgg = mmd_mat(real_g, real_g, remove_diagonal=True)
        rfg = mmd_mat(real_g, real_f)

        d_loss_r = -(rrr - 2. * rrf + rff)
        d_loss_f = -(frr - 2. * frf + fff)
        d_loss = -(d_loss_f + d_loss_r)

        #g_loss = -2. * rfg + rgg
        g_loss = (Ff.mean() - Rf.mean()) ** 2
    elif vral_type == 'fisher':
        d_loss_r, g_loss_r, rr, fr, wr, br = f_divergence(measure, real_r, real_f, boundary_seek=boundary_seek)
        d_loss_f, g_loss_f, rf, ff, wf, bf = f_divergence(measure, fake_r, fake_f, boundary_seek=boundary_seek)
        d_loss = lam * (g_loss_r + g_loss_f) + Ff.mean()
        t_loss = (g_loss_r + g_loss_f) + Ff.mean()

        g_loss = -Ff.mean()
    else:
        d_loss_r, g_loss_r, rr, fr, wr, br = f_divergence(measure, real_r, real_f, boundary_seek=boundary_seek)
        d_loss_f, g_loss_f, rf, ff, wf, bf = f_divergence(measure, fake_r, fake_f, boundary_seek=boundary_seek)

        d_loss = g_loss_r + g_loss_f
        t_loss = d_loss
        g_loss = (Ff.mean() - Rf.mean()) ** 2

    results = dict(g_loss=g_loss.data[0], d_loss=d_loss.data[0], rd_loss=d_loss_r.data[0], fd_loss=d_loss_f.data[0], t_loss=t_loss.data[0],
                   real=torch.mean(Rf).data[0], fake=torch.mean(Ff).data[0])
    samples = dict(images=dict(generated=0.5 * (gen_out + 1.).data, real=0.5 * (X + 1.).data),
                   histograms=dict(discriminator_output=dict(fake=Ff.view(-1).data, real=Rf.view(-1).data)))

    if meta_penalty:
        p_term_r = apply_penalty(data_handler, real_discriminator, Rr, Rf, measure)
        p_term_f = apply_penalty(data_handler, fake_discriminator, Fr, Ff, measure)

        d_loss_r += meta_penalty * p_term_r.mean()
        d_loss_f += meta_penalty * p_term_f.mean()
        results['real gradient penalty'] = p_term_r.mean()
        results['fake gradient penalty'] = p_term_f.mean()

    if penalty:
        real = Variable(X.data.cuda(), requires_grad=True)
        fake = Variable(gen_out.data.cuda(), requires_grad=True)
        real_out = discriminator(real)
        fake_out = discriminator(fake)

        real_out = topnet(real_out)
        fake_out = topnet(fake_out)

        g_r = autograd.grad(outputs=real_out, inputs=real, grad_outputs=torch.ones(real_out.size()).cuda(),
                            create_graph=True, retain_graph=True, only_inputs=True)[0]

        g_f = autograd.grad(outputs=fake_out, inputs=fake, grad_outputs=torch.ones(fake_out.size()).cuda(),
                            create_graph=True, retain_graph=True, only_inputs=True)[0]

        g_r = g_r.view(g_r.size()[0], -1)
        g_f = g_f.view(g_f.size()[0], -1)

        g_r = (g_r ** 2).sum(1)
        g_f = (g_f ** 2).sum(1)

        g_p = 0.5 * (g_r.mean() + g_f.mean())

        d_loss += penalty * g_p
        results['gradient penalty'] = g_p.data[0]

    loss = dict(generator=g_loss, real_discriminator=d_loss_r, fake_discriminator=d_loss_f, discriminator=d_loss,
                topnet=t_loss)
    return loss, results, samples, 'boundary'

def build_model(data_handler, model_type='resnet', dim_h=64, dim_d=1, dim_e=1,
                discriminator_args=None, generator_args=None):
    discriminator_args = discriminator_args or {}
    generator_args = generator_args or {}
    shape = data_handler.get_dims('x', 'y', 'c')
    dim_z = data_handler.get_dims('z')[0]

    if model_type == 'resnet':
        from .modules.resnets import ResEncoder as Discriminator
        from .modules.resnets import ResDecoder as Generator
        discriminator_args_ = resnet_discriminator_args_
        generator_args_ = resnet_generator_args_
    elif model_type == 'dcgan':
        from .modules.conv_decoders import SimpleConvDecoder as Generator
        from .modules.convnets import SimpleConvEncoder as Discriminator
        discriminator_args_ = dcgan_discriminator_args_
        generator_args_ = dcgan_generator_args_
    elif model_type == 'mnist':
        from .modules.conv_decoders import SimpleConvDecoder as Generator
        from .modules.convnets import SimpleConvEncoder as Discriminator
        discriminator_args_ = mnist_discriminator_args_
        generator_args_ = mnist_generator_args_
    else:
        raise NotImplementedError(model_type)

    discriminator_args_.update(**discriminator_args)
    generator_args_.update(**generator_args)

    if shape[0] == 64:
        discriminator_args_['n_steps'] = 4
        generator_args_['n_steps'] = 4
    elif shape[0] == 128:
        discriminator_args_['n_steps'] = 5
        generator_args_['n_steps'] = 5

    discriminator = Discriminator(shape, dim_out=dim_h, **discriminator_args_)
    topnet = DenseNet(dim_h, dim_h=[], dim_out=dim_d)
    generator = Generator(shape, dim_in=dim_z, **generator_args_)
    real_discriminator = DenseNet(dim_d, dim_h=[64], dim_out=dim_e, nonlinearity='LeakyReLU', batch_norm=False,
                                  layer_norm=False)
    fake_discriminator = DenseNet(dim_d, dim_h=[64], dim_out=dim_e, nonlinearity='LeakyReLU', batch_norm=False,
                                  layer_norm=False)
    logger.debug(discriminator)
    logger.debug(generator)
    logger.debug(real_discriminator)
    logger.debug(fake_discriminator)

    return dict(generator=generator, discriminator=discriminator, real_discriminator=real_discriminator,
                fake_discriminator=fake_discriminator, topnet=topnet), vral

<commit_msg>removed defauly gp for meta discriminators in vral<commit_after>'''Simple GAN model

'''

import logging

import torch
from torch import autograd
from torch.autograd import Variable
import torch.nn.functional as fun

from .gan import apply_penalty, f_divergence
from .modules.densenet import DenseNet

logger = logging.getLogger('cortex.models' + __name__)

resnet_discriminator_args_ = dict(dim_h=64, batch_norm=True, f_size=3, n_steps=3)
resnet_generator_args_ = dict(dim_h=64, batch_norm=True, f_size=3, n_steps=3)

mnist_discriminator_args_ = dict(dim_h=64, batch_norm=True, f_size=5, pad=2, stride=2, min_dim=7,
                                 nonlinearity='LeakyReLU')
mnist_generator_args_ = dict(dim_h=64, batch_norm=True, f_size=4, pad=1, stride=2, n_steps=2)

dcgan_discriminator_args_ = dict(dim_h=64, batch_norm=True, n_steps=3, nonlinearity='ReLU')
dcgan_generator_args_ = dict(dim_h=64, batch_norm=True, n_steps=3)

DEFAULTS = dict(
    data=dict(batch_size=dict(train=64, test=640), skip_last_batch=True,
              noise_variables=dict(z=('normal', 128), r=('normal', 1), f=('normal', 1))),
    optimizer=dict(
        optimizer='Adam',
        learning_rate=1e-4,
        clipping=dict(real_discriminator=0.1, fake_discriminator=0.1),
        updates_per_model=dict(discriminator=1, generator=1, real_discriminator=1, fake_discriminator=1, topnet=1)
    ),
    model=dict(model_type='dcgan', dim_d=1, dim_e=1, discriminator_args=None, generator_args=None),
    procedures=dict(measure='proxy_gan', boundary_seek=False, penalty=1.0, meta_penalty=0.0),
    train=dict(
        epochs=200,
        summary_updates=100,
        archive_every=10
    )
)

def setup(model=None, data=None, procedure=None, **kwargs):
    data['noise_variables']['r'] = ('normal', model['dim_d'])
    data['noise_variables']['f'] = ('normal', model['dim_d'])

def vral(nets, data_handler, measure=None, boundary_seek=False, penalty=None,
         real_mu=1.0, fake_mu=0.0, lam=100., vral_type=None, meta_penalty=None):
    # Variables
    X = data_handler['images']
    Z = data_handler['z']
    Rr = data_handler['r']
    Fr = data_handler['f']

    # Nets
    discriminator = nets['discriminator']
    generator = nets['generator']
    topnet = nets['topnet']
    real_discriminator = nets['real_discriminator']
    fake_discriminator = nets['fake_discriminator']
    gen_out = generator(Z, nonlinearity=fun.tanh)

    Rfphi = discriminator(X)
    Ffphi = discriminator(gen_out)
    Rf = topnet(Rfphi)
    Ff = topnet(Ffphi)

    if vral_type == 'fisher':
        real_r = real_discriminator(Rr)
        real_f = real_discriminator(Rf)
        fake_r = fake_discriminator(Fr)
        fake_f = fake_discriminator(Ff - Ff.mean())
        real_g = real_discriminator(Ff - real_mu)
    else:
        real_r = real_discriminator(Rr)
        real_f = real_discriminator(Rf - real_mu)
        fake_r = fake_discriminator(Fr)
        fake_f = fake_discriminator(Ff - fake_mu)
        real_g = real_discriminator(Ff - real_mu)

    if measure == 'mmd':
        assert False, 'Doesnt work right now'
        B, D = real_r.size()
        eye_B = Variable(torch.eye(B).cuda())
        def mmd_mat(x, y, remove_diagonal=False):
            mat = x.unsqueeze(2).expand(B, D, D) - y.unsqueeze(2).expand(B, D, D)
            term = torch.exp(-(mat ** 2).sum(1).sum(1))
            if remove_diagonal:
                term = ((1. - eye_B) * term).sum()
                return term / (B * (B - 1.))
            else:
                return term.mean()

        rrr = mmd_mat(real_r, real_r, remove_diagonal=True)
        rff = mmd_mat(real_f, real_f, remove_diagonal=True)
        rrf = mmd_mat(real_r, real_f)

        frr = mmd_mat(fake_r, fake_r, remove_diagonal=True)
        fff = mmd_mat(fake_f, fake_f, remove_diagonal=True)
        frf = mmd_mat(fake_r, fake_f)

        rgg = mmd_mat(real_g, real_g, remove_diagonal=True)
        rfg = mmd_mat(real_g, real_f)

        d_loss_r = -(rrr - 2. * rrf + rff)
        d_loss_f = -(frr - 2. * frf + fff)
        d_loss = -(d_loss_f + d_loss_r)

        #g_loss = -2. * rfg + rgg
        g_loss = (Ff.mean() - Rf.mean()) ** 2
    elif vral_type == 'fisher':
        d_loss_r, g_loss_r, rr, fr, wr, br = f_divergence(measure, real_r, real_f, boundary_seek=boundary_seek)
        d_loss_f, g_loss_f, rf, ff, wf, bf = f_divergence(measure, fake_r, fake_f, boundary_seek=boundary_seek)
        d_loss = lam * (g_loss_r + g_loss_f) + Ff.mean()
        t_loss = (g_loss_r + g_loss_f) + Ff.mean()

        g_loss = -Ff.mean()
    else:
        d_loss_r, g_loss_r, rr, fr, wr, br = f_divergence(measure, real_r, real_f, boundary_seek=boundary_seek)
        d_loss_f, g_loss_f, rf, ff, wf, bf = f_divergence(measure, fake_r, fake_f, boundary_seek=boundary_seek)

        d_loss = g_loss_r + g_loss_f
        t_loss = d_loss
        g_loss = (Ff.mean() - Rf.mean()) ** 2

    results = dict(g_loss=g_loss.data[0], d_loss=d_loss.data[0], rd_loss=d_loss_r.data[0], fd_loss=d_loss_f.data[0], t_loss=t_loss.data[0],
                   real=torch.mean(Rf).data[0], fake=torch.mean(Ff).data[0])
    samples = dict(images=dict(generated=0.5 * (gen_out + 1.).data, real=0.5 * (X + 1.).data),
                   histograms=dict(discriminator_output=dict(fake=Ff.view(-1).data, real=Rf.view(-1).data)))

    if meta_penalty:
        p_term_r = apply_penalty(data_handler, real_discriminator, Rr, Rf, measure)
        p_term_f = apply_penalty(data_handler, fake_discriminator, Fr, Ff, measure)

        d_loss_r += meta_penalty * p_term_r.mean()
        d_loss_f += meta_penalty * p_term_f.mean()
        results['real gradient penalty'] = p_term_r.mean()
        results['fake gradient penalty'] = p_term_f.mean()

    if penalty:
        real = Variable(X.data.cuda(), requires_grad=True)
        fake = Variable(gen_out.data.cuda(), requires_grad=True)
        real_out = discriminator(real)
        fake_out = discriminator(fake)

        real_out = topnet(real_out)
        fake_out = topnet(fake_out)

        g_r = autograd.grad(outputs=real_out, inputs=real, grad_outputs=torch.ones(real_out.size()).cuda(),
                            create_graph=True, retain_graph=True, only_inputs=True)[0]

        g_f = autograd.grad(outputs=fake_out, inputs=fake, grad_outputs=torch.ones(fake_out.size()).cuda(),
                            create_graph=True, retain_graph=True, only_inputs=True)[0]

        g_r = g_r.view(g_r.size()[0], -1)
        g_f = g_f.view(g_f.size()[0], -1)

        g_r = (g_r ** 2).sum(1)
        g_f = (g_f ** 2).sum(1)

        g_p = 0.5 * (g_r.mean() + g_f.mean())

        d_loss += penalty * g_p
        results['gradient penalty'] = g_p.data[0]

    loss = dict(generator=g_loss, real_discriminator=d_loss_r, fake_discriminator=d_loss_f, discriminator=d_loss,
                topnet=t_loss)
    return loss, results, samples, 'boundary'

def build_model(data_handler, model_type='resnet', dim_h=64, dim_d=1, dim_e=1,
                discriminator_args=None, generator_args=None):
    discriminator_args = discriminator_args or {}
    generator_args = generator_args or {}
    shape = data_handler.get_dims('x', 'y', 'c')
    dim_z = data_handler.get_dims('z')[0]

    if model_type == 'resnet':
        from .modules.resnets import ResEncoder as Discriminator
        from .modules.resnets import ResDecoder as Generator
        discriminator_args_ = resnet_discriminator_args_
        generator_args_ = resnet_generator_args_
    elif model_type == 'dcgan':
        from .modules.conv_decoders import SimpleConvDecoder as Generator
        from .modules.convnets import SimpleConvEncoder as Discriminator
        discriminator_args_ = dcgan_discriminator_args_
        generator_args_ = dcgan_generator_args_
    elif model_type == 'mnist':
        from .modules.conv_decoders import SimpleConvDecoder as Generator
        from .modules.convnets import SimpleConvEncoder as Discriminator
        discriminator_args_ = mnist_discriminator_args_
        generator_args_ = mnist_generator_args_
    else:
        raise NotImplementedError(model_type)

    discriminator_args_.update(**discriminator_args)
    generator_args_.update(**generator_args)

    if shape[0] == 64:
        discriminator_args_['n_steps'] = 4
        generator_args_['n_steps'] = 4
    elif shape[0] == 128:
        discriminator_args_['n_steps'] = 5
        generator_args_['n_steps'] = 5

    discriminator = Discriminator(shape, dim_out=dim_h, **discriminator_args_)
    topnet = DenseNet(dim_h, dim_h=[], dim_out=dim_d)
    generator = Generator(shape, dim_in=dim_z, **generator_args_)
    real_discriminator = DenseNet(dim_d, dim_h=[64], dim_out=dim_e, nonlinearity='LeakyReLU', batch_norm=False,
                                  layer_norm=False)
    fake_discriminator = DenseNet(dim_d, dim_h=[64], dim_out=dim_e, nonlinearity='LeakyReLU', batch_norm=False,
                                  layer_norm=False)
    logger.debug(discriminator)
    logger.debug(generator)
    logger.debug(real_discriminator)
    logger.debug(fake_discriminator)

    return dict(generator=generator, discriminator=discriminator, real_discriminator=real_discriminator,
                fake_discriminator=fake_discriminator, topnet=topnet), vral

<|endoftext|>