<commit_before>""" Tests the 'ETSConfig' configuration object. """

# Standard library imports.
import contextlib
import os
import shutil
import sys
import tempfile
import time
import unittest

# Enthought library imports.
from traits.etsconfig.api import ETSConfig

@contextlib.contextmanager
def temporary_home_directory():
    """
    Context manager that temporarily remaps HOME / APPDATA
    to a temporary directory.

    """
    temp_home = tempfile.mkdtemp()
    try:
        # Use the same recipe as in ETSConfig._initialize_application_data
        # to determine the home directory.
        if sys.platform == 'win32':
            home_var = 'APPDATA'
        else:
            home_var = 'HOME'

        old_home = os.environ.get(home_var, None)
        try:
            os.environ[home_var] = temp_home
            yield temp_home
        finally:
            if old_home is None:
                os.environ.pop(home_var, None)
            else:
                os.environ[home_var] = old_home
    finally:
        shutil.rmtree(temp_home)

class ETSConfigTestCase(unittest.TestCase):
    """ Tests the 'ETSConfig' configuration object. """

    ###########################################################################
    # 'TestCase' interface.
    ###########################################################################

    #### public methods #######################################################

    def setUp(self):
        """
        Prepares the test fixture before each test method is called.

        """
        # Make a fresh instance each time.
        self.ETSConfig = type(ETSConfig)()

    def run(self, result=None):
        # Extend TestCase.run to use a temporary home directory.
        with temporary_home_directory():
            super(ETSConfigTestCase, self).run(result)

    ###########################################################################
    # 'ETSConfigTestCase' interface.
    ###########################################################################

    #### public methods #######################################################

    def test_application_data(self):
        """
        application data

        """

        dirname = self.ETSConfig.application_data

        self.assertEqual(os.path.exists(dirname), True)
        self.assertEqual(os.path.isdir(dirname), True)

        return

    def test_set_application_data(self):
        """
        set application data

        """

        old = self.ETSConfig.application_data

        self.ETSConfig.application_data = 'foo'
        self.assertEqual('foo', self.ETSConfig.application_data)

        self.ETSConfig.application_data = old
        self.assertEqual(old, self.ETSConfig.application_data)

        return

    def test_application_data_is_idempotent(self):
        """
        application data is idempotent

        """

        # Just do the previous test again!
        self.test_application_data()
        self.test_application_data()

        return

    def test_write_to_application_data_directory(self):
        """
        write to application data directory

        """

        self.ETSConfig.company = 'Blah'
        dirname = self.ETSConfig.application_data

        path = os.path.join(dirname, 'dummy.txt')
        data = str(time.time())

        f = open(path, 'w')
        f.write(data)
        f.close()

        self.assertEqual(os.path.exists(path), True)

        f = open(path)
        result = f.read()
        f.close()

        os.remove(path)

        self.assertEqual(data, result)

        return

    def test_default_company(self):
        """
        default company

        """

        self.assertEqual(self.ETSConfig.company, 'Enthought')

        return

    def test_set_company(self):
        """
        set company

        """

        old = self.ETSConfig.company

        self.ETSConfig.company = 'foo'
        self.assertEqual('foo', self.ETSConfig.company)

        self.ETSConfig.company = old
        self.assertEqual(old, self.ETSConfig.company)

        return

    def _test_default_application_home(self):
        """
        application home

        """

        # This test is only valid when run with the 'main' at the end of this
        # file: "python app_dat_locator_test_case.py", in which case the
        # app_name will be the directory this file is in ('tests').
        app_home = self.ETSConfig.application_home
        (dirname, app_name) = os.path.split(app_home)

        self.assertEqual(dirname, self.ETSConfig.application_data)
        self.assertEqual(app_name, 'tests')

    def test_user_data(self):
        """
        user data

        """

        dirname = self.ETSConfig.user_data

        self.assertEqual(os.path.exists(dirname), True)
        self.assertEqual(os.path.isdir(dirname), True)

        return

    def test_set_user_data(self):
        """
        set user data

        """

        old = self.ETSConfig.user_data

        self.ETSConfig.user_data = 'foo'
        self.assertEqual('foo', self.ETSConfig.user_data)

        self.ETSConfig.user_data = old
        self.assertEqual(old, self.ETSConfig.user_data)

        return

    def test_user_data_is_idempotent(self):
        """
        user data is idempotent

        """

        # Just do the previous test again!
        self.test_user_data()

        return

    def test_write_to_user_data_directory(self):
        """
        write to user data directory

        """

        self.ETSConfig.company = 'Blah'
        dirname = self.ETSConfig.user_data

        path = os.path.join(dirname, 'dummy.txt')
        data = str(time.time())

        f = open(path, 'w')
        f.write(data)
        f.close()

        self.assertEqual(os.path.exists(path), True)

        f = open(path)
        result = f.read()
        f.close()

        os.remove(path)

        self.assertEqual(data, result)

        return

# For running as an individual set of tests.
if __name__ == '__main__':

    # Add the non-default test of application_home...non-default because it
    # must be run using this module as a script to be valid.
    suite = unittest.TestLoader().loadTestsFromTestCase(ETSConfigTestCase)
    suite.addTest(ETSConfigTestCase('_test_default_application_home'))

    unittest.TextTestRunner(verbosity=2).run(suite)

#### EOF ######################################################################
<commit_msg>Revert unrelated spacing change.<commit_after>""" Tests the 'ETSConfig' configuration object. """

# Standard library imports.
import contextlib
import os
import shutil
import sys
import tempfile
import time
import unittest

# Enthought library imports.
from traits.etsconfig.api import ETSConfig

@contextlib.contextmanager
def temporary_home_directory():
    """
    Context manager that temporarily remaps HOME / APPDATA
    to a temporary directory.

    """
    temp_home = tempfile.mkdtemp()
    try:
        # Use the same recipe as in ETSConfig._initialize_application_data
        # to determine the home directory.
        if sys.platform == 'win32':
            home_var = 'APPDATA'
        else:
            home_var = 'HOME'

        old_home = os.environ.get(home_var, None)
        try:
            os.environ[home_var] = temp_home
            yield temp_home
        finally:
            if old_home is None:
                os.environ.pop(home_var, None)
            else:
                os.environ[home_var] = old_home
    finally:
        shutil.rmtree(temp_home)

class ETSConfigTestCase(unittest.TestCase):
    """ Tests the 'ETSConfig' configuration object. """

    ###########################################################################
    # 'TestCase' interface.
    ###########################################################################

    #### public methods #######################################################

    def setUp(self):
        """
        Prepares the test fixture before each test method is called.

        """

        # Make a fresh instance each time.
        self.ETSConfig = type(ETSConfig)()

    def run(self, result=None):
        # Extend TestCase.run to use a temporary home directory.
        with temporary_home_directory():
            super(ETSConfigTestCase, self).run(result)

    ###########################################################################
    # 'ETSConfigTestCase' interface.
    ###########################################################################

    #### public methods #######################################################

    def test_application_data(self):
        """
        application data

        """

        dirname = self.ETSConfig.application_data

        self.assertEqual(os.path.exists(dirname), True)
        self.assertEqual(os.path.isdir(dirname), True)

        return

    def test_set_application_data(self):
        """
        set application data

        """

        old = self.ETSConfig.application_data

        self.ETSConfig.application_data = 'foo'
        self.assertEqual('foo', self.ETSConfig.application_data)

        self.ETSConfig.application_data = old
        self.assertEqual(old, self.ETSConfig.application_data)

        return

    def test_application_data_is_idempotent(self):
        """
        application data is idempotent

        """

        # Just do the previous test again!
        self.test_application_data()
        self.test_application_data()

        return

    def test_write_to_application_data_directory(self):
        """
        write to application data directory

        """

        self.ETSConfig.company = 'Blah'
        dirname = self.ETSConfig.application_data

        path = os.path.join(dirname, 'dummy.txt')
        data = str(time.time())

        f = open(path, 'w')
        f.write(data)
        f.close()

        self.assertEqual(os.path.exists(path), True)

        f = open(path)
        result = f.read()
        f.close()

        os.remove(path)

        self.assertEqual(data, result)

        return

    def test_default_company(self):
        """
        default company

        """

        self.assertEqual(self.ETSConfig.company, 'Enthought')

        return

    def test_set_company(self):
        """
        set company

        """

        old = self.ETSConfig.company

        self.ETSConfig.company = 'foo'
        self.assertEqual('foo', self.ETSConfig.company)

        self.ETSConfig.company = old
        self.assertEqual(old, self.ETSConfig.company)

        return

    def _test_default_application_home(self):
        """
        application home

        """

        # This test is only valid when run with the 'main' at the end of this
        # file: "python app_dat_locator_test_case.py", in which case the
        # app_name will be the directory this file is in ('tests').
        app_home = self.ETSConfig.application_home
        (dirname, app_name) = os.path.split(app_home)

        self.assertEqual(dirname, self.ETSConfig.application_data)
        self.assertEqual(app_name, 'tests')

    def test_user_data(self):
        """
        user data

        """

        dirname = self.ETSConfig.user_data

        self.assertEqual(os.path.exists(dirname), True)
        self.assertEqual(os.path.isdir(dirname), True)

        return

    def test_set_user_data(self):
        """
        set user data

        """

        old = self.ETSConfig.user_data

        self.ETSConfig.user_data = 'foo'
        self.assertEqual('foo', self.ETSConfig.user_data)

        self.ETSConfig.user_data = old
        self.assertEqual(old, self.ETSConfig.user_data)

        return

    def test_user_data_is_idempotent(self):
        """
        user data is idempotent

        """

        # Just do the previous test again!
        self.test_user_data()

        return

    def test_write_to_user_data_directory(self):
        """
        write to user data directory

        """

        self.ETSConfig.company = 'Blah'
        dirname = self.ETSConfig.user_data

        path = os.path.join(dirname, 'dummy.txt')
        data = str(time.time())

        f = open(path, 'w')
        f.write(data)
        f.close()

        self.assertEqual(os.path.exists(path), True)

        f = open(path)
        result = f.read()
        f.close()

        os.remove(path)

        self.assertEqual(data, result)

        return

# For running as an individual set of tests.
if __name__ == '__main__':

    # Add the non-default test of application_home...non-default because it
    # must be run using this module as a script to be valid.
    suite = unittest.TestLoader().loadTestsFromTestCase(ETSConfigTestCase)
    suite.addTest(ETSConfigTestCase('_test_default_application_home'))

    unittest.TextTestRunner(verbosity=2).run(suite)

#### EOF ######################################################################
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

import cgi
import json
import os
import re
import SimpleHTTPServer
import SocketServer
import subprocess
import sys
import threading

VERSION = 'v0.7.2'

hostname = ''
try:
    command = "bash -c '[[ $(dig +short $HOSTNAME) ]] && echo $HOSTNAME || wget -q -O - icanhazip.com'"
    hostname = subprocess.check_output(command, shell=True)
    if ':' in hostname:
        hostname = ''
except subprocess.CalledProcessError:
    pass

key_file = os.getenv('KEY_FILE', '/root/.ssh/authorized_keys')

admin_keys = []
if os.path.isfile(key_file):
    try:
        command = "cat {0}".format(key_file)
        admin_keys = subprocess.check_output(command, shell=True).strip().split("\n")
    except subprocess.CalledProcessError:
        pass

def check_boot():
    if 'onboot' not in sys.argv:
        return
    init_dir = os.getenv('INIT_DIR', '/etc/init')
    systemd_dir = os.getenv('SYSTEMD_DIR', '/etc/systemd/system')
    nginx_dir = os.getenv('NGINX_DIR', '/etc/nginx/conf.d')

    if os.path.exists(init_dir):
        with open('{0}/dokku-installer.conf'.format(init_dir), 'w') as f:
            f.write("start on runlevel [2345]\n")
            f.write("exec {0} selfdestruct\n".format(os.path.abspath(__file__)))
    if os.path.exists(systemd_dir):
        with open('{0}/dokku-installer.service'.format(systemd_dir), 'w') as f:
            f.write("[Unit]\n")
            f.write("Description=Dokku web-installer\n")
            f.write("\n")
            f.write("[Service]\n")
            f.write("ExecStart={0} selfdestruct\n".format(os.path.abspath(__file__)))
            f.write("\n")
            f.write("[Install]\n")
            f.write("WantedBy=multi-user.target\n")
            f.write("WantedBy=graphical.target\n")
    if os.path.exists(nginx_dir):
        with open('{0}/dokku-installer.conf'.format(nginx_dir), 'w') as f:
            f.write("upstream dokku-installer { server 127.0.0.1:2000; }\n")
            f.write("server {\n")
            f.write("  listen      80;\n")
            f.write("  location    / {\n")
            f.write("    proxy_pass  http://dokku-installer;\n")
            f.write("  }\n")
            f.write("}\n")

    subprocess.call('rm -f /etc/nginx/sites-enabled/*', shell=True)
    sys.exit(0)

class GetHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
    def do_GET(self):
        content = PAGE.replace('{VERSION}', VERSION)
        content = content.replace('{HOSTNAME}', hostname)
        content = content.replace('{ADMIN_KEYS}', "\n".join(admin_keys))
        self.send_response(200)
        self.end_headers()
        self.wfile.write(content)

    def do_POST(self):
        if self.path not in ['/setup', '/setup/']:
            return

        params = cgi.FieldStorage(fp=self.rfile,
                                  headers=self.headers,
                                  environ={
                                      'REQUEST_METHOD': 'POST',
                                      'CONTENT_TYPE': self.headers['Content-Type']})

        vhost_enable = 'false'
        dokku_root = os.getenv('DOKKU_ROOT', '/home/dokku')
        if 'vhost' in params and params['vhost'].value == 'true':
            vhost_enable = 'true'
            with open('{0}/VHOST'.format(dokku_root), 'w') as f:
                f.write(params['hostname'].value)
        else:
            try:
                os.remove('{0}/VHOST'.format(dokku_root))
            except OSError:
                pass
        with open('{0}/HOSTNAME'.format(dokku_root), 'w') as f:
            f.write(params['hostname'].value)

        for (index, key) in enumerate(params['keys'].value.splitlines(), 1):
            user = 'admin'
            if self.admin_user_exists() is not None:
                user = 'web-admin'
                if self.web_admin_user_exists() is not None:
                    index = int(self.web_admin_user_exists()) + 1
                elif self.web_admin_user_exists() is None:
                    index = 1
            elif self.admin_user_exists() is None:
                pass
            else:
                index = int(self.admin_user_exists()) + 1
            user = user + str(index)
            command = ['sshcommand', 'acl-add', 'dokku', user]
            proc = subprocess.Popen(command, stdin=subprocess.PIPE)
            proc.stdin.write(key)
            proc.stdin.close()
            proc.wait()

        set_debconf_selection('boolean', 'skip_key_file', 'true')
        set_debconf_selection('boolean', 'vhost_enable', vhost_enable)
        set_debconf_selection('boolean', 'web_config', 'false')
        set_debconf_selection('string', 'hostname', params['hostname'].value)

        if 'selfdestruct' in sys.argv:
            DeleteInstallerThread()

        self.send_response(200)
        self.end_headers()
        self.wfile.write(json.dumps({'status': 'ok'}))

    def web_admin_user_exists(self):
    return self.user_exists('web-admin(\d+)')

    def admin_user_exists(self):
    return self.user_exists('admin(\d+)')

    def user_exists(self, name):
    command = 'dokku ssh-keys:list'
    pattern = re.compile(r'NAME="' + name + '"')
    proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
    max_num = 0
    exists = False
    for line in proc.stdout:
        m = pattern.search(line)
        if m:
            # User of the form `user` or `user#` exists
            exists = True
            max_num = max(max_num, m.group(1))
    if exists:
        return max_num
    else:
        return None

def set_debconf_selection(debconf_type, key, value):
    found = False
    with open('/etc/os-release', 'r') as f:
        for line in f:
            if 'debian' in line:
                found = True

    if not found:
        return

    ps = subprocess.Popen(['echo', 'dokku dokku/{0} {1} {2}'.format(
        key, debconf_type, value
    )], stdout=subprocess.PIPE)

    try:
        subprocess.check_output(['debconf-set-selections'], stdin=ps.stdout)
    except subprocess.CalledProcessError:
        pass

    ps.wait()

class DeleteInstallerThread(object):
    def __init__(self, interval=1):
        thread = threading.Thread(target=self.run, args=())
        thread.daemon = True
        thread.start()

    def run(self):
        command = "rm /etc/nginx/conf.d/dokku-installer.conf && /etc/init.d/nginx stop && /etc/init.d/nginx start"
        try:
            subprocess.call(command, shell=True)
        except:
            pass

        command = "rm -f /etc/init/dokku-installer.conf /etc/systemd/system/dokku-installer.service && (stop dokku-installer || systemctl stop dokku-installer.service)"
        try:
            subprocess.call(command, shell=True)
        except:
            pass

def main():
    check_boot()

    port = int(os.getenv('PORT', 2000))
    httpd = SocketServer.TCPServer(("", port), GetHandler)
    print "Listening on 0.0.0.0:{0}, CTRL+C to stop".format(port)
    httpd.serve_forever()

PAGE = """

  <title>Dokku Setup</title>
  <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.min.css" />

  <form id="form" role="form">

Dokku Setup {VERSION}

Admin Access

      Public Key

      <textarea class="form-control" name="keys" rows="7" id="key">{ADMIN_KEYS}</textarea>

Hostname Configuration

        Hostname
        <input class="form-control" type="text" id="hostname" name="hostname" value="{HOSTNAME}" />

        <input id="vhost" name="vhost" type="checkbox" value="true"> Use virtualhost naming for apps

Your app URLs will look like:

http://hostname:port

    <button type="button" onclick="setup()" class="btn btn-primary">Finish Setup</button>
  </form>

"""

if __name__ == "__main__":
    main()
<commit_msg>Correct indentation<commit_after>#!/usr/bin/env python2.7

import cgi
import json
import os
import re
import SimpleHTTPServer
import SocketServer
import subprocess
import sys
import threading

VERSION = 'v0.7.2'

hostname = ''
try:
    command = "bash -c '[[ $(dig +short $HOSTNAME) ]] && echo $HOSTNAME || wget -q -O - icanhazip.com'"
    hostname = subprocess.check_output(command, shell=True)
    if ':' in hostname:
        hostname = ''
except subprocess.CalledProcessError:
    pass

key_file = os.getenv('KEY_FILE', '/root/.ssh/authorized_keys')

admin_keys = []
if os.path.isfile(key_file):
    try:
        command = "cat {0}".format(key_file)
        admin_keys = subprocess.check_output(command, shell=True).strip().split("\n")
    except subprocess.CalledProcessError:
        pass

def check_boot():
    if 'onboot' not in sys.argv:
        return
    init_dir = os.getenv('INIT_DIR', '/etc/init')
    systemd_dir = os.getenv('SYSTEMD_DIR', '/etc/systemd/system')
    nginx_dir = os.getenv('NGINX_DIR', '/etc/nginx/conf.d')

    if os.path.exists(init_dir):
        with open('{0}/dokku-installer.conf'.format(init_dir), 'w') as f:
            f.write("start on runlevel [2345]\n")
            f.write("exec {0} selfdestruct\n".format(os.path.abspath(__file__)))
    if os.path.exists(systemd_dir):
        with open('{0}/dokku-installer.service'.format(systemd_dir), 'w') as f:
            f.write("[Unit]\n")
            f.write("Description=Dokku web-installer\n")
            f.write("\n")
            f.write("[Service]\n")
            f.write("ExecStart={0} selfdestruct\n".format(os.path.abspath(__file__)))
            f.write("\n")
            f.write("[Install]\n")
            f.write("WantedBy=multi-user.target\n")
            f.write("WantedBy=graphical.target\n")
    if os.path.exists(nginx_dir):
        with open('{0}/dokku-installer.conf'.format(nginx_dir), 'w') as f:
            f.write("upstream dokku-installer { server 127.0.0.1:2000; }\n")
            f.write("server {\n")
            f.write("  listen      80;\n")
            f.write("  location    / {\n")
            f.write("    proxy_pass  http://dokku-installer;\n")
            f.write("  }\n")
            f.write("}\n")

    subprocess.call('rm -f /etc/nginx/sites-enabled/*', shell=True)
    sys.exit(0)

class GetHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
    def do_GET(self):
        content = PAGE.replace('{VERSION}', VERSION)
        content = content.replace('{HOSTNAME}', hostname)
        content = content.replace('{ADMIN_KEYS}', "\n".join(admin_keys))
        self.send_response(200)
        self.end_headers()
        self.wfile.write(content)

    def do_POST(self):
        if self.path not in ['/setup', '/setup/']:
            return

        params = cgi.FieldStorage(fp=self.rfile,
                                  headers=self.headers,
                                  environ={
                                      'REQUEST_METHOD': 'POST',
                                      'CONTENT_TYPE': self.headers['Content-Type']})

        vhost_enable = 'false'
        dokku_root = os.getenv('DOKKU_ROOT', '/home/dokku')
        if 'vhost' in params and params['vhost'].value == 'true':
            vhost_enable = 'true'
            with open('{0}/VHOST'.format(dokku_root), 'w') as f:
                f.write(params['hostname'].value)
        else:
            try:
                os.remove('{0}/VHOST'.format(dokku_root))
            except OSError:
                pass
        with open('{0}/HOSTNAME'.format(dokku_root), 'w') as f:
            f.write(params['hostname'].value)

        for (index, key) in enumerate(params['keys'].value.splitlines(), 1):
            user = 'admin'
            if self.admin_user_exists() is not None:
                user = 'web-admin'
                if self.web_admin_user_exists() is not None:
                    index = int(self.web_admin_user_exists()) + 1
                elif self.web_admin_user_exists() is None:
                    index = 1
            elif self.admin_user_exists() is None:
                pass
            else:
                index = int(self.admin_user_exists()) + 1
            user = user + str(index)
            command = ['sshcommand', 'acl-add', 'dokku', user]
            proc = subprocess.Popen(command, stdin=subprocess.PIPE)
            proc.stdin.write(key)
            proc.stdin.close()
            proc.wait()

        set_debconf_selection('boolean', 'skip_key_file', 'true')
        set_debconf_selection('boolean', 'vhost_enable', vhost_enable)
        set_debconf_selection('boolean', 'web_config', 'false')
        set_debconf_selection('string', 'hostname', params['hostname'].value)

        if 'selfdestruct' in sys.argv:
            DeleteInstallerThread()

        self.send_response(200)
        self.end_headers()
        self.wfile.write(json.dumps({'status': 'ok'}))

    def web_admin_user_exists(self):
        return self.user_exists('web-admin(\d+)')

    def admin_user_exists(self):
        return self.user_exists('admin(\d+)')

    def user_exists(self, name):
        command = 'dokku ssh-keys:list'
        pattern = re.compile(r'NAME="' + name + '"')
        proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
        max_num = 0
        exists = False
        for line in proc.stdout:
            m = pattern.search(line)
            if m:
                # User of the form `user` or `user#` exists
                exists = True
                max_num = max(max_num, m.group(1))
        if exists:
            return max_num
        else:
            return None

def set_debconf_selection(debconf_type, key, value):
    found = False
    with open('/etc/os-release', 'r') as f:
        for line in f:
            if 'debian' in line:
                found = True

    if not found:
        return

    ps = subprocess.Popen(['echo', 'dokku dokku/{0} {1} {2}'.format(
        key, debconf_type, value
    )], stdout=subprocess.PIPE)

    try:
        subprocess.check_output(['debconf-set-selections'], stdin=ps.stdout)
    except subprocess.CalledProcessError:
        pass

    ps.wait()

class DeleteInstallerThread(object):
    def __init__(self, interval=1):
        thread = threading.Thread(target=self.run, args=())
        thread.daemon = True
        thread.start()

    def run(self):
        command = "rm /etc/nginx/conf.d/dokku-installer.conf && /etc/init.d/nginx stop && /etc/init.d/nginx start"
        try:
            subprocess.call(command, shell=True)
        except:
            pass

        command = "rm -f /etc/init/dokku-installer.conf /etc/systemd/system/dokku-installer.service && (stop dokku-installer || systemctl stop dokku-installer.service)"
        try:
            subprocess.call(command, shell=True)
        except:
            pass

def main():
    check_boot()

    port = int(os.getenv('PORT', 2000))
    httpd = SocketServer.TCPServer(("", port), GetHandler)
    print "Listening on 0.0.0.0:{0}, CTRL+C to stop".format(port)
    httpd.serve_forever()

PAGE = """

  <title>Dokku Setup</title>
  <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.min.css" />

  <form id="form" role="form">

Dokku Setup {VERSION}

Admin Access

      Public Key

      <textarea class="form-control" name="keys" rows="7" id="key">{ADMIN_KEYS}</textarea>

Hostname Configuration

        Hostname
        <input class="form-control" type="text" id="hostname" name="hostname" value="{HOSTNAME}" />

        <input id="vhost" name="vhost" type="checkbox" value="true"> Use virtualhost naming for apps

Your app URLs will look like:

http://hostname:port

    <button type="button" onclick="setup()" class="btn btn-primary">Finish Setup</button>
  </form>

"""

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#! /usr/bin/env python
#
# Mixpanel, Inc. -- http://mixpanel.com/
#
# Python API client library to consume mixpanel.com analytics data.

import os, sys
import pygal
import hashlib
import urllib
import time
try:
    import json
except ImportError:
    import simplejson as json

class Mixpanel(object):

    ENDPOINT = 'http://mixpanel.com/api'
    VERSION = '2.0'

    def __init__(self, api_key, api_secret):
        self.api_key = api_key
        self.api_secret = api_secret

    def request(self, methods, params, format='json'):
        """
            methods - List of methods to be joined, e.g. ['events', 'properties', 'values']
                      will give us http://mixpanel.com/api/2.0/events/properties/values/
            params - Extra parameters associated with method
        """
        params['api_key'] = self.api_key
        params['expire'] = int(time.time()) + 600   # Grant this request 10 minutes.
        params['format'] = format
        if 'sig' in params: del params['sig']
        params['sig'] = self.hash_args(params)

        request_url = '/'.join([self.ENDPOINT, str(self.VERSION)] + methods) + '/?' + self.unicode_urlencode(params)

        request = urllib.urlopen(request_url)
        data = request.read()

        return json.loads(data)

    def unicode_urlencode(self, params):
        """
            Convert lists to JSON encoded strings, and correctly handle any
            unicode URL parameters.
        """
        if isinstance(params, dict):
            params = params.items()
        for i, param in enumerate(params):
            if isinstance(param[1], list):
                params[i] = (param[0], json.dumps(param[1]),)

        return urllib.urlencode(
            [(k, isinstance(v, unicode) and v.encode('utf-8') or v) for k, v in params]
        )

    def hash_args(self, args, secret=None):
        """
            Hashes arguments by joining key=value pairs, appending a secret, and
            then taking the MD5 hex digest.
        """
        for a in args:
            if isinstance(args[a], list): args[a] = json.dumps(args[a])

        args_joined = ''
        for a in sorted(args.keys()):
            if isinstance(a, unicode):
                args_joined += a.encode('utf-8')
            else:
                args_joined += str(a)

            args_joined += '='

            if isinstance(args[a], unicode):
                args_joined += args[a].encode('utf-8')
            else:
                args_joined += str(args[a])

        hash = hashlib.md5(args_joined)

        if secret:
            hash.update(secret)
        elif self.api_secret:
            hash.update(self.api_secret)
        return hash.hexdigest()

def check_for_api_error(data):
    if 'error' in data:
        sys.exit('Error: ' + data['error'])

if __name__ == '__main__':
    api = Mixpanel(
        api_key = os.environ.get('MP_API_KEY'),
        api_secret = os.environ.get('MP_API_SECRET')
    )
    eventNames = api.request(['events', 'names'], {
        'type': 'general'
    })
    data = api.request(['events'], {
        'event' : eventNames,
        'unit' : 'day',
        'interval' : 31,
        'type': 'general'
    })
    print json.dumps(data, indent=4, sort_keys=True)
<commit_msg>Create 'charts/' directory for SVG files<commit_after>#! /usr/bin/env python
#
# Mixpanel, Inc. -- http://mixpanel.com/
#
# Python API client library to consume mixpanel.com analytics data.

import os, sys
import pygal
import hashlib
import urllib
import time
try:
    import json
except ImportError:
    import simplejson as json

class Mixpanel(object):

    ENDPOINT = 'http://mixpanel.com/api'
    VERSION = '2.0'

    def __init__(self, api_key, api_secret):
        self.api_key = api_key
        self.api_secret = api_secret

    def request(self, methods, params, format='json'):
        """
            methods - List of methods to be joined, e.g. ['events', 'properties', 'values']
                      will give us http://mixpanel.com/api/2.0/events/properties/values/
            params - Extra parameters associated with method
        """
        params['api_key'] = self.api_key
        params['expire'] = int(time.time()) + 600   # Grant this request 10 minutes.
        params['format'] = format
        if 'sig' in params: del params['sig']
        params['sig'] = self.hash_args(params)

        request_url = '/'.join([self.ENDPOINT, str(self.VERSION)] + methods) + '/?' + self.unicode_urlencode(params)

        request = urllib.urlopen(request_url)
        data = request.read()

        return json.loads(data)

    def unicode_urlencode(self, params):
        """
            Convert lists to JSON encoded strings, and correctly handle any
            unicode URL parameters.
        """
        if isinstance(params, dict):
            params = params.items()
        for i, param in enumerate(params):
            if isinstance(param[1], list):
                params[i] = (param[0], json.dumps(param[1]),)

        return urllib.urlencode(
            [(k, isinstance(v, unicode) and v.encode('utf-8') or v) for k, v in params]
        )

    def hash_args(self, args, secret=None):
        """
            Hashes arguments by joining key=value pairs, appending a secret, and
            then taking the MD5 hex digest.
        """
        for a in args:
            if isinstance(args[a], list): args[a] = json.dumps(args[a])

        args_joined = ''
        for a in sorted(args.keys()):
            if isinstance(a, unicode):
                args_joined += a.encode('utf-8')
            else:
                args_joined += str(a)

            args_joined += '='

            if isinstance(args[a], unicode):
                args_joined += args[a].encode('utf-8')
            else:
                args_joined += str(args[a])

        hash = hashlib.md5(args_joined)

        if secret:
            hash.update(secret)
        elif self.api_secret:
            hash.update(self.api_secret)
        return hash.hexdigest()

def check_for_api_error(data):
    if 'error' in data:
        sys.exit('Error: ' + data['error'])

if __name__ == '__main__':
    api = Mixpanel(
        api_key = os.environ.get('MP_API_KEY'),
        api_secret = os.environ.get('MP_API_SECRET')
    )
    SVG_DIR = 'charts/'
    if not os.path.exists(SVG_DIR):
        os.makedirs(SVG_DIR)
    eventNames = api.request(['events', 'names'], {
        'type': 'general'
    })
    data = api.request(['events'], {
        'event' : eventNames,
        'unit' : 'day',
        'interval' : 31,
        'type': 'general'
    })
    print json.dumps(data, indent=4, sort_keys=True)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2011 The Bitcoin developers
# Distributed under the MIT/X11 software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#

import time
import json
import pprint
import hashlib
import struct
import re
import base64
import httplib
import sys
from multiprocessing import Process

ERR_SLEEP = 15
MAX_NONCE = 1000000L

settings = {}
pp = pprint.PrettyPrinter(indent=4)

class BitcoinRPC:
	OBJID = 1

	def __init__(self, host, port, username, password):
		authpair = "%s:%s" % (username, password)
		self.authhdr = "Basic %s" % (base64.b64encode(authpair))
		self.conn = httplib.HTTPConnection(host, port, False, 30)
	def rpc(self, method, params=None):
		self.OBJID += 1
		obj = { 'version' : '1.1',
			'method' : method,
			'id' : self.OBJID }
		if params is None:
			obj['params'] = []
		else:
			obj['params'] = params
		self.conn.request('POST', '/', json.dumps(obj),
			{ 'Authorization' : self.authhdr,
			  'Content-type' : 'application/json' })

		resp = self.conn.getresponse()
		if resp is None:
			print "JSON-RPC: no response"
			return None

		body = resp.read()
		resp_obj = json.loads(body)
		if resp_obj is None:
			print "JSON-RPC: cannot JSON-decode body"
			return None
		if 'error' in resp_obj and resp_obj['error'] != None:
			return resp_obj['error']
		if 'result' not in resp_obj:
			print "JSON-RPC: no result in object"
			return None

		return resp_obj['result']
	def getblockcount(self):
		return self.rpc('getblockcount')
	def getwork(self, data=None):
		return self.rpc('getwork', data)

def uint32(x):
	return x & 0xffffffffL

def bytereverse(x):
	return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
			(((x) >> 8) & 0x0000ff00) | ((x) >> 24) ))

def bufreverse(in_buf):
	out_words = []
	for i in range(0, len(in_buf), 4):
		word = struct.unpack('@I', in_buf[i:i+4])[0]
		out_words.append(struct.pack('@I', bytereverse(word)))
	return ''.join(out_words)

def wordreverse(in_buf):
	out_words = []
	for i in range(0, len(in_buf), 4):
		out_words.append(in_buf[i:i+4])
	out_words.reverse()
	return ''.join(out_words)

class Miner:
	def __init__(self, id):
		self.id = id
		self.max_nonce = MAX_NONCE

	def work(self, datastr, targetstr):
		# decode work data hex string to binary
		static_data = datastr.decode('hex')
		static_data = bufreverse(static_data)

		# the first 76b of 80b do not change
		blk_hdr = static_data[:76]

		# decode 256-bit target value
		targetbin = targetstr.decode('hex')
		targetbin = targetbin[::-1]	# byte-swap and dword-swap
		targetbin_str = targetbin.encode('hex')
		target = long(targetbin_str, 16)

		# pre-hash first 76b of block header
		static_hash = hashlib.sha256()
		static_hash.update(blk_hdr)

		for nonce in xrange(self.max_nonce):

			# encode 32-bit nonce value
			nonce_bin = struct.pack("<I", nonce)

			# hash final 4b, the nonce value
			hash1_o = static_hash.copy()
			hash1_o.update(nonce_bin)
			hash1 = hash1_o.digest()

			# sha256 hash of sha256 hash
			hash_o = hashlib.sha256()
			hash_o.update(hash1)
			hash = hash_o.digest()

			# quick test for winning solution: high 32 bits zero?
			if hash[-4:] != '\0\0\0\0':
				continue

			# convert binary hash to 256-bit Python long
			hash = bufreverse(hash)
			hash = wordreverse(hash)

			hash_str = hash.encode('hex')
			l = long(hash_str, 16)

			# proof-of-work test:  hash < target
			if l < target:
				print time.asctime(), "PROOF-OF-WORK found: %064x" % (l,)
				return (nonce + 1, nonce_bin)
			else:
				print time.asctime(), "PROOF-OF-WORK false positive %064x" % (l,)
#				return (nonce + 1, nonce_bin)

		return (nonce + 1, None)

	def submit_work(self, rpc, original_data, nonce_bin):
		nonce_bin = bufreverse(nonce_bin)
		nonce = nonce_bin.encode('hex')
		solution = original_data[:152] + nonce + original_data[160:256]
		param_arr = [ solution ]
		result = rpc.getwork(param_arr)
		print time.asctime(), "--> Upstream RPC result:", result

	def iterate(self, rpc):
		work = rpc.getwork()
		if work is None:
			time.sleep(ERR_SLEEP)
			return
		if 'data' not in work or 'target' not in work:
			time.sleep(ERR_SLEEP)
			return

		time_start = time.time()

		(hashes_done, nonce_bin) = self.work(work['data'],
						     work['target'])

		time_end = time.time()
		time_diff = time_end - time_start

		self.max_nonce = long(
			(hashes_done * settings['scantime']) / time_diff)
		if self.max_nonce > 0xfffffffaL:
			self.max_nonce = 0xfffffffaL

		if settings['hashmeter']:
			print "HashMeter(%d): %d hashes, %.2f Khash/sec" % (
			      self.id, hashes_done,
			      (hashes_done / 1000.0) / time_diff)

		if nonce_bin is not None:
			self.submit_work(rpc, work['data'], nonce_bin)

	def loop(self):
		rpc = BitcoinRPC(settings['host'], settings['port'],
				 settings['rpcuser'], settings['rpcpass'])
		if rpc is None:
			return

		while True:
			self.iterate(rpc)

def miner_thread(id):
	miner = Miner(id)
	miner.loop()

if __name__ == '__main__':
	if len(sys.argv) != 2:
		print "Usage: pyminer.py CONFIG-FILE"
		sys.exit(1)

	f = open(sys.argv[1])
	for line in f:
		# skip comment lines
		m = re.search('^\s*#', line)
		if m:
			continue

		# parse key=value lines
		m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
		if m is None:
			continue
		settings[m.group(1)] = m.group(2)
	f.close()

	if 'host' not in settings:
		settings['host'] = '127.0.0.1'
	if 'port' not in settings:
		settings['port'] = 8332
	if 'threads' not in settings:
		settings['threads'] = 1
	if 'hashmeter' not in settings:
		settings['hashmeter'] = 0
	if 'scantime' not in settings:
		settings['scantime'] = 30L
	if 'rpcuser' not in settings or 'rpcpass' not in settings:
		print "Missing username and/or password in cfg file"
		sys.exit(1)

	settings['port'] = int(settings['port'])
	settings['threads'] = int(settings['threads'])
	settings['hashmeter'] = int(settings['hashmeter'])
	settings['scantime'] = long(settings['scantime'])

	thr_list = []
	for thr_id in range(settings['threads']):
		p = Process(target=miner_thread, args=(thr_id,))
		p.start()
		thr_list.append(p)
		time.sleep(1)			# stagger threads

	print settings['threads'], "mining threads started"

	print time.asctime(), "Miner Starts - %s:%s" % (settings['host'], settings['port'])
	try:
		for thr_proc in thr_list:
			thr_proc.join()
	except KeyboardInterrupt:
		pass
	print time.asctime(), "Miner Stops - %s:%s" % (settings['host'], settings['port'])

<commit_msg>pyminer: Fix memory leak, refactor to be more pythonic, maintainable, and possibly faster<commit_after>#!/usr/bin/python
#
# Copyright (c) 2011 The Bitcoin developers
# Distributed under the MIT/X11 software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#

import sys
from multiprocessing import Process
import time
import struct
import hashlib
import base64
import re
import httplib
import json

ERR_SLEEP = 15
MAX_NONCE = 1000000L

settings = {}

class BitcoinRPC:
    object_id = 1

    def __init__(self, host, port, username, password):
        authpair = "{0}:{1}".format(username, password)
        self.authhdr = "Basic {0}".format(base64.b64encode(authpair))
        self.conn = httplib.HTTPConnection(host, port, strict=False, timeout=30)

    def rpc(self, method, params=None):
        self.object_id += 1
        obj = {'version' : '1.1',
               'method' : method,
               'id' : self.object_id,
               'params' : params or []}

        self.conn.request('POST', '/', json.dumps(obj),
            { 'Authorization' : self.authhdr,
              'Content-type' : 'application/json' })

        resp = self.conn.getresponse()

        if resp is None:
            print("JSON-RPC: no response")
            return None

        body = resp.read()
        resp_obj = json.loads(body)

        if resp_obj is None:
            print("JSON-RPC: cannot JSON-decode body")
            return None

        if 'error' in resp_obj and resp_obj['error'] != None:
            return resp_obj['error']

        if 'result' not in resp_obj:
            print("JSON-RPC: no result in object")
            return None

        return resp_obj['result']

    def getblockcount(self):
        return self.rpc('getblockcount')

    def getwork(self, data=None):
        return self.rpc('getwork', data)

def uint32(x):
    return x & 0xffffffffL

def bytereverse(x):
    return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
            (((x) >> 8) & 0x0000ff00) | ((x) >> 24) ))

def bufreverse(in_buf):
    out_words = []

    for i in range(0, len(in_buf), 4):
        word = struct.unpack('@I', in_buf[i:i+4])[0]
        out_words.append(struct.pack('@I', bytereverse(word)))

    return ''.join(out_words)

def wordreverse(in_buf):
    out_words = []

    for i in range(0, len(in_buf), 4):
        out_words.append(in_buf[i:i+4])

    out_words.reverse()

    return ''.join(out_words)

class Miner:
    def __init__(self, id):
        self.id = id
        self.max_nonce = MAX_NONCE

    def work(self, datastr, targetstr):
        # decode work data hex string to binary
        static_data = datastr.decode('hex')
        static_data = bufreverse(static_data)

        # the first 76b of 80b do not change
        blk_hdr = static_data[:76]

        # decode 256-bit target value
        targetbin = targetstr.decode('hex')
        targetbin = targetbin[::-1]    # byte-swap and dword-swap
        targetbin_str = targetbin.encode('hex')
        target = long(targetbin_str, 16)

        # pre-hash first 76b of block header
        static_hash = hashlib.sha256()
        static_hash.update(blk_hdr)

        for nonce in xrange(self.max_nonce):

            # encode 32-bit nonce value
            nonce_bin = struct.pack("<I", nonce)

            # hash final 4b, the nonce value
            hash1_o = static_hash.copy()
            hash1_o.update(nonce_bin)
            hash1 = hash1_o.digest()

            # sha256 hash of sha256 hash
            hash_o = hashlib.sha256()
            hash_o.update(hash1)
            hash = hash_o.digest()

            # quick test for winning solution: high 32 bits zero?
            if hash[-4:] != '\0\0\0\0':
                continue

            # convert binary hash to 256-bit Python long
            hash = bufreverse(hash)
            hash = wordreverse(hash)

            hash_str = hash.encode('hex')
            long_hash = long(hash_str, 16)

            # proof-of-work test:  hash < target
            if long_hash < target:
                print(time.asctime(), "PROOF-OF-WORK found: "
                                      "{0:064x}".format(long_hash))
                return (nonce + 1, nonce_bin)
            else:
                print(time.asctime(), "PROOF-OF-WORK false"
                                      "positive {0:064x}".format(long_hash))

        return (nonce + 1, None)

    def submit_work(self, rpc, original_data, nonce_bin):
        nonce_bin = bufreverse(nonce_bin)
        nonce = nonce_bin.encode('hex')
        solution = original_data[:152] + nonce + original_data[160:256]
        param_arr = [ solution ]
        result = rpc.getwork(param_arr)

        print(time.asctime(), "--> Upstream RPC result:", result)

    def iterate(self, rpc):
        work = rpc.getwork()

        if work is None:
            time.sleep(ERR_SLEEP)
            return

        if 'data' not in work or 'target' not in work:
            time.sleep(ERR_SLEEP)
            return

        time_start = time.time()

        (hashes_done, nonce_bin) = self.work(work['data'],
                             work['target'])

        time_end = time.time()
        time_diff = time_end - time_start

        self.max_nonce = long(
            (hashes_done * settings['scantime']) / time_diff)

        if self.max_nonce > 0xfffffffaL:
            self.max_nonce = 0xfffffffaL

        if settings['hashmeter']:
            print("HashMeter({:d}): {:d} hashes, {:.2f} Khash/sec".format(
                  self.id, hashes_done, (hashes_done / 1000.0) / time_diff))

        if nonce_bin is not None:
            self.submit_work(rpc, work['data'], nonce_bin)

    def loop(self):
        rpc = BitcoinRPC(settings['host'], settings['port'],
                 settings['rpcuser'], settings['rpcpass'])

        if rpc is not None:

            while True:
                self.iterate(rpc)

        self.conn.close()

def miner_thread(id):
    miner = Miner(id)
    miner.loop()

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("Usage: pyminer.py CONFIG-FILE")
        sys.exit(1)

    with open(sys.argv[1]) as f:

        for line in f:
            # skip comment lines
            m = re.search('^\s*#', line)
            if m:
                continue

            # parse key=value lines
            m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
            if m is None:
                continue

            settings[m.group(1)] = m.group(2)

    settings.setdefault('host', '127.0.0.1')
    settings.setdefault('port', 8332)
    settings.setdefault('threads', 1)
    settings.setdefault('hashmeter', 0)
    settings.setdefault('scantime', 30L)

    if 'rpcuser' not in settings or 'rpcpass' not in settings:
        print("Missing username and/or password in cfg file")
        sys.exit(1)

    settings['port'] = int(settings['port'])
    settings['threads'] = int(settings['threads'])
    settings['hashmeter'] = int(settings['hashmeter'])
    settings['scantime'] = long(settings['scantime'])

    thread_list = []

    for thread_id in range(settings['threads']):
        p = Process(target=miner_thread, args=(thread_id,))
        p.start()
        thread_list.append(p)
        time.sleep(1) # stagger threads

    print(settings['threads'], "mining threads started")

    print(time.asctime(), "Miner Starts - {0}:{1}".format(settings['host'],
                                                          settings['port']))
    try:
        for thread_process in thread_list:
            thread_process.join()
    except KeyboardInterrupt:
        pass

    print(time.asctime(), "Miner Stops - {0}:{1}".format(settings['host'],
                                                         settings['port']))
<|endoftext|>


<commit_before>from itertools import filterfalse

import pandas as pd
import numpy as np
from numpy.polynomial import Polynomial

def fit_hipf(reference_sample, controls_individuals, controls_households, maxiter):
    """Hierarchical Iterative Proportional Fitting.

    Algorithm taken from
    Müller and Axhausen 2011: "Hierarchical IPF: Generating a synthetic population for Switzerland"

    Can be used to fit a reference sample of households and individuals to control variables
    simultaneously. The algorithm supports only one dimensional control variables, and hence
    multi-dimensional control variables must be transformed to one dimensional ones first.

    Parameters:
        reference_sample:     The reference sample to be fited to the controls. Must be a pandas
                              DataFrame where the index is a multi index of (household_id,
                              person_id), and each colum represents either a household category
                              or a category of an individual.
        controls_individuals: The control variables for individuals. Must be a dict from control
                              name to a dict of its values.
                              e.g. {'age': {'below_50': 45, '50_or_older'}: 55}
        controls_households:  The control variables for households. Must be in the same format as
                              the controls for individuals.
        maxiter:              Maximum number of iterations.
    """

    weights = pd.Series(
        index=_household_groups(reference_sample).count().index.get_level_values(0),
        data=1.0,
        dtype=np.float64
    )
    for i in range(0, maxiter):
        next_weights = _fit_households(reference_sample, weights, controls_households)
        weights_person = _expand_weights_to_person(next_weights, reference_sample.index)
        weights_person = _fit_person(reference_sample, weights_person, controls_individuals)
        next_weights = _aggregate_person_weights_to_household(weights_person)
        next_weights = _rescale_weights(reference_sample, next_weights)
        weights = next_weights
    return weights

def _household_groups(reference_sample):
    return reference_sample.groupby(reference_sample.index.get_level_values(0))

def _fit_households(reference_sample, weights, controls_households):
    new_weights = weights.copy()
    for control_name, control_values in controls_households.items():
        for household_id in new_weights.index:
            hh_value = reference_sample.loc[(household_id, 1), control_name]
            mask = _household_groups(reference_sample)[control_name].first() == hh_value
            new_weights[household_id] = (weights.loc[household_id] *
                                         control_values[hh_value] / weights[mask].sum())
    return new_weights

def _fit_person(reference_sample, weights, controls_individuals):
    new_weights = weights.copy()
    for control_name, control_values in controls_individuals.items():
        for person_id in weights.index:
            value = reference_sample.loc[person_id, control_name]
            mask = reference_sample[control_name] == value
            new_weights.loc[person_id] = (weights.loc[person_id] *
                                          control_values[value] / weights[mask].sum())
    return new_weights

def _aggregate_person_weights_to_household(person_weights):
    return person_weights.groupby(person_weights.index.get_level_values(0)).mean()

def _expand_weights_to_person(weights, person_index):
    person_weights = weights.copy()
    person_weights = pd.DataFrame(person_weights)
    person_weights['person_id'] = 1
    person_weights.set_index('person_id', append=True, inplace=True)
    person_weights = person_weights.reindex(person_index)
    person_weights.fillna(method='ffill', inplace=True)
    return person_weights.iloc[:, 0] # return series not dataframe

def _rescale_weights(reference_sample, weights):
    household_sizes = _household_groups(reference_sample)[reference_sample.columns[0]].count()
    largest_household_size = household_sizes.max()
    Fp = [weights[household_sizes == p].sum()
          for p in range(0, largest_household_size + 1)]
    polynom = [(190 / 434 * p - 1) * Fp[p] for p in range(0, largest_household_size + 1)]
    roots = Polynomial(polynom).roots()
    dx = list(filter(lambda x: np.real(x) > 0, filterfalse(lambda x: np.iscomplex(x), roots)))
    assert len(dx) == 1
    d = np.real(dx[0])
    c = 190 / sum([Fp[p] * d ** p for p in range(1, largest_household_size + 1)])
    fhprime_by_fh = {p: c * d ** p for p in range(1, largest_household_size + 1)}

    new_weights = weights.copy()
    for household_id in weights.index:
        household_size = household_sizes[household_id]
        new_weights[household_id] = fhprime_by_fh[household_size] * weights[household_id]
    return new_weights
<commit_msg>Performance improvement HIPF<commit_after>from itertools import filterfalse

import pandas as pd
import numpy as np
from numpy.polynomial import Polynomial

def fit_hipf(reference_sample, controls_individuals, controls_households, maxiter):
    """Hierarchical Iterative Proportional Fitting.

    Algorithm taken from
    Müller and Axhausen 2011: "Hierarchical IPF: Generating a synthetic population for Switzerland"

    Can be used to fit a reference sample of households and individuals to control variables
    simultaneously. The algorithm supports only one dimensional control variables, and hence
    multi-dimensional control variables must be transformed to one dimensional ones first.

    Parameters:
        reference_sample:     The reference sample to be fited to the controls. Must be a pandas
                              DataFrame where the index is a multi index of (household_id,
                              person_id), and each colum represents either a household category
                              or a category of an individual.
        controls_individuals: The control variables for individuals. Must be a dict from control
                              name to a dict of its values.
                              e.g. {'age': {'below_50': 45, '50_or_older'}: 55}
        controls_households:  The control variables for households. Must be in the same format as
                              the controls for individuals.
        maxiter:              Maximum number of iterations.
    """

    weights = pd.Series(
        index=_household_groups(reference_sample).count().index.get_level_values(0),
        data=1.0,
        dtype=np.float64
    )
    for i in range(0, maxiter):
        next_weights = _fit(_household_groups(reference_sample).first(), weights,
                            controls_households)
        weights_person = _expand_weights_to_person(next_weights, reference_sample.index)
        weights_person = _fit(reference_sample, weights_person, controls_individuals)
        next_weights = _aggregate_person_weights_to_household(weights_person)
        next_weights = _rescale_weights(reference_sample, next_weights)
        weights = next_weights
    return weights

def _household_groups(reference_sample):
    return reference_sample.groupby(reference_sample.index.get_level_values(0))

def _fit(reference_sample, weights, controls):
    new_weights = weights.copy()
    for control_name, control_values in controls.items():
        summed_weights = {key: weights[reference_sample[control_name] == key].sum()
                          for key, value in control_values.items()}
        df = pd.DataFrame( # temporary data frame to work on
            index=weights.index,
            data={'weight': weights, 'value': reference_sample[control_name]}
        )
        new_weights = df.apply(
            lambda row: row.weight * control_values[row.value] / summed_weights[row.value],
            axis='columns'
        )
    return new_weights

def _aggregate_person_weights_to_household(person_weights):
    return person_weights.groupby(person_weights.index.get_level_values(0)).mean()

def _expand_weights_to_person(weights, person_index):
    person_weights = weights.copy()
    person_weights = pd.DataFrame(person_weights)
    person_weights['person_id'] = 1
    person_weights.set_index('person_id', append=True, inplace=True)
    person_weights = person_weights.reindex(person_index)
    person_weights.fillna(method='ffill', inplace=True)
    return person_weights.iloc[:, 0] # return series not dataframe

def _rescale_weights(reference_sample, weights):
    household_sizes = _household_groups(reference_sample)[reference_sample.columns[0]].count()
    largest_household_size = household_sizes.max()
    Fp = [weights[household_sizes == p].sum()
          for p in range(0, largest_household_size + 1)]
    polynom = [(190 / 434 * p - 1) * Fp[p] for p in range(0, largest_household_size + 1)]
    roots = Polynomial(polynom).roots()
    dx = list(filter(lambda x: np.real(x) > 0, filterfalse(lambda x: np.iscomplex(x), roots)))
    assert len(dx) == 1
    d = np.real(dx[0])
    c = 190 / sum([Fp[p] * d ** p for p in range(1, largest_household_size + 1)])
    fhprime_by_fh = {p: c * d ** p for p in range(1, largest_household_size + 1)}

    new_weights = weights.copy()
    for household_id in weights.index:
        household_size = household_sizes[household_id]
        new_weights[household_id] = fhprime_by_fh[household_size] * weights[household_id]
    return new_weights
<|endoftext|>


<commit_before>from lacli.exceptions import PauseEvent
from lacli.pool import MPConnection, MPUpload, MPFile
from contextlib import contextmanager
from lacli.log import LogHandler, getLogger
from lacli.control import ControlHandler
from lacli.worker import WorkerPool
from twisted.internet import defer, threads
from itertools import count
import json
import errno
import signal

class UploadState(object):
    states = {}

    @classmethod
    def init(cls, cache):
        cls.cache = cache
        uploads = cache._get_uploads()
        a = cache._for_adf('archives')
        sz = lambda f: a[f]['archive'].meta.size
        cls.states = {k: cls(k, sz(k), **v)
            for k, v in uploads.iteritems()}

    @classmethod
    def get(cls, fname, size=None, capsule=None):
        if fname in cls.states:
            if size is not None:
                cls.states[fname].size = size
            if capsule is not None:
                cls.states[fname].capsule = capsule
            return cls.states[fname]
        cls.states[fname] = UploadState(fname, size, capsule=capsule)
        return cls.states[fname]

    @classmethod
    def reset(cls, fname):
        if fname not in cls.states:
            raise ValueError("Upload doesn't exist!")
        cls.cache._del_upload(fname)
        return cls.states.pop(fname)

    def __init__(self, archive, size, uri=None, keys=[], capsule=None):
        self.cache = type(self).cache
        self.archive = archive
        self.logfile = self.control = None
        self.keys = keys
        self._progress = 0
        self.size = size
        self.pausing = False
        self.uri = uri
        self.exc = None
        self.capsule = capsule

    def append(self, key):
        self.keys.append(key)

    @property
    def progress(self):
        f = lambda x, y: x + y['size']
        return self._progress + reduce(f, self.keys, 0)

    def __enter__(self):
        try:
            self.exc = None
            self.control = ControlHandler()
            self.logfile = self.cache._upload_open(self.archive, mode='r+')
            getLogger().debug("Found state file for %s", self.archive)
        except IOError as e:
            if e.errno == errno.ENOENT:
                getLogger().debug("Creating state file for %s", self.archive)
                self.logfile = self.cache._upload_open(self.archive, mode='w+')
            else:
                raise e
        # update keys from file
        upload = self.cache._validate_upload(self.logfile)
        self.uri = upload.get('uri', self.uri)
        self.keys = upload.get('keys', self.keys)
        return self

    def __exit__(self, type, value, traceback):
        self.logfile.close()
        self.logfile = self.control = None
        self._progress = 0

    def keydone(self, key, size):
        assert self.logfile is not None, "Log not open"
        new = { 'name': key, 'size': size}
        self.logfile.write(json.dumps(new)+"\n")
        self.logfile.flush()
        self.append(new)
        self._progress = 0

    def update(self, progress):
        self._progress += progress

    @property
    def seq(self):
        return len(self.keys)

    def pause(self):
        if self.control is not None:
            self.control.pause()

    def signal(self, sig, frame):
        getLogger().debug("Got interrupt")
        if sig == signal.SIGINT:
            getLogger().debug("Pausing")
            if self.pausing is True:
                raise SystemExit("Interrupted")
            self.pausing = True
            self.control.pause()

    def save_op(self, op):
        assert self.uri is None, "Can't change URI for upload state"
        if op.uri is None:
            return
        new = { 'uri': op.uri }
        if self.capsule is not None:
            new['capsule'] = self.capsule
        self.logfile.write(json.dumps(new)+"\n")
        self.logfile.flush()
        self.uri = op.uri

    def error(self, exc):
        self.exc = exc

class Upload(object):
    def __init__(self, session, nprocs, debug, state):
        self.prefs = {
            'nprocs': nprocs,
            'debugworker': debug > 2
        }
        self.log = LogHandler()
        self.state = state

    @contextmanager
    def _workers(self, progq):
        with self.log as logq:
            with self.state.control as ctrlq:
                pool = WorkerPool(
                    self.prefs, logq, progq, ctrlq)
                try:
                    yield pool
                finally:
                    pool.terminate()
                    pool.join()

    def upload_temp(self, token, source, etags, pool, seq):
        key = "{prefix}temp-archive-{seq}".format(
            prefix=token['prefix'], seq=seq)
        connection = MPConnection(token)
        with MPUpload(connection, source, key) as uploader:
            etags[key], source = uploader.get_result(
                uploader.submit_job(pool))
        return source

    @defer.inlineCallbacks
    def upload(self, fname, upload, progq):
        with self._workers(progq) as pool:
            etags = {}
            source = MPFile(fname, self.state.progress)

            for seq in count(start=self.state.seq):
                token = yield upload.status
                try:
                    source = yield threads.deferToThread(
                        self.upload_temp, token, source, etags, pool, seq)
                except PauseEvent:
                    getLogger().debug("paused after uploading %d temporary keys", seq)
                    raise
                if source is None:
                    getLogger().debug("uploaded entire archive")
                    progq.put({'complete': True})
                    break

            getLogger().debug("uploaded %d temp keys", len(etags))
            for key, tag in etags.iteritems():
                getLogger().debug("key: %s (etag: %s)", key, tag)
<commit_msg>handle deleted archives better<commit_after>from lacli.exceptions import PauseEvent
from lacli.pool import MPConnection, MPUpload, MPFile
from contextlib import contextmanager
from lacli.log import LogHandler, getLogger
from lacli.control import ControlHandler
from lacli.worker import WorkerPool
from twisted.internet import defer, threads
from itertools import count
import json
import errno
import signal

class UploadState(object):
    states = {}

    @classmethod
    def init(cls, cache):
        cls.cache = cache
        uploads = cache._get_uploads()
        a = cache._for_adf('archives')
        sz = lambda f: a[f]['archive'].meta.size
        cls.states = {k: cls(k, sz(k), **v)
            for k, v in uploads.iteritems()
            if k in a}

    @classmethod
    def get(cls, fname, size=None, capsule=None):
        if fname in cls.states:
            if size is not None:
                cls.states[fname].size = size
            if capsule is not None:
                cls.states[fname].capsule = capsule
            return cls.states[fname]
        cls.states[fname] = UploadState(fname, size, capsule=capsule)
        return cls.states[fname]

    @classmethod
    def reset(cls, fname):
        if fname not in cls.states:
            raise ValueError("Upload doesn't exist!")
        cls.cache._del_upload(fname)
        return cls.states.pop(fname)

    def __init__(self, archive, size, uri=None, keys=[], capsule=None):
        self.cache = type(self).cache
        self.archive = archive
        self.logfile = self.control = None
        self.keys = keys
        self._progress = 0
        self.size = size
        self.pausing = False
        self.uri = uri
        self.exc = None
        self.capsule = capsule

    def append(self, key):
        self.keys.append(key)

    @property
    def progress(self):
        f = lambda x, y: x + y['size']
        return self._progress + reduce(f, self.keys, 0)

    def __enter__(self):
        try:
            self.exc = None
            self.control = ControlHandler()
            self.logfile = self.cache._upload_open(self.archive, mode='r+')
            getLogger().debug("Found state file for %s", self.archive)
        except IOError as e:
            if e.errno == errno.ENOENT:
                getLogger().debug("Creating state file for %s", self.archive)
                self.logfile = self.cache._upload_open(self.archive, mode='w+')
            else:
                raise e
        # update keys from file
        upload = self.cache._validate_upload(self.logfile)
        self.uri = upload.get('uri', self.uri)
        self.keys = upload.get('keys', self.keys)
        return self

    def __exit__(self, type, value, traceback):
        self.logfile.close()
        self.logfile = self.control = None
        self._progress = 0

    def keydone(self, key, size):
        assert self.logfile is not None, "Log not open"
        new = { 'name': key, 'size': size}
        self.logfile.write(json.dumps(new)+"\n")
        self.logfile.flush()
        self.append(new)
        self._progress = 0

    def update(self, progress):
        self._progress += progress

    @property
    def seq(self):
        return len(self.keys)

    def pause(self):
        if self.control is not None:
            self.control.pause()

    def signal(self, sig, frame):
        getLogger().debug("Got interrupt")
        if sig == signal.SIGINT:
            getLogger().debug("Pausing")
            if self.pausing is True:
                raise SystemExit("Interrupted")
            self.pausing = True
            self.control.pause()

    def save_op(self, op):
        assert self.uri is None, "Can't change URI for upload state"
        if op.uri is None:
            return
        new = { 'uri': op.uri }
        if self.capsule is not None:
            new['capsule'] = self.capsule
        self.logfile.write(json.dumps(new)+"\n")
        self.logfile.flush()
        self.uri = op.uri

    def error(self, exc):
        self.exc = exc

class Upload(object):
    def __init__(self, session, nprocs, debug, state):
        self.prefs = {
            'nprocs': nprocs,
            'debugworker': debug > 2
        }
        self.log = LogHandler()
        self.state = state

    @contextmanager
    def _workers(self, progq):
        with self.log as logq:
            with self.state.control as ctrlq:
                pool = WorkerPool(
                    self.prefs, logq, progq, ctrlq)
                try:
                    yield pool
                finally:
                    pool.terminate()
                    pool.join()

    def upload_temp(self, token, source, etags, pool, seq):
        key = "{prefix}temp-archive-{seq}".format(
            prefix=token['prefix'], seq=seq)
        connection = MPConnection(token)
        with MPUpload(connection, source, key) as uploader:
            etags[key], source = uploader.get_result(
                uploader.submit_job(pool))
        return source

    @defer.inlineCallbacks
    def upload(self, fname, upload, progq):
        with self._workers(progq) as pool:
            etags = {}
            source = MPFile(fname, self.state.progress)

            for seq in count(start=self.state.seq):
                token = yield upload.status
                try:
                    source = yield threads.deferToThread(
                        self.upload_temp, token, source, etags, pool, seq)
                except PauseEvent:
                    getLogger().debug("paused after uploading %d temporary keys", seq)
                    raise
                if source is None:
                    getLogger().debug("uploaded entire archive")
                    progq.put({'complete': True})
                    break

            getLogger().debug("uploaded %d temp keys", len(etags))
            for key, tag in etags.iteritems():
                getLogger().debug("key: %s (etag: %s)", key, tag)
<|endoftext|>


<commit_before>
import IPython
import sys, os
sys.path.append(os.path.join(sys.path[0],'..'))
from src import feed, utils
from src import ladder_network as ldr
from tensorflow.examples.tutorials.mnist import input_data
import csv
import tensorflow as tf

def test_data_balancing():
    # Test mnist data balancing z
    mnist = input_data.read_data_sets(sys.path[0]+'/../data/mnist/', one_hot=True)
    sf = feed.Balanced(mnist.validation.images, mnist.validation.labels, 100, None)
    IPython.embed()

def get_testing_mode_params(gamma=True, num_labeled=100, seed=1):
    params = utils.process_cli_params(utils.get_cli_params())

    # Alter from default to simplify
    params.do_not_save = True
    params.decay_start_epoch = 1
    params.end_epoch = 2
    params.train_flag = True
    params.gamma_flag = gamma
    params.num_labeled = num_labeled
    params.seed = seed

    return params

def test_similarity(file1, file2, verbose=True):
    """Test line-by-line similarity of two tsv files """
    with open(file1, newline='') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t')
        data1 = list(reader)

    with open(file2, newline='') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t')
        data2 = list(reader)

    sim = []
    for i, l in enumerate(data1):
        sim.append(1 if (data2[i]==l) else 0)

        print(i, sim[-1])
        if verbose:
            if sim[-1]==0:
                print(">>>>> MISMATCH START <<<<<")
                print(l)
                print(data2[i])
                print("<<<<< MISMATCH STOP  >>>>>")

    return sim

def test_repr():
    """
    Small test to check reproducibility with seeds.

    Results
    -------
        100% consistency over 2 epochs using same seed.
        31% consistency over 2 epochs using seeds 1 and 2.

    """
    params = get_testing_mode_params()
    # params.seed = 1

    for run in [1, 2]:
        tf.reset_default_graph()
        # run = 2
        params.seed = run
        params.id = "repr_" + str(run)
        params.write_to = "tests/" + params.id
        ldr.main(params)

    sim = test_similarity("tests/" + "repr_" + str(1), "tests/" + "repr_" +
                          str(2))
    print(sum(sim)/len(sim))

def test_fully_supervised_with_labeled_subsets():
    """
    Test to check that the fully supervised case (i.e. no unsupervised examples)
    works for e.g. all of MNIST, 100 labels, etc.

    Easiest way is to set the num_labeled to desired, then all rc_weights to
    zero. This is not very efficient, but allows the code to be used as is.

    Results
    -------
    Overwrote when using labeled epochs. Turns out labeled epochs are shorter
    than print interval for 100, 1000 labels.
    Using unlabeled epochs, found expected patterns of overfitting.

    To Do
    -----
    This suggests an experiment that can be written up showing the
    overfitting tendency on 100 and how ssl effectively regularises using
    information unsupervised. Could do a comparison with alternative
    regularisation techniques or see if anyone else has tried this. As in,
    someone must have tried it -- but did they do it in a form where I could
    copy it into my writeup?

    """

    params = get_testing_mode_params()

    params.seed = 1
    params.rc_weights = [0 for x in params.rc_weights]
    params.use_labeled_epochs = True

    for num_labeled in [100, 1000, 55000]:
        tf.reset_default_graph()
        params.num_labeled = num_labeled
        params.id = "fully_sup_" + str(num_labeled)
        params.write_to = "tests/" + params.id
        ldr.main(params)

def test_fully_supervised_with_all_labeled():
    params = get_testing_mode_params(num_labeled=55000)
    params.seed = 1
    params.rc_weights = [0 for x in params.rc_weights]
    params.use_labeled_epochs = True
    tf.reset_default_graph()
    params.id = "fully_sup"
    params.write_to = "tests/" + params.id
    ldr.main(params)

def test_if_zero_rc_is_dummy():
    """Weights for each reconstruction cost are specified using a dictionary
    with keys from 0 to L supposed to match the layer keys, but quite
    possibly the very first entry (0th layer) RC isn't actually used."""

    params = get_testing_mode_params(gamma=False)

    # Use only the RC, not supervised
    params.sc_weight = 0

    # Gamma setting
    rc_weights = [0, 0, 0, 0, 0, 0, 1]
    params.rc_weights = dict(zip(range(len(rc_weights)), rc_weights))
    tf.reset_default_graph()

    params.id = "rc_zero_base"
    params.write_to = "tests/" + params.id
    ldr.main(params)

    # Gamma setting
    rc_weights = [1, 0, 0, 0, 0, 0, 1]
    params.rc_weights = dict(zip(range(len(rc_weights)), rc_weights))
    tf.reset_default_graph()

    params.id = "rc_zero_one"
    params.write_to = "tests/" + params.id
    ldr.main(params)

    sim = test_similarity("tests/rc_zero_base", "tests/rc_zero_one")
    print(sum(sim) / len(sim))

def test_gamma_equivalence():
    """Compare gamma decoder and ladder decoder with weight only at top

    Results
    -------
    Differed only by time stamp

    """

    tf.reset_default_graph()
    params = get_testing_mode_params(gamma=True, seed=1)
    params.id = 'gamma_equiv_gamma'
    params.write_to = 'tests/' + params.id
    ldr.main(params)
    test_me = (params.write_to, )

    tf.reset_default_graph()
    params.gamma = False
    params.rc_weights = [0, 0, 0, 0, 0, 0, 1]
    params.id = 'gamma_equiv_ladder'
    params.write_to = 'tests/' + params.id
    ldr.main(params)
    test_me += (params.write_to, )

    sim = test_similarity(*test_me)
    print(sum(sim) / len(sim))

def check_layer_sizes():
    """Various checks for the sizes of things in the model"""
    params = get_testing_mode_params(gamma=False)
    ladder = ldr.Ladder(params)

    # print(ladder.noisy.layer_sizes)

    mnist = input_data.read_data_sets(sys.path[0] + '/../data/mnist/',
                                      one_hot=True)
    x, y = mnist.validation.next_batch(params.labeled_batch_size)
    test_dict = {ladder.x: x, ladder.y: y}

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())

    print("No. layers in encoder: {}".format(ladder.noisy.n_layers))

    print("===== Noisy encoder activations ===== ")
    for k,v in ladder.noisy.z.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Clean encoder activations ===== ")
    for k,v in ladder.clean.z.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Combinator Outputs ===== ")
    for k, u in ladder.decoder.combinators.items():
        v = u.outputs
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Reconstructions ===== ")
    for k, v in ladder.decoder.reconstructions.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Decoder activations ===== ")
    for k,v in ladder.decoder.activations.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)
        # print(v_eval)
        # print("=====")

    print("===== Decoder costs ===== ")
    for k,v in ladder.decoder.rc_cost.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Reconstruction cost weights ===== ")
    for k, v in params.rc_weights.items():
        print(k, v)

    sess.close()

def test_only_unsupervised():
    """
    Test unsupervised reconstruction costs and weight effects on overall loss.

    Result
    ------
    (RESOLVED: see test_similarity) Except for difference in timing, results
    are identical, meaning that there is no zero layer.

    Is there supposed to be a reconstruction cost associated with the final
    output label? Well, the final output label is an argmax. So as long as
    there is a cost associated with the ten probabilities layer, all good.

    """
    params = get_testing_mode_params()

    params.seed = 1
    params.sc_weight = 0

    # Test sequences for weights
    # Key:      0 1 2 3 4 5 6
    # Default:  0-0-0-0-0-0-1
    rc_weights = [0, 0, 0, 0, 0, 0, 1]
    rc_weights = [1, 0, 0, 0, 0, 0, 1]

    params.rc_weights = dict(zip(range(len(rc_weights)), rc_weights))

    for num_labeled in [100, 1000, 55000]:
        tf.reset_default_graph()
        params.num_labeled = num_labeled
        params.id = "fully_sup_" + str(num_labeled)
        params.write_to = "tests/" + params.id
        ldr.main(params)

if __name__ == '__main__':
    # test_gamma_equivalence()
    # check_layer_sizes()
    # test_if_zero_rc_is_dummy()
    # sim = test_similarity("tests/gamma_equiv_gamma", "tests/gamma_equiv_ladder")
    # print(sum(sim) / len(sim))
    test_fully_supervised_with_all_labeled()<commit_msg>Fix parameters for fully supervised test<commit_after>
import IPython
import sys, os
sys.path.append(os.path.join(sys.path[0],'..'))
from src import feed, utils
from src import ladder_network as ldr
from tensorflow.examples.tutorials.mnist import input_data
import csv
import tensorflow as tf

def test_data_balancing():
    # Test mnist data balancing z
    mnist = input_data.read_data_sets(sys.path[0]+'/../data/mnist/', one_hot=True)
    sf = feed.Balanced(mnist.validation.images, mnist.validation.labels, 100, None)
    IPython.embed()

def get_testing_mode_params(gamma=True, num_labeled=100, seed=1):
    params = utils.process_cli_params(utils.get_cli_params())

    # Alter from default to simplify
    params.do_not_save = True
    params.decay_start_epoch = 1
    params.end_epoch = 2
    params.train_flag = True
    params.gamma_flag = gamma
    params.num_labeled = num_labeled
    params.seed = seed

    return params

def test_similarity(file1, file2, verbose=True):
    """Test line-by-line similarity of two tsv files """
    with open(file1, newline='') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t')
        data1 = list(reader)

    with open(file2, newline='') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t')
        data2 = list(reader)

    sim = []
    for i, l in enumerate(data1):
        sim.append(1 if (data2[i]==l) else 0)

        print(i, sim[-1])
        if verbose:
            if sim[-1]==0:
                print(">>>>> MISMATCH START <<<<<")
                print(l)
                print(data2[i])
                print("<<<<< MISMATCH STOP  >>>>>")

    return sim

def test_repr():
    """
    Small test to check reproducibility with seeds.

    Results
    -------
        100% consistency over 2 epochs using same seed.
        31% consistency over 2 epochs using seeds 1 and 2.

    """
    params = get_testing_mode_params()
    # params.seed = 1

    for run in [1, 2]:
        tf.reset_default_graph()
        # run = 2
        params.seed = run
        params.id = "repr_" + str(run)
        params.write_to = "tests/" + params.id
        ldr.main(params)

    sim = test_similarity("tests/" + "repr_" + str(1), "tests/" + "repr_" +
                          str(2))
    print(sum(sim)/len(sim))

def test_fully_supervised_with_labeled_subsets():
    """
    Test to check that the fully supervised case (i.e. no unsupervised examples)
    works for e.g. all of MNIST, 100 labels, etc.

    Easiest way is to set the num_labeled to desired, then all rc_weights to
    zero. This is not very efficient, but allows the code to be used as is.

    Results
    -------
    Overwrote when using labeled epochs. Turns out labeled epochs are shorter
    than print interval for 100, 1000 labels.
    Using unlabeled epochs, found expected patterns of overfitting.

    To Do
    -----
    This suggests an experiment that can be written up showing the
    overfitting tendency on 100 and how ssl effectively regularises using
    information unsupervised. Could do a comparison with alternative
    regularisation techniques or see if anyone else has tried this. As in,
    someone must have tried it -- but did they do it in a form where I could
    copy it into my writeup?

    """

    params = get_testing_mode_params()

    params.seed = 1
    params.rc_weights = [0 for x in params.rc_weights]
    params.use_labeled_epochs = True

    for num_labeled in [100, 1000, 55000]:
        tf.reset_default_graph()
        params.num_labeled = num_labeled
        params.id = "fully_sup_" + str(num_labeled)
        params.write_to = "tests/" + params.id
        ldr.main(params)

def test_fully_supervised_with_all_labeled():
    params = utils.process_cli_params(utils.get_cli_params())

    # Alter from default to simplify
    params.do_not_save = True
    # params.decay_start_epoch = 1
    # params.end_epoch = 2
    params.train_flag = True
    params.gamma_flag = False
    params.num_labeled = 55000
    params.seed = 1
    params.rc_weights = [0 for x in params.rc_weights]
    params.use_labeled_epochs = True

    tf.reset_default_graph()
    params.id = "fully_sup"
    params.write_to = "tests/" + params.id
    ldr.main(params)

def test_if_zero_rc_is_dummy():
    """Weights for each reconstruction cost are specified using a dictionary
    with keys from 0 to L supposed to match the layer keys, but quite
    possibly the very first entry (0th layer) RC isn't actually used."""

    params = get_testing_mode_params(gamma=False)

    # Use only the RC, not supervised
    params.sc_weight = 0

    # Gamma setting
    rc_weights = [0, 0, 0, 0, 0, 0, 1]
    params.rc_weights = dict(zip(range(len(rc_weights)), rc_weights))
    tf.reset_default_graph()

    params.id = "rc_zero_base"
    params.write_to = "tests/" + params.id
    ldr.main(params)

    # Gamma setting
    rc_weights = [1, 0, 0, 0, 0, 0, 1]
    params.rc_weights = dict(zip(range(len(rc_weights)), rc_weights))
    tf.reset_default_graph()

    params.id = "rc_zero_one"
    params.write_to = "tests/" + params.id
    ldr.main(params)

    sim = test_similarity("tests/rc_zero_base", "tests/rc_zero_one")
    print(sum(sim) / len(sim))

def test_gamma_equivalence():
    """Compare gamma decoder and ladder decoder with weight only at top

    Results
    -------
    Differed only by time stamp

    """

    tf.reset_default_graph()
    params = get_testing_mode_params(gamma=True, seed=1)
    params.id = 'gamma_equiv_gamma'
    params.write_to = 'tests/' + params.id
    ldr.main(params)
    test_me = (params.write_to, )

    tf.reset_default_graph()
    params.gamma = False
    params.rc_weights = [0, 0, 0, 0, 0, 0, 1]
    params.id = 'gamma_equiv_ladder'
    params.write_to = 'tests/' + params.id
    ldr.main(params)
    test_me += (params.write_to, )

    sim = test_similarity(*test_me)
    print(sum(sim) / len(sim))

def check_layer_sizes():
    """Various checks for the sizes of things in the model"""
    params = get_testing_mode_params(gamma=False)
    ladder = ldr.Ladder(params)

    # print(ladder.noisy.layer_sizes)

    mnist = input_data.read_data_sets(sys.path[0] + '/../data/mnist/',
                                      one_hot=True)
    x, y = mnist.validation.next_batch(params.labeled_batch_size)
    test_dict = {ladder.x: x, ladder.y: y}

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())

    print("No. layers in encoder: {}".format(ladder.noisy.n_layers))

    print("===== Noisy encoder activations ===== ")
    for k,v in ladder.noisy.z.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Clean encoder activations ===== ")
    for k,v in ladder.clean.z.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Combinator Outputs ===== ")
    for k, u in ladder.decoder.combinators.items():
        v = u.outputs
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Reconstructions ===== ")
    for k, v in ladder.decoder.reconstructions.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Decoder activations ===== ")
    for k,v in ladder.decoder.activations.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)
        # print(v_eval)
        # print("=====")

    print("===== Decoder costs ===== ")
    for k,v in ladder.decoder.rc_cost.items():
        v_eval = sess.run(v, feed_dict=test_dict)
        print(k, v, v_eval.shape)

    print("===== Reconstruction cost weights ===== ")
    for k, v in params.rc_weights.items():
        print(k, v)

    sess.close()

def test_only_unsupervised():
    """
    Test unsupervised reconstruction costs and weight effects on overall loss.

    Result
    ------
    (RESOLVED: see test_similarity) Except for difference in timing, results
    are identical, meaning that there is no zero layer.

    Is there supposed to be a reconstruction cost associated with the final
    output label? Well, the final output label is an argmax. So as long as
    there is a cost associated with the ten probabilities layer, all good.

    """
    params = get_testing_mode_params()

    params.seed = 1
    params.sc_weight = 0

    # Test sequences for weights
    # Key:      0 1 2 3 4 5 6
    # Default:  0-0-0-0-0-0-1
    rc_weights = [0, 0, 0, 0, 0, 0, 1]
    rc_weights = [1, 0, 0, 0, 0, 0, 1]

    params.rc_weights = dict(zip(range(len(rc_weights)), rc_weights))

    for num_labeled in [100, 1000, 55000]:
        tf.reset_default_graph()
        params.num_labeled = num_labeled
        params.id = "fully_sup_" + str(num_labeled)
        params.write_to = "tests/" + params.id
        ldr.main(params)

if __name__ == '__main__':
    # test_gamma_equivalence()
    # check_layer_sizes()
    # test_if_zero_rc_is_dummy()
    # sim = test_similarity("tests/gamma_equiv_gamma", "tests/gamma_equiv_ladder")
    # print(sum(sim) / len(sim))
    test_fully_supervised_with_all_labeled()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Test links (random.bin):
# http://www13.zippyshare.com/v/18665333/file.html

import re

from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo

class ZippyshareCom(SimpleHoster):
    __name__ = "ZippyshareCom"
    __type__ = "hoster"
    __pattern__ = r'(?P<HOST>http://www\d{0,2}\.zippyshare.com)/v(?:/|iew.jsp.*key=)(?P<KEY>\d+)'
    __version__ = "0.48"
    __description__ = """Zippyshare.com hoster plugin"""
    __author_name__ = ("spoob", "zoidberg", "stickell", "skylab")
    __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz", "l.stickell@yahoo.it", "development@sky-lab.de")

    FILE_NAME_PATTERN = r'>Name:\s*]*>(?P<N>[^<]+)
'
    FILE_SIZE_PATTERN = r'>Size:\s*]*>(?P<S>[0-9.,]+) (?P[kKMG]+)i?B
'
    FILE_INFO_PATTERN = r'document\.getElementById\(\'dlbutton\'\)\.href = "[^;]*/(?P<N>[^"]+)";'
    FILE_OFFLINE_PATTERN = r'>File does not exist on this server
'

    SH_COOKIES = [('zippyshare.com', 'ziplocale', 'en')]

    def setup(self):
        self.multiDL = True

    def handleFree(self):
        url = self.get_file_url()
        if not url:
            self.fail("Download URL not found.")
        self.logDebug("Download URL: %s" % url)
        self.download(url)

    def get_file_url(self):
        """returns the absolute downloadable filepath"""
        url_parts = re.search(r'(addthis:url="(http://www(\d+).zippyshare.com/v/(\d*)/file.html))', self.html)
        number = url_parts.group(4)
        check = re.search(r'


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re, subprocess, tempfile, os
from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp
from module.plugins.ReCaptcha import ReCaptcha
from module.common.json_layer import json_loads

class ZippyshareCom(SimpleHoster):
    __name__ = "ZippyshareCom"
    __type__ = "hoster"
    __pattern__ = r"(?P<HOST>http://www\d{0,2}\.zippyshare.com)/v(?:/|iew.jsp.*key=)(?P<KEY>\d+)"
    __version__ = "0.37"
    __description__ = """Zippyshare.com Download Hoster"""
    __author_name__ = ("spoob", "zoidberg")
    __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz")
    __config__ = [("swfdump_path", "string", "Path to swfdump", "")]

    FILE_NAME_PATTERN = r'>Name:\s*]*>(?P<N>[^<]+)
'
    FILE_SIZE_PATTERN = r'>Size:\s*]*>(?P<S>[0-9.,]+) (?P[kKMG]+)i?B
'
    FILE_INFO_PATTERN = r'document\.getElementById\(\'dlbutton\'\)\.href = "[^;]*/(?P<N>[^"]+)";'
    FILE_OFFLINE_PATTERN = r'>File does not exist on this server
'

    DOWNLOAD_URL_PATTERN = r"