<commit_before>from bs4 import BeautifulSoup
from requests import get, put, exceptions
from subprocess import call
import hashlib
import base64
import sys
import os
import time
import datetime

def _pull(url, json=False):
    """
    :param url:
    :param json:
    :return:
    """
    tries = 0
    while tries < 3:
        if json:
            try:
                response = get(url, headers={'Accept': 'application/json'}, timeout=1.501)
                return response
            except exceptions.Timeout:
                print('Timed out when trying to pull...')
        else:
            try:
                response = get(url, timeout=1.501)
                if response.status_code == 200:
                    response.objectified = BeautifulSoup(response.content, 'xml')
                    return response
            except exceptions.Timeout:
                print('Timed out when trying to pull...')
        tries += 1
        print('Timed out when trying to pull.')
        print('Sleeping for 10 then trying again...')
        time.sleep(10)
    return False

def pull_package(title, version, url, json=False):
    """
    :param title:
    :param version:
    :param url:
    :param json:
    :return:
    """
    return _pull(''.join([url, '(Id=\'', title, '\',Version=\'', version, '\')']), json=json)

def pull_packages(url, json=False):
    """
    :param url:
    :param json:
    :return:
    """
    return _pull(url, json=json)

def pull_updates(url):
    """
    :param url:
    :return:
    """
    return _pull(url, json=False)

def push_package_dotnet(package_path, repo_url, api_key, dotnet):
    """
    :param package_path:
    :param repo_url:
    :param api_key:
    :param dotnet:
    :return:
    """
    cmd = ' '.join([dotnet, 'nuget', 'push', package_path, '-s', repo_url, '-k', api_key])
    return call(cmd, shell=True)

def push_package_native(package_path, repo_url, api_key):
    """
    :param package_path:
    :param repo_url:
    :param api_key:
    :return:
    """
    f = open(package_path, mode='rb')
    files = {'package': ('package', f, 'application/octet-stream')}
    headers = {'X-NuGet-ApiKey': api_key}
    return put(repo_url, files=files, headers=headers)

def download_file(url, save_to):
    """
    :param url:
    :param save_to:
    :return:
    """
    count = 0
    sys.stdout.write('Downloading.')
    sys.stdout.flush()
    # Does the file already exist?
    if os.path.isfile(save_to):
        # If the file exists remove it since we forced
        os.remove(save_to)
    # Get the file and stream it to the disk
    r = get(url, stream=True)
    with open(save_to, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                # Count the chunks
                count += 1
                if count >= 1024:
                    # Write a dot every 1024 chunks
                    sys.stdout.write('.')
                    sys.stdout.flush()
                    count = 0
                # Write to the file when the chunk gets to 1024
                f.write(chunk)
                f.flush()
    print(' Done!')
    return True if os.path.isfile(save_to) else False

def sha512sum(file_path, block_size=65536):
    """
    :param file_path:
    :param block_size:
    :return:
    """
    return base64.encodestring(
        __hash_byte_str_iter(__file_as_block_iter(open(file_path, 'rb'), block_size), hashlib.sha512(), False)
    ).replace("\n", '')

def sha256sum(file_path, block_size=65536):
    """
    :param file_path:
    :param block_size:
    :return:
    """
    return base64.encodestring(
        __hash_byte_str_iter(__file_as_block_iter(open(file_path, 'rb'), block_size), hashlib.sha256(), False)
    ).replace("\n", '')

def sha1sum(file_path, block_size=65536):
    """
    :param file_path:
    :param block_size:
    :return:
    """
    return base64.encodestring(
        __hash_byte_str_iter(__file_as_block_iter(open(file_path, 'rb'), block_size), hashlib.sha1, False)
    ).replace("\n", '')

def hashes_match(hash_1, hash_2):
    """
    :param hash_1:
    :param hash_2:
    :return:
    """
    if str(hash_1) == str(hash_2):
        return True
    else:
        return False

def verify_hash(file_path, target_hash, message='Verifying package hash.', hash_method='sha512'):
    """
    :param file_path:
    :param target_hash:
    :param message:
    :param hash_method:
    :return:
    """
    sys.stdout.write(message)
    sys.stdout.flush()
    hash_method.lower()

    if hash_method == 'sha512':
        sys.stdout.write('.')
        local_hash = sha512sum(file_path)
        sys.stdout.write('.')
    elif hash_method == 'sha256':
        local_hash = sha256sum(file_path)
        sys.stdout.write('.')
    elif hash_method == 'sha1':
        sys.stdout.write('.')
        local_hash = sha1sum(file_path)
    else:
        sys.stdout.write('.')
        local_hash = sha512sum(file_path)

    print('. Done!')
    return hashes_match(local_hash, target_hash)

def utc_to_epoch(time_stamp, time_format='%Y-%m-%dT%H:%M:%SZ'):
    """
    :param time_stamp:
    :param time_format:
    :return:
    """
    return int(time.mktime(time.strptime(time_stamp, time_format)))

def epoch_to_utc(epoch=0, time_format='%Y-%m-%dT%H:%M:%SZ'):
    """
    :param epoch:
    :param time_format:
    :return:
    """
    return time.strftime(time_format, time.gmtime(epoch))

def now_as_epoch():
    """
    :return:
    """
    return int(time.time())

def first_epoch():
    """
    :return:
    """
    return int(0)

def now_as_utc(time_format='%Y-%m-%dT%H:%M:%SZ'):
    """
    :param time_format:
    :return:
    """
    return datetime.datetime.now().strftime(time_format)

def touch(file_path):
    """
    :param file_path:
    :return:
    """
    with open(file_path, 'a'):
        os.utime(file_path, None)

def store_delta(delta, file_path='storage/sync.delta'):
    """
    :param delta:
    :param file_path:
    :return:
    """
    touch(file_path)
    with open(file_path, 'w') as f:
        f.write(''.join([delta, "\n"]))

def read_delta(file_path='storage/sync.delta'):
    """
    :param file_path:
    :return:
    """
    touch(file_path)
    try:
        with open(file_path, 'rb') as f:
            f.seek(-21, 2)
            return str(f.readlines()[-1].decode().rstrip("\n"))
    except IOError:
        return None

def __hash_byte_str_iter(bytes_iter, hasher, as_hex_str=False):
    """
    :param bytes_iter:
    :param hasher:
    :param as_hex_str:
    :return:
    """
    for block in bytes_iter:
        hasher.update(block)
    return hasher.hexdigest() if as_hex_str else hasher.digest()

def __file_as_block_iter(f, block_size=65536):
    """
    :param f:
    :param block_size:
    :return:
    """
    with f:
        block = f.read(block_size)
        while len(block) > 0:
            yield block
            block = f.read(block_size)
<commit_msg>increased timeout and sleep when pulling<commit_after>from bs4 import BeautifulSoup
from requests import get, put, exceptions
from subprocess import call
import hashlib
import base64
import sys
import os
import time
import datetime

def _pull(url, json=False):
    """
    :param url:
    :param json:
    :return:
    """
    tries = 0
    while tries < 3:
        if json:
            try:
                response = get(url, headers={'Accept': 'application/json'}, timeout=5.501)
                return response
            except exceptions.Timeout:
                print('Timed out when trying to pull...')
        else:
            try:
                response = get(url, timeout=5.501)
                if response.status_code == 200:
                    response.objectified = BeautifulSoup(response.content, 'xml')
                    return response
            except exceptions.Timeout:
                print('Timed out when trying to pull...')
        tries += 1
        print('Timed out when trying to pull.')
        print('Sleeping for 10 then trying again...')
        time.sleep(25)
    return False

def pull_package(title, version, url, json=False):
    """
    :param title:
    :param version:
    :param url:
    :param json:
    :return:
    """
    return _pull(''.join([url, '(Id=\'', title, '\',Version=\'', version, '\')']), json=json)

def pull_packages(url, json=False):
    """
    :param url:
    :param json:
    :return:
    """
    return _pull(url, json=json)

def pull_updates(url):
    """
    :param url:
    :return:
    """
    return _pull(url, json=False)

def push_package_dotnet(package_path, repo_url, api_key, dotnet):
    """
    :param package_path:
    :param repo_url:
    :param api_key:
    :param dotnet:
    :return:
    """
    cmd = ' '.join([dotnet, 'nuget', 'push', package_path, '-s', repo_url, '-k', api_key])
    return call(cmd, shell=True)

def push_package_native(package_path, repo_url, api_key):
    """
    :param package_path:
    :param repo_url:
    :param api_key:
    :return:
    """
    f = open(package_path, mode='rb')
    files = {'package': ('package', f, 'application/octet-stream')}
    headers = {'X-NuGet-ApiKey': api_key}
    return put(repo_url, files=files, headers=headers)

def download_file(url, save_to):
    """
    :param url:
    :param save_to:
    :return:
    """
    count = 0
    sys.stdout.write('Downloading.')
    sys.stdout.flush()
    # Does the file already exist?
    if os.path.isfile(save_to):
        # If the file exists remove it since we forced
        os.remove(save_to)
    # Get the file and stream it to the disk
    r = get(url, stream=True)
    with open(save_to, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                # Count the chunks
                count += 1
                if count >= 1024:
                    # Write a dot every 1024 chunks
                    sys.stdout.write('.')
                    sys.stdout.flush()
                    count = 0
                # Write to the file when the chunk gets to 1024
                f.write(chunk)
                f.flush()
    print(' Done!')
    return True if os.path.isfile(save_to) else False

def sha512sum(file_path, block_size=65536):
    """
    :param file_path:
    :param block_size:
    :return:
    """
    return base64.encodestring(
        __hash_byte_str_iter(__file_as_block_iter(open(file_path, 'rb'), block_size), hashlib.sha512(), False)
    ).replace("\n", '')

def sha256sum(file_path, block_size=65536):
    """
    :param file_path:
    :param block_size:
    :return:
    """
    return base64.encodestring(
        __hash_byte_str_iter(__file_as_block_iter(open(file_path, 'rb'), block_size), hashlib.sha256(), False)
    ).replace("\n", '')

def sha1sum(file_path, block_size=65536):
    """
    :param file_path:
    :param block_size:
    :return:
    """
    return base64.encodestring(
        __hash_byte_str_iter(__file_as_block_iter(open(file_path, 'rb'), block_size), hashlib.sha1, False)
    ).replace("\n", '')

def hashes_match(hash_1, hash_2):
    """
    :param hash_1:
    :param hash_2:
    :return:
    """
    if str(hash_1) == str(hash_2):
        return True
    else:
        return False

def verify_hash(file_path, target_hash, message='Verifying package hash.', hash_method='sha512'):
    """
    :param file_path:
    :param target_hash:
    :param message:
    :param hash_method:
    :return:
    """
    sys.stdout.write(message)
    sys.stdout.flush()
    hash_method.lower()

    if hash_method == 'sha512':
        sys.stdout.write('.')
        local_hash = sha512sum(file_path)
        sys.stdout.write('.')
    elif hash_method == 'sha256':
        local_hash = sha256sum(file_path)
        sys.stdout.write('.')
    elif hash_method == 'sha1':
        sys.stdout.write('.')
        local_hash = sha1sum(file_path)
    else:
        sys.stdout.write('.')
        local_hash = sha512sum(file_path)

    print('. Done!')
    return hashes_match(local_hash, target_hash)

def utc_to_epoch(time_stamp, time_format='%Y-%m-%dT%H:%M:%SZ'):
    """
    :param time_stamp:
    :param time_format:
    :return:
    """
    return int(time.mktime(time.strptime(time_stamp, time_format)))

def epoch_to_utc(epoch=0, time_format='%Y-%m-%dT%H:%M:%SZ'):
    """
    :param epoch:
    :param time_format:
    :return:
    """
    return time.strftime(time_format, time.gmtime(epoch))

def now_as_epoch():
    """
    :return:
    """
    return int(time.time())

def first_epoch():
    """
    :return:
    """
    return int(0)

def now_as_utc(time_format='%Y-%m-%dT%H:%M:%SZ'):
    """
    :param time_format:
    :return:
    """
    return datetime.datetime.now().strftime(time_format)

def touch(file_path):
    """
    :param file_path:
    :return:
    """
    with open(file_path, 'a'):
        os.utime(file_path, None)

def store_delta(delta, file_path='storage/sync.delta'):
    """
    :param delta:
    :param file_path:
    :return:
    """
    touch(file_path)
    with open(file_path, 'w') as f:
        f.write(''.join([delta, "\n"]))

def read_delta(file_path='storage/sync.delta'):
    """
    :param file_path:
    :return:
    """
    touch(file_path)
    try:
        with open(file_path, 'rb') as f:
            f.seek(-21, 2)
            return str(f.readlines()[-1].decode().rstrip("\n"))
    except IOError:
        return None

def __hash_byte_str_iter(bytes_iter, hasher, as_hex_str=False):
    """
    :param bytes_iter:
    :param hasher:
    :param as_hex_str:
    :return:
    """
    for block in bytes_iter:
        hasher.update(block)
    return hasher.hexdigest() if as_hex_str else hasher.digest()

def __file_as_block_iter(f, block_size=65536):
    """
    :param f:
    :param block_size:
    :return:
    """
    with f:
        block = f.read(block_size)
        while len(block) > 0:
            yield block
            block = f.read(block_size)
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2013 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.compute import base
from tempest.common import tempest_fixtures as fixtures
from tempest.test import attr

class HostsAdminTestJSON(base.BaseV2ComputeAdminTest):

    """
    Tests hosts API using admin privileges.
    """

    _interface = 'json'

    @classmethod
    def setUpClass(cls):
        super(HostsAdminTestJSON, cls).setUpClass()
        cls.client = cls.os_adm.hosts_client

    def _get_host_name(self):
        resp, hosts = self.client.list_hosts()
        self.assertEqual(200, resp.status)
        self.assertTrue(len(hosts) >= 1)
        hostname = hosts[0]['host_name']
        return hostname

    @attr(type='gate')
    def test_list_hosts(self):
        resp, hosts = self.client.list_hosts()
        self.assertEqual(200, resp.status)
        self.assertTrue(len(hosts) >= 2)

    @attr(type='gate')
    def test_list_hosts_with_zone(self):
        self.useFixture(fixtures.LockFixture('availability_zone'))
        resp, hosts = self.client.list_hosts()
        host = hosts[0]
        zone_name = host['zone']
        params = {'zone': zone_name}
        resp, hosts = self.client.list_hosts(params)
        self.assertEqual(200, resp.status)
        self.assertTrue(len(hosts) >= 1)
        self.assertIn(host, hosts)

    @attr(type='gate')
    def test_list_hosts_with_a_blank_zone(self):
        # If send the request with a blank zone, the request will be successful
        # and it will return all the hosts list
        params = {'zone': ''}
        resp, hosts = self.client.list_hosts(params)
        self.assertNotEqual(0, len(hosts))
        self.assertEqual(200, resp.status)

    @attr(type='gate')
    def test_list_hosts_with_nonexistent_zone(self):
        # If send the request with a nonexistent zone, the request will be
        # successful and no hosts will be retured
        params = {'zone': 'xxx'}
        resp, hosts = self.client.list_hosts(params)
        self.assertEqual(0, len(hosts))
        self.assertEqual(200, resp.status)

    @attr(type='gate')
    def test_show_host_detail(self):
        hostname = self._get_host_name()

        resp, resources = self.client.show_host_detail(hostname)
        self.assertEqual(200, resp.status)
        self.assertTrue(len(resources) >= 1)
        host_resource = resources[0]['resource']
        self.assertIsNotNone(host_resource)
        self.assertIsNotNone(host_resource['cpu'])
        self.assertIsNotNone(host_resource['disk_gb'])
        self.assertIsNotNone(host_resource['memory_mb'])
        self.assertIsNotNone(host_resource['project'])
        self.assertEqual(hostname, host_resource['host'])

class HostsAdminTestXML(HostsAdminTestJSON):
    _interface = 'xml'
<commit_msg>Make test_show_host_detail work with multi host installation<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2013 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.compute import base
from tempest.common import tempest_fixtures as fixtures
from tempest.test import attr

class HostsAdminTestJSON(base.BaseV2ComputeAdminTest):

    """
    Tests hosts API using admin privileges.
    """

    _interface = 'json'

    @classmethod
    def setUpClass(cls):
        super(HostsAdminTestJSON, cls).setUpClass()
        cls.client = cls.os_adm.hosts_client

    @attr(type='gate')
    def test_list_hosts(self):
        resp, hosts = self.client.list_hosts()
        self.assertEqual(200, resp.status)
        self.assertTrue(len(hosts) >= 2)

    @attr(type='gate')
    def test_list_hosts_with_zone(self):
        self.useFixture(fixtures.LockFixture('availability_zone'))
        resp, hosts = self.client.list_hosts()
        host = hosts[0]
        zone_name = host['zone']
        params = {'zone': zone_name}
        resp, hosts = self.client.list_hosts(params)
        self.assertEqual(200, resp.status)
        self.assertTrue(len(hosts) >= 1)
        self.assertIn(host, hosts)

    @attr(type='gate')
    def test_list_hosts_with_a_blank_zone(self):
        # If send the request with a blank zone, the request will be successful
        # and it will return all the hosts list
        params = {'zone': ''}
        resp, hosts = self.client.list_hosts(params)
        self.assertNotEqual(0, len(hosts))
        self.assertEqual(200, resp.status)

    @attr(type='gate')
    def test_list_hosts_with_nonexistent_zone(self):
        # If send the request with a nonexistent zone, the request will be
        # successful and no hosts will be retured
        params = {'zone': 'xxx'}
        resp, hosts = self.client.list_hosts(params)
        self.assertEqual(0, len(hosts))
        self.assertEqual(200, resp.status)

    @attr(type='gate')
    def test_show_host_detail(self):
        resp, hosts = self.client.list_hosts()
        self.assertEqual(200, resp.status)

        hosts = [host for host in hosts if host['service'] == 'compute']
        self.assertTrue(len(hosts) >= 1)

        for host in hosts:
            hostname = host['host_name']
            resp, resources = self.client.show_host_detail(hostname)
            self.assertEqual(200, resp.status)
            self.assertTrue(len(resources) >= 1)
            host_resource = resources[0]['resource']
            self.assertIsNotNone(host_resource)
            self.assertIsNotNone(host_resource['cpu'])
            self.assertIsNotNone(host_resource['disk_gb'])
            self.assertIsNotNone(host_resource['memory_mb'])
            self.assertIsNotNone(host_resource['project'])
            self.assertEqual(hostname, host_resource['host'])

class HostsAdminTestXML(HostsAdminTestJSON):
    _interface = 'xml'
<|endoftext|>


<commit_before>import numpy as np
import os
import os.path as op
import sys

rl = sys.argv[1]
PRD = os.environ['PRD']
FS = os.environ['FS']
SUBJ_ID = os.environ['SUBJ_ID']

def read_annot(fname):

    """Read a Freesurfer annotation from a .annot file.
    Note : Copied from PySurfer
    Parameters
    ----------
    fname : str
        Path to annotation file
    Returns
    -------
    annot : numpy array, shape=(n_verts)
        Annotation id at each vertex
    ctab : numpy array, shape=(n_entries, 5)
        RGBA + label id colortable array
    names : list of str
        List of region names as stored in the annot file
    """
    if not op.isfile(fname):
        dir_name = op.split(fname)[0]
        if not op.isdir(dir_name):
            raise IOError('Directory for annotation does not exist: %s',
                          fname)
        cands = os.listdir(dir_name)
        cands = [c for c in cands if '.annot' in c]
        if len(cands) == 0:
            raise IOError('No such file %s, no candidate parcellations '
                          'found in directory' % fname)
        else:
            raise IOError('No such file %s, candidate parcellations in '
                          'that directory: %s' % (fname, ', '.join(cands)))
    with open(fname, "rb") as fid:
        n_verts = np.fromfile(fid, '>i4', 1)[0]
        data = np.fromfile(fid, '>i4', n_verts * 2).reshape(n_verts, 2)
        annot = data[data[:, 0], 1]
        ctab_exists = np.fromfile(fid, '>i4', 1)[0]
        if not ctab_exists:
            raise Exception('Color table not found in annotation file')
        n_entries = np.fromfile(fid, '>i4', 1)[0]
        if n_entries > 0:
            length = np.fromfile(fid, '>i4', 1)[0]
            orig_tab = np.fromfile(fid, '>c', length)
            orig_tab = orig_tab[:-1]
            names = list()
            ctab = np.zeros((n_entries, 5), np.int)
            for i in range(n_entries):
                name_length = np.fromfile(fid, '>i4', 1)[0]
                name = np.fromfile(fid, "|S%d" % name_length, 1)[0]
                names.append(name)
                ctab[i, :4] = np.fromfile(fid, '>i4', 4)
                ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
                              ctab[i, 2] * (2 ** 16) +
                              ctab[i, 3] * (2 ** 24))
        else:
            ctab_version = -n_entries
            if ctab_version != 2:
                raise Exception('Color table version not supported')
            n_entries = np.fromfile(fid, '>i4', 1)[0]
            ctab = np.zeros((n_entries, 5), np.int)
            length = np.fromfile(fid, '>i4', 1)[0]
            np.fromfile(fid, "|S%d" % length, 1)  # Orig table path
            entries_to_read = np.fromfile(fid, '>i4', 1)[0]
            names = list()
            for i in range(entries_to_read):
                np.fromfile(fid, '>i4', 1)  # Structure
                name_length = np.fromfile(fid, '>i4', 1)[0]
                name = np.fromfile(fid, "|S%d" % name_length, 1)[0]
                names.append(name)
                ctab[i, :4] = np.fromfile(fid, '>i4', 4)
                ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
                              ctab[i, 2] * (2 ** 16))
        # convert to more common alpha value
        ctab[:, 3] = 255 - ctab[:, 3]
    return annot, ctab, names

L, _, _ = read_annot(os.path.join(FS, SUBJ_ID, 'label', rl + '.aparc.annot'))
ref_table = np.loadtxt(rl + '_ref_table.txt')
os.chdir(os.path.join(PRD, 'surface'))
vl = np.loadtxt(rl + '_vertices_low.txt')  # vertices low
vh = np.loadtxt(rl + '_vertices_high.txt')  # vertices high
reg_map = []
for vli in vl:
    pos = np.argmin(np.sum(np.abs(vh - vli), 1))
    find_tab = np.nonzero(ref_table[:, 5] == L[pos])[0][0]
    reg_map.append(ref_table[find_tab, 4])
np.savetxt(rl + '_region_mapping_low_not_corrected.txt', reg_map)
<commit_msg>release 3.0<commit_after>import numpy as np
import os
import os.path as op
import sys

rl = sys.argv[1]
PRD = os.environ['PRD']
FS = os.environ['FS']
SUBJ_ID = os.environ['SUBJ_ID']

def read_annot(fname):

    """Read a Freesurfer annotation from a .annot file.
    Note : Copied from nibabel
    Parameters
    ----------
    fname : str
        Path to annotation file
    Returns
    -------
    annot : numpy array, shape=(n_verts)
        Annotation id at each vertex
    ctab : numpy array, shape=(n_entries, 5)
        RGBA + label id colortable array
    names : list of str
        List of region names as stored in the annot file
    """
    if not op.isfile(fname):
        dir_name = op.split(fname)[0]
        if not op.isdir(dir_name):
            raise IOError('Directory for annotation does not exist: %s',
                          fname)
        cands = os.listdir(dir_name)
        cands = [c for c in cands if '.annot' in c]
        if len(cands) == 0:
            raise IOError('No such file %s, no candidate parcellations '
                          'found in directory' % fname)
        else:
            raise IOError('No such file %s, candidate parcellations in '
                          'that directory: %s' % (fname, ', '.join(cands)))
    with open(fname, "rb") as fid:
        n_verts = np.fromfile(fid, '>i4', 1)[0]
        data = np.fromfile(fid, '>i4', n_verts * 2).reshape(n_verts, 2)
        annot = data[data[:, 0], 1]
        ctab_exists = np.fromfile(fid, '>i4', 1)[0]
        if not ctab_exists:
            raise Exception('Color table not found in annotation file')
        n_entries = np.fromfile(fid, '>i4', 1)[0]
        if n_entries > 0:
            length = np.fromfile(fid, '>i4', 1)[0]
            orig_tab = np.fromfile(fid, '>c', length)
            orig_tab = orig_tab[:-1]
            names = list()
            ctab = np.zeros((n_entries, 5), np.int)
            for i in range(n_entries):
                name_length = np.fromfile(fid, '>i4', 1)[0]
                name = np.fromfile(fid, "|S%d" % name_length, 1)[0]
                names.append(name)
                ctab[i, :4] = np.fromfile(fid, '>i4', 4)
                ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
                              ctab[i, 2] * (2 ** 16) +
                              ctab[i, 3] * (2 ** 24))
        else:
            ctab_version = -n_entries
            if ctab_version != 2:
                raise Exception('Color table version not supported')
            n_entries = np.fromfile(fid, '>i4', 1)[0]
            ctab = np.zeros((n_entries, 5), np.int)
            length = np.fromfile(fid, '>i4', 1)[0]
            np.fromfile(fid, "|S%d" % length, 1)  # Orig table path
            entries_to_read = np.fromfile(fid, '>i4', 1)[0]
            names = list()
            for i in range(entries_to_read):
                np.fromfile(fid, '>i4', 1)  # Structure
                name_length = np.fromfile(fid, '>i4', 1)[0]
                name = np.fromfile(fid, "|S%d" % name_length, 1)[0]
                names.append(name)
                ctab[i, :4] = np.fromfile(fid, '>i4', 4)
                ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
                              ctab[i, 2] * (2 ** 16))
        # convert to more common alpha value
        ctab[:, 3] = 255 - ctab[:, 3]
    return annot, ctab, names

L, _, _ = read_annot(os.path.join(FS, SUBJ_ID, 'label', rl + '.aparc.annot'))
ref_table = np.loadtxt(rl + '_ref_table.txt')
os.chdir(os.path.join(PRD, 'surface'))
vl = np.loadtxt(rl + '_vertices_low.txt')  # vertices low
vh = np.loadtxt(rl + '_vertices_high.txt')  # vertices high
reg_map = []
for vli in vl:
    pos = np.argmin(np.sum(np.abs(vh - vli), 1))
    find_tab = np.nonzero(ref_table[:, 5] == L[pos])[0][0]
    reg_map.append(ref_table[find_tab, 4])
np.savetxt(rl + '_region_mapping_low_not_corrected.txt', reg_map)
<|endoftext|>


<commit_before>"""
Solver method for GP-based optimization which uses an inner-loop optimizer to
maximize some acquisition function, generally given as a simple function of the
posterior sufficient statistics.
"""

# future imports
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

# global imports
import numpy as np
import pygp
import inspect
import functools

# update a recarray at the end of solve_bayesopt.
from numpy.lib.recfunctions import append_fields

# local imports
from ..utils.random import rstate

# each method/class defined exported by these modules will be exposed as a
# string to the solve_bayesopt method so that we can swap in/out different
# components for the "meta" solver.
from . import inits
from . import solvers
from . import policies
from . import recommenders

# exported symbols
__all__ = ['solve_bayesopt']

### SOLVER COMPONENTS #########################################################

def get_components(init, policy, solver, recommender, rng):
    """
    Return model components for Bayesian optimization of the correct form given
    string identifiers.
    """
    def get_func(key, value, module, lstrip):
        if isinstance(value, (list, tuple)):
            try:
                value, kwargs = value
                if not isinstance(kwargs, dict):
                    raise ValueError
            except ValueError:
                raise ValueError('invalid arguments for component %r' % key)
        else:
            kwargs = {}

        if hasattr(value, '__call__'):
            func = value
        else:
            for fname in module.__all__:
                func = getattr(module, fname)
                if fname.startswith(lstrip):
                    fname = fname[len(lstrip):]
                fname = fname.lower()
                if fname == value:
                    break
            else:
                raise ValueError('invalid identifier for component %r' % key)

        kwarg_set = set(kwargs.keys())
        valid_set = getattr(func, '_params', set())

        if not kwarg_set.issubset(valid_set):
            raise ValueError('unknown parameters for component %r: %r' %
                             (key, list(kwarg_set - valid_set)))

        if 'rng' in inspect.getargspec(func).args:
            kwargs['rng'] = rng

        if len(kwargs) > 0:
            func = functools.partial(func, **kwargs)

        return func

    return (get_func('init', init, inits, lstrip='init_'),
            get_func('policy', policy, policies, lstrip=''),
            get_func('solver', solver, solvers, lstrip='solve_'),
            get_func('recommender', recommender, recommenders, lstrip='best_'))

### THE BAYESOPT META SOLVER ##################################################

def solve_bayesopt(f,
                   bounds,
                   T=100,
                   init='middle',
                   policy='ei',
                   solver='lbfgs',
                   recommender='latent',
                   model=None,
                   noisefree=False,
                   ftrue=None,
                   rng=None,
                   callback=None):
    """
    Maximize the given function using Bayesian Optimization.
    """
    # make sure the bounds are a 2d-array.
    bounds = np.array(bounds, dtype=float, ndmin=2)

    # see if the query object itself defines ground truth.
    if (ftrue is None) and hasattr(f, 'get_f'):
        ftrue = f.get_f

    # initialize the random number generator.
    rng = rstate(rng)

    # get the model components.
    init, policy, solver, recommender = \
        get_components(init, policy, solver, recommender, rng)

    # create a list of initial points to query.
    X = init(bounds)
    Y = [f(x) for x in X]

    if model is None:
        # initialize parameters of a simple GP model.
        sf = np.std(Y) if (len(Y) > 1) else 10.
        mu = np.mean(Y)
        ell = bounds[:, 1] - bounds[:, 0]

        # FIXME: this may not be a great setting for the noise parameter; if
        # we're noisy it may not be so bad... but for "noisefree" models this
        # sets it fixed to 1e-6, which may be too big.
        sn = 1e-6 if noisefree else 1e-3

        # specify a hyperprior for the GP.
        prior = {
            'sn': (
                None if noisefree else
                pygp.priors.Horseshoe(scale=0.1, min=1e-6)),
            'sf': pygp.priors.LogNormal(mu=np.log(sf), sigma=1., min=1e-6),
            'ell': pygp.priors.Uniform(ell / 100, ell * 2),
            'mu': pygp.priors.Gaussian(mu, sf)}

        # create the GP model (with hyperprior).
        model = pygp.BasicGP(sn, sf, ell, mu, kernel='matern5')
        model = pygp.meta.MCMC(model, prior, n=10, burn=100, rng=rng)

    # add any initial data to our model.
    model.add_data(X, Y)

    # allocate a datastructure containing "convergence" info.
    info = np.zeros(T, [('x', np.float, (len(bounds),)),
                        ('y', np.float),
                        ('xbest', np.float, (len(bounds),))])

    # initialize the data.
    info['x'][:len(X)] = X
    info['y'][:len(Y)] = Y
    info['xbest'][:len(Y)] = [X[np.argmax(Y[:i+1])] for i in xrange(len(Y))]

    for i in xrange(model.ndata, T):
        # get the next point to evaluate.
        index = policy(model)
        x, _ = solver(index, bounds)

        # deal with any visualization.
        if callback is not None:
            callback(model, bounds, info[:i], x, index, ftrue)

        # make an observation and record it.
        y = f(x)
        model.add_data(x, y)

        # record everything.
        info[i] = (x, y, recommender(model, bounds))

    if ftrue is not None:
        fbest = ftrue(info['xbest'])
        info = append_fields(info, 'fbest', fbest, usemask=False)

    return info
<commit_msg>Simplify a few things; update docstring.<commit_after>"""
Solver method for GP-based optimization which uses an inner-loop optimizer to
maximize some acquisition function, generally given as a simple function of the
posterior sufficient statistics.
"""

# future imports
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

# global imports
import numpy as np
import pygp
import inspect
import functools

# update a recarray at the end of solve_bayesopt.
from numpy.lib.recfunctions import append_fields

# local imports
from ..utils.random import rstate

# each method/class defined exported by these modules will be exposed as a
# string to the solve_bayesopt method so that we can swap in/out different
# components for the "meta" solver.
from . import inits
from . import solvers
from . import policies
from . import recommenders

# exported symbols
__all__ = ['solve_bayesopt']

### SOLVER COMPONENTS #########################################################

def get_components(init, policy, solver, recommender, rng):
    """
    Return model components for Bayesian optimization of the correct form given
    string identifiers.
    """
    def get_func(key, value, module, lstrip):
        """
        Construct the model component if the given value is either a function
        or a string identifying a function in the given module (after stripping
        extraneous text). The value can also be passed as a 2-tuple where the
        second element includes kwargs. Partially apply any kwargs and the rng
        before returning the function.
        """
        if isinstance(value, (list, tuple)):
            try:
                value, kwargs = value
                kwargs = dict(kwargs)
            except (ValueError, TypeError):
                raise ValueError('invalid arguments for component %r' % key)
        else:
            kwargs = {}

        if hasattr(value, '__call__'):
            func = value
        else:
            for fname in module.__all__:
                func = getattr(module, fname)
                if fname.startswith(lstrip):
                    fname = fname[len(lstrip):]
                fname = fname.lower()
                if fname == value:
                    break
            else:
                raise ValueError('invalid identifier for component %r' % key)

        kwarg_set = set(kwargs.keys())
        valid_set = getattr(func, '_params', set())

        if not kwarg_set.issubset(valid_set):
            raise ValueError('unknown parameters for component %r: %r' %
                             (key, list(kwarg_set - valid_set)))

        if 'rng' in inspect.getargspec(func).args:
            kwargs['rng'] = rng

        if len(kwargs) > 0:
            func = functools.partial(func, **kwargs)

        return func

    return (get_func('init', init, inits, lstrip='init_'),
            get_func('policy', policy, policies, lstrip=''),
            get_func('solver', solver, solvers, lstrip='solve_'),
            get_func('recommender', recommender, recommenders, lstrip='best_'))

### THE BAYESOPT META SOLVER ##################################################

def solve_bayesopt(f,
                   bounds,
                   T=100,
                   init='middle',
                   policy='ei',
                   solver='lbfgs',
                   recommender='latent',
                   model=None,
                   noisefree=False,
                   ftrue=None,
                   rng=None,
                   callback=None):
    """
    Maximize the given function using Bayesian Optimization.

    Args:
        f: function handle representing the objective function.
        bounds: bounds of the search space as a (d,2)-array.
        T: horizon for optimization.
        init: the initialization component.
        policy: the acquisition component.
        solver: the inner-loop solver component.
        recommender: the recommendation component.
        model: the Bayesian model instantiation.
        noisefree: a boolean denoting that the model is noisefree; this only
                   applies if a default model is used (ie. it is ignored if the
                   model argument is used).
        ftrue: a ground-truth function (for evaluation).
        rng: either an RandomState object or an integer used to seed the state;
             this will be fed to each component that requests randomness.
        callback: a function to call on each iteration for visualization.

    Note that the modular way in which this function has been written allows
    one to also pass parameters directly to some of the components. This works
    for the `init`, `policy`, `solver`, and `recommender` inputs. These
    components can be passed as either a string, a function, or a 2-tuple where
    the first item is a string/function and the second is a dictionary of
    additional arguments to pass to the component.

    Returns:
        A numpy record array containing a trace of the optimization process.
        The fields of this array are `x`, `y`, and `xbest` corresponding to the
        query locations, outputs, and recommendations at each iteration. If
        ground-truth is known an additional field `fbest` will be included.
    """
    # make sure the bounds are a 2d-array.
    bounds = np.array(bounds, dtype=float, ndmin=2)

    # see if the query object itself defines ground truth.
    if (ftrue is None) and hasattr(f, 'get_f'):
        ftrue = f.get_f

    # initialize the random number generator.
    rng = rstate(rng)

    # get the model components.
    init, policy, solver, recommender = \
        get_components(init, policy, solver, recommender, rng)

    # create a list of initial points to query.
    X = init(bounds)
    Y = [f(x) for x in X]

    if model is None:
        # initialize parameters of a simple GP model.
        sf = np.std(Y) if (len(Y) > 1) else 10.
        mu = np.mean(Y)
        ell = bounds[:, 1] - bounds[:, 0]

        # FIXME: this may not be a great setting for the noise parameter; if
        # we're noisy it may not be so bad... but for "noisefree" models this
        # sets it fixed to 1e-6, which may be too big.
        sn = 1e-6 if noisefree else 1e-3

        # specify a hyperprior for the GP.
        prior = {
            'sn': (
                None if noisefree else
                pygp.priors.Horseshoe(scale=0.1, min=1e-6)),
            'sf': pygp.priors.LogNormal(mu=np.log(sf), sigma=1., min=1e-6),
            'ell': pygp.priors.Uniform(ell / 100, ell * 2),
            'mu': pygp.priors.Gaussian(mu, sf)}

        # create the GP model (with hyperprior).
        model = pygp.BasicGP(sn, sf, ell, mu, kernel='matern5')
        model = pygp.meta.MCMC(model, prior, n=10, burn=100, rng=rng)

    # add any initial data to our model.
    model.add_data(X, Y)

    # allocate a datastructure containing "convergence" info.
    info = np.zeros(T, [('x', np.float, (len(bounds),)),
                        ('y', np.float),
                        ('xbest', np.float, (len(bounds),))])

    # initialize the data.
    info['x'][:len(X)] = X
    info['y'][:len(Y)] = Y
    info['xbest'][:len(Y)] = [X[np.argmax(Y[:i+1])] for i in xrange(len(Y))]

    for i in xrange(model.ndata, T):
        # get the next point to evaluate.
        index = policy(model)
        x, _ = solver(index, bounds)

        # deal with any visualization.
        if callback is not None:
            callback(model, bounds, info[:i], x, index, ftrue)

        # make an observation and record it.
        y = f(x)
        model.add_data(x, y)

        # record everything.
        info[i] = (x, y, recommender(model, bounds))

    if ftrue is not None:
        fbest = ftrue(info['xbest'])
        info = append_fields(info, 'fbest', fbest, usemask=False)

    return info
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import logging as logging
logger = logging.getLogger("vaex.kld")
import numpy as np
import math
import functools
import vaex.vaexfast
import vaex.utils
from vaex.utils import Timer

def mutual_information(data):
	Q = vaex.utils.disjoined(data)
	P = data

	P = P / P.sum()
	Q = Q / Q.sum()
	mask = (P > 0) & (Q>0)
	information = np.sum(P[mask] * np.log(P[mask]/Q[mask]))# * np.sum(dx)
	return information

class KlDivergenceShuffle(object):
	def __init__(self, dataset, pairs, gridsize=128):
		self.dataset = dataset
		self.pairs = pairs
		self.dimension = len(self.pairs[0])
		self.logger = logger.getLogger('kld')
		self.gridsize = gridsize
		logger.debug("dimension: %d, pairs: %s" % (self.dimension, self.pairs))

	def get_jobs(self):
		def job(pair):
			pass

def to_disjoined(counts):
	shape = counts.shape
	assert len(counts.shape) == 2
	counts_0 = counts.sum(axis=1).reshape((shape[0], 1))
	counts_1 = counts.sum(axis=0).reshape((1, shape[1]))
	counts_disjoined = counts_0 * counts_1
	return counts_disjoined

def kld_shuffled(columns, Ngrid=128, datamins=None, datamaxes=None, offset=1):
	if datamins is None:
		datamins = np.array([np.nanmin(column) for column in columns])
	if datamaxes is None:
		datamaxes = np.array([np.nanmax(column) for column in columns])
	dim = len(columns)
	counts = np.zeros((Ngrid, ) * dim, dtype=np.float64)
	counts_shuffled = np.zeros((Ngrid, ) * dim, dtype=np.float64)
	D_kl = -1
	if len(columns) == 2:
		x, y = columns
		#print x
		#print y
		print((x, y, counts, counts_shuffled, datamins[0], datamaxes[0], datamins[1], datamaxes[1], offset))
		try:
			vaex.histogram.hist2d_and_shuffled(x, y, counts, counts_shuffled, datamins[0], datamaxes[0], datamins[1], datamaxes[1], offset)
		except:
			args = [x, y, counts, counts_shuffled, datamins[0], datamaxes[0], datamins[1], datamaxes[1], offset]
			sig = [numba.dispatcher.typeof_pyval(a) for a in args]
			print(sig)
			raise

		print(("counts", sum(counts)))
		deltax = [float(datamaxes[i] - datamins[i]) for i in range(dim)]
		dx = np.array([deltax[d]/counts.shape[d] for d in range(dim)])
		density = counts/np.sum(counts)# * np.sum(dx)
		density_shuffled = counts_shuffled / np.sum(counts_shuffled)# * np.sum(dx)
		mask = (density_shuffled > 0) & (density>0)
		#print density
		D_kl = np.sum(density[mask] * np.log(density[mask]/density_shuffled[mask]))# * np.sum(dx)
		#if D_kl < 0:
		#	import pdb
		#	pdb.set_trace()
	return D_kl

def kld_shuffled_grouped(dataset, range_map, pairs, feedback=None, size_grid=32, use_mask=True, bytes_max=int(1024**3/2)):
	dimension = len(pairs[0])
	bytes_per_grid = size_grid ** dimension * 8 # 8==sizeof (double)
	grids_per_iteration = min(128, int(bytes_max/bytes_per_grid))
	iterations = int(math.ceil(len(pairs) * 1. / grids_per_iteration))
	jobsManager = vaex.dataset.JobsManager()
	ranges = [None] * len(pairs)
	D_kls = []

	class Wrapper(object):
		pass
	wrapper = Wrapper()
	wrapper.N_done = 0

	counts = np.zeros((grids_per_iteration,) + (size_grid,) * dimension, dtype=np.float64)
	#counts_shuffled = np.zeros((grids_per_iteration,) + (size_grid,) * dimension, dtype=np.float64)

	N_total = len(pairs) * len(dataset) * dimension + len(pairs) *counts.size

	logger.debug("{iterations} iterations with {grids_per_iteration} grids per iteration".format(**locals()))
	for part in range(iterations):
		if part > 0: # next iterations reset the counts
			counts.reshape(-1)[:] = 0
			#counts_shuffled.reshape(-1)[:] = 0
		i1, i2 = part * grids_per_iteration, (part+1)*grids_per_iteration
		if i2 > len(pairs):
			i2 = len(pairs)
		n_grids = i2 - i1
		logger.debug("part {part} of {iterations}, from {i1} to {i2}".format(**locals()))
		def grid(info, *blocks, **kwargs):
			index = kwargs["index"]
			if use_mask and dataset.mask is not None:
				mask = dataset.mask[info.i1:info.i2]
				blocks = [block[mask] for block in blocks]
			else:
				mask = None
			ranges = []
			minima = []
			maxima = []
			for dim in range(dimension):
				ranges += list(range_map[pairs[index][dim]])
				mi, ma = range_map[pairs[index][dim]]
				minima.append(mi)
				maxima.append(ma)
			if len(blocks) == 2:
				print(("mask", mask))
				vaex.vaexfast.histogram2d(blocks[0], blocks[1], None, counts[index], *(ranges + [0, 0]))
				#vaex.vaexfast.histogram2d(blocks[0], blocks[1], None, counts_shuffled[index], *(ranges + [1, 0]))
			if len(blocks) == 3:
				vaex.vaexfast.histogram3d(blocks[0], blocks[1], blocks[2], None, counts[index], *(ranges + [0,0,0]))
				#vaex.vaexfast.histogramNd(list(blocks), None, counts[index], minima, maxima)
				#for i in range(5):
				#	vaex.vaexfast.histogram3d(blocks[0], blocks[1], blocks[2], None, counts_shuffled[index], *(ranges + [2+i,1+i,0]))
			if len(blocks) > 3:
				vaex.vaexfast.histogramNd(list(blocks), None, counts[index], minima, maxima)
				#for i in range(5):
				#	vaex.vaexfast.histogram3d(blocks[0], blocks[1], blocks[2], None, counts_shuffled[index], *(ranges + [2+i,1+i,0]))
			if feedback:
				wrapper.N_done += len(blocks[0]) * dimension
				if feedback:
					cancel = feedback(wrapper.N_done*100./N_total)
					if cancel:
						raise Exception("cancelled")

		for index, pair in zip(list(range(i1, i2)), pairs[i1:i2]):
			logger.debug("add job %r %r" % (index, pair))
			jobsManager.addJob(0, functools.partial(grid, index=index-i1), dataset, *pair)
		jobsManager.execute()

		with Timer("D_kl"):
			for i in range(n_grids):
				if 0:
					deltax = [float(range_map[pairs[i][d]][1] - range_map[pairs[i][d]][0]) for d in range(dimension)]
					dx = np.array([deltax[d]/counts[i].shape[d] for d in range(dimension)])
					density = counts[i]/np.sum(counts[i])# * np.sum(dx)
					counts_shuffled = to_disjoined(counts[i])
					density_shuffled = counts_shuffled / np.sum(counts_shuffled)# * np.sum(dx)
					mask = (density_shuffled > 0) & (density>0)
					print(("mask sum", np.sum(mask)))
					print(("mask sum", np.sum((counts_shuffled > 0) & (counts[i]>0))))
					#print density
					D_kl = np.sum(density[mask] * np.log(density[mask]/density_shuffled[mask]))# * np.sum(dx)
				else:
					D_kl = mutual_information(counts[i])
				D_kls.append(D_kl)
				if feedback:
					wrapper.N_done += counts.size
					cancel = feedback(wrapper.N_done*100./N_total)
					if cancel:
						return None
						#raise Exception, "cancelled"
	return D_kls

if __name__ == "__main__":
	import vaex.dataset
	import vaex.files
	from optparse import OptionParser
	parser = OptionParser() #usage="")

	#parser.add_option("-n", "--name",
	#                 help="dataset name [default=%default]", default="data", type=str)
	#parser.add_option("-o", "--output",
	#                 help="dataset output filename [by default the suffix of input filename will be replaced by hdf5]", default=None, type=str)
	(options, args) = parser.parse_args()
	if len(args) > 0:
		filename = args[0]
	else:
		filename = "gaussian4d-1e7.hdf5"

	path = vaex.files.get_datafile(filename)
	dataset = vaex.dataset.Hdf5MemoryMapped(path)
	#for column_name in dataset.column_names:
	#	print column_name

	subspace = dataset.subspace(1)[:3] * dataset.subspace(1)[:3]
	#subspace = Subspace(dataset, [("x",), ("y",), ("z",)]<commit_msg>kld function added<commit_after># -*- coding: utf-8 -*-

import logging as logging
logger = logging.getLogger("vaex.kld")
import numpy as np
import math
import functools
import vaex.vaexfast
import vaex.utils
from vaex.utils import Timer

def mutual_information(data):
	Q = vaex.utils.disjoined(data)
	P = data

	P = P / P.sum()
	Q = Q / Q.sum()
	mask = (P > 0) & (Q>0)
	information = np.sum(P[mask] * np.log(P[mask]/Q[mask]))# * np.sum(dx)
	return information

def kl_divergence(P, Q, axis=None):
    P = P / P.sum(axis=axis)
    Q = Q / Q.sum(axis=axis)
    #mask = (P > 0) & (Q>0)
    #information = np.sum(P[mask] * np.log(P[mask]/Q[mask]), axis=axis)# * np.sum(dx)
    #mask = (P > 0) & (Q>0)
    information = np.sum(P * np.log(P/Q), axis=axis)# * np.sum(dx)
    return information

class KlDivergenceShuffle(object):
	def __init__(self, dataset, pairs, gridsize=128):
		self.dataset = dataset
		self.pairs = pairs
		self.dimension = len(self.pairs[0])
		self.logger = logger.getLogger('kld')
		self.gridsize = gridsize
		logger.debug("dimension: %d, pairs: %s" % (self.dimension, self.pairs))

	def get_jobs(self):
		def job(pair):
			pass

def to_disjoined(counts):
	shape = counts.shape
	assert len(counts.shape) == 2
	counts_0 = counts.sum(axis=1).reshape((shape[0], 1))
	counts_1 = counts.sum(axis=0).reshape((1, shape[1]))
	counts_disjoined = counts_0 * counts_1
	return counts_disjoined

def kld_shuffled(columns, Ngrid=128, datamins=None, datamaxes=None, offset=1):
	if datamins is None:
		datamins = np.array([np.nanmin(column) for column in columns])
	if datamaxes is None:
		datamaxes = np.array([np.nanmax(column) for column in columns])
	dim = len(columns)
	counts = np.zeros((Ngrid, ) * dim, dtype=np.float64)
	counts_shuffled = np.zeros((Ngrid, ) * dim, dtype=np.float64)
	D_kl = -1
	if len(columns) == 2:
		x, y = columns
		#print x
		#print y
		print((x, y, counts, counts_shuffled, datamins[0], datamaxes[0], datamins[1], datamaxes[1], offset))
		try:
			vaex.histogram.hist2d_and_shuffled(x, y, counts, counts_shuffled, datamins[0], datamaxes[0], datamins[1], datamaxes[1], offset)
		except:
			args = [x, y, counts, counts_shuffled, datamins[0], datamaxes[0], datamins[1], datamaxes[1], offset]
			sig = [numba.dispatcher.typeof_pyval(a) for a in args]
			print(sig)
			raise

		print(("counts", sum(counts)))
		deltax = [float(datamaxes[i] - datamins[i]) for i in range(dim)]
		dx = np.array([deltax[d]/counts.shape[d] for d in range(dim)])
		density = counts/np.sum(counts)# * np.sum(dx)
		density_shuffled = counts_shuffled / np.sum(counts_shuffled)# * np.sum(dx)
		mask = (density_shuffled > 0) & (density>0)
		#print density
		D_kl = np.sum(density[mask] * np.log(density[mask]/density_shuffled[mask]))# * np.sum(dx)
		#if D_kl < 0:
		#	import pdb
		#	pdb.set_trace()
	return D_kl

def kld_shuffled_grouped(dataset, range_map, pairs, feedback=None, size_grid=32, use_mask=True, bytes_max=int(1024**3/2)):
	dimension = len(pairs[0])
	bytes_per_grid = size_grid ** dimension * 8 # 8==sizeof (double)
	grids_per_iteration = min(128, int(bytes_max/bytes_per_grid))
	iterations = int(math.ceil(len(pairs) * 1. / grids_per_iteration))
	jobsManager = vaex.dataset.JobsManager()
	ranges = [None] * len(pairs)
	D_kls = []

	class Wrapper(object):
		pass
	wrapper = Wrapper()
	wrapper.N_done = 0

	counts = np.zeros((grids_per_iteration,) + (size_grid,) * dimension, dtype=np.float64)
	#counts_shuffled = np.zeros((grids_per_iteration,) + (size_grid,) * dimension, dtype=np.float64)

	N_total = len(pairs) * len(dataset) * dimension + len(pairs) *counts.size

	logger.debug("{iterations} iterations with {grids_per_iteration} grids per iteration".format(**locals()))
	for part in range(iterations):
		if part > 0: # next iterations reset the counts
			counts.reshape(-1)[:] = 0
			#counts_shuffled.reshape(-1)[:] = 0
		i1, i2 = part * grids_per_iteration, (part+1)*grids_per_iteration
		if i2 > len(pairs):
			i2 = len(pairs)
		n_grids = i2 - i1
		logger.debug("part {part} of {iterations}, from {i1} to {i2}".format(**locals()))
		def grid(info, *blocks, **kwargs):
			index = kwargs["index"]
			if use_mask and dataset.mask is not None:
				mask = dataset.mask[info.i1:info.i2]
				blocks = [block[mask] for block in blocks]
			else:
				mask = None
			ranges = []
			minima = []
			maxima = []
			for dim in range(dimension):
				ranges += list(range_map[pairs[index][dim]])
				mi, ma = range_map[pairs[index][dim]]
				minima.append(mi)
				maxima.append(ma)
			if len(blocks) == 2:
				print(("mask", mask))
				vaex.vaexfast.histogram2d(blocks[0], blocks[1], None, counts[index], *(ranges + [0, 0]))
				#vaex.vaexfast.histogram2d(blocks[0], blocks[1], None, counts_shuffled[index], *(ranges + [1, 0]))
			if len(blocks) == 3:
				vaex.vaexfast.histogram3d(blocks[0], blocks[1], blocks[2], None, counts[index], *(ranges + [0,0,0]))
				#vaex.vaexfast.histogramNd(list(blocks), None, counts[index], minima, maxima)
				#for i in range(5):
				#	vaex.vaexfast.histogram3d(blocks[0], blocks[1], blocks[2], None, counts_shuffled[index], *(ranges + [2+i,1+i,0]))
			if len(blocks) > 3:
				vaex.vaexfast.histogramNd(list(blocks), None, counts[index], minima, maxima)
				#for i in range(5):
				#	vaex.vaexfast.histogram3d(blocks[0], blocks[1], blocks[2], None, counts_shuffled[index], *(ranges + [2+i,1+i,0]))
			if feedback:
				wrapper.N_done += len(blocks[0]) * dimension
				if feedback:
					cancel = feedback(wrapper.N_done*100./N_total)
					if cancel:
						raise Exception("cancelled")

		for index, pair in zip(list(range(i1, i2)), pairs[i1:i2]):
			logger.debug("add job %r %r" % (index, pair))
			jobsManager.addJob(0, functools.partial(grid, index=index-i1), dataset, *pair)
		jobsManager.execute()

		with Timer("D_kl"):
			for i in range(n_grids):
				if 0:
					deltax = [float(range_map[pairs[i][d]][1] - range_map[pairs[i][d]][0]) for d in range(dimension)]
					dx = np.array([deltax[d]/counts[i].shape[d] for d in range(dimension)])
					density = counts[i]/np.sum(counts[i])# * np.sum(dx)
					counts_shuffled = to_disjoined(counts[i])
					density_shuffled = counts_shuffled / np.sum(counts_shuffled)# * np.sum(dx)
					mask = (density_shuffled > 0) & (density>0)
					print(("mask sum", np.sum(mask)))
					print(("mask sum", np.sum((counts_shuffled > 0) & (counts[i]>0))))
					#print density
					D_kl = np.sum(density[mask] * np.log(density[mask]/density_shuffled[mask]))# * np.sum(dx)
				else:
					D_kl = mutual_information(counts[i])
				D_kls.append(D_kl)
				if feedback:
					wrapper.N_done += counts.size
					cancel = feedback(wrapper.N_done*100./N_total)
					if cancel:
						return None
						#raise Exception, "cancelled"
	return D_kls

if __name__ == "__main__":
	import vaex.dataset
	import vaex.files
	from optparse import OptionParser
	parser = OptionParser() #usage="")

	#parser.add_option("-n", "--name",
	#                 help="dataset name [default=%default]", default="data", type=str)
	#parser.add_option("-o", "--output",
	#                 help="dataset output filename [by default the suffix of input filename will be replaced by hdf5]", default=None, type=str)
	(options, args) = parser.parse_args()
	if len(args) > 0:
		filename = args[0]
	else:
		filename = "gaussian4d-1e7.hdf5"

	path = vaex.files.get_datafile(filename)
	dataset = vaex.dataset.Hdf5MemoryMapped(path)
	#for column_name in dataset.column_names:
	#	print column_name

	subspace = dataset.subspace(1)[:3] * dataset.subspace(1)[:3]
	#subspace = Subspace(dataset, [("x",), ("y",), ("z",)]<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import json
import uuid

from sqlalchemy import Text
from sqlalchemy.orm import relationship, backref, class_mapper
from sqlalchemy.types import MutableType, TypeDecorator
from sqlalchemy import event
from sqlalchemy.engine import reflection
from sqlalchemy.schema import (
    MetaData,
    Table,
    DropTable,
    ForeignKeyConstraint,
    DropConstraint,
    )

import logging
from time import time

try:
    import cPickle as pickle
except ImportError:  # pragma: no cover
    import pickle

log = logging.getLogger(__name__)

class DomainObject(object):

    def dictize(self):
        out = {}
        for col in self.__table__.c:
            out[col.name] = getattr(self, col.name)
        return out

    @classmethod
    def undictize(cls, dict_):
        raise NotImplementedError()

    def __str__(self):  # pragma: no cover
        return self.__unicode__().encode('utf8')

    def __unicode__(self): # pragma: no cover
        repr = u'<%s' % self.__class__.__name__
        table = class_mapper(self.__class__).mapped_table
        for col in table.c:
            try:
                repr += u' %s=%s' % (col.name, getattr(self, col.name))
            except Exception, inst:
                repr += u' %s=%s' % (col.name, inst)

        repr += '>'
        return repr

class JSONType(MutableType, TypeDecorator):
    '''Additional Database Type for handling JSON values.
    '''
    impl = Text

    def __init__(self):
        super(JSONType, self).__init__()

    def process_bind_param(self, value, dialect):
        return json.dumps(value)

    def process_result_value(self, value, dialiect):
        return json.loads(value)

    def copy_value(self, value):
        return json.loads(json.dumps(value))

def make_timestamp():
    now = datetime.datetime.utcnow()
    return now.isoformat()

def make_uuid():
    return str(uuid.uuid4())

def rebuild_db():
    # See this SQLAlchemy recipe: http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything
    inspector = reflection.Inspector.from_engine(db.engine)
    # gather all data first before dropping anything.
    # some DBs lock after things have been dropped in
    # a transaction.

    metadata = MetaData()

    tbs = []
    all_fks = []

    for table_name in inspector.get_table_names():
        fks = []
        for fk in inspector.get_foreign_keys(table_name):
            if not fk['name']: # pragma: no cover
                continue
            fks.append(
                ForeignKeyConstraint((),(),name=fk['name'])
                )
        t = Table(table_name,metadata,*fks)
        tbs.append(t)
        all_fks.extend(fks)

    for fkc in all_fks:
        db.engine.execute(DropConstraint(fkc))

    for table in tbs:
        db.engine.execute(DropTable(table))

    db.session.commit()
    db.create_all()

def update_redis(obj):
    """Add domain object to update feed in Redis."""
    from pybossa.core import sentinel
    p = sentinel.master.pipeline()
    tmp = pickle.dumps(obj)
    p.zadd('pybossa_feed', time(), tmp)
    p.execute()
<commit_msg>Import sentinel in the top<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import json
import uuid

from sqlalchemy import Text
from sqlalchemy.orm import relationship, backref, class_mapper
from sqlalchemy.types import MutableType, TypeDecorator
from sqlalchemy import event
from sqlalchemy.engine import reflection
from sqlalchemy.schema import (
    MetaData,
    Table,
    DropTable,
    ForeignKeyConstraint,
    DropConstraint,
    )

import logging
from time import time

try:
    import cPickle as pickle
except ImportError:  # pragma: no cover
    import pickle

from pybossa.core import sentinel

log = logging.getLogger(__name__)

class DomainObject(object):

    def dictize(self):
        out = {}
        for col in self.__table__.c:
            out[col.name] = getattr(self, col.name)
        return out

    @classmethod
    def undictize(cls, dict_):
        raise NotImplementedError()

    def __str__(self):  # pragma: no cover
        return self.__unicode__().encode('utf8')

    def __unicode__(self): # pragma: no cover
        repr = u'<%s' % self.__class__.__name__
        table = class_mapper(self.__class__).mapped_table
        for col in table.c:
            try:
                repr += u' %s=%s' % (col.name, getattr(self, col.name))
            except Exception, inst:
                repr += u' %s=%s' % (col.name, inst)

        repr += '>'
        return repr

class JSONType(MutableType, TypeDecorator):
    '''Additional Database Type for handling JSON values.
    '''
    impl = Text

    def __init__(self):
        super(JSONType, self).__init__()

    def process_bind_param(self, value, dialect):
        return json.dumps(value)

    def process_result_value(self, value, dialiect):
        return json.loads(value)

    def copy_value(self, value):
        return json.loads(json.dumps(value))

def make_timestamp():
    now = datetime.datetime.utcnow()
    return now.isoformat()

def make_uuid():
    return str(uuid.uuid4())

def rebuild_db():
    # See this SQLAlchemy recipe: http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything
    inspector = reflection.Inspector.from_engine(db.engine)
    # gather all data first before dropping anything.
    # some DBs lock after things have been dropped in
    # a transaction.

    metadata = MetaData()

    tbs = []
    all_fks = []

    for table_name in inspector.get_table_names():
        fks = []
        for fk in inspector.get_foreign_keys(table_name):
            if not fk['name']: # pragma: no cover
                continue
            fks.append(
                ForeignKeyConstraint((),(),name=fk['name'])
                )
        t = Table(table_name,metadata,*fks)
        tbs.append(t)
        all_fks.extend(fks)

    for fkc in all_fks:
        db.engine.execute(DropConstraint(fkc))

    for table in tbs:
        db.engine.execute(DropTable(table))

    db.session.commit()
    db.create_all()

def update_redis(obj):
    """Add domain object to update feed in Redis."""
    p = sentinel.master.pipeline()
    tmp = pickle.dumps(obj)
    p.zadd('pybossa_feed', time(), tmp)
    p.execute()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import angus

conn = angus.connect()
service = conn.services.get_service('agegender_estimation', version=1)
job = service.process({'image': open('./macgyver.jpg')})
print job.result['faces']
<commit_msg>fixing age and gender estimation service name<commit_after>#!/usr/bin/env python

import angus

conn = angus.connect()
service = conn.services.get_service('age_and_gender_estimation', version=1)
job = service.process({'image': open('./macgyver.jpg')})
print job.result
<|endoftext|>


<commit_before># Copyright 2007 Andrey Golovizin
#
# This file is part of pybtex.
#
# pybtex is free software; you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# pybtex is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pybtex; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
# USA

"""Built-in functions for BibTeX interpreter.

CAUTION: functions should PUSH results, not RETURN
"""

import interpreter

class Builtin(object):
    def __init__(self, f):
        self.f = f
    def execute(self, interpreter):
        self.f(interpreter)
    def __repr__(self):
        return '<builtin %s>' % self.f.__name__

def builtin(f):
    return Builtin(f)

@builtin
def operator_more(i):
    arg1 = i.pop()
    arg2 = i.pop()
    if arg2 > arg1:
        i.push(1)
    else:
        i.push(0)

@builtin
def operator_less(i):
    arg1 = i.pop()
    arg2 = i.pop()
    if arg2 < arg1:
        i.push(1)
    else:
        i.push(0)

@builtin
def operator_equals(i):
    arg1 = i.pop()
    arg2 = i.pop()
    if arg2 == arg1:
        i.push(1)
    else:
        i.push(0)

@builtin
def operator_asterisk(i):
    arg1 = i.pop()
    arg2 = i.pop()
    i.push(arg2 + arg1)

@builtin
def operator_assign(i):
    var = i.pop()
    value = i.pop()
    var.set(value)

@builtin
def operator_plus(i):
    arg1 = i.pop()
    arg2 = i.pop()
    i.push(arg2 + arg1)

@builtin
def operator_minus(i):
    arg1 = i.pop()
    arg2 = i.pop()
    i.push(arg2 - arg1)

@builtin
def add_period(i):
    s = i.pop()
    if s and not s.rstrip('}')[-1] in '.?!':
        s += '.'
    i.push(s)

@builtin
def call_type(i):
    type = i.current_entry.type
    i.vars[type].execute(i)

@builtin
def change_case(i):
    mode = i.pop()
    s = i.pop()
    #FIXME stub
    i.push(s)

@builtin
def cite(i):
    i.push(i.current_entry_key)

@builtin
def duplicate(i):
    tmp = i.pop()
    i.push(tmp)
    i.push(tmp)

@builtin
def empty(i):
    #FIXME error checking
    s = i.pop()
    if s and not s.isspace():
        i.push(0)
    else:
        i.push(1)

@builtin
def format_name(i):
    #FIXME stub
    format = i.pop()
    n = i.pop()
    list = i.pop()
    i.push(list)

@builtin
def if_(i):
    f1 = i.pop()
    f2 = i.pop()
    p = i.pop()
    if p > 0:
        f2.execute(i)
    else:
        f1.execute(i)

@builtin
def int_to_str(i):
    i.push(str(i.pop()))

@builtin
def missing(i):
    f = i.pop()
    if isinstance(f, interpreter.MissingField):
        i.push(1)
    else:
        i.push(0)

@builtin
def newline(i):
    # FIXME bibtex does some automatic line breaking
    # needs more investigation
    i.output('\n')

@builtin
def num_names(i):
    #FIXME stub
    i.pop()
    i.push(1)

@builtin
def pop(i):
    i.pop()

@builtin
def preamble(i):
    #FIXME stub
    i.push('')

@builtin
def skip(i):
    pass

@builtin
def substring(i):
    len = i.pop()
    start = i.pop()
    s = i.pop()
    if start > 0:
        i.push(s[start - 1:start - 1 + len])
    elif start < 0:
        i.push(s[-start - len:-start])
    else:
        raise BibTeXError('start=0 passed to substring$')

@builtin
def swap(i):
    tmp1 = i.pop()
    tmp2 = i.pop()
    i.push(tmp1)
    i.push(tmp2)

@builtin
def text_length(i):
    # FIXME special characters and braces
    s = i.pop()
    i.push(len(s))

@builtin
def warning(i):
    #FIXME stub
    msg = i.pop()
    print 'WARNING:', msg

@builtin
def while_(i):
    p = i.pop()
    f = i.pop()
    while True:
        p.execute(i)
        if i.pop() > 0:
            break
        f.execute(i)

@builtin
def width(i):
    #FIXME need to investigate bibtex' source
    s = i.pop()
    i.push(len(s))

@builtin
def write(i):
    #FIXME encodings
    s = i.pop()
    i.output(s.encode('UTF-8'))

builtins = {
        '>': operator_more,
        '<': operator_less,
        '=': operator_equals,
        '+': operator_plus,
        '-': operator_minus,
        '*': operator_asterisk,
        ':=': operator_assign,
        'add.period$': add_period,
        'call.type$': call_type,
        'change.case$': change_case,
        'cite$': cite,
        'duplicate$': duplicate,
        'empty$': empty,
        'format.name$': format_name,
        'if$': if_,
        'int.to.str$': int_to_str,
        'missing$': missing,
        'newline$': newline,
        'num.names$': num_names,
        'pop$': pop,
        'preamble$': preamble,
        'skip$': skip,
        'substring$': substring,
        'swap$': swap,
        'text.length$': text_length,
        'warning$': warning,
        'while$': while_,
        'width$': width,
        'write$': write,
}
<commit_msg>finally fixed while$ (after reading the friendly bibtex' source)<commit_after># Copyright 2007 Andrey Golovizin
#
# This file is part of pybtex.
#
# pybtex is free software; you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# pybtex is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pybtex; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
# USA

"""Built-in functions for BibTeX interpreter.

CAUTION: functions should PUSH results, not RETURN
"""

import interpreter

class Builtin(object):
    def __init__(self, f):
        self.f = f
    def execute(self, interpreter):
        self.f(interpreter)
    def __repr__(self):
        return '<builtin %s>' % self.f.__name__

def builtin(f):
    return Builtin(f)

@builtin
def operator_more(i):
    arg1 = i.pop()
    arg2 = i.pop()
    if arg2 > arg1:
        i.push(1)
    else:
        i.push(0)

@builtin
def operator_less(i):
    arg1 = i.pop()
    arg2 = i.pop()
    if arg2 < arg1:
        i.push(1)
    else:
        i.push(0)

@builtin
def operator_equals(i):
    arg1 = i.pop()
    arg2 = i.pop()
    if arg2 == arg1:
        i.push(1)
    else:
        i.push(0)

@builtin
def operator_asterisk(i):
    arg1 = i.pop()
    arg2 = i.pop()
    i.push(arg2 + arg1)

@builtin
def operator_assign(i):
    var = i.pop()
    value = i.pop()
    var.set(value)

@builtin
def operator_plus(i):
    arg1 = i.pop()
    arg2 = i.pop()
    i.push(arg2 + arg1)

@builtin
def operator_minus(i):
    arg1 = i.pop()
    arg2 = i.pop()
    i.push(arg2 - arg1)

@builtin
def add_period(i):
    s = i.pop()
    if s and not s.rstrip('}')[-1] in '.?!':
        s += '.'
    i.push(s)

@builtin
def call_type(i):
    type = i.current_entry.type
    i.vars[type].execute(i)

@builtin
def change_case(i):
    mode = i.pop()
    s = i.pop()
    #FIXME stub
    i.push(s)

@builtin
def cite(i):
    i.push(i.current_entry_key)

@builtin
def duplicate(i):
    tmp = i.pop()
    i.push(tmp)
    i.push(tmp)

@builtin
def empty(i):
    #FIXME error checking
    s = i.pop()
    if s and not s.isspace():
        i.push(0)
    else:
        i.push(1)

@builtin
def format_name(i):
    #FIXME stub
    format = i.pop()
    n = i.pop()
    list = i.pop()
    i.push(list)

@builtin
def if_(i):
    f1 = i.pop()
    f2 = i.pop()
    p = i.pop()
    if p > 0:
        f2.execute(i)
    else:
        f1.execute(i)

@builtin
def int_to_str(i):
    i.push(str(i.pop()))

@builtin
def missing(i):
    f = i.pop()
    if isinstance(f, interpreter.MissingField):
        i.push(1)
    else:
        i.push(0)

@builtin
def newline(i):
    # FIXME bibtex does some automatic line breaking
    # needs more investigation
    i.output('\n')

@builtin
def num_names(i):
    #FIXME stub
    i.pop()
    i.push(1)

@builtin
def pop(i):
    i.pop()

@builtin
def preamble(i):
    #FIXME stub
    i.push('')

@builtin
def skip(i):
    pass

@builtin
def substring(i):
    len = i.pop()
    start = i.pop()
    s = i.pop()
    if start > 0:
        i.push(s[start - 1:start - 1 + len])
    elif start < 0:
        i.push(s[-start - len:-start])
    else:
        raise BibTeXError('start=0 passed to substring$')

@builtin
def swap(i):
    tmp1 = i.pop()
    tmp2 = i.pop()
    i.push(tmp1)
    i.push(tmp2)

@builtin
def text_length(i):
    # FIXME special characters and braces
    s = i.pop()
    i.push(len(s))

@builtin
def warning(i):
    #FIXME stub
    msg = i.pop()
    print 'WARNING:', msg

@builtin
def while_(i):
    f = i.pop()
    p = i.pop()
    while True:
        p.execute(i)
        if i.pop() <= 0:
            break
        f.execute(i)

@builtin
def width(i):
    #FIXME need to investigate bibtex' source
    s = i.pop()
    i.push(len(s))

@builtin
def write(i):
    #FIXME encodings
    s = i.pop()
    i.output(s.encode('UTF-8'))

builtins = {
        '>': operator_more,
        '<': operator_less,
        '=': operator_equals,
        '+': operator_plus,
        '-': operator_minus,
        '*': operator_asterisk,
        ':=': operator_assign,
        'add.period$': add_period,
        'call.type$': call_type,
        'change.case$': change_case,
        'cite$': cite,
        'duplicate$': duplicate,
        'empty$': empty,
        'format.name$': format_name,
        'if$': if_,
        'int.to.str$': int_to_str,
        'missing$': missing,
        'newline$': newline,
        'num.names$': num_names,
        'pop$': pop,
        'preamble$': preamble,
        'skip$': skip,
        'substring$': substring,
        'swap$': swap,
        'text.length$': text_length,
        'warning$': warning,
        'while$': while_,
        'width$': width,
        'write$': write,
}
<|endoftext|>