<commit_before># -*- coding: utf-8 -*-

"""
zenfig.util
~~~~~~~~

Utilities

:copyright: (c) 2016 by Alejandro Ricoveri
:license: MIT, see LICENSE for more details.

"""
import os
import inspect

from functools import wraps
from time import time

from . import log
from . import __name__ as pkg_name

def memoize(func):
    """
    A simple memcache decorator
    """
    # Cache as dict, all entries go in here
    cache = {}

    # Wrapper function
    @wraps(func)
    def _wrapper(key, *args, **kwargs):
        if isinstance(key, str):
            if key in cache:
                return cache[key]
            cache[key] = func(key, *args, **kwargs)
            return cache[key]
        return func(key, *args, **kwargs)

    # give that wrapper
    return _wrapper

# Based on: http://stackoverflow.com/questions/10973362/python-logging-function-name-file-name-line-number-using-a-single-file
def autolog(func):
    """
    Decorator for automatically log the current function details.
    """

    # Wrapper function
    @wraps(func)
    def _log_wrapper(*args, **kwargs):
        # measure its execution time
        start_time = time()
        r = func(*args, **kwargs)
        dt = time() - start_time

        # Dump the message + the name of this function to the log.
        log.msg_debug("{} {:.3f} ms".format(func.__name__, dt*1000))

        # return whatever func has thrown
        return r
    return _log_wrapper

@autolog
def get_xdg_cache_home():
    """Get XDG_CACHE_HOME"""

    # XDG_CACHE_HOME/zenfig/templates is inside
    # the template search path
    xdg_cache_home = os.getenv('XDG_CACHE_HOME')
    if xdg_cache_home is None:
        xdg_cache_home = "{}/.cache/{}".format(os.getenv("HOME"), pkg_name)
    return xdg_cache_home

@autolog
def get_xdg_data_home():
    """Get XDG_DATA_HOME"""

    # XDG_DATA_HOME/zenfig/templates is inside
    # the template search path
    xdg_data_home = os.getenv('XDG_DATA_HOME')
    if xdg_data_home is None:
        xdg_data_home = "{}/.local/share/{}".format(os.getenv("HOME"), pkg_name)
    return xdg_data_home

<commit_msg>A fancier format for autolog functions<commit_after># -*- coding: utf-8 -*-

"""
zenfig.util
~~~~~~~~

Utilities

:copyright: (c) 2016 by Alejandro Ricoveri
:license: MIT, see LICENSE for more details.

"""
import os
import inspect

from functools import wraps
from time import time

from . import log
from . import __name__ as pkg_name

def memoize(func):
    """
    A simple memcache decorator
    """
    # Cache as dict, all entries go in here
    cache = {}

    # Wrapper function
    @wraps(func)
    def _wrapper(key, *args, **kwargs):
        if isinstance(key, str):
            if key in cache:
                return cache[key]
            cache[key] = func(key, *args, **kwargs)
            return cache[key]
        return func(key, *args, **kwargs)

    # give that wrapper
    return _wrapper

# Based on: http://stackoverflow.com/questions/10973362/python-logging-function-name-file-name-line-number-using-a-single-file
def autolog(func):
    """
    Decorator for automatically log the current function details.
    """

    # Wrapper function
    @wraps(func)
    def _log_wrapper(*args, **kwargs):
        # measure its execution time
        start_time = time()
        r = func(*args, **kwargs)
        dt = time() - start_time

        # Dump the message + the name of this function to the log.
        log.msg_debug("{} (t = {:.3f} ms)".format(func.__name__, dt*1000))

        # return whatever func has thrown
        return r
    return _log_wrapper

@autolog
def get_xdg_cache_home():
    """Get XDG_CACHE_HOME"""

    # XDG_CACHE_HOME/zenfig/templates is inside
    # the template search path
    xdg_cache_home = os.getenv('XDG_CACHE_HOME')
    if xdg_cache_home is None:
        xdg_cache_home = "{}/.cache/{}".format(os.getenv("HOME"), pkg_name)
    return xdg_cache_home

@autolog
def get_xdg_data_home():
    """Get XDG_DATA_HOME"""

    # XDG_DATA_HOME/zenfig/templates is inside
    # the template search path
    xdg_data_home = os.getenv('XDG_DATA_HOME')
    if xdg_data_home is None:
        xdg_data_home = "{}/.local/share/{}".format(os.getenv("HOME"), pkg_name)
    return xdg_data_home

<|endoftext|>


<commit_before>import time
import unittest
import os, os.path

from usecases.TestUtils import *

class testRunVmFromPersistentDisk(unittest.TestCase):

    vm_image_info = getVmImageInfo()
    ttylinux_url = vm_image_info['ttylinux']['url']
    marketplaceId = vm_image_info['ubuntu']['id']

    vm_id = None
    vm_id_ttylinux = None

    diskSize = 1

    def setUp(self):
        self.uuid = stratusCreateVolume(self.diskSize)
        self.vm_id, self.vm_ip = stratusRunInstance(self.marketplaceId)

    def tearDown(self):
        stratusKillInstance(self.vm_id)
        stratusKillInstance(self.vm_id_ttylinux)

    def test_usecase(self):
        waitVmRunningOrTimeout(self.vm_id)
        sshConnectionOrTimeout(self.vm_ip, timeout=(4*60))

        # Ensure kernel module is available for dynamic disk attachment
        ssh(ip=self.vm_ip, cmd='modprobe acpiphp')

        # Attach disk to machine.
        stratusAttachVolume(self.vm_id, self.uuid)

        # Copy ttylinux image onto persistent disk
        ssh(ip=self.vm_ip, cmd='apt-get install -y curl')
        ssh(ip=self.vm_ip, cmd='cat /proc/partitions')
        ssh(ip=self.vm_ip, cmd='curl -o /tmp/ttylinux.img.gz %s' % self.ttylinux_url)
        ssh(ip=self.vm_ip, cmd='gunzip /tmp/ttylinux.img.gz')
        ssh(ip=self.vm_ip, cmd='dd if=/tmp/ttylinux.img of=/dev/vda')

        # Detach disk from machine.
        stratusDetachVolume(self.vm_id, self.uuid)
        time.sleep(5)

        # Mark this as a live machine image.
        stratusUpdateVolume(self.uuid, ['--machine-image-live'])

        # Run ttylinux from persistent disk
        self.vm_id_ttylinux, self.vm_ip_ttylinux = stratusRunInstance(self.uuid)
        waitVmRunningOrTimeout(self.vm_id_ttylinux)
        sshConnectionOrTimeout(self.vm_ip_ttylinux)

        # Kill the instance; sleep to be sure the clean up is done.
        stratusKillInstance(self.vm_id_ttylinux)
        time.sleep(5)

        # Delete the volume to be sure it still belongs to us.
        stratusDeleteVolume(self.uuid)

def suite():
    return unittest.TestLoader().loadTestsFromTestCase(testRunVmFromPersistentDisk)
<commit_msg>try increasing timeout<commit_after>import time
import unittest
import os, os.path

from usecases.TestUtils import *

class testRunVmFromPersistentDisk(unittest.TestCase):

    vm_image_info = getVmImageInfo()
    ttylinux_url = vm_image_info['ttylinux']['url']
    marketplaceId = vm_image_info['ubuntu']['id']

    vm_id = None
    vm_id_ttylinux = None

    diskSize = 1

    def setUp(self):
        self.uuid = stratusCreateVolume(self.diskSize)
        self.vm_id, self.vm_ip = stratusRunInstance(self.marketplaceId)

    def tearDown(self):
        stratusKillInstance(self.vm_id)
        stratusKillInstance(self.vm_id_ttylinux)

    def test_usecase(self):
        waitVmRunningOrTimeout(self.vm_id)
        sshConnectionOrTimeout(self.vm_ip, timeout=(4*60))

        # Ensure kernel module is available for dynamic disk attachment
        ssh(ip=self.vm_ip, cmd='modprobe acpiphp')

        # Attach disk to machine.
        stratusAttachVolume(self.vm_id, self.uuid)

        # Copy ttylinux image onto persistent disk
        ssh(ip=self.vm_ip, cmd='apt-get install -y curl')
        ssh(ip=self.vm_ip, cmd='cat /proc/partitions')
        ssh(ip=self.vm_ip, cmd='curl -o /tmp/ttylinux.img.gz %s' % self.ttylinux_url)
        ssh(ip=self.vm_ip, cmd='gunzip /tmp/ttylinux.img.gz')
        ssh(ip=self.vm_ip, cmd='dd if=/tmp/ttylinux.img of=/dev/vda')

        # Detach disk from machine.
        stratusDetachVolume(self.vm_id, self.uuid)
        time.sleep(5)

        # Mark this as a live machine image.
        stratusUpdateVolume(self.uuid, ['--machine-image-live'])

        # Run ttylinux from persistent disk
        self.vm_id_ttylinux, self.vm_ip_ttylinux = stratusRunInstance(self.uuid)
        waitVmRunningOrTimeout(self.vm_id_ttylinux)
        sshConnectionOrTimeout(self.vm_ip_ttylinux)

        # Kill the instance; sleep to be sure the clean up is done.
        stratusKillInstance(self.vm_id_ttylinux)

        # Wait for the disk to be unmounted and freed.
        time.sleep(10)

        # Delete the volume to be sure it still belongs to us.
        stratusDeleteVolume(self.uuid)

def suite():
    return unittest.TestLoader().loadTestsFromTestCase(testRunVmFromPersistentDisk)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from optparse import OptionParser
import re
import os
import subprocess
import sys
import common
import time

#*********************************************************--
# main script start
#*********************************************************--
this_directory = os.path.dirname(os.path.realpath(__file__)) + "/"

parser = OptionParser()
parser.add_option("-l", "--logfile", dest="logfile",
                  help="The logfile the status is based on. "+\
                        "By default, we will base it on the latest simulations launched.\n" +\
                        "specify \"all\" to use all the simulation logfiles in the directory",
                  default="")
parser.add_option("-N", "--sim_name", dest="sim_name",
                  help="If you are launchign run_simulations.py with the \"-N\" option" +\
                       " then you can run ./job_status.py with \"-N\" and it will" + \
                       " give you the status of the latest run with that name."+ \
                       " if you want older runs from this name, then just point it directly at the"+\
                       " logfile with \"-l\"", default="")

(options, args) = parser.parse_args()
options.logfile = options.logfile.strip()
options.sim_name = options.sim_name.strip()

jobstatus_out_filename = "/tmp/job_status_out.txt"

while True:
    jobstatus_out_file = open(jobstatus_out_filename, 'w+')
    if subprocess.call(["./job_status.py" ,"-l", options.logfile, "-N", options.sim_name],
        stdout=jobstatus_out_file, stderr=jobstatus_out_file) < 0:
            exit("Error Launching job_status.py")
    else:
        jobstatus_out_file.seek(0)
        jobStatusCol = None
        num_passed = 0
        num_not_done = 0
        num_else = 0
        for line in jobstatus_out_file.readlines():
            if jobStatusCol == None:
                name_line_match = re.match("(.*)JobStatus.*", line)
                if name_line_match != None:
                    jobStatusCol = name_line_match.group(1).count('\t')
                    continue
            else:
                tokens = line.split('\t')
                if len(tokens) > jobStatusCol:
                    status = tokens[jobStatusCol].strip()
                    if status == "FUNC_TEST_PASSED":
                        num_passed += 1
                    elif status == "RUNNING" or status == "WAITING_TO_RUN":
                        num_not_done += 1
                    else:
                        num_else += 1
                elif re.match("failed job log.*", line):
                    failed_job_file = line

        jobstatus_out_file.close()
        os.remove(jobstatus_out_filename)

    total = num_passed + num_not_done + num_else
    print "Passed:{0}/{1}, Not passed:{2}/{1}, Not done:{3}/{1}"\
        .format(num_passed, total, num_else, num_not_done)
    if num_else > 0:
            print failed_job_file
    if num_not_done == 0:
        print "All {0} Tests Done.".format(total)
        if num_else == 0:
            print "Congratulations! All Tests Pass!"
        else:
            print "Something did not pass."
        break
    else:
        print "Sleeping for 30s"
        time.sleep(30)
<commit_msg>Adding right exit codes when things don't pass and setting the right path for the job_status script<commit_after>#!/usr/bin/env python

from optparse import OptionParser
import re
import os
import subprocess
import sys
import common
import time

#*********************************************************--
# main script start
#*********************************************************--
this_directory = os.path.dirname(os.path.realpath(__file__)) + "/"

parser = OptionParser()
parser.add_option("-l", "--logfile", dest="logfile",
                  help="The logfile the status is based on. "+\
                        "By default, we will base it on the latest simulations launched.\n" +\
                        "specify \"all\" to use all the simulation logfiles in the directory",
                  default="")
parser.add_option("-N", "--sim_name", dest="sim_name",
                  help="If you are launchign run_simulations.py with the \"-N\" option" +\
                       " then you can run ./job_status.py with \"-N\" and it will" + \
                       " give you the status of the latest run with that name."+ \
                       " if you want older runs from this name, then just point it directly at the"+\
                       " logfile with \"-l\"", default="")

(options, args) = parser.parse_args()
options.logfile = options.logfile.strip()
options.sim_name = options.sim_name.strip()

jobstatus_out_filename = "/tmp/job_status_out.txt"

while True:
    jobstatus_out_file = open(jobstatus_out_filename, 'w+')
    if subprocess.call([os.path.join(this_directory, "job_status.py") ,"-l", options.logfile, "-N", options.sim_name],
        stdout=jobstatus_out_file, stderr=jobstatus_out_file) < 0:
            exit("Error Launching job_status.py")
    else:
        jobstatus_out_file.seek(0)
        jobStatusCol = None
        num_passed = 0
        num_not_done = 0
        num_else = 0
        for line in jobstatus_out_file.readlines():
            if jobStatusCol == None:
                name_line_match = re.match("(.*)JobStatus.*", line)
                if name_line_match != None:
                    jobStatusCol = name_line_match.group(1).count('\t')
                    continue
            else:
                tokens = line.split('\t')
                if len(tokens) > jobStatusCol:
                    status = tokens[jobStatusCol].strip()
                    if status == "FUNC_TEST_PASSED":
                        num_passed += 1
                    elif status == "RUNNING" or status == "WAITING_TO_RUN":
                        num_not_done += 1
                    else:
                        num_else += 1
                elif re.match("failed job log.*", line):
                    failed_job_file = line

        jobstatus_out_file.close()
        os.remove(jobstatus_out_filename)

    total = num_passed + num_not_done + num_else
    print "Passed:{0}/{1}, Not passed:{2}/{1}, Not done:{3}/{1}"\
        .format(num_passed, total, num_else, num_not_done)
    if num_else > 0:
            print failed_job_file

    if num_not_done == 0:
        print "All {0} Tests Done.".format(total)
        if num_else == 0:
            print "Congratulations! All Tests Pass!"
            exit(0)
        else:
            print "Something did not pass."
            exit(1)
    else:
        print "Sleeping for 30s"
        time.sleep(30)
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''Prepare a code coverage artifact.

- Collate raw profiles into one indexed profile.
- Delete the raw profiles.
- Copy the coverage mappings in the binaries directory.
'''

import argparse
import glob
import os
import subprocess
import sys

def merge_raw_profiles(host_llvm_profdata, profile_data_dir):
    print ':: Merging raw profiles...',
    sys.stdout.flush()
    raw_profiles = glob.glob(os.path.join(profile_data_dir, '*.profraw'))
    manifest_path = os.path.join(profile_data_dir, 'profiles.manifest')
    profdata_path = os.path.join(profile_data_dir, 'Coverage.profdata')
    with open(manifest_path, 'w') as manifest:
        manifest.write('\n'.join(raw_profiles))
    subprocess.check_call([host_llvm_profdata, 'merge', '-sparse', '-f',
                           manifest_path, '-o', profdata_path])
    for raw_profile in raw_profiles:
        os.remove(raw_profile)
    print 'Done!'

def extract_covmappings(host_llvm_cov, profile_data_dir, llvm_bin_dir):
    print ':: Extracting covmappings...',
    sys.stdout.flush()
    for prog in os.listdir(llvm_bin_dir):
        if prog == 'llvm-lit':
            continue
        covmapping_path = os.path.join(profile_data_dir,
                                       os.path.basename(prog) + '.covmapping')
        subprocess.check_call([host_llvm_cov, 'convert-for-testing',
                               os.path.join(llvm_bin_dir, prog), '-o',
                               covmapping_path])
    print 'Done!'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('host_llvm_profdata', help='Path to llvm-profdata')
    parser.add_argument('host_llvm_cov', help='Path to llvm-cov')
    parser.add_argument('profile_data_dir',
                       help='Path to the directory containing the raw profiles')
    parser.add_argument('llvm_bin_dir',
                       help='Path to the directory containing llvm binaries')
    args = parser.parse_args()

    merge_raw_profiles(args.host_llvm_profdata, args.profile_data_dir)
    extract_covmappings(args.host_llvm_cov, args.profile_data_dir,
                        args.llvm_bin_dir)
<commit_msg>[utils] Generate html reports with the code coverage utility script<commit_after>#!/usr/bin/env python

'''Prepare a code coverage artifact.

- Collate raw profiles into one indexed profile.
- Delete the raw profiles.
- Generate html reports for llvm binaries.
'''

import argparse
import glob
import os
import subprocess
import sys

def merge_raw_profiles(host_llvm_profdata, profile_data_dir, preserve_profiles):
    print ':: Merging raw profiles...',
    sys.stdout.flush()
    raw_profiles = glob.glob(os.path.join(profile_data_dir, '*.profraw'))
    manifest_path = os.path.join(profile_data_dir, 'profiles.manifest')
    profdata_path = os.path.join(profile_data_dir, 'Coverage.profdata')
    with open(manifest_path, 'w') as manifest:
        manifest.write('\n'.join(raw_profiles))
    subprocess.check_call([host_llvm_profdata, 'merge', '-sparse', '-f',
                           manifest_path, '-o', profdata_path])
    if not preserve_profiles:
        for raw_profile in raw_profiles:
            os.remove(raw_profile)
    os.remove(manifest_path)
    print 'Done!'

def prepare_html_report(host_llvm_cov, profile_data_dir, report_dir, binary):
    print ':: Preparing html report for {0}...'.format(binary),
    sys.stdout.flush()
    binary_report_dir = os.path.join(report_dir, os.path.basename(binary))
    profile = os.path.join(profile_data_dir, 'Coverage.profdata')
    subprocess.check_call([host_llvm_cov, 'show', binary, '-format', 'html',
                           '-instr-profile', profile, '-o', binary_report_dir,
                           '-show-line-counts-or-regions',
                           '-Xdemangler', 'c++filt', '-Xdemangler', '-n'])
    with open(os.path.join(binary_report_dir, 'summary.txt'), 'wb') as Summary:
        subprocess.check_call([host_llvm_cov, 'report', binary,
                               '-instr-profile', profile], stdout=Summary)
    print 'Done!'

def prepare_html_reports(host_llvm_cov, profile_data_dir, report_dir, binaries):
    for binary in binaries:
        prepare_html_report(host_llvm_cov, profile_data_dir, report_dir, binary)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('host_llvm_profdata', help='Path to llvm-profdata')
    parser.add_argument('host_llvm_cov', help='Path to llvm-cov')
    parser.add_argument('profile_data_dir',
                       help='Path to the directory containing the raw profiles')
    parser.add_argument('report_dir',
                       help='Path to the output directory for html reports')
    parser.add_argument('binaries', metavar='B', type=str, nargs='+',
                       help='Path to an instrumented binary')
    parser.add_argument('--preserve-profiles',
                       help='Do not delete raw profiles', action='store_true')
    args = parser.parse_args()

    merge_raw_profiles(args.host_llvm_profdata, args.profile_data_dir,
                       args.preserve_profiles)
    prepare_html_reports(args.host_llvm_cov, args.profile_data_dir,
                         args.report_dir, args.binaries)
<|endoftext|>


<commit_before># (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from six import iteritems, string_types

import os

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook.role.definition import RoleDefinition

__all__ = ['RoleRequirement']

class RoleRequirement(RoleDefinition):

    """
    FIXME: document various ways role specs can be specified
    """

    def __init__(self):
        pass

    def _get_valid_spec_keys(self):
        return (
            'name',
            'role',
            'scm',
            'src',
            'version',
        )

    def parse(self, ds):
        '''
        FIXME: docstring
        '''

        assert type(ds) == dict or isinstance(ds, string_types)

        role_name    = ''
        role_params  = dict()
        new_ds       = dict()

        if isinstance(ds, string_types):
            role_name = ds
        else:
            ds = self._preprocess_role_spec(ds)
            (new_ds, role_params) = self._split_role_params(ds)

            # pull the role name out of the ds
            role_name = new_ds.get('role_name')
            del ds['role_name']

        return (new_ds, role_name, role_params)

    def _preprocess_role_spec(self, ds):
        if 'role' in ds:
            # Old style: {role: "galaxy.role,version,name", other_vars: "here" }
            role_info = self._role_spec_parse(ds['role'])
            if isinstance(role_info, dict):
                # Warning: Slight change in behaviour here.  name may be being
                # overloaded.  Previously, name was only a parameter to the role.
                # Now it is both a parameter to the role and the name that
                # ansible-galaxy will install under on the local system.
                if 'name' in ds and 'name' in role_info:
                    del role_info['name']
                ds.update(role_info)
        else:
            # New style: { src: 'galaxy.role,version,name', other_vars: "here" }
            if 'github.com' in ds["src"] and 'http' in ds["src"] and '+' not in ds["src"] and not ds["src"].endswith('.tar.gz'):
                ds["src"] = "git+" + ds["src"]

            if '+' in ds["src"]:
                (scm, src) = ds["src"].split('+')
                ds["scm"] = scm
                ds["src"] = src

            if 'name' in role:
                ds["role"] = ds["name"]
                del ds["name"]
            else:
                ds["role"] = self._repo_url_to_role_name(ds["src"])

            # set some values to a default value, if none were specified
            ds.setdefault('version', '')
            ds.setdefault('scm', None)

        return ds

    def _repo_url_to_role_name(self, repo_url):
        # gets the role name out of a repo like
        # http://git.example.com/repos/repo.git" => "repo"

        if '://' not in repo_url and '@' not in repo_url:
            return repo_url
        trailing_path = repo_url.split('/')[-1]
        if trailing_path.endswith('.git'):
            trailing_path = trailing_path[:-4]
        if trailing_path.endswith('.tar.gz'):
            trailing_path = trailing_path[:-7]
        if ',' in trailing_path:
            trailing_path = trailing_path.split(',')[0]
        return trailing_path

    def _role_spec_parse(self, role_spec):
        # takes a repo and a version like
        # git+http://git.example.com/repos/repo.git,v1.0
        # and returns a list of properties such as:
        # {
        #   'scm': 'git',
        #   'src': 'http://git.example.com/repos/repo.git',
        #   'version': 'v1.0',
        #   'name': 'repo'
        # }

        default_role_versions = dict(git='master', hg='tip')

        role_spec = role_spec.strip()
        role_version = ''
        if role_spec == "" or role_spec.startswith("#"):
            return (None, None, None, None)

        tokens = [s.strip() for s in role_spec.split(',')]

        # assume https://github.com URLs are git+https:// URLs and not
        # tarballs unless they end in '.zip'
        if 'github.com/' in tokens[0] and not tokens[0].startswith("git+") and not tokens[0].endswith('.tar.gz'):
            tokens[0] = 'git+' + tokens[0]

        if '+' in tokens[0]:
            (scm, role_url) = tokens[0].split('+')
        else:
            scm = None
            role_url = tokens[0]

        if len(tokens) >= 2:
            role_version = tokens[1]

        if len(tokens) == 3:
            role_name = tokens[2]
        else:
            role_name = self._repo_url_to_role_name(tokens[0])

        if scm and not role_version:
            role_version = default_role_versions.get(scm, '')

        return dict(scm=scm, src=role_url, version=role_version, role_name=role_name)

<commit_msg>Fix variable name<commit_after># (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from six import iteritems, string_types

import os

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook.role.definition import RoleDefinition

__all__ = ['RoleRequirement']

class RoleRequirement(RoleDefinition):

    """
    FIXME: document various ways role specs can be specified
    """

    def __init__(self):
        pass

    def _get_valid_spec_keys(self):
        return (
            'name',
            'role',
            'scm',
            'src',
            'version',
        )

    def parse(self, ds):
        '''
        FIXME: docstring
        '''

        assert type(ds) == dict or isinstance(ds, string_types)

        role_name    = ''
        role_params  = dict()
        new_ds       = dict()

        if isinstance(ds, string_types):
            role_name = ds
        else:
            ds = self._preprocess_role_spec(ds)
            (new_ds, role_params) = self._split_role_params(ds)

            # pull the role name out of the ds
            role_name = new_ds.get('role_name')
            del ds['role_name']

        return (new_ds, role_name, role_params)

    def _preprocess_role_spec(self, ds):
        if 'role' in ds:
            # Old style: {role: "galaxy.role,version,name", other_vars: "here" }
            role_info = self._role_spec_parse(ds['role'])
            if isinstance(role_info, dict):
                # Warning: Slight change in behaviour here.  name may be being
                # overloaded.  Previously, name was only a parameter to the role.
                # Now it is both a parameter to the role and the name that
                # ansible-galaxy will install under on the local system.
                if 'name' in ds and 'name' in role_info:
                    del role_info['name']
                ds.update(role_info)
        else:
            # New style: { src: 'galaxy.role,version,name', other_vars: "here" }
            if 'github.com' in ds["src"] and 'http' in ds["src"] and '+' not in ds["src"] and not ds["src"].endswith('.tar.gz'):
                ds["src"] = "git+" + ds["src"]

            if '+' in ds["src"]:
                (scm, src) = ds["src"].split('+')
                ds["scm"] = scm
                ds["src"] = src

            if 'name' in ds:
                ds["role"] = ds["name"]
                del ds["name"]
            else:
                ds["role"] = self._repo_url_to_role_name(ds["src"])

            # set some values to a default value, if none were specified
            ds.setdefault('version', '')
            ds.setdefault('scm', None)

        return ds

    def _repo_url_to_role_name(self, repo_url):
        # gets the role name out of a repo like
        # http://git.example.com/repos/repo.git" => "repo"

        if '://' not in repo_url and '@' not in repo_url:
            return repo_url
        trailing_path = repo_url.split('/')[-1]
        if trailing_path.endswith('.git'):
            trailing_path = trailing_path[:-4]
        if trailing_path.endswith('.tar.gz'):
            trailing_path = trailing_path[:-7]
        if ',' in trailing_path:
            trailing_path = trailing_path.split(',')[0]
        return trailing_path

    def _role_spec_parse(self, role_spec):
        # takes a repo and a version like
        # git+http://git.example.com/repos/repo.git,v1.0
        # and returns a list of properties such as:
        # {
        #   'scm': 'git',
        #   'src': 'http://git.example.com/repos/repo.git',
        #   'version': 'v1.0',
        #   'name': 'repo'
        # }

        default_role_versions = dict(git='master', hg='tip')

        role_spec = role_spec.strip()
        role_version = ''
        if role_spec == "" or role_spec.startswith("#"):
            return (None, None, None, None)

        tokens = [s.strip() for s in role_spec.split(',')]

        # assume https://github.com URLs are git+https:// URLs and not
        # tarballs unless they end in '.zip'
        if 'github.com/' in tokens[0] and not tokens[0].startswith("git+") and not tokens[0].endswith('.tar.gz'):
            tokens[0] = 'git+' + tokens[0]

        if '+' in tokens[0]:
            (scm, role_url) = tokens[0].split('+')
        else:
            scm = None
            role_url = tokens[0]

        if len(tokens) >= 2:
            role_version = tokens[1]

        if len(tokens) == 3:
            role_name = tokens[2]
        else:
            role_name = self._repo_url_to_role_name(tokens[0])

        if scm and not role_version:
            role_version = default_role_versions.get(scm, '')

        return dict(scm=scm, src=role_url, version=role_version, role_name=role_name)

<|endoftext|>


<commit_before>"""
Generic utility routines for number handling and calculating (specific)
variances used by the TKP sourcefinder.
"""

import numpy
from numpy.ma import MaskedArray
from scipy.special import erf
from scipy.optimize import fsolve

from .utils import calculate_correlation_lengths

# CODE & NUMBER HANDLING ROUTINES
#
def find_true_std(sigma, clip_limit, clipped_std):
    help1 = clip_limit/(sigma*numpy.sqrt(2))
    help2 = numpy.sqrt(2*numpy.pi)*erf(help1)
    return sigma**2*(help2-2*numpy.sqrt(2)*help1*numpy.exp(-help1**2))-clipped_std**2*help2

def indep_pixels(N, beam):
    corlengthlong, corlengthshort = calculate_correlation_lengths(
        beam[0], beam[1])
    correlated_area = 0.25 * numpy.pi * corlengthlong * corlengthshort
    return N / correlated_area

def sigma_clip(data, kappa=2.0, max_iter=100,
               centref=numpy.median, distf=numpy.var, my_iterations=0, limit=None):
    """Iterative clipping

    By default, this performs clipping of the standard deviation about the
    median of the data. But by tweaking centref/distf, it could be much
    more general.

    max_iter sets the maximum number of iterations used.

    my_iterations is a counter for recursive operation of the code; leave it
    alone unless you really want to pretend to jump into the middle of a loop.

    sigma is subtle: if a callable is given, it is passed a copy of the data
    array and can calculate a clipping limit. See, for e.g., unbiased_sigma()
    defined above. However, if it isn't callable, sigma is assumed to just set
    a hard limit.

    To do: Improve documentation
            -Returns???
            -How does it make use of the beam? (It estimates the noise correlation)
    """
    if my_iterations >= max_iter:
        # Exceeded maximum number of iterations; return
        return data, my_iterations

    # Numpy 1.1 breaks std() for MaskedArray: see
    # <http://www.scipy.org/scipy/numpy/wiki/MaskedArray>.
    # MaskedArray.compressed() returns a 1-D array of non-masked data.
    if isinstance(data, MaskedArray):
        data = data.compressed()
    centre = centref(data)
    N = numpy.size(data)
    if N < 1:
        # This chunk is too small for processing; return an empty array.
        return numpy.array([]), 0, 0, 0

    clipped_var = distf(data)

    std = numpy.sqrt(clipped_var)

    if limit is not None:
        std_corr_for_clipping_bias = fsolve(find_true_std, std,
                              args=(limit, std))[0]
    else:
        std_corr_for_clipping_bias = std

    limit = kappa * std_corr_for_clipping_bias

    newdata = data.compress(abs(data - centre) <= limit)

    if len(newdata) != len(data) and len(newdata) > 0:
        my_iterations += 1
        return sigma_clip(newdata, kappa, max_iter, centref, distf,
                          my_iterations, limit=limit)
    else:
        return newdata, std_corr_for_clipping_bias, centre, my_iterations
<commit_msg>Fixed indentation<commit_after>"""
Generic utility routines for number handling and calculating (specific)
variances used by the TKP sourcefinder.
"""

import numpy
from numpy.ma import MaskedArray
from scipy.special import erf
from scipy.optimize import fsolve

from .utils import calculate_correlation_lengths

# CODE & NUMBER HANDLING ROUTINES
#
def find_true_std(sigma, clip_limit, clipped_std):
    help1 = clip_limit/(sigma*numpy.sqrt(2))
    help2 = numpy.sqrt(2*numpy.pi)*erf(help1)
    return sigma**2*(help2-2*numpy.sqrt(2)*help1*numpy.exp(-help1**2))-clipped_std**2*help2

def indep_pixels(N, beam):
    corlengthlong, corlengthshort = calculate_correlation_lengths(
        beam[0], beam[1])
    correlated_area = 0.25 * numpy.pi * corlengthlong * corlengthshort
    return N / correlated_area

def sigma_clip(data, kappa=2.0, max_iter=100,
               centref=numpy.median, distf=numpy.var, my_iterations=0, limit=None):
    """Iterative clipping

    By default, this performs clipping of the standard deviation about the
    median of the data. But by tweaking centref/distf, it could be much
    more general.

    max_iter sets the maximum number of iterations used.

    my_iterations is a counter for recursive operation of the code; leave it
    alone unless you really want to pretend to jump into the middle of a loop.

    sigma is subtle: if a callable is given, it is passed a copy of the data
    array and can calculate a clipping limit. See, for e.g., unbiased_sigma()
    defined above. However, if it isn't callable, sigma is assumed to just set
    a hard limit.

    To do: Improve documentation
            -Returns???
            -How does it make use of the beam? (It estimates the noise correlation)
    """
    if my_iterations >= max_iter:
        # Exceeded maximum number of iterations; return
        return data, my_iterations

    # Numpy 1.1 breaks std() for MaskedArray: see
    # <http://www.scipy.org/scipy/numpy/wiki/MaskedArray>.
    # MaskedArray.compressed() returns a 1-D array of non-masked data.
    if isinstance(data, MaskedArray):
        data = data.compressed()
    centre = centref(data)
    N = numpy.size(data)
    if N < 1:
        # This chunk is too small for processing; return an empty array.
        return numpy.array([]), 0, 0, 0

    clipped_var = distf(data)

    std = numpy.sqrt(clipped_var)

    if limit is not None:
        std_corr_for_clipping_bias = fsolve(find_true_std, std,
                                            args=(limit, std))[0]
    else:
        std_corr_for_clipping_bias = std

    limit = kappa * std_corr_for_clipping_bias

    newdata = data.compress(abs(data - centre) <= limit)

    if len(newdata) != len(data) and len(newdata) > 0:
        my_iterations += 1
        return sigma_clip(newdata, kappa, max_iter, centref, distf,
                          my_iterations, limit=limit)
    else:
        return newdata, std_corr_for_clipping_bias, centre, my_iterations
<|endoftext|>


<commit_before>"""
sourcemap
~~~~~~~~~

:copyright: (c) 2013 by Matt Robenolt
:license: BSD, see LICENSE for more details.
"""
from .exceptions import SourceMapDecodeError  #NOQA
from .decoder import SourceMapDecoder

__version__ = '0.1.7'

def load(fp, cls=None):
    "Parse a sourcemap from a file-like object"
    return loads(fp.read(), cls)

def loads(source, cls=None):
    "Parse a sourcemap from a string"
    cls = cls or SourceMapDecoder
    return cls().decode(source)

def discover(source):
    "Given a JavaScript file, find the sourceMappingURL line"
    source = source.splitlines()
    # Source maps are only going to exist at either the top or bottom of the document.
    # Technically, there isn't anything indicating *where* it should exist, so we
    # are generous and assume it's somewhere either in the first or last 5 lines.
    # If it's somewhere else in the document, you're probably doing it wrong.
    if len(source) > 10:
        possibilities = source[:5] + source[-5:]
    else:
        possibilities = source

    for line in set(possibilities):
        pragma = line[:21]
        if pragma == '//# sourceMappingURL=' or pragma == '//@ sourceMappingURL=':
            # We want everything AFTER the pragma, which is 21 chars long
            return line[21:].rstrip()
    # XXX: Return None or raise an exception?
    return None
<commit_msg>v0.1.8<commit_after>"""
sourcemap
~~~~~~~~~

:copyright: (c) 2013 by Matt Robenolt
:license: BSD, see LICENSE for more details.
"""
from .exceptions import SourceMapDecodeError  #NOQA
from .decoder import SourceMapDecoder

__version__ = '0.1.8'

def load(fp, cls=None):
    "Parse a sourcemap from a file-like object"
    return loads(fp.read(), cls)

def loads(source, cls=None):
    "Parse a sourcemap from a string"
    cls = cls or SourceMapDecoder
    return cls().decode(source)

def discover(source):
    "Given a JavaScript file, find the sourceMappingURL line"
    source = source.splitlines()
    # Source maps are only going to exist at either the top or bottom of the document.
    # Technically, there isn't anything indicating *where* it should exist, so we
    # are generous and assume it's somewhere either in the first or last 5 lines.
    # If it's somewhere else in the document, you're probably doing it wrong.
    if len(source) > 10:
        possibilities = source[:5] + source[-5:]
    else:
        possibilities = source

    for line in set(possibilities):
        pragma = line[:21]
        if pragma == '//# sourceMappingURL=' or pragma == '//@ sourceMappingURL=':
            # We want everything AFTER the pragma, which is 21 chars long
            return line[21:].rstrip()
    # XXX: Return None or raise an exception?
    return None
<|endoftext|>


<commit_before>"""
Parsing module for models.py files. Extracts information in a more reliable
way than inspect + regexes.
Now only used as a fallback when introspection and the South custom hook both fail.
"""

import re
import inspect
import parser
import symbol
import token
import keyword
import datetime

from django.db import models
from django.contrib.contenttypes import generic
from django.utils.datastructures import SortedDict
from django.core.exceptions import ImproperlyConfigured

def name_that_thing(thing):
    "Turns a symbol/token int into its name."
    for name in dir(symbol):
        if getattr(symbol, name) == thing:
            return "symbol.%s" % name
    for name in dir(token):
        if getattr(token, name) == thing:
            return "token.%s" % name
    return str(thing)

def thing_that_name(name):
    "Turns a name of a symbol/token into its integer value."
    if name in dir(symbol):
        return getattr(symbol, name)
    if name in dir(token):
        return getattr(token, name)
    raise ValueError("Cannot convert '%s'" % name)

def prettyprint(tree, indent=0, omit_singles=False):
    "Prettyprints the tree, with symbol/token names. For debugging."
    if omit_singles and isinstance(tree, tuple) and len(tree) == 2:
        return prettyprint(tree[1], indent, omit_singles)
    if isinstance(tree, tuple):
        return " (\n%s\n" % "".join([prettyprint(x, indent+1) for x in tree]) + \
            (" " * indent) + ")"
    elif isinstance(tree, int):
        return (" " * indent) + name_that_thing(tree)
    else:
        return " " + repr(tree)

def isclass(obj):
    "Simple test to see if something is a class."
    return issubclass(type(obj), type)

def aliased_models(module):
    """
    Given a models module, returns a dict mapping all alias imports of models
    (e.g. import Foo as Bar) back to their original names. Bug #134.
    """
    aliases = {}
    for name, obj in module.__dict__.items():
        if isclass(obj) and issubclass(obj, models.Model) and obj is not models.Model:
            # Test to see if this has a different name to what it should
            if name != obj._meta.object_name:
                aliases[name] = obj._meta.object_name
    return aliases

class STTree(object):

    "A syntax tree wrapper class."

    def __init__(self, tree):
        self.tree = tree

    def __eq__(self, other):
        return other.tree == self.tree

    def __hash__(self):
        return hash(self.tree)

    @property
    def root(self):
        return self.tree[0]

    @property
    def value(self):
        return self.tree

    def walk(self, recursive=True):
        """
        Yields (symbol, subtree) for the entire subtree.
        Comes out with node 1, node 1's children, node 2, etc.
        """
        stack = [self.tree]
        done_outer = False
        while stack:
            atree = stack.pop()
            if isinstance(atree, tuple):
                if done_outer:
                    yield atree[0], STTree(atree)
                if recursive or not done_outer:
                    for bit in reversed(atree[1:]):
                        stack.append(bit)
                    done_outer = True

    def flatten(self):
        "Yields the tokens/symbols in the tree only, in order."
        bits = []
        for sym, subtree in self.walk():
            if sym in token_map:
                bits.append(sym)
            elif sym == token.NAME:
                bits.append(subtree.value)
            elif sym == token.STRING:
                bits.append(subtree.value)
            elif sym == token.NUMBER:
                bits.append(subtree.value)
        return bits

    def reform(self):
        "Prints how the tree's input probably looked."
        return reform(self.flatten())

    def findAllType(self, ntype, recursive=True):
        "Returns all nodes with the given type in the tree."
        for symbol, subtree in self.walk(recursive=recursive):
            if symbol == ntype:
                yield subtree

    def find(self, selector):
        """
        Searches the syntax tree with a CSS-like selector syntax.
        You can use things like 'suite simple_stmt', 'suite, simple_stmt'
        or 'suite > simple_stmt'. Not guaranteed to return in order.
        """
        # Split up the overall parts
        patterns = [x.strip() for x in selector.split(",")]
        results = []
        for pattern in patterns:
            # Split up the parts
            parts = re.split(r'(?:[\s]|(>))+', pattern)
            # Take the first part, use it for results
            if parts[0] == "^":
                subresults = [self]
            else:
                subresults = list(self.findAllType(thing_that_name(parts[0])))
            recursive = True
            # For each remaining part, do something
            for part in parts[1:]:
                if not subresults:
                    break
                if part == ">":
                    recursive = False
                elif not part:
                    pass
                else:
                    thing = thing_that_name(part)
                    newresults = [
                        list(tree.findAllType(thing, recursive))
                        for tree in subresults
                    ]
                    subresults = []
                    for stuff in newresults:
                        subresults.extend(stuff)
                    recursive = True
            results.extend(subresults)
        return results

    def __str__(self):
        return prettyprint(self.tree)
    __repr__ = __str__

def get_model_tree(model):
    # Get the source of the model's file
    try:
        source = inspect.getsource(model).replace("\r\n", "\n").replace("\r","\n") + "\n"
    except IOError:
        return None
    tree = STTree(parser.suite(source).totuple())
    # Now, we have to find it
    for poss in tree.find("compound_stmt"):
        if poss.value[1][0] == symbol.classdef and \
           poss.value[1][2][1].lower() == model.__name__.lower():
            # This is the tree
            return poss

token_map = {
    token.DOT: ".",
    token.LPAR: "(",
    token.RPAR: ")",
    token.EQUAL: "=",
    token.EQEQUAL: "==",
    token.COMMA: ",",
    token.LSQB: "[",
    token.RSQB: "]",
    token.AMPER: "&",
    token.BACKQUOTE: "`",
    token.CIRCUMFLEX: "^",
    token.CIRCUMFLEXEQUAL: "^=",
    token.COLON: ":",
    token.DOUBLESLASH: "//",
    token.DOUBLESLASHEQUAL: "//=",
    token.DOUBLESTAR: "**",
    token.DOUBLESLASHEQUAL: "**=",
    token.GREATER: ">",
    token.LESS: "<",
    token.GREATEREQUAL: ">=",
    token.LESSEQUAL: "<=",
    token.LBRACE: "{",
    token.RBRACE: "}",
    token.SEMI: ";",
    token.PLUS: "+",
    token.MINUS: "-",
    token.STAR: "*",
    token.SLASH: "/",
    token.VBAR: "|",
    token.PERCENT: "%",
    token.TILDE: "~",
    token.AT: "@",
    token.NOTEQUAL: "!=",
    token.LEFTSHIFT: "<<",
    token.RIGHTSHIFT: ">>",
    token.LEFTSHIFTEQUAL: "<<=",
    token.RIGHTSHIFTEQUAL: ">>=",
    token.PLUSEQUAL: "+=",
    token.MINEQUAL: "-=",
    token.STAREQUAL: "*=",
    token.SLASHEQUAL: "/=",
    token.VBAREQUAL: "|=",
    token.PERCENTEQUAL: "%=",
    token.AMPEREQUAL: "&=",
}

def reform(bits):
    "Returns the string that the list of tokens/symbols 'bits' represents"
    output = ""
    for bit in bits:
        if bit in token_map:
            output += token_map[bit]
        elif bit[0] in [token.NAME, token.STRING, token.NUMBER]:
            if keyword.iskeyword(bit[1]):
                output += " %s " % bit[1]
            else:
                if bit[1] not in symbol.sym_name:
                    output += bit[1]
    return output

def parse_arguments(argstr):
    """
    Takes a string representing arguments and returns the positional and
    keyword argument list and dict respectively.
    All the entries in these are python source, except the dict keys.
    """
    # Get the tree
    tree = STTree(parser.suite(argstr).totuple())

    # Initialise the lists
    curr_kwd = None
    args = []
    kwds = {}

    # Walk through, assigning things
    testlists = tree.find("testlist")
    for i, testlist in enumerate(testlists):
        # BTW: A testlist is to the left or right of an =.
        items = list(testlist.walk(recursive=False))
        for j, item in enumerate(items):
            if item[0] == symbol.test:
                if curr_kwd:
                    kwds[curr_kwd] = item[1].reform()
                    curr_kwd = None
                elif j == len(items)-1 and i != len(testlists)-1:
                    # Last item in a group must be a keyword, unless it's last overall
                    curr_kwd = item[1].reform()
                else:
                    args.append(item[1].reform())
    return args, kwds

def extract_field(tree):
    # Collapses the tree and tries to parse it as a field def
    bits = tree.flatten()
    ## Check it looks right:
    # Second token should be equals
    if len(bits) < 2 or bits[1] != token.EQUAL:
        return
    ## Split into meaningful sections
    name = bits[0][1]
    declaration = bits[2:]
    # Find the first LPAR; stuff before that is the class.
    try:
        lpar_at = declaration.index(token.LPAR)
    except ValueError:
        return
    clsname = reform(declaration[:lpar_at])
    # Now, inside that, find the last RPAR, and we'll take the stuff between
    # them as the arguments
    declaration.reverse()
    rpar_at = (len(declaration) - 1) - declaration.index(token.RPAR)
    declaration.reverse()
    args = declaration[lpar_at+1:rpar_at]
    # Now, extract the arguments as a list and dict
    try:
        args, kwargs = parse_arguments(reform(args))
    except SyntaxError:
        return
    # OK, extract and reform it
    return name, clsname, args, kwargs

def get_model_fields(model, m2m=False):
    """
    Given a model class, will return the dict of name: field_constructor
    mappings.
    """
    tree = get_model_tree(model)
    if tree is None:
        return None
    possible_field_defs = tree.find("^ > classdef > suite > stmt > simple_stmt > small_stmt > expr_stmt")
    field_defs = {}

    # Get aliases, ready for alias fixing (#134)
    try:
        aliases = aliased_models(models.get_app(model._meta.app_label))
    except ImproperlyConfigured:
        aliases = {}

    # Go through all the found defns, and try to parse them
    for pfd in possible_field_defs:
        field = extract_field(pfd)
        if field:
            field_defs[field[0]] = field[1:]

    inherited_fields = {}
    # Go through all bases (that are themselves models, but not Model)
    for base in model.__bases__:
        if base != models.Model and issubclass(base, models.Model):
            inherited_fields.update(get_model_fields(base, m2m))

    # Now, go through all the fields and try to get their definition
    source = model._meta.local_fields[:]
    if m2m:
        source += model._meta.local_many_to_many
    fields = SortedDict()
    for field in source:
        # Get its name
        fieldname = field.name
        if isinstance(field, (models.related.RelatedObject, generic.GenericRel)):
            continue
        # Now, try to get the defn
        if fieldname in field_defs:
            fields[fieldname] = field_defs[fieldname]
        # Try the South definition workaround?
        elif hasattr(field, 'south_field_triple'):
            fields[fieldname] = field.south_field_triple()
        elif hasattr(field, 'south_field_definition'):
            print "Your custom field %s provides the outdated south_field_definition method.\nPlease consider implementing south_field_triple too; it's more reliably evaluated." % field
            fields[fieldname] = field.south_field_definition()
        # Try a parent?
        elif fieldname in inherited_fields:
            fields[fieldname] = inherited_fields[fieldname]
        # Is it a _ptr?
        elif fieldname.endswith("_ptr"):
            fields[fieldname] = ("models.OneToOneField", ["orm['%s.%s']" % (field.rel.to._meta.app_label, field.rel.to._meta.object_name)], {})
        # Try a default for 'id'.
        elif fieldname == "id":
            fields[fieldname] = ("models.AutoField", [], {"primary_key": "True"})
        else:
            fields[fieldname] = None

    # Now, try seeing if we can resolve the values of defaults, and fix aliases.
    for field, defn in fields.items():

        if not isinstance(defn, (list, tuple)):
            continue # We don't have a defn for this one, or it's a string

        # Fix aliases if we can (#134)
        for i, arg in enumerate(defn[1]):
            if arg in aliases:
                defn[1][i] = aliases[arg]

        # Fix defaults if we can
        for arg, val in defn[2].items():
            if arg in ['default']:
                try:
                    # Evaluate it in a close-to-real fake model context
                    real_val = eval(val, __import__(model.__module__, {}, {}, ['']).__dict__, model.__dict__)
                # If we can't resolve it, stick it in verbatim
                except:
                    pass # TODO: Raise nice error here?
                # Hm, OK, we got a value. Callables are not frozen (see #132, #135)
                else:
                    if callable(real_val):
                        # HACK
                        # However, if it's datetime.now, etc., that's special
                        for datetime_key in datetime.datetime.__dict__.keys():
                            # No, you can't use __dict__.values. It's different.
                            dtm = getattr(datetime.datetime, datetime_key)
                            if real_val == dtm:
                                if not val.startswith("datetime.datetime"):
                                    defn[2][arg] = "datetime." + val
                                break
                    else:
                        defn[2][arg] = repr(real_val)

    return fields
<commit_msg>Be gone, modelsparser.<commit_after><|endoftext|>


<commit_before>from typing import Optional, List, Dict
from wasabi import Printer
from pathlib import Path
import re
import srsly
from thinc.api import fix_random_seed

from ..training import Corpus
from ..tokens import Doc
from ._util import app, Arg, Opt, setup_gpu, import_code
from ..scorer import Scorer
from .. import util
from .. import displacy

@app.command("evaluate")
def evaluate_cli(
    # fmt: off
    model: str = Arg(..., help="Model name or path"),
    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
    output: Optional[Path] = Opt(None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False),
    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
    gold_preproc: bool = Opt(False, "--gold-preproc", "-G", help="Use gold preprocessing"),
    displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
    displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
    # fmt: on
):
    """
    Evaluate a trained pipeline. Expects a loadable spaCy pipeline and evaluation
    data in the binary .spacy format. The --gold-preproc option sets up the
    evaluation examples with gold-standard sentences and tokens for the
    predictions. Gold preprocessing helps the annotations align to the
    tokenization, and may result in sequences of more consistent length. However,
    it may reduce runtime accuracy due to train/test skew. To render a sample of
    dependency parses in a HTML file, set as output directory as the
    displacy_path argument.

    DOCS: https://nightly.spacy.io/api/cli#evaluate
    """
    import_code(code_path)
    evaluate(
        model,
        data_path,
        output=output,
        use_gpu=use_gpu,
        gold_preproc=gold_preproc,
        displacy_path=displacy_path,
        displacy_limit=displacy_limit,
        silent=False,
    )

def evaluate(
    model: str,
    data_path: Path,
    output: Optional[Path] = None,
    use_gpu: int = -1,
    gold_preproc: bool = False,
    displacy_path: Optional[Path] = None,
    displacy_limit: int = 25,
    silent: bool = True,
) -> Scorer:
    msg = Printer(no_print=silent, pretty=not silent)
    fix_random_seed()
    setup_gpu(use_gpu)
    data_path = util.ensure_path(data_path)
    output_path = util.ensure_path(output)
    displacy_path = util.ensure_path(displacy_path)
    if not data_path.exists():
        msg.fail("Evaluation data not found", data_path, exits=1)
    if displacy_path and not displacy_path.exists():
        msg.fail("Visualization output directory not found", displacy_path, exits=1)
    corpus = Corpus(data_path, gold_preproc=gold_preproc)
    nlp = util.load_model(model)
    dev_dataset = list(corpus(nlp))
    scores = nlp.evaluate(dev_dataset)
    metrics = {
        "TOK": "token_acc",
        "TAG": "tag_acc",
        "POS": "pos_acc",
        "MORPH": "morph_acc",
        "LEMMA": "lemma_acc",
        "UAS": "dep_uas",
        "LAS": "dep_las",
        "NER P": "ents_p",
        "NER R": "ents_r",
        "NER F": "ents_f",
        "TEXTCAT": "cats_score",
        "SENT P": "sents_p",
        "SENT R": "sents_r",
        "SENT F": "sents_f",
        "SPEED": "speed",
    }
    results = {}
    for metric, key in metrics.items():
        if key in scores:
            if key == "cats_score":
                metric = metric + " (" + scores.get("cats_score_desc", "unk") + ")"
            if key == "speed":
                results[metric] = f"{scores[key]:.0f}"
            else:
                results[metric] = f"{scores[key]*100:.2f}"
    data = {re.sub(r"[\s/]", "_", k.lower()): v for k, v in results.items()}

    msg.table(results, title="Results")

    if "ents_per_type" in scores:
        if scores["ents_per_type"]:
            print_ents_per_type(msg, scores["ents_per_type"])
    if "cats_f_per_type" in scores:
        if scores["cats_f_per_type"]:
            print_textcats_f_per_cat(msg, scores["cats_f_per_type"])
    if "cats_auc_per_type" in scores:
        if scores["cats_auc_per_type"]:
            print_textcats_auc_per_cat(msg, scores["cats_auc_per_type"])

    if displacy_path:
        factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names]
        docs = [ex.predicted for ex in dev_dataset]
        render_deps = "parser" in factory_names
        render_ents = "ner" in factory_names
        render_parses(
            docs,
            displacy_path,
            model_name=model,
            limit=displacy_limit,
            deps=render_deps,
            ents=render_ents,
        )
        msg.good(f"Generated {displacy_limit} parses as HTML", displacy_path)

    if output_path is not None:
        srsly.write_json(output_path, data)
        msg.good(f"Saved results to {output_path}")
    return data

def render_parses(
    docs: List[Doc],
    output_path: Path,
    model_name: str = "",
    limit: int = 250,
    deps: bool = True,
    ents: bool = True,
):
    docs[0].user_data["title"] = model_name
    if ents:
        html = displacy.render(docs[:limit], style="ent", page=True)
        with (output_path / "entities.html").open("w", encoding="utf8") as file_:
            file_.write(html)
    if deps:
        html = displacy.render(
            docs[:limit], style="dep", page=True, options={"compact": True}
        )
        with (output_path / "parses.html").open("w", encoding="utf8") as file_:
            file_.write(html)

def print_ents_per_type(msg: Printer, scores: Dict[str, Dict[str, float]]) -> None:
    data = [
        (k, f"{v['p']*100:.2f}", f"{v['r']*100:.2f}", f"{v['f']*100:.2f}")
        for k, v in scores.items()
    ]
    msg.table(
        data,
        header=("", "P", "R", "F"),
        aligns=("l", "r", "r", "r"),
        title="NER (per type)",
    )

def print_textcats_f_per_cat(msg: Printer, scores: Dict[str, Dict[str, float]]) -> None:
    data = [
        (k, f"{v['p']*100:.2f}", f"{v['r']*100:.2f}", f"{v['f']*100:.2f}")
        for k, v in scores.items()
    ]
    msg.table(
        data,
        header=("", "P", "R", "F"),
        aligns=("l", "r", "r", "r"),
        title="Textcat F (per label)",
    )

def print_textcats_auc_per_cat(
    msg: Printer, scores: Dict[str, Dict[str, float]]
) -> None:
    msg.table(
        [(k, f"{v:.2f}") for k, v in scores.items()],
        header=("", "ROC AUC"),
        aligns=("l", "r"),
        title="Textcat ROC AUC (per label)",
    )
<commit_msg>Include per-type/feat scores in evaluate output<commit_after>from typing import Optional, List, Dict
from wasabi import Printer
from pathlib import Path
import re
import srsly
from thinc.api import fix_random_seed

from ..training import Corpus
from ..tokens import Doc
from ._util import app, Arg, Opt, setup_gpu, import_code
from ..scorer import Scorer
from .. import util
from .. import displacy

@app.command("evaluate")
def evaluate_cli(
    # fmt: off
    model: str = Arg(..., help="Model name or path"),
    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
    output: Optional[Path] = Opt(None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False),
    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
    gold_preproc: bool = Opt(False, "--gold-preproc", "-G", help="Use gold preprocessing"),
    displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
    displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
    # fmt: on
):
    """
    Evaluate a trained pipeline. Expects a loadable spaCy pipeline and evaluation
    data in the binary .spacy format. The --gold-preproc option sets up the
    evaluation examples with gold-standard sentences and tokens for the
    predictions. Gold preprocessing helps the annotations align to the
    tokenization, and may result in sequences of more consistent length. However,
    it may reduce runtime accuracy due to train/test skew. To render a sample of
    dependency parses in a HTML file, set as output directory as the
    displacy_path argument.

    DOCS: https://nightly.spacy.io/api/cli#evaluate
    """
    import_code(code_path)
    evaluate(
        model,
        data_path,
        output=output,
        use_gpu=use_gpu,
        gold_preproc=gold_preproc,
        displacy_path=displacy_path,
        displacy_limit=displacy_limit,
        silent=False,
    )

def evaluate(
    model: str,
    data_path: Path,
    output: Optional[Path] = None,
    use_gpu: int = -1,
    gold_preproc: bool = False,
    displacy_path: Optional[Path] = None,
    displacy_limit: int = 25,
    silent: bool = True,
) -> Scorer:
    msg = Printer(no_print=silent, pretty=not silent)
    fix_random_seed()
    setup_gpu(use_gpu)
    data_path = util.ensure_path(data_path)
    output_path = util.ensure_path(output)
    displacy_path = util.ensure_path(displacy_path)
    if not data_path.exists():
        msg.fail("Evaluation data not found", data_path, exits=1)
    if displacy_path and not displacy_path.exists():
        msg.fail("Visualization output directory not found", displacy_path, exits=1)
    corpus = Corpus(data_path, gold_preproc=gold_preproc)
    nlp = util.load_model(model)
    dev_dataset = list(corpus(nlp))
    scores = nlp.evaluate(dev_dataset)
    metrics = {
        "TOK": "token_acc",
        "TAG": "tag_acc",
        "POS": "pos_acc",
        "MORPH": "morph_acc",
        "LEMMA": "lemma_acc",
        "UAS": "dep_uas",
        "LAS": "dep_las",
        "NER P": "ents_p",
        "NER R": "ents_r",
        "NER F": "ents_f",
        "TEXTCAT": "cats_score",
        "SENT P": "sents_p",
        "SENT R": "sents_r",
        "SENT F": "sents_f",
        "SPEED": "speed",
    }
    results = {}
    for metric, key in metrics.items():
        if key in scores:
            if key == "cats_score":
                metric = metric + " (" + scores.get("cats_score_desc", "unk") + ")"
            if key == "speed":
                results[metric] = f"{scores[key]:.0f}"
            else:
                results[metric] = f"{scores[key]*100:.2f}"
    data = {re.sub(r"[\s/]", "_", k.lower()): v for k, v in results.items()}

    msg.table(results, title="Results")

    if "morph_per_feat" in scores:
        if scores["morph_per_feat"]:
            print_morph_per_feat(msg, scores["morph_per_feat"])
            data["morph_per_feat"] = scores["morph_per_feat"]
    if "ents_per_type" in scores:
        if scores["ents_per_type"]:
            print_ents_per_type(msg, scores["ents_per_type"])
            data["ents_per_type"] = scores["ents_per_type"]
    if "cats_f_per_type" in scores:
        if scores["cats_f_per_type"]:
            print_textcats_f_per_cat(msg, scores["cats_f_per_type"])
            data["cats_f_per_type"] = scores["cats_f_per_type"]
    if "cats_auc_per_type" in scores:
        if scores["cats_auc_per_type"]:
            print_textcats_auc_per_cat(msg, scores["cats_auc_per_type"])
            data["cats_auc_per_type"] = scores["cats_auc_per_type"]

    if displacy_path:
        factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names]
        docs = [ex.predicted for ex in dev_dataset]
        render_deps = "parser" in factory_names
        render_ents = "ner" in factory_names
        render_parses(
            docs,
            displacy_path,
            model_name=model,
            limit=displacy_limit,
            deps=render_deps,
            ents=render_ents,
        )
        msg.good(f"Generated {displacy_limit} parses as HTML", displacy_path)

    if output_path is not None:
        srsly.write_json(output_path, data)
        msg.good(f"Saved results to {output_path}")
    return data

def render_parses(
    docs: List[Doc],
    output_path: Path,
    model_name: str = "",
    limit: int = 250,
    deps: bool = True,
    ents: bool = True,
):
    docs[0].user_data["title"] = model_name
    if ents:
        html = displacy.render(docs[:limit], style="ent", page=True)
        with (output_path / "entities.html").open("w", encoding="utf8") as file_:
            file_.write(html)
    if deps:
        html = displacy.render(
            docs[:limit], style="dep", page=True, options={"compact": True}
        )
        with (output_path / "parses.html").open("w", encoding="utf8") as file_:
            file_.write(html)

def print_morph_per_feat(msg: Printer, scores: Dict[str, Dict[str, float]]) -> None:
    data = [
        (k, f"{v['p']*100:.2f}", f"{v['r']*100:.2f}", f"{v['f']*100:.2f}")
        for k, v in scores.items()
    ]
    msg.table(
        data,
        header=("", "P", "R", "F"),
        aligns=("l", "r", "r", "r"),
        title="MORPH (per feat)",
    )

def print_ents_per_type(msg: Printer, scores: Dict[str, Dict[str, float]]) -> None:
    data = [
        (k, f"{v['p']*100:.2f}", f"{v['r']*100:.2f}", f"{v['f']*100:.2f}")
        for k, v in scores.items()
    ]
    msg.table(
        data,
        header=("", "P", "R", "F"),
        aligns=("l", "r", "r", "r"),
        title="NER (per type)",
    )

def print_textcats_f_per_cat(msg: Printer, scores: Dict[str, Dict[str, float]]) -> None:
    data = [
        (k, f"{v['p']*100:.2f}", f"{v['r']*100:.2f}", f"{v['f']*100:.2f}")
        for k, v in scores.items()
    ]
    msg.table(
        data,
        header=("", "P", "R", "F"),
        aligns=("l", "r", "r", "r"),
        title="Textcat F (per label)",
    )

def print_textcats_auc_per_cat(
    msg: Printer, scores: Dict[str, Dict[str, float]]
) -> None:
    msg.table(
        [(k, f"{v:.2f}") for k, v in scores.items()],
        header=("", "ROC AUC"),
        aligns=("l", "r"),
        title="Textcat ROC AUC (per label)",
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Nicolas P. Rougier
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------
"""
A collection is a container for several items having the same data
structure (dtype). Each data type can be declared as local (it specific to a
vertex), shared (it is shared among an item vertices) or global (it is shared
by all vertices). It is based on the BaseCollection but offers a more intuitive
interface.
"""

import numpy as np
from ... import gloo
from . util import fetchcode
from . base_collection import BaseCollection
from ..shaders import ModularProgram
from ...util.event import EventEmitter

class Collection(BaseCollection):

    """
    A collection is a container for several items having the same data
    structure (dtype). Each data type can be declared as local (it is specific
    to a vertex), shared (it is shared among item vertices) or global (it is
    shared by all items). It is based on the BaseCollection but offers a more
    intuitive interface.

    Parameters
    ----------

    dtype: list
        Data individual types as (name, dtype, scope, default)

    itype: np.dtype or None
        Indices data type

    mode : GL_ENUM
        GL_POINTS, GL_LINES, GL_LINE_STRIP, GL_LINE_LOOP,
        GL_TRIANGLES, GL_TRIANGLE_STRIP, GL_TRIANGLE_FAN

    vertex: str or tuple of str
       Vertex shader to use to draw this collection

    fragment:  str or tuple of str
       Fragment shader to use to draw this collection

    kwargs: str
        Scope can also be specified using keyword argument,
        where parameter name must be one of the dtype.
    """

    _gtypes = {('float32', 1): "float",
               ('float32', 2): "vec2",
               ('float32', 3): "vec3",
               ('float32', 4): "vec4",
               ('int32', 1): "int",
               ('int32', 2): "ivec2",
               ('int32', 3): "ivec3",
               ('int32', 4): "ivec4"}

    def __init__(self, dtype, itype, mode, vertex, fragment, program=None,
                 **kwargs):
        """
        """

        self._uniforms = {}
        self._attributes = {}
        self._varyings = {}
        self._mode = mode
        vtype = []
        utype = []

        self.update = EventEmitter(source=self, type='collection_update')

        # Build vtype and utype according to parameters
        declarations = {"uniforms": "",
                        "attributes": "",
                        "varyings": ""}
        defaults = {}
        for item in dtype:
            name, (basetype, count), scope, default = item
            basetype = np.dtype(basetype).name
            if scope[0] == "!":
                scope = scope[1:]
            else:
                scope = kwargs.pop(name, scope)
            defaults[name] = default
            gtype = Collection._gtypes[(basetype, count)]
            if scope == "local":
                vtype.append((name, basetype, count))
                declarations[
                    "attributes"] += "attribute %s %s;\n" % (gtype, name)
            elif scope == "shared":
                utype.append((name, basetype, count))
                declarations["varyings"] += "varying %s %s;\n" % (gtype, name)
            else:
                declarations["uniforms"] += "uniform %s %s;\n" % (gtype, name)
                self._uniforms[name] = None

        if len(kwargs) > 0:
            raise NameError("Invalid keyword argument(s): %s" %
                            list(kwargs.keys()))

        vtype = np.dtype(vtype)
        itype = np.dtype(itype) if itype else None
        utype = np.dtype(utype) if utype else None

        BaseCollection.__init__(self, vtype=vtype, utype=utype, itype=itype)
        self._declarations = declarations
        self._defaults = defaults

        # Build program (once base collection is built)
        saved = vertex
        vertex = ""

        if self.utype is not None:
            vertex += fetchcode(self.utype) + vertex
        else:
            vertex += "void fetch_uniforms(void) { }\n" + vertex
        vertex += self._declarations["uniforms"]
        vertex += self._declarations["attributes"]
        vertex += saved

        self._vertex = vertex
        self._fragment = fragment

        if program is None:
            program = ModularProgram(vertex, fragment)
        else:
            program.vert = vertex
            program.frag = fragment
        if hasattr(program, 'changed'):
            program.changed.connect(self.update)
        self._programs.append(program)

        # Initialize uniforms
        for name in self._uniforms.keys():
            self._uniforms[name] = self._defaults.get(name)
            program[name] = self._uniforms[name]

    def view(self, transform, viewport=None):
        """ Return a view on the collection using provided transform """

        return CollectionView(self, transform, viewport)

        # program = gloo.Program(self._vertex, self._fragment)
        # if "transform" in program.hooks:
        #     program["transform"] = transform
        # if "viewport" in program.hooks:
        #     if viewport is not None:
        #         program["viewport"] = viewport
        #     else:
        #         program["viewport"] = Viewport()
        # self._programs.append(program)
        # program.bind(self._vertices_buffer)
        # for name in self._uniforms.keys():
        #     program[name] = self._uniforms[name]
        # #if self._uniforms_list is not None:
        # #    program["uniforms"] = self._uniforms_texture
        # #    program["uniforms_shape"] = self._ushape

        # # Piggy backing
        # def draw():
        #     if self._need_update:
        #         self._update()
        #         program.bind(self._vertices_buffer)
        #         if self._uniforms_list is not None:
        #             program["uniforms"] = self._uniforms_texture
        #             program["uniforms_shape"] = self._ushape

        #     if self._indices_list is not None:
        #         Program.draw(program, self._mode, self._indices_buffer)
        #     else:
        #         Program.draw(program, self._mode)

        # program.draw = draw
        # return program

    def __getitem__(self, key):

        program = self._programs[0]
        for name, (storage, _, _) in program._code_variables.items():
            if name == key and storage == 'uniform':
                return program[key]
        return BaseCollection.__getitem__(self, key)

    def __setitem__(self, key, value):
        try:
            BaseCollection.__setitem__(self, key, value)
        except IndexError:
            for program in self._programs:
                program[key] = value

    def draw(self, mode=None):
        """ Draw collection """

        if self._need_update:
            self._update()

        program = self._programs[0]

        mode = mode or self._mode
        if self._indices_list is not None:
            program.draw(mode, self._indices_buffer)
        else:
            program.draw(mode)

class CollectionView(object):

    def __init__(self, collection, transform=None, viewport=None):

        vertex = collection._vertex
        fragment = collection._fragment
        program = gloo.Program(vertex, fragment)

#        if "transform" in program.hooks and transform is not None:
#            program["transform"] = transform
#        if "viewport" in program.hooks and viewport is not None:
#            program["viewport"] = viewport

        program.bind(collection._vertices_buffer)
        for name in collection._uniforms.keys():
            program[name] = collection._uniforms[name]

        collection._programs.append(program)
        self._program = program
        self._collection = collection

    def __getitem__(self, key):
        return self._program[key]

    def __setitem__(self, key, value):
        self._program[key] = value

    def draw(self):

        program = self._program
        collection = self._collection
        mode = collection._mode

        if collection._need_update:
            collection._update()
            # self._program.bind(self._vertices_buffer)
            if collection._uniforms_list is not None:
                program["uniforms"] = collection._uniforms_texture
                program["uniforms_shape"] = collection._ushape

        if collection._indices_list is not None:
            program.draw(mode, collection._indices_buffer)
        else:
            program.draw(mode)
<commit_msg>Fix additional numpy warnings when dealing with dtypes of size 1<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Nicolas P. Rougier
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------
"""
A collection is a container for several items having the same data
structure (dtype). Each data type can be declared as local (it specific to a
vertex), shared (it is shared among an item vertices) or global (it is shared
by all vertices). It is based on the BaseCollection but offers a more intuitive
interface.
"""

import numpy as np
from ... import gloo
from . util import fetchcode
from . base_collection import BaseCollection
from ..shaders import ModularProgram
from ...util.event import EventEmitter

class Collection(BaseCollection):

    """
    A collection is a container for several items having the same data
    structure (dtype). Each data type can be declared as local (it is specific
    to a vertex), shared (it is shared among item vertices) or global (it is
    shared by all items). It is based on the BaseCollection but offers a more
    intuitive interface.

    Parameters
    ----------

    dtype: list
        Data individual types as (name, dtype, scope, default)

    itype: np.dtype or None
        Indices data type

    mode : GL_ENUM
        GL_POINTS, GL_LINES, GL_LINE_STRIP, GL_LINE_LOOP,
        GL_TRIANGLES, GL_TRIANGLE_STRIP, GL_TRIANGLE_FAN

    vertex: str or tuple of str
       Vertex shader to use to draw this collection

    fragment:  str or tuple of str
       Fragment shader to use to draw this collection

    kwargs: str
        Scope can also be specified using keyword argument,
        where parameter name must be one of the dtype.
    """

    _gtypes = {('float32', 1): "float",
               ('float32', 2): "vec2",
               ('float32', 3): "vec3",
               ('float32', 4): "vec4",
               ('int32', 1): "int",
               ('int32', 2): "ivec2",
               ('int32', 3): "ivec3",
               ('int32', 4): "ivec4"}

    def __init__(self, dtype, itype, mode, vertex, fragment, program=None,
                 **kwargs):
        """
        """

        self._uniforms = {}
        self._attributes = {}
        self._varyings = {}
        self._mode = mode
        vtype = []
        utype = []

        self.update = EventEmitter(source=self, type='collection_update')

        # Build vtype and utype according to parameters
        declarations = {"uniforms": "",
                        "attributes": "",
                        "varyings": ""}
        defaults = {}
        for item in dtype:
            name, (basetype, count), scope, default = item
            basetype = np.dtype(basetype).name
            if scope[0] == "!":
                scope = scope[1:]
            else:
                scope = kwargs.pop(name, scope)
            defaults[name] = default
            gtype = Collection._gtypes[(basetype, count)]
            if scope == "local":
                # numpy dtypes with size 1 are ambiguous, only add size if it is greater than 1
                vtype.append((name, basetype, count) if count != 1 else (name, basetype))
                declarations[
                    "attributes"] += "attribute %s %s;\n" % (gtype, name)
            elif scope == "shared":
                # numpy dtypes with size 1 are ambiguous, only add size if it is greater than 1
                utype.append((name, basetype, count) if count != 1 else (name, basetype))
                declarations["varyings"] += "varying %s %s;\n" % (gtype, name)
            else:
                declarations["uniforms"] += "uniform %s %s;\n" % (gtype, name)
                self._uniforms[name] = None

        if len(kwargs) > 0:
            raise NameError("Invalid keyword argument(s): %s" %
                            list(kwargs.keys()))

        vtype = np.dtype(vtype)
        itype = np.dtype(itype) if itype else None
        utype = np.dtype(utype) if utype else None

        BaseCollection.__init__(self, vtype=vtype, utype=utype, itype=itype)
        self._declarations = declarations
        self._defaults = defaults

        # Build program (once base collection is built)
        saved = vertex
        vertex = ""

        if self.utype is not None:
            vertex += fetchcode(self.utype) + vertex
        else:
            vertex += "void fetch_uniforms(void) { }\n" + vertex
        vertex += self._declarations["uniforms"]
        vertex += self._declarations["attributes"]
        vertex += saved

        self._vertex = vertex
        self._fragment = fragment

        if program is None:
            program = ModularProgram(vertex, fragment)
        else:
            program.vert = vertex
            program.frag = fragment
        if hasattr(program, 'changed'):
            program.changed.connect(self.update)
        self._programs.append(program)

        # Initialize uniforms
        for name in self._uniforms.keys():
            self._uniforms[name] = self._defaults.get(name)
            program[name] = self._uniforms[name]

    def view(self, transform, viewport=None):
        """ Return a view on the collection using provided transform """

        return CollectionView(self, transform, viewport)

        # program = gloo.Program(self._vertex, self._fragment)
        # if "transform" in program.hooks:
        #     program["transform"] = transform
        # if "viewport" in program.hooks:
        #     if viewport is not None:
        #         program["viewport"] = viewport
        #     else:
        #         program["viewport"] = Viewport()
        # self._programs.append(program)
        # program.bind(self._vertices_buffer)
        # for name in self._uniforms.keys():
        #     program[name] = self._uniforms[name]
        # #if self._uniforms_list is not None:
        # #    program["uniforms"] = self._uniforms_texture
        # #    program["uniforms_shape"] = self._ushape

        # # Piggy backing
        # def draw():
        #     if self._need_update:
        #         self._update()
        #         program.bind(self._vertices_buffer)
        #         if self._uniforms_list is not None:
        #             program["uniforms"] = self._uniforms_texture
        #             program["uniforms_shape"] = self._ushape

        #     if self._indices_list is not None:
        #         Program.draw(program, self._mode, self._indices_buffer)
        #     else:
        #         Program.draw(program, self._mode)

        # program.draw = draw
        # return program

    def __getitem__(self, key):

        program = self._programs[0]
        for name, (storage, _, _) in program._code_variables.items():
            if name == key and storage == 'uniform':
                return program[key]
        return BaseCollection.__getitem__(self, key)

    def __setitem__(self, key, value):
        try:
            BaseCollection.__setitem__(self, key, value)
        except IndexError:
            for program in self._programs:
                program[key] = value

    def draw(self, mode=None):
        """ Draw collection """

        if self._need_update:
            self._update()

        program = self._programs[0]

        mode = mode or self._mode
        if self._indices_list is not None:
            program.draw(mode, self._indices_buffer)
        else:
            program.draw(mode)

class CollectionView(object):

    def __init__(self, collection, transform=None, viewport=None):

        vertex = collection._vertex
        fragment = collection._fragment
        program = gloo.Program(vertex, fragment)

#        if "transform" in program.hooks and transform is not None:
#            program["transform"] = transform
#        if "viewport" in program.hooks and viewport is not None:
#            program["viewport"] = viewport

        program.bind(collection._vertices_buffer)
        for name in collection._uniforms.keys():
            program[name] = collection._uniforms[name]

        collection._programs.append(program)
        self._program = program
        self._collection = collection

    def __getitem__(self, key):
        return self._program[key]

    def __setitem__(self, key, value):
        self._program[key] = value

    def draw(self):

        program = self._program
        collection = self._collection
        mode = collection._mode

        if collection._need_update:
            collection._update()
            # self._program.bind(self._vertices_buffer)
            if collection._uniforms_list is not None:
                program["uniforms"] = collection._uniforms_texture
                program["uniforms_shape"] = collection._ushape

        if collection._indices_list is not None:
            program.draw(mode, collection._indices_buffer)
        else:
            program.draw(mode)
<|endoftext|>


<commit_before># -*- test-case-name: vumi.transports.smpp.tests.test_mica -*-

from vumi.config import ConfigInt
from vumi.components.session import SessionManager
from vumi.message import TransportUserMessage
from vumi.transports.smpp.processors import default

from twisted.internet.defer import inlineCallbacks, returnValue

def make_vumi_session_identifier(msisdn, mica_session_identifier):
    return '+'.join(map(unicode, [msisdn, mica_session_identifier]))

class DeliverShortMessageProcessorConfig(
        default.DeliverShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class DeliverShortMessageProcessor(default.DeliverShortMessageProcessor):

    CONFIG_CLASS = DeliverShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(DeliverShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_deliver_sm_ussd(self, pdu, pdu_params, pdu_opts):
        service_op = pdu_opts['ussd_service_op']
        mica_session_identifier = pdu_opts['user_message_reference']
        vumi_session_identifier = make_vumi_session_identifier(
            pdu_params['source_addr'], mica_session_identifier)

        session_event = 'close'
        if service_op == '01':
            # PSSR request. Let's assume it means a new session.
            session_event = 'new'
            ussd_code = pdu_params['destination_addr']
            content = None

            yield self.session_manager.create_session(
                vumi_session_identifier, ussd_code=ussd_code)

        elif service_op == '17':
            # PSSR response. This means session end.
            session_event = 'close'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = None

            yield self.session_manager.clear_session(vumi_session_identifier)

        else:
            session_event = 'continue'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = pdu_params['short_message']

        # This is stashed on the message and available when replying
        # with a `submit_sm`
        session_info = {
            'session_identifier': mica_session_identifier,
        }

        decoded_msg = self.decode_message(content,
                                          pdu_params['data_coding'])

        result = yield self.handle_short_message_content(
            source_addr=pdu_params['source_addr'],
            destination_addr=ussd_code,
            short_message=decoded_msg,
            message_type='ussd',
            session_event=session_event,
            session_info=session_info)
        returnValue(result)

class SubmitShortMessageProcessorConfig(
        default.SubmitShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class SubmitShortMessageProcessor(default.SubmitShortMessageProcessor):

    CONFIG_CLASS = SubmitShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(SubmitShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_outbound_message(self, message, protocol):
        to_addr = message['to_addr']
        from_addr = message['from_addr']
        text = message['content']

        session_event = message['session_event']
        transport_type = message['transport_type']
        optional_parameters = {}

        if transport_type == 'ussd':
            continue_session = (
                session_event != TransportUserMessage.SESSION_CLOSE)
            session_info = message['transport_metadata'].get(
                'session_info', {})
            mica_session_identifier = session_info.get(
                'session_identifier', '')
            vumi_session_identifier = make_vumi_session_identifier(
                to_addr, mica_session_identifier)

            optional_parameters.update({
                'ussd_service_op': ('02' if continue_session else '17'),
                'user_message_reference': mica_session_identifier,
            })

            if not continue_session:
                yield self.session_manager.clear_session(
                    vumi_session_identifier)

        if self.config.send_long_messages:
            resp = yield protocol.submit_sm_long(
                to_addr.encode('ascii'),
                long_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_sar:
            resp = yield protocol.submit_csm_sar(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_udh:
            resp = yield protocol.submit_csm_udh(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )
        else:
            resp = yield protocol.submit_sm(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        returnValue(resp)
<commit_msg>use string formatting instead of unicode & join<commit_after># -*- test-case-name: vumi.transports.smpp.tests.test_mica -*-

from vumi.config import ConfigInt
from vumi.components.session import SessionManager
from vumi.message import TransportUserMessage
from vumi.transports.smpp.processors import default

from twisted.internet.defer import inlineCallbacks, returnValue

def make_vumi_session_identifier(msisdn, mica_session_identifier):
    # NOTE: mica_session_identifier is always an int.
    return '%s+%s' % (msisdn, str(mica_session_identifier))

class DeliverShortMessageProcessorConfig(
        default.DeliverShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class DeliverShortMessageProcessor(default.DeliverShortMessageProcessor):

    CONFIG_CLASS = DeliverShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(DeliverShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_deliver_sm_ussd(self, pdu, pdu_params, pdu_opts):
        service_op = pdu_opts['ussd_service_op']
        mica_session_identifier = pdu_opts['user_message_reference']
        vumi_session_identifier = make_vumi_session_identifier(
            pdu_params['source_addr'], mica_session_identifier)

        session_event = 'close'
        if service_op == '01':
            # PSSR request. Let's assume it means a new session.
            session_event = 'new'
            ussd_code = pdu_params['destination_addr']
            content = None

            yield self.session_manager.create_session(
                vumi_session_identifier, ussd_code=ussd_code)

        elif service_op == '17':
            # PSSR response. This means session end.
            session_event = 'close'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = None

            yield self.session_manager.clear_session(vumi_session_identifier)

        else:
            session_event = 'continue'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = pdu_params['short_message']

        # This is stashed on the message and available when replying
        # with a `submit_sm`
        session_info = {
            'session_identifier': mica_session_identifier,
        }

        decoded_msg = self.decode_message(content,
                                          pdu_params['data_coding'])

        result = yield self.handle_short_message_content(
            source_addr=pdu_params['source_addr'],
            destination_addr=ussd_code,
            short_message=decoded_msg,
            message_type='ussd',
            session_event=session_event,
            session_info=session_info)
        returnValue(result)

class SubmitShortMessageProcessorConfig(
        default.SubmitShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class SubmitShortMessageProcessor(default.SubmitShortMessageProcessor):

    CONFIG_CLASS = SubmitShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(SubmitShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_outbound_message(self, message, protocol):
        to_addr = message['to_addr']
        from_addr = message['from_addr']
        text = message['content']

        session_event = message['session_event']
        transport_type = message['transport_type']
        optional_parameters = {}

        if transport_type == 'ussd':
            continue_session = (
                session_event != TransportUserMessage.SESSION_CLOSE)
            session_info = message['transport_metadata'].get(
                'session_info', {})
            mica_session_identifier = session_info.get(
                'session_identifier', '')
            vumi_session_identifier = make_vumi_session_identifier(
                to_addr, mica_session_identifier)

            optional_parameters.update({
                'ussd_service_op': ('02' if continue_session else '17'),
                'user_message_reference': mica_session_identifier,
            })

            if not continue_session:
                yield self.session_manager.clear_session(
                    vumi_session_identifier)

        if self.config.send_long_messages:
            resp = yield protocol.submit_sm_long(
                to_addr.encode('ascii'),
                long_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_sar:
            resp = yield protocol.submit_csm_sar(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_udh:
            resp = yield protocol.submit_csm_udh(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )
        else:
            resp = yield protocol.submit_sm(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        returnValue(resp)
<|endoftext|>