<commit_before>#!/usr/bin/env/python

import pygame
import numpy as np

from note_file_parser import note_file_parser

def int_max_value(bits,signed=True):
    """Returns the maximum int value of a signed or unsigned integer
    based on used bits.
    Arguments:
    bits -- How many bits, e.g., 16
    signed -- True if a signed int
    Returns:
    max_value -- The maximum int value based on given parameters
    """
    if signed:
        # Signed int
        max_value = pow(2,bits-1)-1
    else:
        # Unsigned int
        max_value = pow(2,bits)-1
    return max_value

# Add the data points to an array (create the wave)
def create_note(bits,sampling_rate,volume,freq,duration):
    """Creates a single note as a sine wave.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volume -- Volume, from 0 to 1 (max)
    freq -- The frequency of the note
    duration -- Duration of the note in seconds
    Returns:
    snd_array -- A list of sine wave values based on the current note
    """
    # Make sure volume between 0 and 1
    # The max int value will be multiplied by this
    if volume<0.0: volume = 0.0
    if volume>1.0: volume = 1.0
    # Calculate the maximum int value based on the used bit value (signed int)
    # Basically: what is the maximum signed int value for the amount of bits selected
    max_value = int_max_value(bits)
    # An empty list
    note = []
    # Fill the list with sine wave values
    for x in range(0,int(sampling_rate*duration)):
        value = volume * max_value * np.sin(2 * np.pi * freq * x / sampling_rate)
        note.append(value)
    return note

def create_melody(bits,sampling_rate,volumes,freqs,durations):
    """Creates and returns a melody consisting of one or more notes.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volumes -- A list of volumes for individual notes
    freqs -- A list of frequencies for individual notes
    durations -- Note durations in a list
    Returns:
    melody -- A list of wave values based on the current melody
    """
    melody = []
    # Go through the list
    for volume,freq,duration in zip(volumes,freqs,durations):
        # Create an individual note
        note = create_note(bits,sampling_rate,volume,freq,duration)
        # Add a note to the melody list
        melody.extend(note)
    return melody

def calculate_note_freq(note):
    """Calculates the frequency of a give note based on it's name and frequency of A4=440Hz.
    All notes and octaves are supported. Not case sensitive.
    Arguments:
    note -- The note name and octave, e.g., 'C4' (string)
    Returns:
    The frequency of the given note
    """
    # The name of the note, e.g., 'A'
    note_name = note[0].upper()
    # The octave of the note, e.g., 4
    note_octave = int(note[1])
    # The note name list
    notes = ['C','C#','D','D#','E','F','F#','G','G#','A','A#','B']
    # The index of A in the list
    baseind = 9
    # The index of the note in the list
    noteind = notes.index(note_name)
    # The index difference (how many semitones difference from A)
    n = noteind - baseind
    # Take into account the octave (relative to A4, that's why we have 4 here)
    # Now n is the number of semitones difference from the note and A4
    n = n + (4-note_octave)*12
    # Approximately 2^(1/12)
    a = 1.059463094359
    # Frequency of A4
    f0 = 440
    # The frequency calculation
    freq = f0 * pow(a,n)
    return freq

def main():
    # Set some variables
    sampling_rate = 8000
    bits = 16
    # In this case: mono
    channels = 1
    filename = 'tests/notefile.txt'
    notes,durations = note_file_parser(filename)
    # The duration this program is alive, right now the same as note duration
    wait_duration = sum(durations)
    # In this case all volume values are 0.5
    volumes = []
    for i in range(len(notes)):
        volumes.append(0.5)
    # Note frequencies
    freqs = []
    for note in notes:
        freqs.append(calculate_note_freq(note))

    # Initialize pygame mixer
    pygame.mixer.pre_init(sampling_rate, -bits, channels)
    pygame.init()

    # Create the wave
    melody = create_melody(bits,sampling_rate,volumes,freqs,durations)
    # Create a numpy array of the list, needed later.
    # Note: We don't create a numpy array earlier, because when
    # appending values to it, a new array is always created.
    # That is not efficient.
    # Also, in this phase the whole array is converted to integers.
    # Floats cannot be used for pygame sndarray.
    melody = np.array(melody).astype(np.int16)
    # Get the sound based on the array
    sound = pygame.sndarray.make_sound(melody)
    # Play and loop
    sound.play()
    # Stop after <duration>
    pygame.time.delay(int(wait_duration*1000))
    # Stop playing
    sound.stop()

if __name__=="__main__":
    main()
<commit_msg>Fixes #1 , volumes now read from the notefile<commit_after>#!/usr/bin/env/python

import pygame
import numpy as np

from note_file_parser import note_file_parser

def int_max_value(bits,signed=True):
    """Returns the maximum int value of a signed or unsigned integer
    based on used bits.
    Arguments:
    bits -- How many bits, e.g., 16
    signed -- True if a signed int
    Returns:
    max_value -- The maximum int value based on given parameters
    """
    if signed:
        # Signed int
        max_value = pow(2,bits-1)-1
    else:
        # Unsigned int
        max_value = pow(2,bits)-1
    return max_value

# Add the data points to an array (create the wave)
def create_note(bits,sampling_rate,volume,freq,duration):
    """Creates a single note as a sine wave.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volume -- Volume, from 0 to 1 (max)
    freq -- The frequency of the note
    duration -- Duration of the note in seconds
    Returns:
    snd_array -- A list of sine wave values based on the current note
    """
    # Make sure volume between 0 and 1
    # The max int value will be multiplied by this
    if volume<0.0: volume = 0.0
    if volume>1.0: volume = 1.0
    # Calculate the maximum int value based on the used bit value (signed int)
    # Basically: what is the maximum signed int value for the amount of bits selected
    max_value = int_max_value(bits)
    # An empty list
    note = []
    # Fill the list with sine wave values
    for x in range(0,int(sampling_rate*duration)):
        value = volume * max_value * np.sin(2 * np.pi * freq * x / sampling_rate)
        note.append(value)
    return note

def create_melody(bits,sampling_rate,volumes,freqs,durations):
    """Creates and returns a melody consisting of one or more notes.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volumes -- A list of volumes for individual notes
    freqs -- A list of frequencies for individual notes
    durations -- Note durations in a list
    Returns:
    melody -- A list of wave values based on the current melody
    """
    melody = []
    # Go through the list
    for volume,freq,duration in zip(volumes,freqs,durations):
        # Create an individual note
        note = create_note(bits,sampling_rate,volume,freq,duration)
        # Add a note to the melody list
        melody.extend(note)
    return melody

def calculate_note_freq(note):
    """Calculates the frequency of a give note based on it's name and frequency of A4=440Hz.
    All notes and octaves are supported. Not case sensitive.
    Arguments:
    note -- The note name and octave, e.g., 'C4' (string)
    Returns:
    The frequency of the given note
    """
    # The name of the note, e.g., 'A'
    note_name = note[0].upper()
    # The octave of the note, e.g., 4
    note_octave = int(note[1])
    # The note name list
    notes = ['C','C#','D','D#','E','F','F#','G','G#','A','A#','B']
    # The index of A in the list
    baseind = 9
    # The index of the note in the list
    noteind = notes.index(note_name)
    # The index difference (how many semitones difference from A)
    n = noteind - baseind
    # Take into account the octave (relative to A4, that's why we have 4 here)
    # Now n is the number of semitones difference from the note and A4
    n = n + (4-note_octave)*12
    # Approximately 2^(1/12)
    a = 1.059463094359
    # Frequency of A4
    f0 = 440
    # The frequency calculation
    freq = f0 * pow(a,n)
    return freq

def main():
    # Set some variables
    sampling_rate = 8000
    bits = 16
    # In this case: mono
    channels = 1
    filename = 'tests/notefile.txt'
    notes,durations,volumes = note_file_parser(filename)
    # The duration this program is alive, right now the same as note duration
    wait_duration = sum(durations)
    # Note frequencies
    freqs = []
    for note in notes:
        freqs.append(calculate_note_freq(note))

    # Initialize pygame mixer
    pygame.mixer.pre_init(sampling_rate, -bits, channels)
    pygame.init()

    # Create the wave
    melody = create_melody(bits,sampling_rate,volumes,freqs,durations)
    # Create a numpy array of the list, needed later.
    # Note: We don't create a numpy array earlier, because when
    # appending values to it, a new array is always created.
    # That is not efficient.
    # Also, in this phase the whole array is converted to integers.
    # Floats cannot be used for pygame sndarray.
    melody = np.array(melody).astype(np.int16)
    # Get the sound based on the array
    sound = pygame.sndarray.make_sound(melody)
    # Play and loop
    sound.play()
    # Stop after <duration>
    pygame.time.delay(int(wait_duration*1000))
    # Stop playing
    sound.stop()

if __name__=="__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Representation of type information for DBCore model fields.
"""
from __future__ import division, absolute_import, print_function

from . import query
from beets.util import str2bool
import six

if not six.PY2:
    buffer = memoryview  # sqlite won't accept memoryview in python 2

# Abstract base.

class Type(object):
    """An object encapsulating the type of a model field. Includes
    information about how to store, query, format, and parse a given
    field.
    """

    sql = u'TEXT'
    """The SQLite column type for the value.
    """

    query = query.SubstringQuery
    """The `Query` subclass to be used when querying the field.
    """

    model_type = six.text_type
    """The Python type that is used to represent the value in the model.

    The model is guaranteed to return a value of this type if the field
    is accessed.  To this end, the constructor is used by the `normalize`
    and `from_sql` methods and the `default` property.
    """

    @property
    def null(self):
        """The value to be exposed when the underlying value is None.
        """
        return self.model_type()

    def format(self, value):
        """Given a value of this type, produce a Unicode string
        representing the value. This is used in template evaluation.
        """
        if value is None:
            value = self.null
        # `self.null` might be `None`
        if value is None:
            value = u''
        if isinstance(value, bytes):
            value = value.decode('utf-8', 'ignore')

        return six.text_type(value)

    def parse(self, string):
        """Parse a (possibly human-written) string and return the
        indicated value of this type.
        """
        try:
            return self.model_type(string)
        except ValueError:
            return self.null

    def normalize(self, value):
        """Given a value that will be assigned into a field of this
        type, normalize the value to have the appropriate type. This
        base implementation only reinterprets `None`.
        """
        if value is None:
            return self.null
        else:
            # TODO This should eventually be replaced by
            # `self.model_type(value)`
            return value

    def from_sql(self, sql_value):
        """Receives the value stored in the SQL backend and return the
        value to be stored in the model.

        For fixed fields the type of `value` is determined by the column
        type affinity given in the `sql` property and the SQL to Python
        mapping of the database adapter. For more information see:
        http://www.sqlite.org/datatype3.html
        https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types

        Flexible fields have the type affinity `TEXT`. This means the
        `sql_value` is either a `buffer`/`memoryview` or a `unicode` object`
        and the method must handle these in addition.
        """
        if isinstance(sql_value, buffer):
            sql_value = bytes(sql_value).decode('utf-8', 'ignore')
        if isinstance(sql_value, six.text_type):
            return self.parse(sql_value)
        else:
            return self.normalize(sql_value)

    def to_sql(self, model_value):
        """Convert a value as stored in the model object to a value used
        by the database adapter.
        """
        return model_value

# Reusable types.

class Default(Type):
    null = None

class Integer(Type):
    """A basic integer type.
    """
    sql = u'INTEGER'
    query = query.NumericQuery
    model_type = int

class PaddedInt(Integer):
    """An integer field that is formatted with a given number of digits,
    padded with zeroes.
    """
    def __init__(self, digits):
        self.digits = digits

    def format(self, value):
        return u'{0:0{1}d}'.format(value or 0, self.digits)

class ScaledInt(Integer):
    """An integer whose formatting operation scales the number by a
    constant and adds a suffix. Good for units with large magnitudes.
    """
    def __init__(self, unit, suffix=u''):
        self.unit = unit
        self.suffix = suffix

    def format(self, value):
        return u'{0}{1}'.format((value or 0) // self.unit, self.suffix)

class Id(Integer):
    """An integer used as the row id or a foreign key in a SQLite table.
    This type is nullable: None values are not translated to zero.
    """
    null = None

    def __init__(self, primary=True):
        if primary:
            self.sql = u'INTEGER PRIMARY KEY'

class Float(Type):
    """A basic floating-point type.
    """
    sql = u'REAL'
    query = query.NumericQuery
    model_type = float

    def format(self, value):
        return u'{0:.1f}'.format(value or 0.0)

class NullFloat(Float):
    """Same as `Float`, but does not normalize `None` to `0.0`.
    """
    null = None

class String(Type):
    """A Unicode string type.
    """
    sql = u'TEXT'
    query = query.SubstringQuery

class Boolean(Type):
    """A boolean type.
    """
    sql = u'INTEGER'
    query = query.BooleanQuery
    model_type = bool

    def format(self, value):
        return six.text_type(bool(value))

    def parse(self, string):
        return str2bool(string)

# Shared instances of common types.
DEFAULT = Default()
INTEGER = Integer()
PRIMARY_ID = Id(True)
FOREIGN_ID = Id(False)
FLOAT = Float()
NULL_FLOAT = NullFloat()
STRING = String()
BOOLEAN = Boolean()
<commit_msg>Add a new NullPaddedInt type<commit_after># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Representation of type information for DBCore model fields.
"""
from __future__ import division, absolute_import, print_function

from . import query
from beets.util import str2bool
import six

if not six.PY2:
    buffer = memoryview  # sqlite won't accept memoryview in python 2

# Abstract base.

class Type(object):
    """An object encapsulating the type of a model field. Includes
    information about how to store, query, format, and parse a given
    field.
    """

    sql = u'TEXT'
    """The SQLite column type for the value.
    """

    query = query.SubstringQuery
    """The `Query` subclass to be used when querying the field.
    """

    model_type = six.text_type
    """The Python type that is used to represent the value in the model.

    The model is guaranteed to return a value of this type if the field
    is accessed.  To this end, the constructor is used by the `normalize`
    and `from_sql` methods and the `default` property.
    """

    @property
    def null(self):
        """The value to be exposed when the underlying value is None.
        """
        return self.model_type()

    def format(self, value):
        """Given a value of this type, produce a Unicode string
        representing the value. This is used in template evaluation.
        """
        if value is None:
            value = self.null
        # `self.null` might be `None`
        if value is None:
            value = u''
        if isinstance(value, bytes):
            value = value.decode('utf-8', 'ignore')

        return six.text_type(value)

    def parse(self, string):
        """Parse a (possibly human-written) string and return the
        indicated value of this type.
        """
        try:
            return self.model_type(string)
        except ValueError:
            return self.null

    def normalize(self, value):
        """Given a value that will be assigned into a field of this
        type, normalize the value to have the appropriate type. This
        base implementation only reinterprets `None`.
        """
        if value is None:
            return self.null
        else:
            # TODO This should eventually be replaced by
            # `self.model_type(value)`
            return value

    def from_sql(self, sql_value):
        """Receives the value stored in the SQL backend and return the
        value to be stored in the model.

        For fixed fields the type of `value` is determined by the column
        type affinity given in the `sql` property and the SQL to Python
        mapping of the database adapter. For more information see:
        http://www.sqlite.org/datatype3.html
        https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types

        Flexible fields have the type affinity `TEXT`. This means the
        `sql_value` is either a `buffer`/`memoryview` or a `unicode` object`
        and the method must handle these in addition.
        """
        if isinstance(sql_value, buffer):
            sql_value = bytes(sql_value).decode('utf-8', 'ignore')
        if isinstance(sql_value, six.text_type):
            return self.parse(sql_value)
        else:
            return self.normalize(sql_value)

    def to_sql(self, model_value):
        """Convert a value as stored in the model object to a value used
        by the database adapter.
        """
        return model_value

# Reusable types.

class Default(Type):
    null = None

class Integer(Type):
    """A basic integer type.
    """
    sql = u'INTEGER'
    query = query.NumericQuery
    model_type = int

class PaddedInt(Integer):
    """An integer field that is formatted with a given number of digits,
    padded with zeroes.
    """
    def __init__(self, digits):
        self.digits = digits

    def format(self, value):
        return u'{0:0{1}d}'.format(value or 0, self.digits)

class NullPaddedInt(PaddedInt):
    """Same as `PaddedInt`, but does not normalize `None` to `0.0`.
    """
    null = None

class ScaledInt(Integer):
    """An integer whose formatting operation scales the number by a
    constant and adds a suffix. Good for units with large magnitudes.
    """
    def __init__(self, unit, suffix=u''):
        self.unit = unit
        self.suffix = suffix

    def format(self, value):
        return u'{0}{1}'.format((value or 0) // self.unit, self.suffix)

class Id(Integer):
    """An integer used as the row id or a foreign key in a SQLite table.
    This type is nullable: None values are not translated to zero.
    """
    null = None

    def __init__(self, primary=True):
        if primary:
            self.sql = u'INTEGER PRIMARY KEY'

class Float(Type):
    """A basic floating-point type.
    """
    sql = u'REAL'
    query = query.NumericQuery
    model_type = float

    def format(self, value):
        return u'{0:.1f}'.format(value or 0.0)

class NullFloat(Float):
    """Same as `Float`, but does not normalize `None` to `0.0`.
    """
    null = None

class String(Type):
    """A Unicode string type.
    """
    sql = u'TEXT'
    query = query.SubstringQuery

class Boolean(Type):
    """A boolean type.
    """
    sql = u'INTEGER'
    query = query.BooleanQuery
    model_type = bool

    def format(self, value):
        return six.text_type(bool(value))

    def parse(self, string):
        return str2bool(string)

# Shared instances of common types.
DEFAULT = Default()
INTEGER = Integer()
PRIMARY_ID = Id(True)
FOREIGN_ID = Id(False)
FLOAT = Float()
NULL_FLOAT = NullFloat()
STRING = String()
BOOLEAN = Boolean()
<|endoftext|>


<commit_before>#!/usr/bin/python

import uncertainties
import numpy
import sys

def filter_outliers(numbers, mean, std):
    return filter(lambda x: abs(x - mean) < 2 * std, numbers)

op_str = {
    'create_files' : 'Create 100 files',
    'rm_files' : 'Unlink 100 files',
    'create_files_parallel' : 'Create 100 files (parallel)',
    'rm_files_parallel' : 'Unlink 100 files (parallel)',
    'ls_files' : 'ls with 1000 files',
    'write_md5' : 'Write 1GB',
    'read_first_byte' : 'Time to 1st byte',
    'read_md5' : 'Read 1GB',
}

outputOrder = [
    'create_files',
    'rm_files',
    'create_files_parallel',
    'rm_files_parallel',
    'ls_files',
    'write_md5',
    'read_md5',
    'read_first_byte',
]

f = sys.argv[1]
data = open(f).readlines()
#print 'operation | goofys |  s3fs  | speedup'
#print '----------| ------ | ------ | -------'

table = [{}, {}, {}]

print '#operation,time'
for l in data:
    dataset = l.strip().split('\t')
    for d in range(0, len(dataset)):
        op, num = dataset[d].split(' ')
        if not op in table[d]:
            table[d][op] = []
        table[d][op] += [float(num)]

for c in outputOrder:
    print op_str[c],
    for d in table:
        mean = numpy.mean(d[c])
        err = numpy.std(d[c])
        x = filter_outliers(d[c], mean, err)
        print "\t%s\t%s\t%s" % (numpy.mean(x), numpy.min(x), numpy.max(x)),
    print

    # op = op_str[nums[0]]

    # for i in range(1, len(nums)):

    # x = map(lambda x: float(x), nums[1].strip().split(' '))
    # y = map(lambda x: float(x), nums[2].strip().split(' '))
    # mean_x = numpy.mean(x)
    # err_x = numpy.std(x)
    # mean_y = numpy.mean(y)
    # err_y = numpy.std(y)
    # fixed_x = fixed_y = ""

    # x2 = filter_outliers(x, mean_x, err_x)
    # y2 = filter_outliers(y, mean_y, err_y)
    # if x != x2:
    #     fixed_x = "*" * abs(len(x) - len(x2))
    #     mean_x = numpy.mean(x2)
    #     err_x = numpy.std(x2)
    # if y != y2:
    #     fixed_y = "*" * abs(len(y) - len(y2))
    #     mean_y = numpy.mean(y2)
    #     err_y = numpy.std(y2)

    # print "%s, %s, %s, %s", op, mean_x, mean_x - err_x, mean_x + err_x
    # # u_x = uncertainties.ufloat(mean_x, err_x)
    # # u_y = uncertainties.ufloat(mean_y, err_y)
    # # delta = u_y/u_x
    # # print "%s | %s%s | %s%s | %sx" % (op, u_x, fixed_x, u_y, fixed_y, delta)
<commit_msg>attempt to graph find results<commit_after>#!/usr/bin/python

import uncertainties
import numpy
import sys

def filter_outliers(numbers, mean, std):
    return filter(lambda x: abs(x - mean) < 2 * std, numbers)

op_str = {
    'create_files' : 'Create 100 files',
    'rm_files' : 'Unlink 100 files',
    'create_files_parallel' : 'Create 100 files (parallel)',
    'rm_files_parallel' : 'Unlink 100 files (parallel)',
    'ls_files' : 'ls with 1000 files',
    'find_files' : "`find' in a nested directory tree with ~1000 files",
    'write_md5' : 'Write 1GB',
    'read_first_byte' : 'Time to 1st byte',
    'read_md5' : 'Read 1GB',
}

outputOrder = [
    'create_files',
    'rm_files',
    'create_files_parallel',
    'rm_files_parallel',
    'ls_files',
    'find_files',
    'write_md5',
    'read_md5',
    'read_first_byte',
]

f = sys.argv[1]
data = open(f).readlines()
#print 'operation | goofys |  s3fs  | speedup'
#print '----------| ------ | ------ | -------'

table = [{}, {}, {}]

print '#operation,time'
for l in data:
    dataset = l.strip().split('\t')
    for d in range(0, len(dataset)):
        op, num = dataset[d].split(' ')
        if not op in table[d]:
            table[d][op] = []
        table[d][op] += [float(num)]

for c in outputOrder:
    print op_str[c],
    for d in table:
        mean = numpy.mean(d[c])
        err = numpy.std(d[c])
        x = filter_outliers(d[c], mean, err)
        print "\t%s\t%s\t%s" % (numpy.mean(x), numpy.min(x), numpy.max(x)),
    print

    # op = op_str[nums[0]]

    # for i in range(1, len(nums)):

    # x = map(lambda x: float(x), nums[1].strip().split(' '))
    # y = map(lambda x: float(x), nums[2].strip().split(' '))
    # mean_x = numpy.mean(x)
    # err_x = numpy.std(x)
    # mean_y = numpy.mean(y)
    # err_y = numpy.std(y)
    # fixed_x = fixed_y = ""

    # x2 = filter_outliers(x, mean_x, err_x)
    # y2 = filter_outliers(y, mean_y, err_y)
    # if x != x2:
    #     fixed_x = "*" * abs(len(x) - len(x2))
    #     mean_x = numpy.mean(x2)
    #     err_x = numpy.std(x2)
    # if y != y2:
    #     fixed_y = "*" * abs(len(y) - len(y2))
    #     mean_y = numpy.mean(y2)
    #     err_y = numpy.std(y2)

    # print "%s, %s, %s, %s", op, mean_x, mean_x - err_x, mean_x + err_x
    # # u_x = uncertainties.ufloat(mean_x, err_x)
    # # u_y = uncertainties.ufloat(mean_y, err_y)
    # # delta = u_y/u_x
    # # print "%s | %s%s | %s%s | %sx" % (op, u_x, fixed_x, u_y, fixed_y, delta)
<|endoftext|>


<commit_before>"""Project

A project in benchbuild is an abstract representation of a software
system that can live in various stages throughout an experiment.
It defines two extension points for an experiment to attach on, the
compile-time phase and the (optional) run-time phase.

An experiment can intercept the compilation phase of a project and perform
any experiment that requires the source artifacts as input.

Furthermore, it is possible to intercept a project's run-time pahse with
a measurement.

The project definition ensures that all experiments run through the same
series of commands in both phases and that all experiments run inside
a separate build directory in isolation of one another.
"""
import copy
import logging
import typing as tp
import uuid
from abc import abstractmethod
from functools import partial
from os import getenv

import attr
from plumbum import ProcessExecutionError, local
from pygtrie import StringTrie

from benchbuild import signals
from benchbuild.extensions import compiler
from benchbuild.extensions import run as ext_run
from benchbuild.settings import CFG
from benchbuild.utils import db, run, unionfs, wrapping
from benchbuild.utils.requirements import Requirement

LOG = logging.getLogger(__name__)

class ProjectRegistry(type):
    """Registry for benchbuild projects."""

    projects = StringTrie()

    def __init__(cls, name, bases, attrs):
        """Register a project in the registry."""
        super(ProjectRegistry, cls).__init__(name, bases, attrs)

        if '' not in {cls.NAME, cls.DOMAIN, cls.GROUP}:
            key = "{name}/{group}".format(name=cls.NAME, group=cls.GROUP)
            ProjectRegistry.projects[key] = cls

class ProjectDecorator(ProjectRegistry):
    """
    Decorate the interface of a project with the in_builddir decorator.

    This is just a small safety net for benchbuild users, because we make
    sure to run every project method in the project's build directory.
    """

    decorated_methods = ["redirect", "compile", "run_tests"]

    def __init__(cls, name, bases, attrs):
        unionfs_deco = None
        if CFG["unionfs"]["enable"]:
            rw_dir = str(CFG["unionfs"]["rw"])
            unionfs_deco = partial(unionfs.unionfs, rw=rw_dir)
        config_deco = run.store_config

        methods = ProjectDecorator.decorated_methods
        for key, value in attrs.items():
            if (key in methods) and hasattr(cls, key):
                wrapped_fun = value
                wrapped_fun = config_deco(wrapped_fun)

                if unionfs_deco is not None:
                    wrapped_fun = unionfs_deco()(wrapped_fun)

                wrapped_fun = run.in_builddir('.')(wrapped_fun)
                setattr(cls, key, wrapped_fun)

        super(ProjectDecorator, cls).__init__(name, bases, attrs)

@attr.s
class Project(metaclass=ProjectDecorator):
    """Abstract class for benchbuild projects.

    A project is an arbitrary software system usable by benchbuild in
    experiments.
    Subclasses of Project are registered automatically by benchbuild, if
    imported in the same interpreter session. For this to happen, you must list
    the in the settings under plugins -> projects.

    A project implementation *must* provide the following method:
        compile: Downloads & Compiles the source.

    A project implementation *may* provide the following functions:
        run_tests: Wrap any binary that has to be run under the
            runtime_extension wrapper and execute an implementation defined
            set of run-time tests.
            Defaults to a call of a binary with the name `run_f` in the
            build directory without arguments.
        clean: Clean the project's build directory. Defaults to
            recursive 'rm' on the build directory and can be disabled
            by setting the environment variable ``BB_CLEAN=false``.

    Raises:
        AttributeError: Class definition raises an attribute error, if
            the implementation does not provide a value for the attributes
            `NAME`, `DOMAIN`, and `GROUP`
        TypeError: Validation of properties may throw a TypeError.

    Attributes:
        experiment (benchbuild.experiment.Experiment):
            The experiment this project is assigned to.
        name (str, optional):
            The name of this project. Defaults to `NAME`.
        domain (str, optional):
            The application domain of this project. Defaults to `DOMAIN`.
        group (str, optional):
            The group this project belongs to. Defaults to `GROUP`.
        src_file (str, optional):
            A main src_file this project is assigned to. Defaults to `SRC_FILE`
        container (benchbuild.utils.container.Container, optional):
            A uchroot compatible container that we can use for this project.
            Defaults to `benchbuild.utils.container.Gentoo`.
        requirements (:obj:`list` of :obj:`Requirement`)
            A list of specific requirements that are used to configure the
            execution environment.
        version (str, optional):
            A version information for this project. Defaults to `VERSION`.
        builddir (str, optional):
            The build directory for this project. Auto generated, if not set.
        testdir (str, optional):
            The location of any additional test-files for this project,
            usually stored out of tree. Auto generated, if not set. Usually a
            project implementation
            will define this itself.
        cflags (:obj:`list` of :obj:`str`, optional)
            A list of cflags used, for compilation of this project.
        ldflags (:obj:`list` of :obj:`str`, optional)
            A list of ldflags used, for compilation of this project.
        run_f (str, optional):
            A filename that points to the binary we want to track.
            Usually a project implementation will define this itself.
        run_uuid (uuid.UUID, optional):
            An UUID that identifies all binaries executed by a single run of
            this project. In the database schema this is named the 'run_group'.
        compiler_extension (Callable[str, iterable[str], RunInfo], optional):
            A composable extension that will be used in place of the real
            compiler. Defaults to running the compiler with a timeout command
            wrapped around it.
        runtime_extension (Callable[str, iterable[str], RunInfo], optional):
            A composable extension that will be used in place of any binary
            this project
            wants to execute. Which binaries to replace is defined by the
            implementation using `benchbuild.utils.wrapping.wrap`.
            Defaults to None.
    """
    VERSION = None
    SRC_FILE = None
    CONTAINER = None
    DOMAIN: tp.ClassVar[str] = ""
    GROUP: tp.ClassVar[str] = ""
    NAME: tp.ClassVar[str] = ""
    REQUIREMENTS: tp.List[Requirement] = []

    def __new__(cls, *args, **kwargs):
        """Create a new project instance and set some defaults."""
        new_self = super(Project, cls).__new__(cls)
        if cls.NAME is None:
            raise AttributeError(
                "{0} @ {1} does not define a NAME class attribute.".format(
                    cls.__name__, cls.__module__))
        if cls.DOMAIN is None:
            raise AttributeError(
                "{0} @ {1} does not define a DOMAIN class attribute.".format(
                    cls.__name__, cls.__module__))
        if cls.GROUP is None:
            raise AttributeError(
                "{0} @ {1} does not define a GROUP class attribute.".format(
                    cls.__name__, cls.__module__))
        return new_self

    experiment = attr.ib()

    name = attr.ib(
        default=attr.Factory(lambda self: type(self).NAME, takes_self=True))

    domain = attr.ib(
        default=attr.Factory(lambda self: type(self).DOMAIN, takes_self=True))

    group = attr.ib(
        default=attr.Factory(lambda self: type(self).GROUP, takes_self=True))

    src_file = attr.ib(
        default=attr.Factory(lambda self: type(self).SRC_FILE, takes_self=True))

    container = attr.ib(default=attr.Factory(lambda self: type(self).CONTAINER,
                                             takes_self=True))

    version = attr.ib(
        default=attr.Factory(lambda self: type(self).VERSION, takes_self=True))

    testdir = attr.ib()

    @testdir.default
    def __default_testdir(self):
        if self.group:
            return local.path(str(
                CFG["test_dir"])) / self.domain / self.group / self.name
        return local.path(str(CFG["test_dir"])) / self.domain / self.name

    cflags = attr.ib(default=attr.Factory(list))

    ldflags = attr.ib(default=attr.Factory(list))

    run_uuid = attr.ib()

    @run_uuid.default
    def __default_run_uuid(self):
        run_group = getenv("BB_DB_RUN_GROUP", None)
        if run_group:
            return uuid.UUID(run_group)
        return uuid.uuid4()

    @run_uuid.validator
    def __check_if_uuid(self, _, value):
        if not isinstance(value, uuid.UUID):
            raise TypeError("{attribute} must be a valid UUID object")

    builddir = attr.ib(default=attr.Factory(lambda self: local.path(
        str(CFG["build_dir"])) / "{}-{}".format(self.experiment.name, self.id),
                                            takes_self=True))

    run_f = attr.ib(default=attr.Factory(
        lambda self: local.path(self.builddir) / self.name, takes_self=True))

    compiler_extension = attr.ib(
        default=attr.Factory(lambda self: ext_run.WithTimeout(
            compiler.RunCompiler(self, self.experiment)),
                             takes_self=True))

    runtime_extension = attr.ib(default=None)

    def __attrs_post_init__(self):
        db.persist_project(self)

    def run_tests(self):
        """
        Run the tests of this project.

        Clients override this method to provide customized run-time tests.

        Args:
            experiment: The experiment we run this project under
            run: A function that takes the run command.
        """
        exp = wrapping.wrap(self.run_f, self)
        _exp = run.watch(exp)
        _exp()

    def run(self):
        """Run the tests of this project.

        This method initializes the default environment and takes care of
        cleaning up the mess we made, after a successfull run.

        Args:
            experiment: The experiment we run this project under
        """
        from benchbuild.utils.run import (begin_run_group, end_run_group,
                                          fail_run_group)
        CFG["experiment"] = self.experiment.name
        CFG["project"] = self.NAME
        CFG["domain"] = self.DOMAIN
        CFG["group"] = self.GROUP
        CFG["version"] = self.VERSION
        CFG["db"]["run_group"] = str(self.run_uuid)

        group, session = begin_run_group(self)
        signals.handlers.register(fail_run_group, group, session)

        try:
            self.run_tests()
            end_run_group(group, session)
        except ProcessExecutionError:
            fail_run_group(group, session)
            raise
        except KeyboardInterrupt:
            fail_run_group(group, session)
            raise
        finally:
            signals.handlers.deregister(fail_run_group)

    def clean(self):
        """Clean the project build directory."""
        builddir_p = local.path(self.builddir)
        builddir_p.delete()

    def prepare(self):
        """Prepare the build diretory."""
        builddir_p = local.path(self.builddir)
        if not builddir_p.exists():
            builddir_p.mkdir()

    @abstractmethod
    def compile(self):
        """Compile the project."""

    def download(self, version=None):
        """Auto-generated by with_* decorators."""
        del version
        LOG.error("Not implemented.")

    def redirect(self):
        """Redirect execution to a containerized benchbuild instance."""
        LOG.error("Redirection not supported by project.")

    def clone(self):
        """Create a deepcopy of ourself."""
        new_p = copy.deepcopy(self)
        new_p.run_uuid = uuid.uuid4()
        return new_p

    @property
    def id(self):
        return "{name}-{group}@{version}_uuid_{id}".format(name=self.name,
                                                           group=self.group,
                                                           version=self.version,
                                                           id=self.run_uuid)

    @classmethod
    def versions(cls):
        """Return all available versions."""
        if cls.VERSION:
            return [cls.VERSION]
        return ["unknown"]

def __split_project_input__(
        project_input: str) -> tp.Tuple[str, tp.Optional[str]]:
    split_input = project_input.rsplit('@', maxsplit=1)
    first = split_input[0]
    second = split_input[1] if len(split_input) > 1 else None

    return (first, second)

def populate(
    projects_to_filter=None,
    group=None
) -> tp.Mapping[str, tp.Tuple[tp.Type[Project], tp.Optional[str]]]:
    """
    Populate the list of projects that belong to this experiment.

    Args:
        projects_to_filter (list(Project)):
            List of projects we want to assign to this experiment.
            We intersect the list of projects with the list of supported
            projects to get the list of projects that belong to this
            experiment.
        group (list(str)):
            In addition to the project filter, we provide a way to filter
            whole groups.
    Returns:
        a dictionary of (project name, project class) pairs.
    """
    if projects_to_filter is None:
        projects_to_filter = []

    import benchbuild.projects as all_projects
    all_projects.discover()

    prjs = ProjectRegistry.projects
    if projects_to_filter:
        prjs = {}

        def single_version_impl(version):
            return lambda: [version]

        for filter_project in set(projects_to_filter):
            project_str, version = __split_project_input__(filter_project)
            try:
                for name, project_type in ProjectRegistry.projects.items(
                        prefix=project_str):
                    if version:
                        project_type.versions = single_version_impl(version)
                    prjs.update({name: project_type})
            except KeyError:
                pass

    if group:
        groupkeys = set(group)
        prjs = {
            name: cls for name, cls in prjs.items() if cls.GROUP in groupkeys
        }

    return {
        x: prjs[x]
        for x in prjs
        if prjs[x].DOMAIN != "debug" or x in projects_to_filter
    }
<commit_msg>project: f-strings<commit_after>"""Project

A project in benchbuild is an abstract representation of a software
system that can live in various stages throughout an experiment.
It defines two extension points for an experiment to attach on, the
compile-time phase and the (optional) run-time phase.

An experiment can intercept the compilation phase of a project and perform
any experiment that requires the source artifacts as input.

Furthermore, it is possible to intercept a project's run-time pahse with
a measurement.

The project definition ensures that all experiments run through the same
series of commands in both phases and that all experiments run inside
a separate build directory in isolation of one another.
"""
import copy
import logging
import typing as tp
import uuid
from abc import abstractmethod
from functools import partial
from os import getenv

import attr
from plumbum import ProcessExecutionError, local
from pygtrie import StringTrie

from benchbuild import signals
from benchbuild.extensions import compiler
from benchbuild.extensions import run as ext_run
from benchbuild.settings import CFG
from benchbuild.utils import db, run, unionfs, wrapping
from benchbuild.utils.requirements import Requirement

LOG = logging.getLogger(__name__)

class ProjectRegistry(type):
    """Registry for benchbuild projects."""

    projects = StringTrie()

    def __init__(cls, name, bases, attrs):
        """Register a project in the registry."""
        super(ProjectRegistry, cls).__init__(name, bases, attrs)

        if '' not in {cls.NAME, cls.DOMAIN, cls.GROUP}:
            key = "{name}/{group}".format(name=cls.NAME, group=cls.GROUP)
            ProjectRegistry.projects[key] = cls

class ProjectDecorator(ProjectRegistry):
    """
    Decorate the interface of a project with the in_builddir decorator.

    This is just a small safety net for benchbuild users, because we make
    sure to run every project method in the project's build directory.
    """

    decorated_methods = ["redirect", "compile", "run_tests"]

    def __init__(cls, name, bases, attrs):
        unionfs_deco = None
        if CFG["unionfs"]["enable"]:
            rw_dir = str(CFG["unionfs"]["rw"])
            unionfs_deco = partial(unionfs.unionfs, rw=rw_dir)
        config_deco = run.store_config

        methods = ProjectDecorator.decorated_methods
        for key, value in attrs.items():
            if (key in methods) and hasattr(cls, key):
                wrapped_fun = value
                wrapped_fun = config_deco(wrapped_fun)

                if unionfs_deco is not None:
                    wrapped_fun = unionfs_deco()(wrapped_fun)

                wrapped_fun = run.in_builddir('.')(wrapped_fun)
                setattr(cls, key, wrapped_fun)

        super(ProjectDecorator, cls).__init__(name, bases, attrs)

@attr.s
class Project(metaclass=ProjectDecorator):
    """Abstract class for benchbuild projects.

    A project is an arbitrary software system usable by benchbuild in
    experiments.
    Subclasses of Project are registered automatically by benchbuild, if
    imported in the same interpreter session. For this to happen, you must list
    the in the settings under plugins -> projects.

    A project implementation *must* provide the following method:
        compile: Downloads & Compiles the source.

    A project implementation *may* provide the following functions:
        run_tests: Wrap any binary that has to be run under the
            runtime_extension wrapper and execute an implementation defined
            set of run-time tests.
            Defaults to a call of a binary with the name `run_f` in the
            build directory without arguments.
        clean: Clean the project's build directory. Defaults to
            recursive 'rm' on the build directory and can be disabled
            by setting the environment variable ``BB_CLEAN=false``.

    Raises:
        AttributeError: Class definition raises an attribute error, if
            the implementation does not provide a value for the attributes
            `NAME`, `DOMAIN`, and `GROUP`
        TypeError: Validation of properties may throw a TypeError.

    Attributes:
        experiment (benchbuild.experiment.Experiment):
            The experiment this project is assigned to.
        name (str, optional):
            The name of this project. Defaults to `NAME`.
        domain (str, optional):
            The application domain of this project. Defaults to `DOMAIN`.
        group (str, optional):
            The group this project belongs to. Defaults to `GROUP`.
        src_file (str, optional):
            A main src_file this project is assigned to. Defaults to `SRC_FILE`
        container (benchbuild.utils.container.Container, optional):
            A uchroot compatible container that we can use for this project.
            Defaults to `benchbuild.utils.container.Gentoo`.
        requirements (:obj:`list` of :obj:`Requirement`)
            A list of specific requirements that are used to configure the
            execution environment.
        version (str, optional):
            A version information for this project. Defaults to `VERSION`.
        builddir (str, optional):
            The build directory for this project. Auto generated, if not set.
        testdir (str, optional):
            The location of any additional test-files for this project,
            usually stored out of tree. Auto generated, if not set. Usually a
            project implementation
            will define this itself.
        cflags (:obj:`list` of :obj:`str`, optional)
            A list of cflags used, for compilation of this project.
        ldflags (:obj:`list` of :obj:`str`, optional)
            A list of ldflags used, for compilation of this project.
        run_f (str, optional):
            A filename that points to the binary we want to track.
            Usually a project implementation will define this itself.
        run_uuid (uuid.UUID, optional):
            An UUID that identifies all binaries executed by a single run of
            this project. In the database schema this is named the 'run_group'.
        compiler_extension (Callable[str, iterable[str], RunInfo], optional):
            A composable extension that will be used in place of the real
            compiler. Defaults to running the compiler with a timeout command
            wrapped around it.
        runtime_extension (Callable[str, iterable[str], RunInfo], optional):
            A composable extension that will be used in place of any binary
            this project
            wants to execute. Which binaries to replace is defined by the
            implementation using `benchbuild.utils.wrapping.wrap`.
            Defaults to None.
    """
    VERSION = None
    SRC_FILE = None
    CONTAINER = None
    DOMAIN: tp.ClassVar[str] = ""
    GROUP: tp.ClassVar[str] = ""
    NAME: tp.ClassVar[str] = ""
    REQUIREMENTS: tp.List[Requirement] = []

    def __new__(cls, *args, **kwargs):
        """Create a new project instance and set some defaults."""
        new_self = super(Project, cls).__new__(cls)
        mod_ident = f'{cls.__name__} @ {cls.__module__}'
        if not cls.NAME:
            raise AttributeError(
                f'{mod_ident} does not define a NAME class attribute.')
        if not cls.DOMAIN:
            raise AttributeError(
                f'{mod_ident} does not define a DOMAIN class attribute.')
        if not cls.GROUP:
            raise AttributeError(
                f'{mod_ident} does not define a GROUP class attribute.')
        return new_self

    experiment = attr.ib()

    name = attr.ib(
        default=attr.Factory(lambda self: type(self).NAME, takes_self=True))

    domain = attr.ib(
        default=attr.Factory(lambda self: type(self).DOMAIN, takes_self=True))

    group = attr.ib(
        default=attr.Factory(lambda self: type(self).GROUP, takes_self=True))

    src_file = attr.ib(
        default=attr.Factory(lambda self: type(self).SRC_FILE, takes_self=True))

    container = attr.ib(default=attr.Factory(lambda self: type(self).CONTAINER,
                                             takes_self=True))

    version = attr.ib(
        default=attr.Factory(lambda self: type(self).VERSION, takes_self=True))

    testdir = attr.ib()

    @testdir.default
    def __default_testdir(self):
        if self.group:
            return local.path(str(
                CFG["test_dir"])) / self.domain / self.group / self.name
        return local.path(str(CFG["test_dir"])) / self.domain / self.name

    cflags = attr.ib(default=attr.Factory(list))

    ldflags = attr.ib(default=attr.Factory(list))

    run_uuid = attr.ib()

    @run_uuid.default
    def __default_run_uuid(self):
        run_group = getenv("BB_DB_RUN_GROUP", None)
        if run_group:
            return uuid.UUID(run_group)
        return uuid.uuid4()

    @run_uuid.validator
    def __check_if_uuid(self, _, value):
        if not isinstance(value, uuid.UUID):
            raise TypeError("{attribute} must be a valid UUID object")

    builddir = attr.ib(default=attr.Factory(lambda self: local.path(
        str(CFG["build_dir"])) / self.experiment.name / self.id,
                                            takes_self=True))

    run_f = attr.ib(default=attr.Factory(
        lambda self: local.path(self.builddir) / self.name, takes_self=True))

    compiler_extension = attr.ib(
        default=attr.Factory(lambda self: ext_run.WithTimeout(
            compiler.RunCompiler(self, self.experiment)),
                             takes_self=True))

    runtime_extension = attr.ib(default=None)

    def __attrs_post_init__(self):
        db.persist_project(self)

    def run_tests(self):
        """
        Run the tests of this project.

        Clients override this method to provide customized run-time tests.

        Args:
            experiment: The experiment we run this project under
            run: A function that takes the run command.
        """
        exp = wrapping.wrap(self.run_f, self)
        _exp = run.watch(exp)
        _exp()

    def run(self):
        """Run the tests of this project.

        This method initializes the default environment and takes care of
        cleaning up the mess we made, after a successfull run.

        Args:
            experiment: The experiment we run this project under
        """
        from benchbuild.utils.run import (begin_run_group, end_run_group,
                                          fail_run_group)
        CFG["experiment"] = self.experiment.name
        CFG["project"] = self.NAME
        CFG["domain"] = self.DOMAIN
        CFG["group"] = self.GROUP
        CFG["version"] = self.VERSION
        CFG["db"]["run_group"] = str(self.run_uuid)

        group, session = begin_run_group(self)
        signals.handlers.register(fail_run_group, group, session)

        try:
            self.run_tests()
            end_run_group(group, session)
        except ProcessExecutionError:
            fail_run_group(group, session)
            raise
        except KeyboardInterrupt:
            fail_run_group(group, session)
            raise
        finally:
            signals.handlers.deregister(fail_run_group)

    def clean(self):
        """Clean the project build directory."""
        builddir_p = local.path(self.builddir)
        builddir_p.delete()

    def prepare(self):
        """Prepare the build diretory."""
        builddir_p = local.path(self.builddir)
        if not builddir_p.exists():
            builddir_p.mkdir()

    @abstractmethod
    def compile(self):
        """Compile the project."""

    def download(self, version=None):
        """Auto-generated by with_* decorators."""
        del version
        LOG.error("Not implemented.")

    def redirect(self):
        """Redirect execution to a containerized benchbuild instance."""
        LOG.error("Redirection not supported by project.")

    def clone(self):
        """Create a deepcopy of ourself."""
        new_p = copy.deepcopy(self)
        new_p.run_uuid = uuid.uuid4()
        return new_p

    @property
    def id(self):
        return "{name}-{group}@{version}_uuid_{id}".format(name=self.name,
                                                           group=self.group,
                                                           version=self.version,
                                                           id=self.run_uuid)

    @classmethod
    def versions(cls):
        """Return all available versions."""
        if cls.VERSION:
            return [cls.VERSION]
        return ["unknown"]

def __split_project_input__(
        project_input: str) -> tp.Tuple[str, tp.Optional[str]]:
    split_input = project_input.rsplit('@', maxsplit=1)
    first = split_input[0]
    second = split_input[1] if len(split_input) > 1 else None

    return (first, second)

def populate(
    projects_to_filter=None,
    group=None
) -> tp.Mapping[str, tp.Tuple[tp.Type[Project], tp.Optional[str]]]:
    """
    Populate the list of projects that belong to this experiment.

    Args:
        projects_to_filter (list(Project)):
            List of projects we want to assign to this experiment.
            We intersect the list of projects with the list of supported
            projects to get the list of projects that belong to this
            experiment.
        group (list(str)):
            In addition to the project filter, we provide a way to filter
            whole groups.
    Returns:
        a dictionary of (project name, project class) pairs.
    """
    if projects_to_filter is None:
        projects_to_filter = []

    import benchbuild.projects as all_projects
    all_projects.discover()

    prjs = ProjectRegistry.projects
    if projects_to_filter:
        prjs = {}

        def single_version_impl(version):
            return lambda: [version]

        for filter_project in set(projects_to_filter):
            project_str, version = __split_project_input__(filter_project)
            try:
                for name, project_type in ProjectRegistry.projects.items(
                        prefix=project_str):
                    if version:
                        project_type.versions = single_version_impl(version)
                    prjs.update({name: project_type})
            except KeyError:
                pass

    if group:
        groupkeys = set(group)
        prjs = {
            name: cls for name, cls in prjs.items() if cls.GROUP in groupkeys
        }

    return {
        x: prjs[x]
        for x in prjs
        if prjs[x].DOMAIN != "debug" or x in projects_to_filter
    }
<|endoftext|>


<commit_before>from .utils import library_macro
from .. import shell
from ..file_types import *
from ..iterutils import iterate, uniques
from ..path import Root

class MsvcCompiler(object):
    def __init__(self, env, lang, name, command, cflags):
        self.platform = env.platform
        self.lang = lang

        self.rule_name = self.command_var = name
        self.command = command

        self.global_args = ['/nologo'] + cflags

    @property
    def flavor(self):
        return 'msvc'

    @property
    def deps_flavor(self):
        return 'msvc'

    def __call__(self, cmd, input, output, deps=None, args=None):
        result = [cmd]
        result.extend(iterate(args))
        if deps:
            result.append('/showIncludes')
        result.extend(['/c', input])
        result.append('/Fo' + output)
        return result

    def output_file(self, name):
        return ObjectFile(name + '.obj', Root.builddir, self.lang)

    @property
    def library_args(self):
        return []

    def include_dir(self, directory):
        return ['/I' + directory.path]

    def link_args(self, name, mode):
        if mode == 'executable':
            return []
        elif mode in ['shared_library', 'static_library']:
            return ['/D' + library_macro(name, mode)]
        else:
            raise ValueError("unknown mode '{}'".format(mode))

class MsvcLinker(object):
    def __init__(self, env, mode, lang, name, command, ldflags, ldlibs):
        self.platform = env.platform
        self.mode = mode
        self.lang = lang

        self.rule_name = self.command_var = 'link_' + name
        self.command = command
        self.link_var = 'ld'

        self.global_args = ['/nologo'] + ldflags
        self.global_libs = ldlibs

    @property
    def flavor(self):
        return 'msvc'

    def __call__(self, cmd, input, output, libs=None, args=None):
        result = [cmd]
        result.extend(iterate(args))
        result.extend(iterate(input))
        result.extend(iterate(libs))
        result.append('/OUT:' + output)
        return result

    def output_file(self, name):
        if self.mode == 'executable':
            return Executable(name + self.platform.executable_ext,
                              Root.builddir, self.lang)
        elif self.mode == 'shared_library':
            return DllLibrary(name + self.platform.executable_ext,
                              name + '.lib', Root.builddir, self.lang)
        else:
            raise ValueError("unknown mode '{}'".format(self.mode))

    @property
    def auto_link(self):
        return True

    @property
    def mode_args(self):
        return ['/DLL'] if self.mode == 'shared_library' else []

    def lib_dirs(self, libraries, target):
        def get_dir(lib):
            return lib.path.parent() if isinstance(lib, Library) else lib
        dirs = uniques(get_dir(i) for i in iterate(libraries))
        return ['/LIBPATH:' + i for i in dirs]

    def link_lib(self, library):
        if isinstance(library, WholeArchive):
            raise ValueError('MSVC does not support whole-archives')
        return [library.link.path.basename()]

    def import_lib(self, library):
        if self.mode != 'shared_library':
            return []
        return ['/IMPLIB:' + library.import_lib.path]

class MsvcStaticLinker(object):
    def __init__(self, env, lang, name, command):
        self.platform = env.platform
        self.mode = 'static_library'
        self.lang = lang

        self.rule_name = self.command_var = 'lib_' + name
        self.command = command
        self.link_var = 'lib'

        self.global_args = shell.split(env.getvar('LIBFLAGS', ''))

    @property
    def flavor(self):
        return 'msvc'

    def __call__(self, cmd, input, output, args=None):
        result = [cmd]
        result.extend(iterate(args))
        result.extend(iterate(input))
        result.append('/OUT:' + output)
        return result

    def output_file(self, name):
        return StaticLibrary(name + '.lib', Root.builddir, self.lang)

    @property
    def mode_args(self):
        return []
<commit_msg>Fix creation of shared libraries with MSVC<commit_after>from .utils import library_macro
from .. import shell
from ..file_types import *
from ..iterutils import iterate, uniques
from ..path import Root

class MsvcCompiler(object):
    def __init__(self, env, lang, name, command, cflags):
        self.platform = env.platform
        self.lang = lang

        self.rule_name = self.command_var = name
        self.command = command

        self.global_args = ['/nologo'] + cflags

    @property
    def flavor(self):
        return 'msvc'

    @property
    def deps_flavor(self):
        return 'msvc'

    def __call__(self, cmd, input, output, deps=None, args=None):
        result = [cmd]
        result.extend(iterate(args))
        if deps:
            result.append('/showIncludes')
        result.extend(['/c', input])
        result.append('/Fo' + output)
        return result

    def output_file(self, name):
        return ObjectFile(name + '.obj', Root.builddir, self.lang)

    @property
    def library_args(self):
        return []

    def include_dir(self, directory):
        return ['/I' + directory.path]

    def link_args(self, name, mode):
        if mode == 'executable':
            return []
        elif mode in ['shared_library', 'static_library']:
            return ['/D' + library_macro(name, mode)]
        else:
            raise ValueError("unknown mode '{}'".format(mode))

class MsvcLinker(object):
    def __init__(self, env, mode, lang, name, command, ldflags, ldlibs):
        self.platform = env.platform
        self.mode = mode
        self.lang = lang

        self.rule_name = self.command_var = 'link_' + name
        self.command = command
        self.link_var = 'ld'

        self.global_args = ['/nologo'] + ldflags
        self.global_libs = ldlibs

    @property
    def flavor(self):
        return 'msvc'

    def __call__(self, cmd, input, output, libs=None, args=None):
        result = [cmd]
        result.extend(iterate(args))
        result.extend(iterate(input))
        result.extend(iterate(libs))
        result.append('/OUT:' + output)
        return result

    def output_file(self, name):
        if self.mode == 'executable':
            return Executable(name + self.platform.executable_ext,
                              Root.builddir, self.lang)
        elif self.mode == 'shared_library':
            return DllLibrary(name + self.platform.shared_library_ext,
                              name + '.lib', Root.builddir, self.lang)
        else:
            raise ValueError("unknown mode '{}'".format(self.mode))

    @property
    def auto_link(self):
        return True

    @property
    def mode_args(self):
        return ['/DLL'] if self.mode == 'shared_library' else []

    def lib_dirs(self, libraries, target):
        def get_dir(lib):
            return lib.path.parent() if isinstance(lib, Library) else lib
        dirs = uniques(get_dir(i) for i in iterate(libraries))
        return ['/LIBPATH:' + i for i in dirs]

    def link_lib(self, library):
        if isinstance(library, WholeArchive):
            raise ValueError('MSVC does not support whole-archives')
        return [library.link.path.basename()]

    def import_lib(self, library):
        if self.mode != 'shared_library':
            return []
        return ['/IMPLIB:' + library.import_lib.path]

class MsvcStaticLinker(object):
    def __init__(self, env, lang, name, command):
        self.platform = env.platform
        self.mode = 'static_library'
        self.lang = lang

        self.rule_name = self.command_var = 'lib_' + name
        self.command = command
        self.link_var = 'lib'

        self.global_args = shell.split(env.getvar('LIBFLAGS', ''))

    @property
    def flavor(self):
        return 'msvc'

    def __call__(self, cmd, input, output, args=None):
        result = [cmd]
        result.extend(iterate(args))
        result.extend(iterate(input))
        result.append('/OUT:' + output)
        return result

    def output_file(self, name):
        return StaticLibrary(name + '.lib', Root.builddir, self.lang)

    @property
    def mode_args(self):
        return []
<|endoftext|>


<commit_before>import numpy as np
import struct
import pandas as pd

def read_ascii(filename,pandas=False):
    """
    Takes a filename containing the text output (with headers) from xlook and
    reads the columns into a rec array or dataframe object for easy data
    processing and access.
    """

    try:
        f = open(filename,'r')
    except:
        print "Error Opening %s" %filename
        return 0

    col_width = 12 # Columns are 12 char wide in header

    # First line of the file is the number of records
    num_recs = f.readline()
    num_recs = int(num_recs.strip('number of records = '))
    print "\nNumber of records: %d" %num_recs

    # Second line is column numbers, we don't care so just count them
    num_cols = f.readline()
    num_cols = num_cols.split('col')
    num_cols = len(num_cols)
    print "Number of columns: %d" %num_cols

    # Third line is the column headings
    col_headings_str = f.readline()
    col_headings_str = col_headings_str[5:-1]
    col_headings = ['row_num'] # Row number the the first (unlabeled) column
    for i in range(len(col_headings_str)/12):
        heading = col_headings_str[12*i:12*i+12].strip()
        col_headings.append(heading)

    # Fourth line is column units
    col_units_str = f.readline()
    col_units_str = col_units_str[5:-1]
    col_units=['.']
    for i in xrange(len(col_units_str)/12):
        heading = col_units_str[12*i:12*i+12].strip()
        col_units.append(heading)
    col_units = [x for x in col_units if x != '\n']  #Remove newlines

    # Fifth line is number of records per column
    col_recs = f.readline()
    col_recs = col_recs.split('recs')
    col_recs = [int(x) for x in col_recs if x != '\n']
    col_recs.insert(0, num_recs)

    # Show column units and headings
    print "\n\n-------------------------------------------------"
    print "|%15s|%15s|%15s|" %('Name','Unit','Records')
    print "-------------------------------------------------"
    for column in zip(col_headings,col_units,col_recs):
        print "|%15s|%15s|%15s|" %(column[0],column[1],column[2])
    print "-------------------------------------------------"

    # Read the data into a numpy recarray
    dtype=[]
    #dtype.append(('row_num','float'))
    for name in col_headings:
        dtype.append((name,'float'))
    dtype = np.dtype(dtype)

    data = np.zeros([num_recs,num_cols])

    i=0
    for row in f:
        row_data = row.split()
        for j in xrange(num_cols):
            data[i,j] = row_data[j]
        i+=1

    f.close()

    if pandas==True:
        # If a pandas object is requested, make a data frame
        # indexed on row number and return it
        dfo  = pd.DataFrame(data,columns=col_headings)
        dfo = dfo.set_index('row_num')
        return dfo

    else:
        # Otherwise return the default (Numpy Recarray)
        data_rec = np.rec.array(data,dtype=dtype)
        return data_rec

def read_binary(filename,dataendianness='little',pandas=False):
    """
    Takes a filename containing the binary output from xlook and
    reads the columns into a rec array or dataframe object for easy
    data processing and access.

    The data section of the file is written in the native format of the machine
    used to produce the file.  Endianness of data is little by default, but may
    be changed to 'big' to accomodate older files or files written on power pc
    chips.
    """

    try:
        f = open(filename,'rb')
    except:
        print "Error Opening %s" %filename
        return 0

    col_headings = []
    col_recs     = []
    col_units    = []

    # Unpack information at the top of the file about the experiment
    name = struct.unpack('20c',f.read(20))
    name = ''.join(str(i) for i in name)
    name = name.split("\0")[0]
    print "\nName: ",name

    # The rest of the header information is written in big endian format

    # Number of records (int)
    num_recs = struct.unpack('>i',f.read(4))
    num_recs = int(num_recs[0])
    print "Number of records: %d" %num_recs

    # Number of columns (int)
    num_cols =  struct.unpack('>i',f.read(4))
    num_cols = int(num_cols[0])
    print "Number of columns: %d" %num_cols

    # Sweep (int) - No longer used
    swp =  struct.unpack('>i',f.read(4))[0]
    print "Swp: ",swp

    # Date/time(int) - No longer used
    dtime =  struct.unpack('>i',f.read(4))[0]
    print "dtime: ",dtime

    # For each possible column (32 maximum columns) unpack its header
    # information and store it.  Only store column headers of columns
    # that contain data.  Use termination at first NUL.
    for i in range(32):

        # Channel name (13 characters)
        chname = struct.unpack('13c',f.read(13))
        chname = ''.join(str(i) for i in chname)
        chname = chname.split("\0")[0]

        # Channel units (13 characters)
        chunits = struct.unpack('13c',f.read(13))
        chunits = ''.join(str(i) for i in chunits)
        chunits = chunits.split("\0")[0]

        # This field is now unused, so we just read past it (int)
        gain = struct.unpack('>i',f.read(4))

        # This field is now unused, so we just read past it (50 characters)
        comment = struct.unpack('50c',f.read(50))

        # Number of elements (int)
        nelem = struct.unpack('>i',f.read(4))
        nelem = int(nelem[0])

        if chname[0:6] == 'no_val':
            continue # Skip Blank Channels
        else:
            col_headings.append(chname)
            col_recs.append(nelem)
            col_units.append(chunits)

    # Show column units and headings
    print "\n\n-------------------------------------------------"
    print "|%15s|%15s|%15s|" %('Name','Unit','Records')
    print "-------------------------------------------------"
    for column in zip(col_headings,col_units,col_recs):
        print "|%15s|%15s|%15s|" %(column[0],column[1],column[2])
    print "-------------------------------------------------"

    # Read the data into a numpy recarray
    dtype=[]
    for name in col_headings:
        dtype.append((name,'double'))
    dtype = np.dtype(dtype)

    data = np.zeros([num_recs,num_cols])

    for col in range(num_cols):
        for row in range(col_recs[col]):
            if dataendianness == 'little':
                data[row,col] = struct.unpack('<d',f.read(8))[0]
            elif dataendianness == 'big':
                data[row,col] = struct.unpack('>d',f.read(8))[0]
            else:
                print "Data endian setting invalid, please check and retry"
                return 0

    data_rec = np.rec.array(data,dtype=dtype)

    f.close()

    if pandas==True:
        # If a pandas object is requested, make a data frame
        # indexed on row number and return it
        dfo  = pd.DataFrame(data,columns=col_headings)
        # Binary didn't give us a row number, so we just let
        # pandas do that and name the index column
        dfo.index.name = 'row_num'
        return dfo

    else:
        # Otherwise return the default (Numpy Recarray)
        data_rec = np.rec.array(data,dtype=dtype)
        return data_rec
<commit_msg>Update for Python 3<commit_after>import numpy as np
import struct
import pandas as pd

def read_ascii(filename, pandas=False):
    """
    Takes a filename containing the text output (with headers) from xlook and
    reads the columns into a rec array or dataframe object for easy data
    processing and access.
    """

    try:
        f = open(filename, 'r')
    except:
        print(f'Error Opening {filename}')
        return 0

    col_width = 12  # Columns are 12 char wide in header

    # First line of the file is the number of records
    num_recs = f.readline()
    num_recs = int(num_recs.strip('number of records = '))
    print(f'\nNumber of records: {num_recs}')

    # Second line is column numbers, we don't care so just count them
    num_cols = f.readline()
    num_cols = num_cols.split('col')
    num_cols = len(num_cols)
    print(f'Number of columns: {num_cols}')

    # Third line is the column headings
    col_headings_str = f.readline()
    col_headings_str = col_headings_str[5:-1]
    col_headings = ['row_num']  # Row number the the first (unlabeled) column

    for i in range(int(len(col_headings_str)/12)):
        heading = col_headings_str[12*i:12*i+12].strip()
        col_headings.append(heading)

    # Fourth line is column units
    col_units_str = f.readline()
    col_units_str = col_units_str[5:-1]
    col_units = ['.']
    for i in range(int(len(col_units_str)/12) + 1):
        heading = col_units_str[12*i:12*i+12].strip()
        col_units.append(heading)
    col_units = [x for x in col_units if x != '\n']  # Remove newlines

    # Fifth line is number of records per column
    col_recs = f.readline()
    col_recs = col_recs.split('recs')
    col_recs = [int(x) for x in col_recs if x != '\n']
    col_recs.insert(0, num_recs)

    # Show column units and headings
    print('\n\n-------------------------------------------------')
    print('|%15s|%15s|%15s|' % ('Name', 'Unit', 'Records'))
    print('-------------------------------------------------')
    for column in zip(col_headings, col_units, col_recs):
        print('|%15s|%15s|%15s|' % (column[0], column[1], column[2]))
    print('-------------------------------------------------')

    # Read the data into a numpy recarray
    dtype = []
    for name in col_headings:
        dtype.append((name, 'float'))
    dtype = np.dtype(dtype)

    data = np.zeros([num_recs, num_cols])

    i = 0
    for row in f:
        row_data = row.split()
        for j in range(num_cols):
            data[i, j] = row_data[j]
        i += 1

    f.close()

    if pandas:
        # If a pandas object is requested, make a data frame
        # indexed on row number and return it
        dfo = pd.DataFrame(data, columns=col_headings)
        dfo = dfo.set_index('row_num')
        return dfo

    else:
        # Otherwise return the default (Numpy Recarray)
        data_rec = np.rec.array(data, dtype=dtype)
        return data_rec

def _binary_tuple_to_string(binary_form):
    binary_form = [c.decode() for c in binary_form]
    return ''.join(binary_form)

def read_binary(filename, dataendianness='little', pandas=False):
    """
    Takes a filename containing the binary output from xlook and
    reads the columns into a rec array or dataframe object for easy
    data processing and access.

    The data section of the file is written in the native format of the machine
    used to produce the file.  Endianness of data is little by default, but may
    be changed to 'big' to accomodate older files or files written on power pc
    chips.
    """

    try:
        f = open(filename, 'rb')
    except:
        print(f'Error Opening {filename}')
        return 0

    col_headings = []
    col_recs = []
    col_units = []

    # Unpack information at the top of the file about the experiment
    name = struct.unpack('20c', f.read(20))
    name = _binary_tuple_to_string(name)
    name = name.split('\0')[0]
    print(f'\nName: {name}')

    # The rest of the header information is written in big endian format

    # Number of records (int)
    num_recs = struct.unpack('>i', f.read(4))
    num_recs = int(num_recs[0])
    print(f'Number of records: {num_recs}')

    # Number of columns (int)
    num_cols = struct.unpack('>i', f.read(4))
    num_cols = int(num_cols[0])
    print(f'Number of columns: {num_cols}')

    # Sweep (int) - No longer used
    swp = struct.unpack('>i', f.read(4))[0]
    print(f'Swp: {swp}')

    # Date/time(int) - No longer used
    dtime = struct.unpack('>i', f.read(4))[0]
    print(f'dtime: {dtime}')

    # For each possible column (32 maximum columns) unpack its header
    # information and store it.  Only store column headers of columns
    # that contain data.  Use termination at first NUL.
    for i in range(32):

        # Channel name (13 characters)
        chname = struct.unpack('13c', f.read(13))
        chname = _binary_tuple_to_string(chname)
        chname = chname.split('\0')[0]

        # Channel units (13 characters)
        chunits = struct.unpack('13c', f.read(13))
        chunits = _binary_tuple_to_string(chunits)
        chunits = chunits.split('\0')[0]

        # This field is now unused, so we just read past it (int)
        gain = struct.unpack('>i', f.read(4))

        # This field is now unused, so we just read past it (50 characters)
        comment = struct.unpack('50c', f.read(50))
        comment = _binary_tuple_to_string(comment)

        # Number of elements (int)
        nelem = struct.unpack('>i', f.read(4))
        nelem = int(nelem[0])

        if chname[0:6] == 'no_val':
            continue  # Skip Blank Channels
        else:
            col_headings.append(chname)
            col_recs.append(nelem)
            col_units.append(chunits)

    # Show column units and headings
    print('\n\n-------------------------------------------------')
    print('|%15s|%15s|%15s|' % ('Name', 'Unit', 'Records'))
    print('-------------------------------------------------')
    for column in zip(col_headings, col_units, col_recs):
        print('|%15s|%15s|%15s|' % (column[0], column[1], column[2]))
    print('-------------------------------------------------')

    # Read the data into a numpy recarray
    dtype = []
    for name in col_headings:
        dtype.append((name, 'double'))
    dtype = np.dtype(dtype)

    data = np.zeros([num_recs, num_cols])

    for col in range(num_cols):
        for row in range(col_recs[col]):
            if dataendianness == 'little':
                data[row, col] = struct.unpack('<d', f.read(8))[0]
            elif dataendianness == 'big':
                data[row, col] = struct.unpack('>d', f.read(8))[0]
            else:
                print('Data endian setting invalid, please check and retry')
                return 0

    data_rec = np.rec.array(data, dtype=dtype)

    f.close()

    if pandas:
        # If a pandas object is requested, make a data frame
        # indexed on row number and return it
        dfo = pd.DataFrame(data, columns=col_headings)
        # Binary didn't give us a row number, so we just let
        # pandas do that and name the index column
        dfo.index.name = 'row_num'
        return dfo

    else:
        # Otherwise return the default (Numpy Recarray)
        data_rec = np.rec.array(data, dtype=dtype)
        return data_rec
<|endoftext|>