<commit_before>#!/usr/bin/env python
# encoding: utf-8

import sys
import urwid
import logging
import thread
from collections import OrderedDict
import unittest
from StringIO import StringIO

def result_state(test_result):
    if test_result.skipped:
        return 'skipped'
    elif test_result.failures:
        return 'failed'
    elif test_result.errors:
        return 'error'

    return 'ok'

def get_tests(suite):
    test_list = {}
    for item in suite:
        if isinstance(item, unittest.suite.TestSuite):
            test_list.update(get_tests(item))
        else:
            test_list[item.id()] = item

    return OrderedDict(sorted(test_list.iteritems()))

def exit_program(button):
    raise urwid.ExitMainLoop()

class TestLine(urwid.Columns):

    def __init__(self, text):
        self._test_result = ' '

        self.w_text = urwid.Text(text)
        self.w_state = urwid.Text(['[', (self._test_result, self._test_result.upper()[0]), ']'])
        super(TestLine, self).__init__([self.w_text, self.w_state])

class TestLine2(urwid.Widget):
    _sizing = frozenset(['flow'])
    _selectable = True

    signals = ["click"]

    def __init__(self, test_id, *args, **kwargs):
        self.test_id = test_id
        self.test_result = ' '
        self._is_running = False
        super(TestLine2, self).__init__(*args, **kwargs)

    def rows(self, size, focus=False):
        return 1

    def render(self, size, focus=False):
        (maxcol,) = size
        attr = []
        main_attr = ('running', maxcol - 13) if self._is_running else (None, maxcol - 13)
        state_attr = (self.test_result.lower(), 10)

        return urwid.TextCanvas(['{} [{:10}]'.format(self.test_id.ljust(maxcol - 13), self.test_result.lower()[:10])],
            maxcol=maxcol, attr=[[main_attr, (None, 2), state_attr, (None, 1)]])

    def keypress(self, size, key):
        if key == 'enter':
            self._emit('click')

        return key

class TestResultWindow(urwid.WidgetWrap):
    _sizing = frozenset(['box', 'flow', 'fixed'])

    def __init__(self, text, escape_method):
        self.escape_method = escape_method
        super(TestResultWindow, self).__init__(urwid.LineBox(urwid.Filler(urwid.Text(text))))

    def keypress(self, size, key):
        if key == 'q':
            self.escape_method()

        return None

    def selectable(self):
        return True

class TestResultWindow2(urwid.LineBox):
    _sizing = frozenset(['box'])

    def __init__(self, text, escape_method):
        self.escape_method = escape_method
        super(TestResultWindow2, self).__init__(urwid.Filler(urwid.Text(text)))

    def keypress(self, size, key):
        if key == 'q':
            self.escape_method()

        return None

    def selectable(self):
        return True

class TestRunner(object):
    palette = [
        ('reversed',    '',           'dark gray'),
        ('edit',        '',           'dark blue',    '', '',     '#008'),
        ('edit_focus',  '',           'light blue',   '', '',     '#00b'),
        ('failed',      'light red',  '',             '', '',     '#b00'),
        ('error',       'brown',      '',             '', '#f88', '#b00'),
        ('skipped',     'light gray', '',             '', '#f88', '#b00'),
        ('running',     'yellow',     '',             '', '',     ''),
        ('ok',          'dark green', '',             '', '',     ''),
    ]

    _test_fail_states = ['failed', 'error', None]

    def __init__(self):
        urwid.set_encoding("UTF-8")
        loader = unittest.TestLoader()
        top_suite = loader.discover('.')
        self.tests = get_tests(top_suite)
        self.test_data = {}
        self._init_main_screen()
        self.main_loop = None

    def _init_main_screen(self):
        self.w_filter_edit = urwid.AttrMap(urwid.Edit('Filter '), 'edit', 'edit_focus')
        urwid.connect_signal(self.w_filter_edit.original_widget, 'change', self.on_filter_change)
        self._init_test_listbox()
        self.w_main = urwid.Padding(
            urwid.Pile(
                [
                 ('pack', urwid.Text(u'Python Urwid Test Runner', align='center')),
                 ('pack', urwid.Divider()),
                 ('pack', self.w_filter_edit),
                 ('pack', urwid.Divider()),
                 self.w_test_listbox
                 ]
            ),
            left=2, right=2
        )

    def _init_test_listbox(self):
        self.w_test_listbox = self.test_listbox(self.tests.keys())

    def on_filter_change(self, filter_widget, key):
        print filter_widget

    def run(self):
        self.main_loop = urwid.MainLoop(self.w_main, palette=self.palette,
                       unhandled_input=self.unhandled_keypress)

        # self._run_tests()
        self.main_loop.run()

    def popup(self, widget):
        self._popup_original = self.main_loop.widget
        self.main_loop.widget = urwid.Overlay(
            widget,
            self._popup_original,
            'center', ('relative', 90), 'middle', ('relative', 90)
        )

    def _get_test_position(self, test_id):
        return self.test_data[test_id]['position']

    def _run_test(self, test_id):

        self.test_data[test_id]['widget']._is_running = True
        self.test_data[test_id]['widget']._invalidate()
        # self.w_test_listbox._invalidate()
        # self.w_main._invalidate()
        self.main_loop.draw_screen()

        _orig_stdout = sys.stdout
        _orig_stderr = sys.stderr
        sys.stdout = StringIO()
        sys.stderr = StringIO()

        suite = self.tests[test_id]
        result = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(suite)
        result_state_str = result_state(result)
        if result_state_str in ['failed', 'error'] and not self._first_failed_focused:
            self.w_test_listbox.set_focus(self._get_test_position(test_id))
            self._first_failed_focused = True

        self.test_data[test_id]['widget'].test_result = result_state_str
        self.test_data[test_id].update({
            'output': sys.stdout.getvalue(),
            'result_state': result_state_str,
        })
        sys.stdout.close()
        sys.stderr.close()

        sys.stdout = _orig_stdout
        sys.stderr = _orig_stderr

        self.test_data[test_id]['widget']._is_running = False
        self.test_data[test_id]['widget']._invalidate()
        # self.w_test_listbox._invalidate()
        # self.w_main._invalidate()
        self.main_loop.draw_screen()

    def _get_failed_tests(self):
        return OrderedDict([(test_id, test) for test_id, test in self.tests.iteritems()
                                  if self.test_data[test_id].get('result_state') in self._test_fail_states])

    def _run_tests(self, failed_only=True):
        self._first_failed_focused = False
        tests = self._get_failed_tests() if failed_only else self.tests

        for test_id, suite in tests.iteritems():
            self._run_test(test_id)

        self.w_test_listbox._invalidate()
        self.w_main._invalidate()
        self.main_loop.draw_screen()

    def show_test_detail(self, widget, choice):
        # if test has already been run
        if 'output' in self.test_data[choice]:
            self.popup(
                TestResultWindow2(self.test_data[choice]['output'], self.popup_close)
            )

    def popup_close(self):
        self.main_loop.widget = self._popup_original

    def test_listbox(self, test_list):
        list_items = []
        for position, test_id in enumerate(test_list):
            test_line = TestLine2(test_id)
            self.test_data[test_id] = {'widget': test_line, 'position': position}
            urwid.connect_signal(test_line, 'click', self.show_test_detail, test_id)
            list_items.append(urwid.AttrMap(test_line, None, focus_map='reversed'))
        return urwid.ListBox(urwid.SimpleFocusListWalker(list_items))

    def unhandled_keypress(self, key):
        if key in ('q', 'Q'):
            exit_program(None)
        elif key == 'shift r':
            self._run_tests()
        elif key == 'r':
            thread.start_new_thread(
                self._run_tests, (True, )
            )

    def set_test_list(self, test_list):
        self._test_list = test_list

if __name__ == '__main__':
    logging.basicConfig()
    runner = TestRunner()
    runner.run()
<commit_msg>Reacting to filter changes<commit_after>#!/usr/bin/env python
# encoding: utf-8

import sys
import urwid
import logging
import thread
from collections import OrderedDict
import unittest
from StringIO import StringIO

def result_state(test_result):
    if test_result.skipped:
        return 'skipped'
    elif test_result.failures:
        return 'failed'
    elif test_result.errors:
        return 'error'

    return 'ok'

def get_tests(suite):
    test_list = {}
    for item in suite:
        if isinstance(item, unittest.suite.TestSuite):
            test_list.update(get_tests(item))
        else:
            test_list[item.id()] = item

    return OrderedDict(sorted(test_list.iteritems()))

def exit_program(button):
    raise urwid.ExitMainLoop()

class TestLine(urwid.Columns):

    def __init__(self, text):
        self._test_result = ' '

        self.w_text = urwid.Text(text)
        self.w_state = urwid.Text(['[', (self._test_result, self._test_result.upper()[0]), ']'])
        super(TestLine, self).__init__([self.w_text, self.w_state])

class TestLine2(urwid.Widget):
    _sizing = frozenset(['flow'])
    _selectable = True

    signals = ["click"]

    def __init__(self, test_id, *args, **kwargs):
        self.test_id = test_id
        self.test_result = ' '
        self._is_running = False
        super(TestLine2, self).__init__(*args, **kwargs)

    def rows(self, size, focus=False):
        return 1

    def render(self, size, focus=False):
        (maxcol,) = size
        attr = []
        main_attr = ('running', maxcol - 13) if self._is_running else (None, maxcol - 13)
        state_attr = (self.test_result.lower(), 10)

        return urwid.TextCanvas(['{} [{:10}]'.format(self.test_id.ljust(maxcol - 13), self.test_result.lower()[:10])],
            maxcol=maxcol, attr=[[main_attr, (None, 2), state_attr, (None, 1)]])

    def keypress(self, size, key):
        if key == 'enter':
            self._emit('click')

        return key

class TestResultWindow(urwid.WidgetWrap):
    _sizing = frozenset(['box', 'flow', 'fixed'])

    def __init__(self, text, escape_method):
        self.escape_method = escape_method
        super(TestResultWindow, self).__init__(urwid.LineBox(urwid.Filler(urwid.Text(text))))

    def keypress(self, size, key):
        if key == 'q':
            self.escape_method()

        return None

    def selectable(self):
        return True

class TestResultWindow2(urwid.LineBox):
    _sizing = frozenset(['box'])

    def __init__(self, text, escape_method):
        self.escape_method = escape_method
        super(TestResultWindow2, self).__init__(urwid.Filler(urwid.Text(text)))

    def keypress(self, size, key):
        if key == 'q':
            self.escape_method()

        return None

    def selectable(self):
        return True

class TestRunner(object):
    palette = [
        ('reversed',    '',           'dark gray'),
        ('edit',        '',           'dark blue',    '', '',     '#008'),
        ('edit_focus',  '',           'light blue',   '', '',     '#00b'),
        ('failed',      'light red',  '',             '', '',     '#b00'),
        ('error',       'brown',      '',             '', '#f88', '#b00'),
        ('skipped',     'light gray', '',             '', '#f88', '#b00'),
        ('running',     'yellow',     '',             '', '',     ''),
        ('ok',          'dark green', '',             '', '',     ''),
    ]

    _test_fail_states = ['failed', 'error', None]

    def __init__(self):
        urwid.set_encoding("UTF-8")
        loader = unittest.TestLoader()
        top_suite = loader.discover('.')
        self.tests = get_tests(top_suite)
        self.current_test_list = self.tests
        self.test_data = {}
        self._init_main_screen()
        self.main_loop = None

    def _init_main_screen(self):
        self.w_filter_edit = urwid.AttrMap(urwid.Edit('Filter '), 'edit', 'edit_focus')
        urwid.connect_signal(self.w_filter_edit.original_widget, 'change', self.on_filter_change)
        self._init_test_listbox()
        self.w_main = urwid.Padding(
            urwid.Pile(
                [
                 ('pack', urwid.Text(u'Python Urwid Test Runner', align='center')),
                 ('pack', urwid.Divider()),
                 ('pack', self.w_filter_edit),
                 ('pack', urwid.Divider()),
                 self.w_test_listbox
                 ]
            ),
            left=2, right=2
        )

    def _init_test_listbox(self):
        self.w_test_listbox = self.test_listbox(self.current_test_list.keys())

    def on_filter_change(self, filter_widget, key):
        filter_value = filter_widget.get_edit_text()
        self.w_main.original_widget.widget_list[0].set_text(filter_value)

        self.current_test_list = {k: v for k, v in self.tests.iteritems() if filter_value in k}
        # import pdb; pdb.set_trace()
        self.w_main.original_widget.widget_list[4] = self.test_listbox(self.current_test_list.keys())
        self.w_main.original_widget._invalidate()
        # self.main_loop.widget._invalidate()
        # self.main_loop.draw_screen()

    def run(self):
        self.main_loop = urwid.MainLoop(self.w_main, palette=self.palette,
                       unhandled_input=self.unhandled_keypress)

        # self._run_tests()
        self.main_loop.run()

    def popup(self, widget):
        self._popup_original = self.main_loop.widget
        self.main_loop.widget = urwid.Overlay(
            widget,
            self._popup_original,
            'center', ('relative', 90), 'middle', ('relative', 90)
        )

    def _get_test_position(self, test_id):
        return self.test_data[test_id]['position']

    def _run_test(self, test_id):

        self.test_data[test_id]['widget']._is_running = True
        self.test_data[test_id]['widget']._invalidate()
        # self.w_test_listbox._invalidate()
        # self.w_main._invalidate()
        self.main_loop.draw_screen()

        _orig_stdout = sys.stdout
        _orig_stderr = sys.stderr
        sys.stdout = StringIO()
        sys.stderr = StringIO()

        suite = self.tests[test_id]
        result = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(suite)
        result_state_str = result_state(result)
        if result_state_str in ['failed', 'error'] and not self._first_failed_focused:
            self.w_test_listbox.set_focus(self._get_test_position(test_id))
            self._first_failed_focused = True

        self.test_data[test_id]['widget'].test_result = result_state_str
        self.test_data[test_id].update({
            'output': sys.stdout.getvalue(),
            'result_state': result_state_str,
        })
        sys.stdout.close()
        sys.stderr.close()

        sys.stdout = _orig_stdout
        sys.stderr = _orig_stderr

        self.test_data[test_id]['widget']._is_running = False
        self.test_data[test_id]['widget']._invalidate()
        # self.w_test_listbox._invalidate()
        # self.w_main._invalidate()
        self.main_loop.draw_screen()

    def _get_failed_tests(self):
        return OrderedDict([(test_id, test) for test_id, test in self.tests.iteritems()
                                  if self.test_data[test_id].get('result_state') in self._test_fail_states])

    def _run_tests(self, failed_only=True):
        self._first_failed_focused = False
        tests = self._get_failed_tests() if failed_only else self.tests

        for test_id, suite in tests.iteritems():
            self._run_test(test_id)

        self.w_test_listbox._invalidate()
        self.w_main._invalidate()
        self.main_loop.draw_screen()

    def show_test_detail(self, widget, choice):
        # if test has already been run
        if 'output' in self.test_data[choice]:
            self.popup(
                TestResultWindow2(self.test_data[choice]['output'], self.popup_close)
            )

    def popup_close(self):
        self.main_loop.widget = self._popup_original

    def test_listbox(self, test_list):
        list_items = []
        for position, test_id in enumerate(test_list):
            test_line = TestLine2(test_id)
            self.test_data[test_id] = {'widget': test_line, 'position': position}
            urwid.connect_signal(test_line, 'click', self.show_test_detail, test_id)
            list_items.append(urwid.AttrMap(test_line, None, focus_map='reversed'))
        return urwid.ListBox(urwid.SimpleFocusListWalker(list_items))

    def unhandled_keypress(self, key):
        if key in ('q', 'Q'):
            exit_program(None)
        elif key == 'shift r':
            self._run_tests()
        elif key == 'r':
            thread.start_new_thread(
                self._run_tests, (True, )
            )

    def set_test_list(self, test_list):
        self._test_list = test_list

if __name__ == '__main__':
    logging.basicConfig()
    runner = TestRunner()
    runner.run()
<|endoftext|>


<commit_before>"""
mixpandas.py

A library to request data from Mixpanel's Raw Data Export API
"""
import datetime
import hashlib
import urllib.request, urllib.parse, urllib.error
import time
try:
    import json
except ImportError:
    import simplejson as json

import pandas as pd

VERSION = '2.0'  # Mixpanel API version
date_format = '%Y-%m-%d'  # Mixpanel's API date format

def read_events(keys, events=None, start=None, end=None,
                where=None, bucket=None, columns=None, exclude_mp=True):
    """
    Request data from Mixpanel's Raw Data Export API and return as a pandas
    DataFrame with event times converted to pandas Timestamp objects

    Parameters
    ----------
    keys : tuple containing (Mixpanel API Key, Mixpanel Secret Key)
    events : event name or list of event names to get, optional
        If not specified or None, all events will be downloaded
    start : start date. String or datetime-like, default 2011-07-10
        The input is converted to a date using pandas.to_datetime which
        accepts a variety of inputs, e.g., '5/6/2013', '2013-05-06',
        'May 6, 2013'.  It can also be a datetime object

        The default comes from the earliest start date allowed by the API
    end: end date. String or datetime-like, default yesterday
        The input is converted in the same manner as start. The latest this
        date can be is yesterday's date, which is the default value.
    where: string, Mixpanel filter expression, optional
        See the documentation:
        https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions
    bucket: string, optional
        See the documentation:
        https://mixpanel.com/docs/api-documentation/displaying-mixpanel-data-to-your-users
    columns: string or list of strings, optional
        Returned DataFrame will only contain the specified parameters
    exclude_mp: Filter out Mixpanel-specific data. default True
        Filter out event properties that begin with '$' or 'mp_'.
        These properties are automatically inserted by Mixpanel and indicate
        things like region, OS, etc.

    For more information, see:
    https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel

    """

    if start is None:
        # This default comes from an error message you'll receive if you
        # try an start date earlier than 7/10/2011
        start = datetime.date(2011, 0o7, 10)
    start = pd.to_datetime(start)
    start_str = start.strftime(date_format)

    if end is None:  # Defaults to yesterday, the latest allowed
        end = datetime.date.today() - datetime.timedelta(1)
    end = pd.to_datetime(end)
    end_str = end.strftime(date_format)

    payload = {
        'from_date' : start_str,
        'to_date' : end_str,
    }

    # Handle when single event passed in as string
    if isinstance(events, str):
        events = [events]

    # Fill the payload with the parameters if they're specified
    params = {'event' : events, 'where' : where, 'bucket' : bucket}
    for k, v in params.items():
        if v is not None:
            payload[k] = v

    data = request(keys, ['export'], payload, data_api=True)

    return _export_to_df(data, columns, exclude_mp)

def _export_to_df(data, columns, exclude_mp):
    # Keep track of the parameters each returned event
    parameters = set()

    # Calls to the data export API do not return JSON.  They return
    # records separated by newlines, where each record is valid JSON.
    # The event parameters are in the properties field
    events = []
    for line in data.split('\n'):
        try:
            event = json.loads(line)
            ev = event['properties']
            ev['event']=event['event']
        except ValueError:  # Not valid JSON
            continue

        parameters.update(list(ev.keys()))
        events.append(ev)

    # If columns is excluded, leave off parameters that start with '$' as
    # these are automatically included in the Mixpanel events and clutter the
    # real data
    if columns is None:
        if exclude_mp:
            columns = [p for p in parameters if not (p.startswith('$') or
                                                     p.startswith('mp_'))]
        else:
            columns = parameters
    elif 'time' not in columns:
        columns.append('time')
    df = pd.DataFrame(events, columns=columns)

    # Make time a datetime.
    df['time'] = df['time'].map(lambda x: datetime.datetime.fromtimestamp(x))

    return df

# The code below is from Mixpanel's Python client for the data export API.
# There are only a few modifications:
#     * Add data_api optional argument
#     * Change the base URL to http://data.mixpanel.com if data_api is set
#     * make it a function that takes keys instead of a class initialized
#       with the keys (this is just a personal preference)
#     * Changed max. line width
# Mixpanel, Inc. -- http://mixpanel.com/
#
# Python API client library to consume mixpanel.com analytics data.
# https://mixpanel.com/site_media//api/v2/mixpanel.py
def request(keys, methods, params, format='json', data_api=False):
    """
        methods - List of methods to be joined,
                      e.g. ['events', 'properties', 'values']
                  will give us
                      http://mixpanel.com/api/2.0/events/properties/values/

        params - Extra parameters associated with method
    """
    api_key, api_secret = keys

    params['api_key'] = api_key
    params['expire'] = int(time.time()) + 600   # Grant this request 10 minutes.
    params['format'] = format
    if 'sig' in params: del params['sig']
    params['sig'] = hash_args(params, api_secret)

    if data_api:
        url_base = r'http://data.mixpanel.com/api'
    else:
        url_base = r'http://mixpanel.com/api'

    request_url = ('/'.join([url_base, str(VERSION)] + methods) + '/?' +
                   unicode_urlencode(params))

    request = urllib.request.urlopen(request_url)
    data = request.read()

    if data_api:
        return data

    return json.loads(data)

def unicode_urlencode(params):
    """
        Convert lists to JSON encoded strings, and correctly handle any
        unicode URL parameters.
    """
    if isinstance(params, dict):
        params = list(params.items())
    for i, param in enumerate(params):
        if isinstance(param[1], list):
            params[i] = (param[0], json.dumps(param[1]),)

    return urllib.parse.urlencode(
        [(k, isinstance(v, str) and v.encode('utf-8') or v)
            for k, v in params]
    )

def hash_args(args, api_secret):
    """
        Hashes arguments by joining key=value pairs, appending a secret, and
        then taking the MD5 hex digest.
    """
    for a in args:
        if isinstance(args[a], list): args[a] = json.dumps(args[a])

    args_joined = b''
    for a in sorted(args.keys()):
        if isinstance(a, str):
            args_joined += a.encode('utf-8')
        else:
            args_joined += str(a).encode('utf-8')

        args_joined += b'='

        if isinstance(args[a], str):
            args_joined += args[a].encode('utf-8')
        else:
            args_joined += str(args[a]).encode('utf-8')

    hash = hashlib.md5(args_joined)

    hash.update(api_secret.encode('utf-8'))
    return hash.hexdigest()
<commit_msg>more encoding<commit_after>"""
mixpandas.py

A library to request data from Mixpanel's Raw Data Export API
"""
import datetime
import hashlib
import urllib.request, urllib.parse, urllib.error
import time
try:
    import json
except ImportError:
    import simplejson as json

import pandas as pd

VERSION = '2.0'  # Mixpanel API version
date_format = '%Y-%m-%d'  # Mixpanel's API date format

def read_events(keys, events=None, start=None, end=None,
                where=None, bucket=None, columns=None, exclude_mp=True):
    """
    Request data from Mixpanel's Raw Data Export API and return as a pandas
    DataFrame with event times converted to pandas Timestamp objects

    Parameters
    ----------
    keys : tuple containing (Mixpanel API Key, Mixpanel Secret Key)
    events : event name or list of event names to get, optional
        If not specified or None, all events will be downloaded
    start : start date. String or datetime-like, default 2011-07-10
        The input is converted to a date using pandas.to_datetime which
        accepts a variety of inputs, e.g., '5/6/2013', '2013-05-06',
        'May 6, 2013'.  It can also be a datetime object

        The default comes from the earliest start date allowed by the API
    end: end date. String or datetime-like, default yesterday
        The input is converted in the same manner as start. The latest this
        date can be is yesterday's date, which is the default value.
    where: string, Mixpanel filter expression, optional
        See the documentation:
        https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions
    bucket: string, optional
        See the documentation:
        https://mixpanel.com/docs/api-documentation/displaying-mixpanel-data-to-your-users
    columns: string or list of strings, optional
        Returned DataFrame will only contain the specified parameters
    exclude_mp: Filter out Mixpanel-specific data. default True
        Filter out event properties that begin with '$' or 'mp_'.
        These properties are automatically inserted by Mixpanel and indicate
        things like region, OS, etc.

    For more information, see:
    https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel

    """

    if start is None:
        # This default comes from an error message you'll receive if you
        # try an start date earlier than 7/10/2011
        start = datetime.date(2011, 0o7, 10)
    start = pd.to_datetime(start)
    start_str = start.strftime(date_format)

    if end is None:  # Defaults to yesterday, the latest allowed
        end = datetime.date.today() - datetime.timedelta(1)
    end = pd.to_datetime(end)
    end_str = end.strftime(date_format)

    payload = {
        'from_date' : start_str,
        'to_date' : end_str,
    }

    # Handle when single event passed in as string
    if isinstance(events, str):
        events = [events]

    # Fill the payload with the parameters if they're specified
    params = {'event' : events, 'where' : where, 'bucket' : bucket}
    for k, v in params.items():
        if v is not None:
            payload[k] = v

    data = request(keys, ['export'], payload, data_api=True)

    return _export_to_df(data, columns, exclude_mp)

def _export_to_df(data, columns, exclude_mp):
    # Keep track of the parameters each returned event
    parameters = set()

    # Calls to the data export API do not return JSON.  They return
    # records separated by newlines, where each record is valid JSON.
    # The event parameters are in the properties field
    events = []
    for line in data.split(b'\n'):
        try:
            event = json.loads(line)
            ev = event['properties']
            ev['event']=event['event']
        except ValueError:  # Not valid JSON
            continue

        parameters.update(list(ev.keys()))
        events.append(ev)

    # If columns is excluded, leave off parameters that start with '$' as
    # these are automatically included in the Mixpanel events and clutter the
    # real data
    if columns is None:
        if exclude_mp:
            columns = [p for p in parameters if not (p.startswith('$') or
                                                     p.startswith('mp_'))]
        else:
            columns = parameters
    elif 'time' not in columns:
        columns.append('time')
    df = pd.DataFrame(events, columns=columns)

    # Make time a datetime.
    df['time'] = df['time'].map(lambda x: datetime.datetime.fromtimestamp(x))

    return df

# The code below is from Mixpanel's Python client for the data export API.
# There are only a few modifications:
#     * Add data_api optional argument
#     * Change the base URL to http://data.mixpanel.com if data_api is set
#     * make it a function that takes keys instead of a class initialized
#       with the keys (this is just a personal preference)
#     * Changed max. line width
# Mixpanel, Inc. -- http://mixpanel.com/
#
# Python API client library to consume mixpanel.com analytics data.
# https://mixpanel.com/site_media//api/v2/mixpanel.py
def request(keys, methods, params, format='json', data_api=False):
    """
        methods - List of methods to be joined,
                      e.g. ['events', 'properties', 'values']
                  will give us
                      http://mixpanel.com/api/2.0/events/properties/values/

        params - Extra parameters associated with method
    """
    api_key, api_secret = keys

    params['api_key'] = api_key
    params['expire'] = int(time.time()) + 600   # Grant this request 10 minutes.
    params['format'] = format
    if 'sig' in params: del params['sig']
    params['sig'] = hash_args(params, api_secret)

    if data_api:
        url_base = r'http://data.mixpanel.com/api'
    else:
        url_base = r'http://mixpanel.com/api'

    request_url = ('/'.join([url_base, str(VERSION)] + methods) + '/?' +
                   unicode_urlencode(params))

    request = urllib.request.urlopen(request_url)
    data = request.read()

    if data_api:
        return data

    return json.loads(data)

def unicode_urlencode(params):
    """
        Convert lists to JSON encoded strings, and correctly handle any
        unicode URL parameters.
    """
    if isinstance(params, dict):
        params = list(params.items())
    for i, param in enumerate(params):
        if isinstance(param[1], list):
            params[i] = (param[0], json.dumps(param[1]),)

    return urllib.parse.urlencode(
        [(k, isinstance(v, str) and v.encode('utf-8') or v)
            for k, v in params]
    )

def hash_args(args, api_secret):
    """
        Hashes arguments by joining key=value pairs, appending a secret, and
        then taking the MD5 hex digest.
    """
    for a in args:
        if isinstance(args[a], list): args[a] = json.dumps(args[a])

    args_joined = b''
    for a in sorted(args.keys()):
        if isinstance(a, str):
            args_joined += a.encode('utf-8')
        else:
            args_joined += str(a).encode('utf-8')

        args_joined += b'='

        if isinstance(args[a], str):
            args_joined += args[a].encode('utf-8')
        else:
            args_joined += str(args[a]).encode('utf-8')

    hash = hashlib.md5(args_joined)

    hash.update(api_secret.encode('utf-8'))
    return hash.hexdigest()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import sys, subprocess, os, multiprocessing, magic, tempfile, getopt, re;
OUTPUT_CUE='audiocd.cue';
OUTPUT_AUDIO='audiocd.wav';

opts, args = getopt.getopt(sys.argv[1:], 'o:');
for k,v in opts:
    if (k == '-o'):
        OUTPUT_CUE = v+'.cue';
        OUTPUT_AUDIO = v+'.wav';

files = args;
tempfiles = [];

mime = magic.open(magic.MIME);
mime.load();
for i in range(len(files)):
    mime_type = mime.file(files[i]).split(';')[0];
    if (mime_type in ('audio/mpeg') or re.search(r'\.mp3$', files[i], re.I)):
        tf = tempfile.NamedTemporaryFile(prefix='%02d-'%(i+1), suffix='.wav', delete=False);
        tf.close();
        tempfiles.append(tf.name);
        plame = subprocess.Popen(['/usr/bin/lame', '--decode', files[i], tf.name]);
        if (0 == plame.wait()):
            files[i] = tf.name;
mime.close();

pwav = subprocess.Popen(['/usr/bin/shntool', 'join'] + files + ['-o', 'wav', '-O', 'always']);
pcue = subprocess.Popen(
    ['/usr/bin/shntool', 'cue']+files,
    stdin=subprocess.PIPE,
    stdout=subprocess.PIPE,
    stderr=subprocess.PIPE
);
stdout, stderr = pcue.communicate();
sys.stderr.write(stderr.decode('UTF-8'));
with open(OUTPUT_CUE, 'w') as f:
    f.write(stdout.decode('UTF-8').replace('joined.wav', OUTPUT_AUDIO).replace('WAVE', 'BINARY'));
pwav.wait();
os.rename('joined.wav', OUTPUT_AUDIO);

for f in tempfiles:
    os.unlink(f);

<commit_msg>mkaudiocd.py updated to generate raw data<commit_after>#!/usr/bin/env python3
import sys, subprocess, os, multiprocessing, magic, tempfile, getopt, re;
OUTPUT_CUE='audiocd.cue';
OUTPUT_AUDIO='audiocd.dat';

opts, args = getopt.getopt(sys.argv[1:], 'o:');
for k,v in opts:
    if (k == '-o'):
        OUTPUT_CUE = v+'.cue';
        OUTPUT_AUDIO = v+'.wav';

files = args;
tempfiles = [];

mime = magic.open(magic.MIME);
mime.load();
for i in range(len(files)):
    mime_type = mime.file(files[i]).split(';')[0];
    if (mime_type in ('audio/mpeg') or re.search(r'\.mp3$', files[i], re.I)):
        tf = tempfile.NamedTemporaryFile(prefix='%02d-'%(i+1), suffix='.wav', delete=False);
        tf.close();
        tempfiles.append(tf.name);
        plame = subprocess.Popen(['/usr/bin/lame', '--decode', files[i], tf.name]);
        if (0 == plame.wait()):
            files[i] = tf.name;
mime.close();

pwav = subprocess.Popen(['/usr/bin/shntool', 'join'] + files + ['-o', 'wav', '-O', 'always']);
pcue = subprocess.Popen(
    ['/usr/bin/shntool', 'cue']+files,
    stdin=subprocess.PIPE,
    stdout=subprocess.PIPE,
    stderr=subprocess.PIPE
);
stdout, stderr = pcue.communicate();
sys.stderr.write(stderr.decode('UTF-8'));
with open(OUTPUT_CUE, 'w') as f:
    f.write(stdout.decode('UTF-8').replace('joined.wav', OUTPUT_AUDIO).replace('WAVE', 'BINARY'));

pwav.wait();
with open(OUTPUT_AUDIO, 'wb') as fp:
    pwav = subprocess.Popen(['/usr/bin/shntool', 'cat', '-e', 'joined.wav'], stdout=fp);
    pwav.wait();

os.unlink('joined.wav');
for f in tempfiles:
    os.unlink(f);

<|endoftext|>


<commit_before>#!/usr/bin/env python3
import threading

__author__ = 'Adaephon'

import json
import queue
import socket
import struct
import subprocess

msgTypes = [
    'command',
    'get_workspaces',
    'subscribe',
    'get_outputs',
    'get_tree',
    'get_marks',
    'get_bar_config',
    'get_version',
]
msgTypesMap = {msgTypes[i]: i for i in range(len(msgTypes))}

eventTypes = [
    'workspace',
    'output',
    'mode',
    'window',
    'barconfig_update',
]
eventTypesMap = {eventTypes[i]: i for i in range(len(eventTypes))}

class Socket:
    magicString = b'i3-ipc'
    headerPacking = bytes('={}sLL'.format(len(magicString)), 'utf-8')
    headerLen = struct.calcsize(headerPacking)

    @staticmethod
    def get_path():
        path = subprocess.check_output(['i3', '--get-socketpath']).rstrip()
        return path

    def __init__(self, socketPath=None):
        self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        self.socket.settimeout(1)
        self.socket.connect(socketPath or self.get_path())

    def _send(self, msgType, msg=b''):
        message = (struct.pack(self.headerPacking, self.magicString,
                               len(msg), msgType) + msg)
        self.socket.sendall(message)

    def _receive(self):
        header = self.socket.recv(self.headerLen)
        _, msgSize, msgType = struct.unpack(self.headerPacking, header)
        data = self.socket.recv(msgSize)
        return msgType, data

    def receive(self):
        type_, data = self._receive()
        isEvent = type_ >> 31
        typeName = (eventTypes[type_ & 0x7f] if isEvent
                    else msgTypes[type_])
        # maybe use data.decode(errors=ignore) -> faulty jetbrains class name
        # see http://bugs.i3wm.org/report/ticket/1347
        parsedData = json.loads(data.decode())
        response = ('event' if isEvent else 'reply',
                    typeName,
                    parsedData)
        return response

    def __getattr__(self, attr):
        # provide method for every msgType
        if attr in msgTypes:
            def func(msg=b''):
                self._send(msgTypesMap[attr], msg)
                return self.receive()
            return func
        else:
            raise AttributeError("'{}' object has no attribute '{}'".format(
                self.__class__.__name__, attr))

class I3Base:
    def __init__(self, socket=None):
        self.socket = socket or Socket()

class EventHandler(I3Base):
    def __init__(self, socket=None):
        super().__init__(socket)
        self.events = [0] * len(eventTypes)
        self._eventqueue = queue.Queue()
        self._subscript_confirmation = queue.Queue()
        self.isrunning = threading.Event()

    def run(self):
        self.isrunning.set()
        handler = threading.Thread(target=self._handle_events)
        reader = threading.Thread(target=self._read_socket)

    def _read_socket(self):
        while self.isrunning.is_set():
            dataType, name, payload = self.socket.receive()
            if dataType == 'event':
                self._eventqueue.put((name, payload))
            elif name == 'subcribe':
                self._subscript_confirmation.put(payload)
            else:
                raise UnexpectedDataError((dataType, name, payload))

    def _handle_events(self):
        while self.isrunning.is_set():
            type_, payload = self._eventqueue.get()
            if type_ == -1:
                break
            self._handle_event(type_, payload)

    def _handle_event(self, type_, payload):
        pass
        print(type_, payload)

    # TODO: something to pause handling (certain) events to avoid recursion
    def pause(self, events=None):
        pass

class Hook:
    def __init__(self, event, change=None, callback=None):
        pass

class UnexpectedDataError(Exception):
    pass

class Item:
    def __init__(self, dict):
        self.__dict__.update(dict)

class WorkspaceHandler(I3Base):
    def __init__(self, socket=None):
        super().__init__(socket)

    @property
    def _workspaces(self):
        _, _, wslist = self.socket.get_workspaces()
        return map(lambda ws: Item(ws), wslist)

    def workspaces(self, visible=None, focused=None, urgent=None,
                   num=None):
        pass

<commit_msg>add convenience shortcut for commands<commit_after>#!/usr/bin/env python3
import threading

__author__ = 'Adaephon'

import json
import queue
import socket
import struct
import subprocess

msgTypes = [
    'command',
    'get_workspaces',
    'subscribe',
    'get_outputs',
    'get_tree',
    'get_marks',
    'get_bar_config',
    'get_version',
]
msgTypesMap = {msgTypes[i]: i for i in range(len(msgTypes))}

eventTypes = [
    'workspace',
    'output',
    'mode',
    'window',
    'barconfig_update',
]
eventTypesMap = {eventTypes[i]: i for i in range(len(eventTypes))}

class Socket:
    magicString = b'i3-ipc'
    headerPacking = bytes('={}sLL'.format(len(magicString)), 'utf-8')
    headerLen = struct.calcsize(headerPacking)

    @staticmethod
    def get_path():
        path = subprocess.check_output(['i3', '--get-socketpath']).rstrip()
        return path

    def __init__(self, socketPath=None):
        self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        self.socket.settimeout(1)
        self.socket.connect(socketPath or self.get_path())

    def _send(self, msgType, msg=b''):
        message = (struct.pack(self.headerPacking, self.magicString,
                               len(msg), msgType) + msg)
        self.socket.sendall(message)

    def _receive(self):
        header = self.socket.recv(self.headerLen)
        _, msgSize, msgType = struct.unpack(self.headerPacking, header)
        data = self.socket.recv(msgSize)
        return msgType, data

    def receive(self):
        type_, data = self._receive()
        isEvent = type_ >> 31
        typeName = (eventTypes[type_ & 0x7f] if isEvent
                    else msgTypes[type_])
        # maybe use data.decode(errors=ignore) -> faulty jetbrains class name
        # see http://bugs.i3wm.org/report/ticket/1347
        parsedData = json.loads(data.decode())
        response = ('event' if isEvent else 'reply',
                    typeName,
                    parsedData)
        return response

    def __getattr__(self, attr):
        # provide method for every msgType
        if attr in msgTypes:
            msgType = attr
        else:
            msgType = msgTypesMap['command']

        def func(msg=b''):
            self._send(msgType, b"%b %b" % (attr.encode(), msg))
            return self.receive()
        # else:
        #     raise AttributeError("'{}' object has no attribute '{}'".format(
        #         self.__class__.__name__, attr))
        return func

class I3Base:
    def __init__(self, socket=None):
        self.socket = socket or Socket()

class EventHandler(I3Base):
    def __init__(self, socket=None):
        super().__init__(socket)
        self.events = [0] * len(eventTypes)
        self._eventqueue = queue.Queue()
        self._subscript_confirmation = queue.Queue()
        self.isrunning = threading.Event()

    def run(self):
        self.isrunning.set()
        handler = threading.Thread(target=self._handle_events)
        reader = threading.Thread(target=self._read_socket)

    def _read_socket(self):
        while self.isrunning.is_set():
            dataType, name, payload = self.socket.receive()
            if dataType == 'event':
                self._eventqueue.put((name, payload))
            elif name == 'subcribe':
                self._subscript_confirmation.put(payload)
            else:
                raise UnexpectedDataError((dataType, name, payload))

    def _handle_events(self):
        while self.isrunning.is_set():
            type_, payload = self._eventqueue.get()
            if type_ == -1:
                break
            self._handle_event(type_, payload)

    def _handle_event(self, type_, payload):
        pass
        print(type_, payload)

    # TODO: something to pause handling (certain) events to avoid recursion
    def pause(self, events=None):
        pass

class Hook:
    def __init__(self, event, change=None, callback=None):
        pass

class UnexpectedDataError(Exception):
    pass

class Item:
    def __init__(self, dict):
        self.__dict__.update(dict)

class WorkspaceHandler(I3Base):
    def __init__(self, socket=None):
        super().__init__(socket)

    @property
    def _workspaces(self):
        _, _, wslist = self.socket.get_workspaces()
        return map(lambda ws: Item(ws), wslist)

    def workspaces(self, visible=None, focused=None, urgent=None,
                   num=None):
        pass

<|endoftext|>


<commit_before>'''
Created on Jun 5, 2013

@author: timmahrt
'''

import os
from os.path import join
import copy

from praatio import tgio
from praatio import praat_scripts
from praatio import dataio

from promo.morph_utils import utils
from promo.morph_utils import audio_scripts
from promo.morph_utils import plot_morphed_data

# This value is used to differentiate a praat interval boundary that marks
# the start of one region and the end of another.
PRAAT_TIME_DIFF = 0.000001

class NoLabeledRegionFoundException(Exception):

    def __init__(self, tgFN):
        super(NoLabeledRegionFoundException, self).__init__()
        self.tgFN = tgFN

    def __str__(self):
        return "No labeled region fitting the specified criteria for tg: " + \
            self.tgFN

def changeDuration(fromWavFN, durationParameters, stepList, outputName,
                   outputMinPitch, outputMaxPitch, praatEXE):
    '''
    Uses praat to morph duration in one file to duration in another

    Praat uses the PSOLA algorithm
    '''

    rootPath = os.path.split(fromWavFN)[0]

    # Prep output directories
    outputPath = join(rootPath, "duration_resynthesized_wavs")
    utils.makeDir(outputPath)

    durationTierPath = join(rootPath, "duration_tiers")
    utils.makeDir(durationTierPath)

    fromWavDuration = audio_scripts.getSoundFileDuration(fromWavFN)

    durationParameters = copy.deepcopy(durationParameters)
    # Pad any gaps with values of 1 (no change in duration)

    # No need to stretch out any pauses at the beginning
    if durationParameters[0][0] != 0:
        tmpVar = (0, durationParameters[0][0] - PRAAT_TIME_DIFF, 1)
        durationParameters.insert(0, tmpVar)

    # Or the end
    if durationParameters[-1][1] < fromWavDuration:
        durationParameters.append((durationParameters[-1][1] + PRAAT_TIME_DIFF,
                                   fromWavDuration, 1))

    # Create the praat script for doing duration manipulation
    for stepAmount in stepList:
        durationPointList = []
        for start, end, ratio in durationParameters:
            percentChange = 1 + (ratio - 1) * stepAmount
            durationPointList.append((start, percentChange))
            durationPointList.append((end, percentChange))

        outputPrefix = "%s_%0.3g" % (outputName, stepAmount)
        durationTierFN = join(durationTierPath,
                              "%s.DurationTier" % outputPrefix)
        outputWavFN = join(outputPath, "%s.wav" % outputPrefix)
        durationTier = dataio.PointObject2D(durationPointList, dataio.DURATION,
                                            0, fromWavDuration)
        durationTier.save(durationTierFN)

        praat_scripts.resynthesizeDuration(praatEXE,
                                           fromWavFN,
                                           durationTierFN,
                                           outputWavFN,
                                           outputMinPitch, outputMaxPitch)

def getBareParameters(wavFN):
    wavDuration = audio_scripts.getSoundFileDuration(wavFN)
    return [(0, wavDuration, ''), ]

def getMorphParameters(fromTGFN, toTGFN, tierName, stepList=None,
                       outputTGName=None,
                       outputImageFN=None,
                       filterFunc=None, useBlanks=False):
    '''
    Get intervals for source and target audio files

    Use this information to find out how much to stretch/shrink each source
    interval
    '''

    if stepList is None:
        stepList = [1, ]  # Complete morph
    if filterFunc is None:
        filterFunc = lambda entry: True  # Everything is accepted
    if outputTGName is not None:
        utils.makeDir(os.path.split(outputTGName)[0])
    if outputImageFN is not None:
        utils.makeDir(os.path.split(outputImageFN)[0])

    fromEntryList = utils.getIntervals(fromTGFN, tierName,
                                       includeUnlabeledRegions=useBlanks)
    toEntryList = utils.getIntervals(toTGFN, tierName,
                                     includeUnlabeledRegions=useBlanks)

    fromEntryList = [entry for entry in fromEntryList if filterFunc(entry)]
    toEntryList = [entry for entry in toEntryList if filterFunc(entry)]

    assert(len(fromEntryList) == len(toEntryList))

    durationParameters = []
    for fromEntry, toEntry in zip(fromEntryList, toEntryList):
        fromStart, fromEnd = fromEntry[:2]
        toStart, toEnd = toEntry[:2]

        # Praat will ignore a second value appearing at the same time as
        # another so we give each start a tiny offset to distinguish intervals
        # that start and end at the same point
        toStart += PRAAT_TIME_DIFF
        fromStart += PRAAT_TIME_DIFF

        ratio = (toEnd - toStart) / float((fromEnd - fromStart))
        durationParameters.append((fromStart, fromEnd, ratio))

    # Create the adjusted textgrids
    if outputTGName is not None:

        for stepFactor in stepList:

            stepDurationParameters = [(start,
                                       end,
                                       1 + (ratio - 1) * stepFactor)
                                      for start, end, ratio
                                      in durationParameters]
            adjustedTG = textgridManipulateDuration(fromTGFN,
                                                    stepDurationParameters)

            outputTGFN = "%s_%0.3g.TextGrid" % (outputTGName, stepFactor)
            adjustedTG.save(outputTGFN)

    # Create the plot of the morph
    if outputImageFN is not None:
        _plotResults(durationParameters, fromTGFN, toTGFN,
                     tierName, stepList, outputImageFN,
                     filterFunc, False)

    return durationParameters

def getManipulatedParamaters(tgFN, tierName, modFunc,
                             includeUnlabeledRegions=False):
    '''
    Get intervals for source and target audio files

    Use this information to find out how much to stretch/shrink each source
    interval.
    '''

    fromExtractInfo = utils.getIntervals(tgFN, tierName, None,
                                         includeUnlabeledRegions)

    durationParameters = []
    for fromInfoTuple in fromExtractInfo:
        fromStart, fromEnd = fromInfoTuple[:2]
        toStart, toEnd = modFunc(fromStart), modFunc(fromEnd)

        # Praat will ignore a second value appearing at the same time as
        # another so we give each start a tiny offset to distinguish intervals
        # that start and end at the same point
        toStart += PRAAT_TIME_DIFF
        fromStart += PRAAT_TIME_DIFF

        ratio = (toEnd - toStart) / float((fromEnd - fromStart))

        ratioTuple = (fromStart, fromEnd, ratio)
        durationParameters.append(ratioTuple)

    return durationParameters

def _plotResults(durationParameters, fromTGFN, toTGFN, tierName,
                 stepList, outputPNGFN, filterFunc,
                 includeUnlabeledRegions):

    # Containers
    fromDurList = []
    toDurList = []
    actDurList = []
    labelList = []

    fromExtractInfo = utils.getIntervals(fromTGFN, tierName, filterFunc,
                                         includeUnlabeledRegions)
    toExtractInfo = utils.getIntervals(toTGFN, tierName, filterFunc,
                                       includeUnlabeledRegions)

    # Get durations
    for fromInfoTuple, toInfoTuple in zip(fromExtractInfo, toExtractInfo):
        fromStart, fromEnd = fromInfoTuple[:2]
        toStart, toEnd = toInfoTuple[:2]

        labelList.append(fromInfoTuple[2])
        fromDurList.append(fromEnd - fromStart)
        toDurList.append(toEnd - toStart)

    # Get iterpolated values
    for stepAmount in stepList:
        tmpDurList = []
        for fromStart, fromEnd, ratio in durationParameters:
            dur = (fromEnd - fromStart)
            percentChange = 1 + (ratio - 1) * stepAmount
            tmpDurList.append(dur * percentChange)

        actDurList.append(tmpDurList)

    # Plot data
    plot_morphed_data.plotDuration(fromDurList, toDurList, actDurList,
                                   labelList, outputPNGFN)

def textgridMorphDuration(fromTGFN, toTGFN):

    fromTG = tgio.openTextGrid(fromTGFN)
    toTG = tgio.openTextGrid(toTGFN)
    adjustedTG = tgio.Textgrid()

    for tierName in fromTG.tierNameList:
        fromTier = fromTG.tierDict[tierName]
        toTier = toTG.tierDict[tierName]
        adjustedTier = fromTier.morph(toTier)
        adjustedTG.addTier(adjustedTier)

    return adjustedTG

def textgridManipulateDuration(tgFN, ratioList):

    tg = tgio.openTextGrid(tgFN)

    adjustedTG = tgio.Textgrid()

    for tierName in tg.tierNameList:
        fromTier = tg.tierDict[tierName]

        adjustedTier = None
        if isinstance(fromTier, tgio.IntervalTier):
            adjustedTier = _morphIntervalTier(fromTier, ratioList)
        elif isinstance(fromTier, tgio.PointTier):
            adjustedTier = _morphPointTier(fromTier, ratioList)

        assert(adjustedTier is not None)
        adjustedTG.addTier(adjustedTier)

    return adjustedTG

def _getTimeDiff(start, stop, ratio):
    '''Returns the time difference between interval and interval*ratio'''
    return (ratio - 1) * (stop - start)

def _morphPointTier(tier, ratioList):

    cumulativeAdjustAmount = 0
    i = 0
    newEntryList = []
    for timestamp, label in tier.entryList:

        # Advance to the manipulation interval that coincides with the
        # current point or appears after it
        while i < len(ratioList) and timestamp > ratioList[i][1]:
            rStart, rStop, ratio = ratioList[i]
            cumulativeAdjustAmount += _getTimeDiff(rStart, rStop, ratio)
            i += 1

        newTime = timestamp + cumulativeAdjustAmount

        # Alter the time if the point is within a manipulation interval
        if i < len(ratioList):
            rStart, rStop, ratio = ratioList[i]
            if timestamp > rStart and timestamp <= rStop:
                newTime += _getTimeDiff(rStart, timestamp, ratio)

        newEntryList.append((newTime, label))

    maxT = tier.maxTimestamp + cumulativeAdjustAmount
    return tier.newTier(entryList=newEntryList, maxTimestamp=maxT)

def _morphIntervalTier(tier, ratioList):

    cumulativeAdjustAmount = 0
    i = 0
    newEntryList = []
    for start, stop, label in tier.entryList:

        # Syncronize the manipulation and data intervals so that they
        # are either overlapping or the manipulation interval is farther
        # in the future.  This accumulates the effect of past
        # manipulations so we know how much to offset timestamps
        # for the current interval.
        while (i < len(ratioList) and start > ratioList[i][0] and
               start >= ratioList[i][1]):
            rStart, rStop, ratio = ratioList[i]
            cumulativeAdjustAmount = _getTimeDiff(rStart, rStop, ratio)
            i += 1

        newStart = start + cumulativeAdjustAmount
        newStop = stop + cumulativeAdjustAmount

        # Manipulate the interval further if there is overlap with a
        # manipulation interval
        while i < len(ratioList):
            rStart, rStop, ratio = ratioList[i]

            # currAdjustAmount is the ratio modified by percent change
            # e.g. if the ratio is a boost of 1.2 but percent change is
            # 0.5, ratio = 1.1 (it loses half of its effect)

            # Adjusting the start position based on overlap with the
            # current adjust interval
            if start <= rStart:
                pass
            elif start > rStart and start <= rStop:
                newStart += _getTimeDiff(start, rStart, ratio)

            # Adjusting the stop position based on overlap with the
            # current adjust interval
            if stop >= ratioList[i][1]:
                newStop += _getTimeDiff(rStart, rStop, ratio)
            elif stop < rStop and stop >= rStart:
                newStop += _getTimeDiff(rStart, stop, ratio)

            # If we are beyond the current manipulation interval,
            # then we need to move to the next one
            if stop >= rStop:
                cumulativeAdjustAmount += _getTimeDiff(rStart, rStop, ratio)
                i += 1
            # Otherwise, we are done manipulating the current interval
            elif stop < rStop:
                break

        newEntryList.append((newStart, newStop, label))

    newMax = tier.maxTimestamp + cumulativeAdjustAmount
    return tier.newTier(entryList=newEntryList, maxTimestamp=newMax)
<commit_msg>REFACTOR: Split out textgrid and plotting code<commit_after>'''
Created on Jun 5, 2013

@author: timmahrt
'''

import os
from os.path import join
import copy

from praatio import tgio
from praatio import praat_scripts
from praatio import dataio

from promo.morph_utils import utils
from promo.morph_utils import audio_scripts
from promo.morph_utils import plot_morphed_data

# This value is used to differentiate a praat interval boundary that marks
# the start of one region and the end of another.
PRAAT_TIME_DIFF = 0.000001

class NoLabeledRegionFoundException(Exception):

    def __init__(self, tgFN):
        super(NoLabeledRegionFoundException, self).__init__()
        self.tgFN = tgFN

    def __str__(self):
        return "No labeled region fitting the specified criteria for tg: " + \
            self.tgFN

def changeDuration(fromWavFN, durationParameters, stepList, outputName,
                   outputMinPitch, outputMaxPitch, praatEXE):
    '''
    Uses praat to morph duration in one file to duration in another

    Praat uses the PSOLA algorithm
    '''

    rootPath = os.path.split(fromWavFN)[0]

    # Prep output directories
    outputPath = join(rootPath, "duration_resynthesized_wavs")
    utils.makeDir(outputPath)

    durationTierPath = join(rootPath, "duration_tiers")
    utils.makeDir(durationTierPath)

    fromWavDuration = audio_scripts.getSoundFileDuration(fromWavFN)

    durationParameters = copy.deepcopy(durationParameters)
    # Pad any gaps with values of 1 (no change in duration)

    # No need to stretch out any pauses at the beginning
    if durationParameters[0][0] != 0:
        tmpVar = (0, durationParameters[0][0] - PRAAT_TIME_DIFF, 1)
        durationParameters.insert(0, tmpVar)

    # Or the end
    if durationParameters[-1][1] < fromWavDuration:
        durationParameters.append((durationParameters[-1][1] + PRAAT_TIME_DIFF,
                                   fromWavDuration, 1))

    # Create the praat script for doing duration manipulation
    for stepAmount in stepList:
        durationPointList = []
        for start, end, ratio in durationParameters:
            percentChange = 1 + (ratio - 1) * stepAmount
            durationPointList.append((start, percentChange))
            durationPointList.append((end, percentChange))

        outputPrefix = "%s_%0.3g" % (outputName, stepAmount)
        durationTierFN = join(durationTierPath,
                              "%s.DurationTier" % outputPrefix)
        outputWavFN = join(outputPath, "%s.wav" % outputPrefix)
        durationTier = dataio.PointObject2D(durationPointList, dataio.DURATION,
                                            0, fromWavDuration)
        durationTier.save(durationTierFN)

        praat_scripts.resynthesizeDuration(praatEXE,
                                           fromWavFN,
                                           durationTierFN,
                                           outputWavFN,
                                           outputMinPitch, outputMaxPitch)

def getBareParameters(wavFN):
    wavDuration = audio_scripts.getSoundFileDuration(wavFN)
    return [(0, wavDuration, ''), ]

def getMorphParameters(fromTGFN, toTGFN, tierName, stepList=None,
                       filterFunc=None, useBlanks=False):
    '''
    Get intervals for source and target audio files

    Use this information to find out how much to stretch/shrink each source
    interval.

    The target values are based on the contents of toTGFN.
    '''

    if stepList is None:
        stepList = [1, ]  # Complete morph
    if filterFunc is None:
        filterFunc = lambda entry: True  # Everything is accepted

    fromEntryList = utils.getIntervals(fromTGFN, tierName,
                                       includeUnlabeledRegions=useBlanks)
    toEntryList = utils.getIntervals(toTGFN, tierName,
                                     includeUnlabeledRegions=useBlanks)

    fromEntryList = [entry for entry in fromEntryList if filterFunc(entry)]
    toEntryList = [entry for entry in toEntryList if filterFunc(entry)]

    assert(len(fromEntryList) == len(toEntryList))

    durationParameters = []
    for fromEntry, toEntry in zip(fromEntryList, toEntryList):
        fromStart, fromEnd = fromEntry[:2]
        toStart, toEnd = toEntry[:2]

        # Praat will ignore a second value appearing at the same time as
        # another so we give each start a tiny offset to distinguish intervals
        # that start and end at the same point
        toStart += PRAAT_TIME_DIFF
        fromStart += PRAAT_TIME_DIFF

        ratio = (toEnd - toStart) / float((fromEnd - fromStart))
        durationParameters.append((fromStart, fromEnd, ratio))

    return durationParameters

def getManipulatedParamaters(tgFN, tierName, modFunc,
                             filterFunc=None, useBlanks=False):
    '''
    Get intervals for source and target audio files

    Use this information to find out how much to stretch/shrink each source
    interval.

    The target values are based on modfunc.
    '''

    fromExtractInfo = utils.getIntervals(tgFN, tierName, filterFunc,
                                         useBlanks)

    durationParameters = []
    for fromInfoTuple in fromExtractInfo:
        fromStart, fromEnd = fromInfoTuple[:2]
        toStart, toEnd = modFunc(fromStart), modFunc(fromEnd)

        # Praat will ignore a second value appearing at the same time as
        # another so we give each start a tiny offset to distinguish intervals
        # that start and end at the same point
        toStart += PRAAT_TIME_DIFF
        fromStart += PRAAT_TIME_DIFF

        ratio = (toEnd - toStart) / float((fromEnd - fromStart))

        ratioTuple = (fromStart, fromEnd, ratio)
        durationParameters.append(ratioTuple)

    return durationParameters

def outputMorphTextgrids(fromTGFN, durationParameters, stepList,
                         outputTGName):

    if outputTGName is not None:
        utils.makeDir(os.path.split(outputTGName)[0])

    # Create the adjusted textgrids
    if outputTGName is not None:

        for stepFactor in stepList:

            stepDurationParameters = [(start,
                                       end,
                                       1 + (ratio - 1) * stepFactor)
                                      for start, end, ratio
                                      in durationParameters]
            adjustedTG = textgridManipulateDuration(fromTGFN,
                                                    stepDurationParameters)

            outputTGFN = "%s_%0.3g.TextGrid" % (outputTGName, stepFactor)
            adjustedTG.save(outputTGFN)

def outputMorphPlot(fromTGFN, toTGFN, tierName, durationParameters, stepList,
                    outputImageFN):

    if outputImageFN is not None:
        utils.makeDir(os.path.split(outputImageFN)[0])

    # Create the plot of the morph
    if outputImageFN is not None:
        _plotResults(durationParameters, fromTGFN, toTGFN,
                     tierName, stepList, outputImageFN,
                     None, False)

def _plotResults(durationParameters, fromTGFN, toTGFN, tierName,
                 stepList, outputPNGFN, filterFunc,
                 includeUnlabeledRegions):

    # Containers
    fromDurList = []
    toDurList = []
    actDurList = []
    labelList = []

    fromExtractInfo = utils.getIntervals(fromTGFN, tierName, filterFunc,
                                         includeUnlabeledRegions)
    toExtractInfo = utils.getIntervals(toTGFN, tierName, filterFunc,
                                       includeUnlabeledRegions)

    # Get durations
    for fromInfoTuple, toInfoTuple in zip(fromExtractInfo, toExtractInfo):
        fromStart, fromEnd = fromInfoTuple[:2]
        toStart, toEnd = toInfoTuple[:2]

        labelList.append(fromInfoTuple[2])
        fromDurList.append(fromEnd - fromStart)
        toDurList.append(toEnd - toStart)

    # Get iterpolated values
    for stepAmount in stepList:
        tmpDurList = []
        for fromStart, fromEnd, ratio in durationParameters:
            dur = (fromEnd - fromStart)
            percentChange = 1 + (ratio - 1) * stepAmount
            tmpDurList.append(dur * percentChange)

        actDurList.append(tmpDurList)

    # Plot data
    plot_morphed_data.plotDuration(fromDurList, toDurList, actDurList,
                                   labelList, outputPNGFN)

def textgridMorphDuration(fromTGFN, toTGFN):
    '''
    A convenience function.  Morphs interval durations of one tg to another.

    This assumes the two textgrids have the same number of segments.
    '''
    fromTG = tgio.openTextGrid(fromTGFN)
    toTG = tgio.openTextGrid(toTGFN)
    adjustedTG = tgio.Textgrid()

    for tierName in fromTG.tierNameList:
        fromTier = fromTG.tierDict[tierName]
        toTier = toTG.tierDict[tierName]
        adjustedTier = fromTier.morph(toTier)
        adjustedTG.addTier(adjustedTier)

    return adjustedTG

def textgridManipulateDuration(tgFN, ratioList):

    tg = tgio.openTextGrid(tgFN)

    adjustedTG = tgio.Textgrid()

    for tierName in tg.tierNameList:
        fromTier = tg.tierDict[tierName]

        adjustedTier = None
        if isinstance(fromTier, tgio.IntervalTier):
            adjustedTier = _morphIntervalTier(fromTier, ratioList)
        elif isinstance(fromTier, tgio.PointTier):
            adjustedTier = _morphPointTier(fromTier, ratioList)

        assert(adjustedTier is not None)
        adjustedTG.addTier(adjustedTier)

    return adjustedTG

def _getTimeDiff(start, stop, ratio):
    '''Returns the time difference between interval and interval*ratio'''
    return (ratio - 1) * (stop - start)

def _morphPointTier(tier, ratioList):

    cumulativeAdjustAmount = 0
    i = 0
    newEntryList = []
    for timestamp, label in tier.entryList:

        # Advance to the manipulation interval that coincides with the
        # current point or appears after it
        while i < len(ratioList) and timestamp > ratioList[i][1]:
            rStart, rStop, ratio = ratioList[i]
            cumulativeAdjustAmount += _getTimeDiff(rStart, rStop, ratio)
            i += 1

        newTime = timestamp + cumulativeAdjustAmount

        # Alter the time if the point is within a manipulation interval
        if i < len(ratioList):
            rStart, rStop, ratio = ratioList[i]
            if timestamp > rStart and timestamp <= rStop:
                newTime += _getTimeDiff(rStart, timestamp, ratio)

        newEntryList.append((newTime, label))

    maxT = tier.maxTimestamp + cumulativeAdjustAmount
    return tier.newTier(entryList=newEntryList, maxTimestamp=maxT)

def _morphIntervalTier(tier, ratioList):

    cumulativeAdjustAmount = 0
    i = 0
    newEntryList = []
    for start, stop, label in tier.entryList:

        # Syncronize the manipulation and data intervals so that they
        # are either overlapping or the manipulation interval is farther
        # in the future.  This accumulates the effect of past
        # manipulations so we know how much to offset timestamps
        # for the current interval.
        while (i < len(ratioList) and start > ratioList[i][0] and
               start >= ratioList[i][1]):
            rStart, rStop, ratio = ratioList[i]
            cumulativeAdjustAmount = _getTimeDiff(rStart, rStop, ratio)
            i += 1

        newStart = start + cumulativeAdjustAmount
        newStop = stop + cumulativeAdjustAmount

        # Manipulate the interval further if there is overlap with a
        # manipulation interval
        while i < len(ratioList):
            rStart, rStop, ratio = ratioList[i]

            # currAdjustAmount is the ratio modified by percent change
            # e.g. if the ratio is a boost of 1.2 but percent change is
            # 0.5, ratio = 1.1 (it loses half of its effect)

            # Adjusting the start position based on overlap with the
            # current adjust interval
            if start <= rStart:
                pass
            elif start > rStart and start <= rStop:
                newStart += _getTimeDiff(start, rStart, ratio)

            # Adjusting the stop position based on overlap with the
            # current adjust interval
            if stop >= ratioList[i][1]:
                newStop += _getTimeDiff(rStart, rStop, ratio)
            elif stop < rStop and stop >= rStart:
                newStop += _getTimeDiff(rStart, stop, ratio)

            # If we are beyond the current manipulation interval,
            # then we need to move to the next one
            if stop >= rStop:
                cumulativeAdjustAmount += _getTimeDiff(rStart, rStop, ratio)
                i += 1
            # Otherwise, we are done manipulating the current interval
            elif stop < rStop:
                break

        newEntryList.append((newStart, newStop, label))

    newMax = tier.maxTimestamp + cumulativeAdjustAmount
    return tier.newTier(entryList=newEntryList, maxTimestamp=newMax)
<|endoftext|>


<commit_before>import ctypes.util
import os
import re
from contextlib import contextmanager

from prophyc import model
from prophyc.generators.cpp import _HppDefinitionsTranslator
from prophyc.six import to_bytes
from .clang import cindex

class SackParserError(Exception):
    pass

class SackModelTree(object):
    def __init__(self):
        self.known = set()
        self.nodes = []

    def add_node(self, node):
        self.known.add(node.name)
        self.nodes.append(node)

    def remove_nodes(self, node_names_to_remove):
        self.nodes = [node for node in self.nodes if node.name not in node_names_to_remove]
        self.known -= set(node_names_to_remove)

class Builder(object):
    unambiguous_builtins = {
        cindex.TypeKind.UCHAR: 'u8',
        cindex.TypeKind.SCHAR: 'i8',
        cindex.TypeKind.CHAR_S: 'i8',
        cindex.TypeKind.POINTER: 'u32',
        cindex.TypeKind.FLOAT: 'r32',
        cindex.TypeKind.DOUBLE: 'r64',
        cindex.TypeKind.BOOL: 'i32'
    }

    def __init__(self, tree_model):
        self.tree = tree_model

    @staticmethod
    def alphanumeric_name(cursor):
        name = cursor.type.spelling.decode()
        if name.startswith('struct '):
            name = name.replace('struct ', '', 1)
        elif name.startswith('enum '):
            name = name.replace('enum ', '', 1)
        elif name.startswith('union '):
            name = name.replace('union ', '', 1)
        return re.sub('[^0-9a-zA-Z_]+', '__', name)

    def get_type_name(self, tp):
        decl = tp.get_declaration()

        def dive_deeper(method):
            name = Builder.alphanumeric_name(decl)
            if name not in self.tree.known:
                method(decl)
            return name

        if tp.kind is cindex.TypeKind.TYPEDEF:
            return self.get_type_name(decl.underlying_typedef_type)

        elif tp.kind in (cindex.TypeKind.UNEXPOSED, cindex.TypeKind.ELABORATED, cindex.TypeKind.RECORD):

            if decl.kind in (cindex.CursorKind.STRUCT_DECL, cindex.CursorKind.CLASS_DECL):
                return dive_deeper(self.add_struct)

            elif decl.kind is cindex.CursorKind.UNION_DECL:
                return dive_deeper(self.add_union)

            elif decl.kind is cindex.CursorKind.ENUM_DECL:
                return self.get_type_name(decl.type)

            elif decl.kind is cindex.CursorKind.TYPEDEF_DECL:
                return self.get_type_name(decl.underlying_typedef_type)

            else:
                raise SackParserError("Unknown declaration, {} {}".format(tp.spelling, decl.kind))

        elif tp.kind in (cindex.TypeKind.CONSTANTARRAY, cindex.TypeKind.INCOMPLETEARRAY):
            return self.get_type_name(tp.element_type)

        elif tp.kind is cindex.TypeKind.ENUM:
            return dive_deeper(self.add_enum)

        if tp.kind in (cindex.TypeKind.USHORT, cindex.TypeKind.UINT, cindex.TypeKind.ULONG, cindex.TypeKind.ULONGLONG):
            return 'u%d' % (tp.get_size() * 8)

        elif tp.kind in (cindex.TypeKind.SHORT, cindex.TypeKind.INT, cindex.TypeKind.LONG, cindex.TypeKind.LONGLONG):
            return 'i%d' % (tp.get_size() * 8)

        return self.unambiguous_builtins[tp.kind]

    def add_enum(self, cursor):
        def enum_member(cursor):
            name = cursor.spelling.decode()
            value = cursor.enum_value
            if value < 0:
                value = "0x%X" % (0x100000000 + value)
            else:
                value = str(value)
            return model.EnumMember(name, value)

        members = [enum_member(x) for x in cursor.get_children()]
        node = model.Enum(Builder.alphanumeric_name(cursor), members)
        self.tree.add_node(node)

    def add_struct(self, cursor):
        def array_length(tp):
            if tp.kind is cindex.TypeKind.CONSTANTARRAY:
                return tp.element_count

        def struct_member(cursor_):
            name = cursor_.spelling.decode()
            type_name = self.get_type_name(cursor_.type)
            array_len = array_length(cursor_.type)
            return model.StructMember(name, type_name, size=array_len)

        members = [struct_member(x) for x in cursor.get_children()
                   if x.kind is cindex.CursorKind.FIELD_DECL and not x.is_bitfield()]
        node = model.Struct(Builder.alphanumeric_name(cursor), members)
        self.tree.add_node(node)

    def add_union(self, cursor):
        def union_member(cursor, disc):
            name = cursor.spelling.decode()
            type_name = self.get_type_name(cursor.type)
            return model.UnionMember(name, type_name, str(disc))

        members = [union_member(x, i) for i, x in enumerate(cursor.get_children())
                   if x.kind is cindex.CursorKind.FIELD_DECL]
        node = model.Union(Builder.alphanumeric_name(cursor), members)
        self.tree.add_node(node)

    def build_model(self, translation_unit):
        for cursor in translation_unit.cursor.get_children():
            if cursor.kind is cindex.CursorKind.UNEXPOSED_DECL:
                for in_cursor in cursor.get_children():
                    if in_cursor.kind is cindex.CursorKind.STRUCT_DECL:
                        if in_cursor.spelling and in_cursor.is_definition():
                            self.add_struct(in_cursor)
            if cursor.spelling and cursor.is_definition():
                if cursor.kind is cindex.CursorKind.STRUCT_DECL:
                    self.add_struct(cursor)
                if cursor.kind is cindex.CursorKind.ENUM_DECL:
                    self.add_enum(cursor)

class SupplementaryDefs(object):

    def __init__(self, include_tree):

        self.include_tree = include_tree
        self.stub_names = [node.name for node in SupplementaryDefs.flatten_nodes(include_tree)]
        self.stub_defs = SupplementaryDefs.prepare_stubs(include_tree)
        self.stubs_lines_count = len(self.stub_defs)

    @staticmethod
    def flatten_nodes(nodes_list):
        for node in nodes_list:
            if isinstance(node, model.Include):
                for node in SupplementaryDefs.flatten_nodes(node.members):
                    yield node
            else:
                yield node

    @staticmethod
    def unique_nodes(include_tree):
        picked = []
        for node in SupplementaryDefs.flatten_nodes(include_tree):
            if node.name not in picked:
                picked.append(node.name)
                yield node

    @staticmethod
    def prepare_stubs(include_tree):
        def create_stub_definition(node):
            if isinstance(node, model.Constant):
                yield "#define {} {}".format(node.name, node.value)
            elif isinstance(node, model.Enum):
                cpp_translator = _HppDefinitionsTranslator()
                for line in cpp_translator.translate_enum(node).split('\n'):
                    # need to have all lines separated to know its count
                    yield line
            else:
                yield "struct {} {{}};".format(node.name)

        flatten_nodes = SupplementaryDefs.unique_nodes(include_tree)
        return [line for node in flatten_nodes for line in create_stub_definition(node)]

    def prepend_stubs(self, content):
        if not self.stub_defs:
            return content
        return '\n'.join(self.stub_defs) + '\n' + content

    @contextmanager
    def implicit_supplementation(self, parsed_content):
        enriched_content = self.prepend_stubs(parsed_content)

        sack_tree = SackModelTree()
        for include_node in self.include_tree:
            sack_tree.add_node(include_node)

        yield enriched_content, sack_tree

        sack_tree.remove_nodes(self.stub_names)

class SackParser(object):

    @staticmethod
    def check():
        class SackParserStatus(object):
            def __init__(self, error=None):
                self.error = error

            def __bool__(self):
                return not bool(self.error)

            __nonzero__ = __bool__

        def _check_libclang():
            testconf = cindex.Config()
            try:
                testconf.get_cindex_library()
                return True
            except cindex.LibclangError:
                return False

        import platform
        if platform.python_implementation() == 'PyPy':
            return SackParserStatus("sack input doesn't work under PyPy due to ctypes incompatibilities")
        if not _check_libclang():
            return SackParserStatus("sack input requires libclang and it's not installed")
        return SackParserStatus()

    def __init__(self, include_dirs=None, warn=None, include_tree=None):
        self.include_dirs = include_dirs or []
        self.warn = warn
        self.supples = SupplementaryDefs(include_tree or [])

    def parse(self, content, path, _):
        args_ = [to_bytes("-I" + x) for x in self.include_dirs]
        index = cindex.Index.create()
        with self.supples.implicit_supplementation(content) as (content_, tree):
            builder = Builder(tree)
            path = path.encode()
            content_ = content_.encode()

            try:
                translation_unit = index.parse(path, args_, unsaved_files=((path, content_),))
            except cindex.TranslationUnitLoadError:
                raise model.ParseError([(path.decode(), 'error parsing translation unit')])

            self.print_diagnostics(path, translation_unit)
            builder.build_model(translation_unit)

        return tree.nodes

    def print_diagnostics(self, path, translation_unit):
        if self.warn:
            for diag in translation_unit.diagnostics:
                spelling = diag.spelling.decode()
                location = self._get_location(diag.location, path)
                self.warn(spelling, location)

    def _get_location(self, location, target_path):
        location_file = location.file.name.decode()
        target_file_name = os.path.basename(target_path.decode())
        if os.path.basename(location_file) == target_file_name:
            stubs_len = self.supples.stubs_lines_count
            is_in_stubs = location.line < stubs_len
            stubs_file = "supplementary_defs_in_{}".format(target_file_name)
            location_line = location.line if is_in_stubs else (location.line - stubs_len)
            location_file = location_file if not is_in_stubs else stubs_file

        else:
            location_line = location.line
        return '%s:%s:%s' % (location_file, location_line, location.column)

def _setup_libclang():
    if os.environ.get('PROPHY_NOCLANG'):
        cindex.Config.set_library_file('prophy_noclang')
        return

    versions = [None, '3.5', '3.4', '3.3', '3.2', '3.6', '3.7', '3.8', '3.9']
    for v in versions:
        name = v and 'clang-' + v or 'clang'
        libname = ctypes.util.find_library(name)
        if libname:
            cindex.Config.set_library_file(libname)
            break

_setup_libclang()
<commit_msg>ci-upgrade: allow finding all compatible libclang versions<commit_after>import ctypes.util
import os
import re
from contextlib import contextmanager

from prophyc import model
from prophyc.generators.cpp import _HppDefinitionsTranslator
from prophyc.six import to_bytes
from .clang import cindex

class SackParserError(Exception):
    pass

class SackModelTree(object):
    def __init__(self):
        self.known = set()
        self.nodes = []

    def add_node(self, node):
        self.known.add(node.name)
        self.nodes.append(node)

    def remove_nodes(self, node_names_to_remove):
        self.nodes = [node for node in self.nodes if node.name not in node_names_to_remove]
        self.known -= set(node_names_to_remove)

class Builder(object):
    unambiguous_builtins = {
        cindex.TypeKind.UCHAR: 'u8',
        cindex.TypeKind.SCHAR: 'i8',
        cindex.TypeKind.CHAR_S: 'i8',
        cindex.TypeKind.POINTER: 'u32',
        cindex.TypeKind.FLOAT: 'r32',
        cindex.TypeKind.DOUBLE: 'r64',
        cindex.TypeKind.BOOL: 'i32'
    }

    def __init__(self, tree_model):
        self.tree = tree_model

    @staticmethod
    def alphanumeric_name(cursor):
        name = cursor.type.spelling.decode()
        if name.startswith('struct '):
            name = name.replace('struct ', '', 1)
        elif name.startswith('enum '):
            name = name.replace('enum ', '', 1)
        elif name.startswith('union '):
            name = name.replace('union ', '', 1)
        return re.sub('[^0-9a-zA-Z_]+', '__', name)

    def get_type_name(self, tp):
        decl = tp.get_declaration()

        def dive_deeper(method):
            name = Builder.alphanumeric_name(decl)
            if name not in self.tree.known:
                method(decl)
            return name

        if tp.kind is cindex.TypeKind.TYPEDEF:
            return self.get_type_name(decl.underlying_typedef_type)

        elif tp.kind in (cindex.TypeKind.UNEXPOSED, cindex.TypeKind.ELABORATED, cindex.TypeKind.RECORD):

            if decl.kind in (cindex.CursorKind.STRUCT_DECL, cindex.CursorKind.CLASS_DECL):
                return dive_deeper(self.add_struct)

            elif decl.kind is cindex.CursorKind.UNION_DECL:
                return dive_deeper(self.add_union)

            elif decl.kind is cindex.CursorKind.ENUM_DECL:
                return self.get_type_name(decl.type)

            elif decl.kind is cindex.CursorKind.TYPEDEF_DECL:
                return self.get_type_name(decl.underlying_typedef_type)

            else:
                raise SackParserError("Unknown declaration, {} {}".format(tp.spelling, decl.kind))

        elif tp.kind in (cindex.TypeKind.CONSTANTARRAY, cindex.TypeKind.INCOMPLETEARRAY):
            return self.get_type_name(tp.element_type)

        elif tp.kind is cindex.TypeKind.ENUM:
            return dive_deeper(self.add_enum)

        if tp.kind in (cindex.TypeKind.USHORT, cindex.TypeKind.UINT, cindex.TypeKind.ULONG, cindex.TypeKind.ULONGLONG):
            return 'u%d' % (tp.get_size() * 8)

        elif tp.kind in (cindex.TypeKind.SHORT, cindex.TypeKind.INT, cindex.TypeKind.LONG, cindex.TypeKind.LONGLONG):
            return 'i%d' % (tp.get_size() * 8)

        return self.unambiguous_builtins[tp.kind]

    def add_enum(self, cursor):
        def enum_member(cursor):
            name = cursor.spelling.decode()
            value = cursor.enum_value
            if value < 0:
                value = "0x%X" % (0x100000000 + value)
            else:
                value = str(value)
            return model.EnumMember(name, value)

        members = [enum_member(x) for x in cursor.get_children()]
        node = model.Enum(Builder.alphanumeric_name(cursor), members)
        self.tree.add_node(node)

    def add_struct(self, cursor):
        def array_length(tp):
            if tp.kind is cindex.TypeKind.CONSTANTARRAY:
                return tp.element_count

        def struct_member(cursor_):
            name = cursor_.spelling.decode()
            type_name = self.get_type_name(cursor_.type)
            array_len = array_length(cursor_.type)
            return model.StructMember(name, type_name, size=array_len)

        members = [struct_member(x) for x in cursor.get_children()
                   if x.kind is cindex.CursorKind.FIELD_DECL and not x.is_bitfield()]
        node = model.Struct(Builder.alphanumeric_name(cursor), members)
        self.tree.add_node(node)

    def add_union(self, cursor):
        def union_member(cursor, disc):
            name = cursor.spelling.decode()
            type_name = self.get_type_name(cursor.type)
            return model.UnionMember(name, type_name, str(disc))

        members = [union_member(x, i) for i, x in enumerate(cursor.get_children())
                   if x.kind is cindex.CursorKind.FIELD_DECL]
        node = model.Union(Builder.alphanumeric_name(cursor), members)
        self.tree.add_node(node)

    def build_model(self, translation_unit):
        for cursor in translation_unit.cursor.get_children():
            if cursor.kind is cindex.CursorKind.UNEXPOSED_DECL:
                for in_cursor in cursor.get_children():
                    if in_cursor.kind is cindex.CursorKind.STRUCT_DECL:
                        if in_cursor.spelling and in_cursor.is_definition():
                            self.add_struct(in_cursor)
            if cursor.spelling and cursor.is_definition():
                if cursor.kind is cindex.CursorKind.STRUCT_DECL:
                    self.add_struct(cursor)
                if cursor.kind is cindex.CursorKind.ENUM_DECL:
                    self.add_enum(cursor)

class SupplementaryDefs(object):

    def __init__(self, include_tree):

        self.include_tree = include_tree
        self.stub_names = [node.name for node in SupplementaryDefs.flatten_nodes(include_tree)]
        self.stub_defs = SupplementaryDefs.prepare_stubs(include_tree)
        self.stubs_lines_count = len(self.stub_defs)

    @staticmethod
    def flatten_nodes(nodes_list):
        for node in nodes_list:
            if isinstance(node, model.Include):
                for node in SupplementaryDefs.flatten_nodes(node.members):
                    yield node
            else:
                yield node

    @staticmethod
    def unique_nodes(include_tree):
        picked = []
        for node in SupplementaryDefs.flatten_nodes(include_tree):
            if node.name not in picked:
                picked.append(node.name)
                yield node

    @staticmethod
    def prepare_stubs(include_tree):
        def create_stub_definition(node):
            if isinstance(node, model.Constant):
                yield "#define {} {}".format(node.name, node.value)
            elif isinstance(node, model.Enum):
                cpp_translator = _HppDefinitionsTranslator()
                for line in cpp_translator.translate_enum(node).split('\n'):
                    # need to have all lines separated to know its count
                    yield line
            else:
                yield "struct {} {{}};".format(node.name)

        flatten_nodes = SupplementaryDefs.unique_nodes(include_tree)
        return [line for node in flatten_nodes for line in create_stub_definition(node)]

    def prepend_stubs(self, content):
        if not self.stub_defs:
            return content
        return '\n'.join(self.stub_defs) + '\n' + content

    @contextmanager
    def implicit_supplementation(self, parsed_content):
        enriched_content = self.prepend_stubs(parsed_content)

        sack_tree = SackModelTree()
        for include_node in self.include_tree:
            sack_tree.add_node(include_node)

        yield enriched_content, sack_tree

        sack_tree.remove_nodes(self.stub_names)

class SackParser(object):

    @staticmethod
    def check():
        class SackParserStatus(object):
            def __init__(self, error=None):
                self.error = error

            def __bool__(self):
                return not bool(self.error)

            __nonzero__ = __bool__

        def _check_libclang():
            testconf = cindex.Config()
            try:
                testconf.get_cindex_library()
                return True
            except cindex.LibclangError:
                return False

        import platform
        if platform.python_implementation() == 'PyPy':
            return SackParserStatus("sack input doesn't work under PyPy due to ctypes incompatibilities")
        if not _check_libclang():
            return SackParserStatus("sack input requires libclang and it's not installed")
        return SackParserStatus()

    def __init__(self, include_dirs=None, warn=None, include_tree=None):
        self.include_dirs = include_dirs or []
        self.warn = warn
        self.supples = SupplementaryDefs(include_tree or [])

    def parse(self, content, path, _):
        args_ = [to_bytes("-I" + x) for x in self.include_dirs]
        index = cindex.Index.create()
        with self.supples.implicit_supplementation(content) as (content_, tree):
            builder = Builder(tree)
            path = path.encode()
            content_ = content_.encode()

            try:
                translation_unit = index.parse(path, args_, unsaved_files=((path, content_),))
            except cindex.TranslationUnitLoadError:
                raise model.ParseError([(path.decode(), 'error parsing translation unit')])

            self.print_diagnostics(path, translation_unit)
            builder.build_model(translation_unit)

        return tree.nodes

    def print_diagnostics(self, path, translation_unit):
        if self.warn:
            for diag in translation_unit.diagnostics:
                spelling = diag.spelling.decode()
                location = self._get_location(diag.location, path)
                self.warn(spelling, location)

    def _get_location(self, location, target_path):
        location_file = location.file.name.decode()
        target_file_name = os.path.basename(target_path.decode())
        if os.path.basename(location_file) == target_file_name:
            stubs_len = self.supples.stubs_lines_count
            is_in_stubs = location.line < stubs_len
            stubs_file = "supplementary_defs_in_{}".format(target_file_name)
            location_line = location.line if is_in_stubs else (location.line - stubs_len)
            location_file = location_file if not is_in_stubs else stubs_file

        else:
            location_line = location.line
        return '%s:%s:%s' % (location_file, location_line, location.column)

def _setup_libclang():
    if os.environ.get('PROPHY_NOCLANG'):
        cindex.Config.set_library_file('prophy_noclang')
        return

    versions = ([None] +
                ['%d' % m for m in tuple(range(14, 3, -1))] +
                ['3.%d' % m for m in tuple(range(9, 1, -1))])
    for v in versions:
        name = v and 'clang-' + v or 'clang'
        libname = ctypes.util.find_library(name)
        if libname:
            cindex.Config.set_library_file(libname)
            break

_setup_libclang()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from docopt import docopt
import sys

from cloudmesh.cm_mongo import cm_mongo
from cloudmesh.config.cm_config import cm_config
from prettytable import PrettyTable

from cloudmesh_common.logger import LOGGER
from tabulate import tabulate

log = LOGGER(__file__)

def shell_command_flavor(arguments):
    """
    Usage:
        flavor
        flavor <cm_cloud>... [--refresh]
	flavor -h | --help
        flavor --version

   Options:
       -h                   help message
       --refresh            refresh flavors of IaaS

    Arguments:
        cm_cloud    Name of the IaaS cloud e.g. india_openstack_grizzly.

    Description:
       flavor command provides list of available flavors. Flavor describes
       virtual hardware configurations such as size of memory, disk, cpu cores.

    Result:

    Examples:
        $ flavor india_openstack_grizzly

    """

    #log.info(arguments)
    cloud_names = arguments['<cm_cloud>']
    # clouds in c.flavors treats None value as a ALL clouds
    if not cloud_names:
        cloud_names = None
    config = cm_config()
    username = config.username()
    c = cm_mongo()
    c.activate(cm_user_id=username)
    if arguments['--refresh']:
        c.refresh(cm_user_id=username, names=cloud_names, types=['flavors'])
    flavors_dict = c.flavors(cm_user_id=username, clouds=cloud_names)

    your_keys = [
        'id',
        'name',
        'vcpus',
        'ram',
        'disk',
        'cm_refresh',
    ]

    flavors = _select_flavors(flavors_dict, your_keys)

    _display(flavors)

def _display(json_data, headers="firstrow", tablefmt="orgtbl"):
    table = tabulate(json_data, headers, tablefmt)
    try:
        separator = table.split("\n")[1].replace("|", "+")
    except:
        separator = "-" * 50
    print separator
    print table
    print separator

def _select_flavors(data, selected_keys, env=[]):
    '''
    {"india_openstack_havana": {"1":
        {"cm_cloud_type": "openstack",
        "disk": 0,
        "name": "m1.tiny",
        "links": [{"href":
            "http://149.165.146.57:8774/v1.1/8bc7e259464944b3bf4d8b050d1ab935/flavors/1",
            "rel": "self"}, {"href": "http://149.165.146.57
            :8774/8bc7e259464944b3bf4d8b050d1ab935/flavors/1", "rel": "bookmark"}],
        "ram": 512,
        "cm_type_version": "havana",
        "id": "1",
        "OS-FLV-DISABLED:disabled": false,
        "cm_id": "india_openstack_havana-flavors-m1-tiny",
        "vcpus": 1,
        "cm_refresh": "2014-06-14T17-21-25Z",
        "swap": "",
        "os-flavor-access:is_public": true,
        "rxtx_factor": 1.0,
        "cm_kind": "flavors"}}}
    '''
    headers = ["cm_cloud"] + selected_keys
    flavors = [headers]
    for cm_cloud, _id in data.iteritems():
        for flavor_name, v in _id.iteritems():
            values = [cm_cloud]
            for k in selected_keys:
                try:
                    values.append(v[k])
                except:
                    values.append(0)
            flavors.append(values)
    return flavors

def main():
    arguments = docopt(shell_command_flavor.__doc__)
    shell_command_flavor(arguments)

if __name__ == "__main__":
    #print sys.argv
    main()
<commit_msg>variable name capitalized like others<commit_after>#!/usr/bin/env python
from docopt import docopt
import sys

from cloudmesh.cm_mongo import cm_mongo
from cloudmesh.config.cm_config import cm_config
from prettytable import PrettyTable

from cloudmesh_common.logger import LOGGER
from tabulate import tabulate

log = LOGGER(__file__)

def shell_command_flavor(arguments):
    """
    Usage:
        flavor
        flavor CLOUD... [--refresh]
	flavor -h | --help
        flavor --version

   Options:
       -h                   help message
       --refresh            refresh flavors of IaaS

    Arguments:
        CLOUD    Name of the IaaS cloud e.g. india_openstack_grizzly.

    Description:
       flavor command provides list of available flavors. Flavor describes
       virtual hardware configurations such as size of memory, disk, cpu cores.

    Result:

    Examples:
        $ flavor india_openstack_grizzly

    """

    #log.info(arguments)
    cloud_names = arguments['CLOUD']
    # clouds in c.flavors treats None value as a ALL clouds
    if not cloud_names:
        cloud_names = None
    config = cm_config()
    username = config.username()
    c = cm_mongo()
    c.activate(cm_user_id=username)
    if arguments['--refresh']:
        c.refresh(cm_user_id=username, names=cloud_names, types=['flavors'])
    flavors_dict = c.flavors(cm_user_id=username, clouds=cloud_names)

    your_keys = [
        'id',
        'name',
        'vcpus',
        'ram',
        'disk',
        'cm_refresh',
    ]

    flavors = _select_flavors(flavors_dict, your_keys)

    _display(flavors)

def _display(json_data, headers="firstrow", tablefmt="orgtbl"):
    table = tabulate(json_data, headers, tablefmt)
    try:
        separator = table.split("\n")[1].replace("|", "+")
    except:
        separator = "-" * 50
    print separator
    print table
    print separator

def _select_flavors(data, selected_keys, env=[]):
    '''
    {"india_openstack_havana": {"1":
        {"cm_cloud_type": "openstack",
        "disk": 0,
        "name": "m1.tiny",
        "links": [{"href":
            "http://149.165.146.57:8774/v1.1/8bc7e259464944b3bf4d8b050d1ab935/flavors/1",
            "rel": "self"}, {"href": "http://149.165.146.57
            :8774/8bc7e259464944b3bf4d8b050d1ab935/flavors/1", "rel": "bookmark"}],
        "ram": 512,
        "cm_type_version": "havana",
        "id": "1",
        "OS-FLV-DISABLED:disabled": false,
        "cm_id": "india_openstack_havana-flavors-m1-tiny",
        "vcpus": 1,
        "cm_refresh": "2014-06-14T17-21-25Z",
        "swap": "",
        "os-flavor-access:is_public": true,
        "rxtx_factor": 1.0,
        "cm_kind": "flavors"}}}
    '''
    headers = ["CLOUD"] + selected_keys
    flavors = [headers]
    for cm_cloud, _id in data.iteritems():
        for flavor_name, v in _id.iteritems():
            values = [cm_cloud]
            for k in selected_keys:
                try:
                    values.append(v[k])
                except:
                    values.append(0)
            flavors.append(values)
    return flavors

def main():
    arguments = docopt(shell_command_flavor.__doc__)
    shell_command_flavor(arguments)

if __name__ == "__main__":
    #print sys.argv
    main()
<|endoftext|>