<commit_before>"""
Machine learning music
"""

import numpy as np
from scipy.io import wavfile
import jack
import peakutils
import tensorflow as tf
import matplotlib
import matplotlib.pyplot

# Datatypes that SciPy can import from .wav
SND_DTYPES = {'int16': 16-1, 'int32': 32-1}
# Colours for cycling in plots
CLRS = 'bgrcmyk'

def jack_client(midi_ins=1, midi_outs=1, name="MuserClient"):
    """Returns an active JACK client with specified number of inputs and outputs
    """
    client = jack.Client(name)
    for j in range(midi_ins):
        client.midi_inports.register("midi_in_{}".format(j))
    for k in range(midi_outs):
        client.midi_outports.register("midi_out_{}".format(k))
    client.activate()
    return client

def get_to_frames(sample_frq):
    """Return function that converts time (is s if sf is Hz) to sample index
       for a defined sampling rate.
    """
    def to_frames(time):
        def to_f(time):
            try:
                return int(time * sample_frq)
            except TypeError: # time not specified or specified badly
                raise TypeError("Real local endpoints must be specified! (t_endp)")
        try:
            return [to_f(t) for t in time]
        except TypeError: # time not iterable
            return [to_f(time)]

    return to_frames

def scale_snd(snd, factor=None):
    """ Scale signal from -1 to 1.
    """
    if factor is None:
        return scale_snd(snd, 2.**SND_DTYPES[snd.dtype.name])
    else:
        return snd / factor

def local_rfft(chs, f_endp, norm="ortho"):
    """ Calculate FFTs for each channel over the interval
    """
    # TODO: Move f0 and f1 out of this function?
    rfft = np.stack(np.fft.rfft(ch[slice(*f_endp)], norm=norm) for ch in chs)
    return rfft

def get_peaks(amp, frq, thres):
    """ """
    peak_i = peakutils.indexes(amp, thres=thres)
    return (frq[peak_i], amp[peak_i])

def get_plot(title="", xlabel="", ylabel="", facecolor='w', bgcolor='w'):
    """ """
    matplotlib.rcParams['font.family'] = 'cmr10'

    plot = matplotlib.pyplot.subplot()
    plot.patch.set_facecolor(facecolor)
    plot.set_axis_bgcolor(bgcolor)
    plot.set_title(title)
    plot.set_xlabel(xlabel)
    plot.set_ylabel(ylabel)

    return plot

def main(wav_name="op28-20.wav", t_endp=(None, None)):

    # t = np.arange(0, x, 1) / sample_frq  # sample points as time points (seconds)

    # import contents of wav file
    sample_frq, snd = wavfile.read(wav_name)
    snd = scale_snd(snd)
    to_frames = get_to_frames(sample_frq)
    f_endp = to_frames(t_endp)
    # total number of frames should be even
    f_endp[1] += sum(f_endp) % 2

    amp = local_rfft(snd.T, f_endp)
    amp_pwr = 10. * np.log10(abs(amp) ** 2) # dB
    amp_out = amp_pwr
    frq = np.fft.fftfreq(f_endp[1] - f_endp[0])
    frq_a = abs(sample_frq * frq)[0:amp.shape[1]]   # Hz

    thres = 0.01  # peak detection threshold as factor of amp_max
    peaks = [get_peaks(amp_j, frq_a, thres) for amp_j in amp_out]
    amp_max = [max(amp_j) for amp_j in amp_out]
    # plot sound intensity (dB) versus frequency (Hz)

    plot_title = u'{}, \u0394t = {}'.format(wav_name, t_endp)
    plot = get_plot(plot_title, 'Hz', 'dB')
    frq_max = 0
    marg = 1.1
    for i, ch in enumerate(amp_out):
        pyplot.plot(frq_a, ch, "{}-".format(CLRS[-i]))
        pyplot.plot(peaks[i][0], peaks[i][1] + i, "{}.".format(CLRS[-i]))
        frq_max_i = max(peaks[i][0])
        if frq_max_i > frq_max:
            frq_max = frq_max_i

    plot.axis([0, frq_max * marg, 0, max(amp_max) * marg])
    pyplot.show()
<commit_msg>Even more functionalization, simplifications of main()<commit_after>"""
Machine learning music
"""

import sys
import numpy as np
import tensorflow as tf
from scipy.io import wavfile
import jack
import matplotlib
import matplotlib.pyplot
import peakutils

# Datatypes that SciPy can import from .wav
SND_DTYPES = {'int16': 16-1, 'int32': 32-1}
# Colours for cycling in plots
CLRS = 'bgrcmyk'
# Set default font for matplotlib
matplotlib.rcParams['font.family'] = 'TeX Gyre Adventor'

def jack_client(midi_ins=1, midi_outs=1, name="MuserClient"):
    """Returns an active JACK client with specified number of inputs and outputs
    """
    client = jack.Client(name)
    for j in range(midi_ins):
        client.midi_inports.register("midi_in_{}".format(j))
    for k in range(midi_outs):
        client.midi_outports.register("midi_out_{}".format(k))
    client.activate()

    return client

def get_to_frames(sample_frq):
    """Return function that converts time (is s if sf is Hz) to sample index
       for a defined sampling rate.
    """
    def to_frames(time):
        def to_f(time):
            try:
                return int(time * sample_frq)
            except TypeError: # time not specified or specified badly
                raise TypeError("Real local endpoints must be specified! (t_endp)")
        try:
            return [to_f(t) for t in time]
        except TypeError: # time not iterable
            return [to_f(time)]

    return to_frames

def scale_snd(snd, factor=None):
    """ Scale signal from -1 to 1.
    """
    if factor is None:
        return scale_snd(snd, 2.**SND_DTYPES[snd.dtype.name])

    return snd / factor

def local_rfft(chs, f_endp, units='', norm='ortho'):
    """ Calculate FFTs for each channel over the interval
    """
    rfft = np.stack(np.fft.rfft(ch[slice(*f_endp)], norm=norm) for ch in chs)
    frq = np.fft.fftfreq(f_endp[1] - f_endp[0])[0:rfft.shape[1]]

    if units == '':
        amp = rfft
    elif units == 'dB':
        amp = 10. * np.log10(abs(rfft) ** 2.)
    elif units == 'sqr':
        amp = abs(rfft) ** 2.

    return amp, frq

def get_peaks(amp, frq, thres):
    """ """
    try:
        iter(amp)
        peaks_idx = [peakutils.indexes(ch, thres=thres) for ch in amp]
        peaks = [(frq[idx], amp[i][idx]) for i, idx in enumerate(peaks_idx)]
    except TypeError:
        return get_peaks([amp], frq, thres=thres)

    return peaks

def get_plot(title="", xlabel="", ylabel="", facecolor='w', bgcolor='w'):
    """ """
    fig = matplotlib.pyplot.figure()
    axes = fig.add_subplot(111)
    axes.patch.set_facecolor(facecolor)
    axes.set_axis_bgcolor(bgcolor)
    axes.set_title(title)
    axes.set_xlabel(xlabel)
    axes.set_ylabel(ylabel)

    return fig, axes

def load_wav(wav_name, scale=True):
    sample_frq, snd = wavfile.read(wav_name)
    if scale: snd = scale_snd(snd)

    return sample_frq, snd

def main(wav_name="op28-20.wav", t_endp=(None, None)):
    # t = np.arange(0, x, 1) / sample_frq  # sample points as time points (seconds)
    # import contents of wav file

    sample_frq, snd = load_wav(wav_name)
    to_frames = get_to_frames(sample_frq)
    f_endp = to_frames(t_endp)
    f_endp[1] += sum(f_endp) % 2  # total number of frames should be even

    amp, frq = local_rfft(snd.T, f_endp, units='dB')
    frq = abs(sample_frq * frq)   # Hz

    thres = 0.01  # peak detection threshold as factor of amp_max
    peaks = get_peaks(amp, frq, thres)
    maxes = zip(*[(max(peak[0]), max(peak[1])) for peak in peaks])

    # axes sound intensity (dB) versus frequency (Hz)
    axes_title = '{}, $\Delta$t = {}'.format(wav_name, t_endp)
    fig, axes = get_plot(axes_title, 'Hz', 'dB')
    marg = 1.1
    for i, ch in enumerate(amp):
        axes.plot(frq, ch, "{}-".format(CLRS[-i]))
        axes.plot(peaks[i][0], peaks[i][1] + i, "{}.".format(CLRS[-i]))

    axes.axis([0, max(maxes[0]) * marg, 0, max(maxes[1]) * marg])
    matplotlib.pyplot.show()

    return peaks

if __name__ == '__main__':
    t_endp = (float(sys.argv[1]), float(sys.argv[2]))
    main(t_endp=t_endp)
<|endoftext|>


<commit_before># pylint: disable=invalid-name
# pylint: disable=missing-docstring
# pylint: disable=too-few-public-methods
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-locals

import json
import os
import re
import subprocess
import sys
import warnings
import yaml

TESTCASES = list()
INVENTORY = 'test/fixtures/hosts'

HERE = os.path.abspath(os.path.dirname(__file__))
ROLE = re.match(
    r'^.*\/ansible-eos-([^/\s]+)\/test/arista-ansible-role-test$', HERE).group(1)
RUN_CONFIG_BACKUP = '_eos_role_test_{}_running'.format(ROLE)
START_CONFIG_BACKUP = '_eos_role_test_{}_startup'.format(ROLE)

EOS_ROLE_PLAYBOOK = 'test/arista-ansible-role-test/eos_role.yml'
EOS_MODULE_PLAYBOOK = 'test/arista-ansible-role-test/eos_module.yml'

LOG_FILE = '{}/roletest.log'.format(HERE)
try:
    os.remove(LOG_FILE)
except OSError:
    pass
LOG = open(LOG_FILE, 'w')

SEPARATOR = '    ' + '*' * 50

class TestCase(object):
    def __init__(self, **kwargs):
        self.name = kwargs['name']
        self.module = kwargs['module']

        self.host = None

        self.inventory = kwargs.get('inventory')
        self.exitcode = kwargs.get('exitcode', 0)
        self.idempotent = kwargs.get('idempotent', True)
        self.changed = kwargs.get('changed', True)
        self.present = kwargs.get('present')
        self.absent = kwargs.get('absent')

        self.arguments = kwargs.get('arguments', list())
        self.variables = dict()

        # optional properties
        self.setup = kwargs.get('setup', list())
        self.teardown = kwargs.get('teardown', list())

    def __str__(self):
        return self.name

class TestModule(object):
    def __init__(self, testcase):
        self.testcase = testcase
        self.description = 'Test [%s]: %s' % (testcase.module, testcase.name)

    def __call__(self):
        self.output('Run first pass')
        response = self.run_module()
        for device in response:
            hostname = device.keys()[0]
            reported = int(device[hostname]['changed'])
            expected = int(self.testcase.changed)
            msg = ("First pass role execution reported {} task change(s), "
                   "expected {}".format(reported, expected))
            self.output(msg)
            assert reported == expected, msg

        if self.testcase.idempotent:
            self.output('Run second pass')
            response = self.run_module()
            for device in response:
                hostname = device.keys()[0]
                reported = int(device[hostname]['changed'])
                msg = (
                    "Second pass role execution reported {} task change(s), "
                    "expected 0".format(reported))
                self.output(msg)
                assert not reported, msg

        if self.testcase.present:
            desc = 'Validate present configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.present, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should contain an empty list of updates
                delim = " ---\n"
                updates = device[hostname]['updates']
                msg = ("{} - Expected configuration\n{}{}\n{}not found "
                       "on device '{}'".format(desc, delim,
                                               '\n'.join(updates), delim,
                                               hostname))
                assert device[hostname]['updates'] == [], msg
                # Result should show no changes
                msg = ("{} - Device '{}' reported no updates, but "
                       "returned 'changed'".format(desc, hostname))
                assert device[hostname]['changed'] == False, msg

        if self.testcase.absent:
            desc = 'Validate absent configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.absent, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should show change has taken place
                msg = (
                    "{} - Entire absent configuration found on device '{}'".
                    format(desc, hostname)
                )
                assert device[hostname]['changed'] == True, msg
                # Compare changes with expected values, sorted at global level
                updates = '\n'.join(device[hostname]['updates'])
                updates = re.split(r'\n(?=\S)', updates)
                updates = '\n'.join(sorted(updates))
                absent = re.split(r'\n(?=\S)', self.testcase.absent.rstrip())
                absent = '\n'.join(sorted(absent))
                msg = ("{} - Some part of absent configuration found "
                       "on device '{}'".format(desc, hostname))
                assert updates == absent, msg

    def setUp(self):
        print("\n{}\n".format(SEPARATOR) +
              "  See run log for complete output:\n  {}".format(LOG_FILE) +
              "\n{}\n".format(SEPARATOR))

        LOG.write("\n\n\n{}\n".format(SEPARATOR) +
                  "  Begin log for {}".format(self.description) +
                  "\n{}\n\n".format(SEPARATOR))

        if self.testcase.setup:
            self.output('Running test case setup commands')
            setup_cmds = self.testcase.setup
            if not isinstance(setup_cmds, list):
                setup_cmds = setup_cmds.splitlines()
            self.output("{}".format(setup_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case setup commands',
                'cmds': ['configure terminal'] + setup_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                LOG.write("Playbook stdout:\n\n{}".format(out))
                LOG.write("Playbook stderr:\n\n{}".format(err))
                raise

    def tearDown(self):
        if self.testcase.teardown:
            self.output('Running test case teardown commands')
            teardown_cmds = self.testcase.teardown
            if not isinstance(teardown_cmds, list):
                teardown_cmds = teardown_cmds.splitlines()
            self.output("{}\n".format(teardown_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case teardown_cmds commands',
                'cmds': ['configure terminal'] + teardown_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                self.output("Playbook stdout:\n\n{}".format(out))
                self.output("Playbook stderr:\n\n{}".format(err))
                warnings.warn("\nError in test case teardown\n\n{}".format(
                    out))

    @classmethod
    def output(cls, text):
        print '>>', str(text)
        LOG.write('++ {}'.format(text) + '\n')

    def run_module(self):
        (retcode, out, _) = self.execute_module()
        out_stripped = re.sub(r'\"config\": \"! Command:.*\\nend\"',
                              '\"config\": \"--- stripped for space ---\"',
                              out)
        LOG.write("PLaybook stdout:\n\n{}".format(out_stripped))
        msg = "Return code: {}, Expected code: {}".format(retcode, self.testcase.exitcode)
        self.output(msg)
        assert retcode == self.testcase.exitcode, msg
        return self.parse_response(out)

    def execute_module(self):
        arguments = [json.dumps(self.testcase.arguments)]
        arguments.append(json.dumps(
            {'rolename': "ansible-eos-{}".format(ROLE)}))
        return ansible_playbook(EOS_ROLE_PLAYBOOK, arguments=arguments)

    def parse_response(self, output, validate=False):
        # Get all the lines after the 'PLAY RECAP ****...' header
        lines = re.sub(r'^.*PLAY RECAP \*+', '', output, 0, re.S).split('\n')
        # Remove any empty lines from the list
        lines = [x for x in lines if x]

        recap = []
        for line in lines:
            match = re.search(r'^(\S+)\s+\:\s+' \
                              r'ok=(\d+)\s+' \
                              r'changed=(\d+)\s+' \
                              r'unreachable=(\d+)\s+' \
                              r'failed=(\d+)', line)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "recap information")
            (name, okcount, changed, unreach, failed) = match.groups()
            recap.append({name: {'ok': okcount,
                                 'changed': changed,
                                 'unreachable': unreach,
                                 'failed': failed}})

        if not validate:
            return recap

        updates = []
        for device in recap:
            hostname = device.keys()[0]
            match = re.search(r'(?<!skipping: )\[%s\] => (\{.*\})' % hostname,
                              output, re.M)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "result validation")
            result = json.loads(match.group(1))
            updates.append({hostname: result})

        return updates

    def run_validation(self, src, desc='Validate configuration'):
        args = {'module': 'eos_template', 'description': desc, 'src': src, }
        arguments = [json.dumps(args)]
        (ret_code, out, _) = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments,
                                              options=['--check'])
        LOG.write(out)
        assert ret_code == 0, "Validation playbook failed execution"
        return self.parse_response(out, validate=True)

def filter_modules(modules, filenames):
    if modules:
        modules = ['{0}.yml'.format(s) for s in modules.split(',')]
        return list(set(modules).intersection(filenames))
    return filenames

def setup():
    print >> sys.stderr, "Test Suite Setup:"

    run_backup = "  Backing up running-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up running-config on node',
        'cmds': [
            'configure terminal',
            'copy running-config {}'.format(RUN_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(str(">> ansible-playbook {} stdout:\n".format(EOS_MODULE_PLAYBOOK), out))
        LOG.write(str(">> ansible-playbook {} stddrr:\n".format(EOS_MODULE_PLAYBOOK), err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    run_backup = "  Backing up startup-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up startup-config on node',
        'cmds': [
            'configure terminal',
            'copy startup-config {}'.format(START_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(str(">> ansible-playbook {} stdout:\n".format(EOS_MODULE_PLAYBOOK), out))
        LOG.write(str(">> ansible-playbook {} stddrr:\n".format(EOS_MODULE_PLAYBOOK), err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    print >> sys.stderr, "  Gathering test cases ..."

    modules = os.environ.get('ANSIBLE_ROLE_TEST_CASES')

    testcases_home = os.path.join(HERE, 'testcases')
    if not os.path.exists(testcases_home):
        print >> sys.stderr, "\n  ***** Testcase directory not found!! *****"
        teardown()
        raise RuntimeError("Testcase path '{}' does not exist".format(testcases_home))

    filenames = os.listdir(testcases_home)

    for module in filter_modules(modules, filenames):
        path = os.path.join(testcases_home, module)
        definition = yaml.load(open(path))

        defaults = definition.get('defaults', {})
        testcases = definition.get('testcases', [])
        if not testcases:
            print >> sys.stderr, ("\n  ***** No testcases defined in "
                                  "module {} *****\n".format(module))
        else:
            for testcase in definition.get('testcases', []):
                kwargs = defaults.copy()
                kwargs.update(testcase)
                TESTCASES.append(TestCase(**kwargs))

    print >> sys.stderr, "  Setup complete\n"

def teardown():
    print >> sys.stderr, "\nTest Suite Teardown:"

    no_teardown = os.environ.get('NO_ANSIBLE_ROLE_TEST_TEARDOWN')

    if no_teardown:
        print >> sys.stderr, ("{}\n"
                              "  Skipping test suite teardown due to "
                              "NO_ANSIBLE_ROLE_TEST_TEARDOWN\n"
                              "  To restore each device to pre-test state "
                              "execute the following commands\n"
                              "  - configure terminal\n"
                              "  - configure replace {}\n"
                              "  - delete {}\n"
                              "  - copy {} startup-config\n"
                              "  - delete {}\n"
                              "{}".format(SEPARATOR, RUN_CONFIG_BACKUP,
                                          RUN_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP, SEPARATOR))
    else:
        # Restore the running-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring running-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore running-config from backup',
            'cmds': [
                'configure terminal',
                'configure replace {}'.format(RUN_CONFIG_BACKUP),
                'delete {}'.format(RUN_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring running-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

        # Restore the startup-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring startup-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore startup-config from backup',
            'cmds': [
                'configure terminal',
                'copy {} startup-config'.format(START_CONFIG_BACKUP),
                'delete {}'.format(START_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring startup-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

    print >> sys.stderr, "  Teardown complete"

def test_module():
    for testcase in TESTCASES:
        yield TestModule(testcase)

def ansible_playbook(playbook, arguments=None, options=None):
    if arguments is None:
        arguments = []
    if options is None:
        options = []

    command = ['ansible-playbook']

    command.append(playbook)
    command.extend(['-i', INVENTORY])
    for arg in arguments:
        command.extend(['-e', arg])
    for opt in options:
        command.append(opt)
    command.append('-vvv')

    # Format the command string for output on error - for easier
    # copy/paste for manual run
    cmdstr = ''
    for segment in command:
        if segment[0] == '{':
            cmdstr = cmdstr + "\'{}\' ".format(segment)
        else:
            cmdstr = cmdstr + "{} ".format(segment)
    LOG.write("-- Ansible playbook command:\n-- {}\n".format(cmdstr))

    stdout = subprocess.PIPE
    stderr = subprocess.PIPE
    proc = subprocess.Popen(command, stdout=stdout, stderr=stderr)
    out, err = proc.communicate()

    return (proc.returncode, out, err)
<commit_msg>Pull in arista-ansible-role-test updates<commit_after># pylint: disable=invalid-name
# pylint: disable=missing-docstring
# pylint: disable=too-few-public-methods
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-locals

import json
import os
import re
import subprocess
import sys
import warnings
import yaml

TESTCASES = list()
INVENTORY = 'test/fixtures/hosts'

HERE = os.path.abspath(os.path.dirname(__file__))
ROLE = re.match(
    r'^.*\/ansible-eos-([^/\s]+)\/test/arista-ansible-role-test$', HERE).group(1)
RUN_CONFIG_BACKUP = '_eos_role_test_{}_running'.format(ROLE)
START_CONFIG_BACKUP = '_eos_role_test_{}_startup'.format(ROLE)

EOS_ROLE_PLAYBOOK = 'test/arista-ansible-role-test/eos_role.yml'
EOS_MODULE_PLAYBOOK = 'test/arista-ansible-role-test/eos_module.yml'

LOG_FILE = '{}/roletest.log'.format(HERE)
try:
    os.remove(LOG_FILE)
except OSError:
    pass
LOG = open(LOG_FILE, 'w')

SEPARATOR = '    ' + '*' * 50

class TestCase(object):
    def __init__(self, **kwargs):
        self.name = kwargs['name']
        self.module = kwargs['module']

        self.host = None

        self.inventory = kwargs.get('inventory')
        self.exitcode = kwargs.get('exitcode', 0)
        self.idempotent = kwargs.get('idempotent', True)
        self.changed = kwargs.get('changed', True)
        self.present = kwargs.get('present')
        self.absent = kwargs.get('absent')

        self.arguments = kwargs.get('arguments', list())
        self.variables = dict()

        # optional properties
        self.setup = kwargs.get('setup', list())
        self.teardown = kwargs.get('teardown', list())

    def __str__(self):
        return self.name

class TestModule(object):
    def __init__(self, testcase):
        self.testcase = testcase
        self.description = 'Test [%s]: %s' % (testcase.module, testcase.name)

    def __call__(self):
        self.output('Run first pass')
        response = self.run_module()
        for device in response:
            hostname = device.keys()[0]
            reported = int(device[hostname]['changed'])
            expected = int(self.testcase.changed)
            msg = ("First pass role execution reported {} task change(s), "
                   "expected {}".format(reported, expected))
            self.output(msg)
            assert reported == expected, msg

        if self.testcase.idempotent:
            self.output('Run second pass')
            response = self.run_module()
            for device in response:
                hostname = device.keys()[0]
                reported = int(device[hostname]['changed'])
                msg = (
                    "Second pass role execution reported {} task change(s), "
                    "expected 0".format(reported))
                self.output(msg)
                assert not reported, msg

        if self.testcase.present:
            desc = 'Validate present configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.present, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should contain an empty list of updates
                delim = " ---\n"
                updates = device[hostname]['updates']
                msg = ("{} - Expected configuration\n{}{}\n{}not found "
                       "on device '{}'".format(desc, delim,
                                               '\n'.join(updates), delim,
                                               hostname))
                assert device[hostname]['updates'] == [], msg
                # Result should show no changes
                msg = ("{} - Device '{}' reported no updates, but "
                       "returned 'changed'".format(desc, hostname))
                assert device[hostname]['changed'] == False, msg

        if self.testcase.absent:
            desc = 'Validate absent configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.absent, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should show change has taken place
                msg = (
                    "{} - Entire absent configuration found on device '{}'".
                    format(desc, hostname)
                )
                assert device[hostname]['changed'] == True, msg
                # Compare changes with expected values, sorted at global level
                updates = '\n'.join(device[hostname]['updates'])
                updates = re.split(r'\n(?=\S)', updates)
                updates = '\n'.join(sorted(updates))
                absent = re.split(r'\n(?=\S)', self.testcase.absent.rstrip())
                absent = '\n'.join(sorted(absent))
                msg = ("{} - Some part of absent configuration found "
                       "on device '{}'".format(desc, hostname))
                assert updates == absent, msg

    def setUp(self):
        print("\n{}\n".format(SEPARATOR) +
              "  See run log for complete output:\n  {}".format(LOG_FILE) +
              "\n{}\n".format(SEPARATOR))

        LOG.write("\n\n\n{}\n".format(SEPARATOR) +
                  "  Begin log for {}".format(self.description) +
                  "\n{}\n\n".format(SEPARATOR))

        if self.testcase.setup:
            self.output('Running test case setup commands')
            setup_cmds = self.testcase.setup
            if not isinstance(setup_cmds, list):
                setup_cmds = setup_cmds.splitlines()
            self.output("{}".format(setup_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case setup commands',
                'cmds': ['configure terminal'] + setup_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                LOG.write("Playbook stdout:\n\n{}".format(out))
                LOG.write("Playbook stderr:\n\n{}".format(err))
                raise

    def tearDown(self):
        if self.testcase.teardown:
            self.output('Running test case teardown commands')
            teardown_cmds = self.testcase.teardown
            if not isinstance(teardown_cmds, list):
                teardown_cmds = teardown_cmds.splitlines()
            self.output("{}\n".format(teardown_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case teardown_cmds commands',
                'cmds': ['configure terminal'] + teardown_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                self.output("Playbook stdout:\n\n{}".format(out))
                self.output("Playbook stderr:\n\n{}".format(err))
                warnings.warn("\nError in test case teardown\n\n{}".format(
                    out))

    @classmethod
    def output(cls, text):
        print '>>', str(text)
        LOG.write('++ {}'.format(text) + '\n')

    def run_module(self):
        (retcode, out, _) = self.execute_module()
        out_stripped = re.sub(r'\"config\": \"! Command:.*\\nend\"',
                              '\"config\": \"--- stripped for space ---\"',
                              out)
        LOG.write("PLaybook stdout:\n\n{}".format(out_stripped))
        msg = "Return code: {}, Expected code: {}".format(retcode, self.testcase.exitcode)
        self.output(msg)
        assert retcode == self.testcase.exitcode, msg
        return self.parse_response(out)

    def execute_module(self):
        arguments = [json.dumps(self.testcase.arguments)]
        arguments.append(json.dumps(
            {'rolename': "ansible-eos-{}".format(ROLE)}))
        return ansible_playbook(EOS_ROLE_PLAYBOOK, arguments=arguments)

    def parse_response(self, output, validate=False):
        # Get all the lines after the 'PLAY RECAP ****...' header
        lines = re.sub(r'^.*PLAY RECAP \*+', '', output, 0, re.S).split('\n')
        # Remove any empty lines from the list
        lines = [x for x in lines if x]

        recap = []
        for line in lines:
            match = re.search(r'^(\S+)\s+\:\s+' \
                              r'ok=(\d+)\s+' \
                              r'changed=(\d+)\s+' \
                              r'unreachable=(\d+)\s+' \
                              r'failed=(\d+)', line)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "recap information")
            (name, okcount, changed, unreach, failed) = match.groups()
            recap.append({name: {'ok': okcount,
                                 'changed': changed,
                                 'unreachable': unreach,
                                 'failed': failed}})

        if not validate:
            return recap

        updates = []
        for device in recap:
            hostname = device.keys()[0]
            match = re.search(r'(?<!skipping: )\[%s\] => (\{.*\})' % hostname,
                              output, re.M)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "result validation")
            result = json.loads(match.group(1))
            updates.append({hostname: result})

        return updates

    def run_validation(self, src, desc='Validate configuration'):
        args = {'module': 'eos_template', 'description': desc, 'src': src, }
        arguments = [json.dumps(args)]
        (ret_code, out, _) = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments,
                                              options=['--check'])
        LOG.write(out)
        assert ret_code == 0, "Validation playbook failed execution"
        return self.parse_response(out, validate=True)

def filter_modules(modules, filenames):
    if modules:
        modules = ['{0}.yml'.format(s) for s in modules.split(',')]
        return list(set(modules).intersection(filenames))
    return filenames

def setup():
    print >> sys.stderr, "Test Suite Setup:"

    run_backup = "  Backing up running-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up running-config on node',
        'cmds': [
            'configure terminal',
            'copy running-config {}'.format(RUN_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(">> ansible-playbook {} stdout:\n{}".format(EOS_MODULE_PLAYBOOK, out))
        LOG.write(">> ansible-playbook {} stddrr:\n{}".format(EOS_MODULE_PLAYBOOK, err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    run_backup = "  Backing up startup-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up startup-config on node',
        'cmds': [
            'configure terminal',
            'copy startup-config {}'.format(START_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(">> ansible-playbook {} stdout:\n{}".format(EOS_MODULE_PLAYBOOK, out))
        LOG.write(">> ansible-playbook {} stddrr:\n{}".format(EOS_MODULE_PLAYBOOK, err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    print >> sys.stderr, "  Gathering test cases ..."

    modules = os.environ.get('ANSIBLE_ROLE_TEST_CASES')

    testcases_home = os.path.join(HERE, 'testcases')
    if not os.path.exists(testcases_home):
        print >> sys.stderr, "\n  ***** Testcase directory not found!! *****"
        teardown()
        raise RuntimeError("Testcase path '{}' does not exist".format(testcases_home))

    filenames = os.listdir(testcases_home)

    for module in filter_modules(modules, filenames):
        path = os.path.join(testcases_home, module)
        definition = yaml.load(open(path))

        defaults = definition.get('defaults', {})
        testcases = definition.get('testcases', [])
        if not testcases:
            print >> sys.stderr, ("\n  ***** No testcases defined in "
                                  "module {} *****\n".format(module))
        else:
            for testcase in definition.get('testcases', []):
                kwargs = defaults.copy()
                kwargs.update(testcase)
                TESTCASES.append(TestCase(**kwargs))

    print >> sys.stderr, "  Setup complete\n"

def teardown():
    print >> sys.stderr, "\nTest Suite Teardown:"

    no_teardown = os.environ.get('NO_ANSIBLE_ROLE_TEST_TEARDOWN')

    if no_teardown:
        print >> sys.stderr, ("{}\n"
                              "  Skipping test suite teardown due to "
                              "NO_ANSIBLE_ROLE_TEST_TEARDOWN\n"
                              "  To restore each device to pre-test state "
                              "execute the following commands\n"
                              "  - configure terminal\n"
                              "  - configure replace {}\n"
                              "  - delete {}\n"
                              "  - copy {} startup-config\n"
                              "  - delete {}\n"
                              "{}".format(SEPARATOR, RUN_CONFIG_BACKUP,
                                          RUN_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP, SEPARATOR))
    else:
        # Restore the running-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring running-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore running-config from backup',
            'cmds': [
                'configure terminal',
                'configure replace {}'.format(RUN_CONFIG_BACKUP),
                'delete {}'.format(RUN_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring running-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

        # Restore the startup-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring startup-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore startup-config from backup',
            'cmds': [
                'configure terminal',
                'copy {} startup-config'.format(START_CONFIG_BACKUP),
                'delete {}'.format(START_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring startup-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

    print >> sys.stderr, "  Teardown complete"

def test_module():
    for testcase in TESTCASES:
        yield TestModule(testcase)

def ansible_playbook(playbook, arguments=None, options=None):
    if arguments is None:
        arguments = []
    if options is None:
        options = []

    command = ['ansible-playbook']

    command.append(playbook)
    command.extend(['-i', INVENTORY])
    for arg in arguments:
        command.extend(['-e', arg])
    for opt in options:
        command.append(opt)
    command.append('-vvv')

    # Format the command string for output on error - for easier
    # copy/paste for manual run
    cmdstr = ''
    for segment in command:
        if segment[0] == '{':
            cmdstr = cmdstr + "\'{}\' ".format(segment)
        else:
            cmdstr = cmdstr + "{} ".format(segment)
    LOG.write("-- Ansible playbook command:\n-- {}\n".format(cmdstr))

    stdout = subprocess.PIPE
    stderr = subprocess.PIPE
    proc = subprocess.Popen(command, stdout=stdout, stderr=stderr)
    out, err = proc.communicate()

    return (proc.returncode, out, err)
<|endoftext|>


<commit_before>import sys
import pep8

from coverage import coverage
from distutils import log
from django.conf import settings
from importlib import import_module
from StringIO import StringIO

def init(test_suite):
    try:
        test_settings = import_module('%s.settings' % test_suite)
    except ImportError:
        log.info('ImportError: Unable to import %s.settings' % test_suite)
        sys.exit(1)

    setting_attrs = {}
    for attr in dir(test_settings):
        if '__' not in attr:
            setting_attrs[attr] = getattr(test_settings, attr)

    if not settings.configured:
        settings.configure(**setting_attrs)

def runpep8(package):
    # Hook into stdout.
    old_stdout = sys.stdout
    sys.stdout = mystdout = StringIO()

    # Run Pep8 checks.
    pep8.options, pep8.args = pep8.process_options()
    pep8.options.repeat = True
    pep8.input_dir(package)

    # Restore stdout.
    sys.stdout = old_stdout

    # Save result to pep8.txt.
    result = mystdout.getvalue()
    output = open('pep8.txt', 'w')
    output.write(result)
    output.close()

    # Return Pep8 result
    if result:
        return result
    else:
        return None

def runtests(test):
    test_suite = test.test_suite
    module = test_suite.split('.')[0]
    init(test_suite)

    from django.test.simple import run_tests
    # Start coverage.
    if test.verbose:
        cov = coverage()
        cov.start()

    # Run tests.
    failures = run_tests((module,), verbosity=1, interactive=True)

    if test.verbose:
        # Stop and generate coverage report.
        cov.stop()
        log.info("\nCoverage Report:")
        cov.report(include=['%s*' % module,], omit=['*tests*',])
        cov.xml_report(include=['%s*' % module,], omit=['*tests*',])

        # Generate PEP8 report.
        pep_result = runpep8(module)
        if pep_result:
            log.info("\nPEP8 Report:")
            log.info(pep_result)

    sys.exit(failures)

<commit_msg>more robust test settings import<commit_after>import sys
import pep8

from coverage import coverage
from distutils import log
from django.conf import settings
from StringIO import StringIO

def init(test_suite):
    try:
        test_settings = __import__('test_settings')
    except ImportError, e:
        log.info('ImportError: Unable to import test settings.')
        sys.exit(1)

    setting_attrs = {}
    for attr in dir(test_settings):
        if '__' not in attr:
            setting_attrs[attr] = getattr(test_settings, attr)

    if not settings.configured:
        settings.configure(**setting_attrs)

def runpep8(package):
    # Hook into stdout.
    old_stdout = sys.stdout
    sys.stdout = mystdout = StringIO()

    # Run Pep8 checks.
    pep8.options, pep8.args = pep8.process_options()
    pep8.options.repeat = True
    pep8.input_dir(package)

    # Restore stdout.
    sys.stdout = old_stdout

    # Save result to pep8.txt.
    result = mystdout.getvalue()
    output = open('pep8.txt', 'w')
    output.write(result)
    output.close()

    # Return Pep8 result
    if result:
        return result
    else:
        return None

def runtests(test):
    test_suite = test.test_suite
    module = test_suite.split('.')[0]
    init(test_suite)

    from django.test.simple import run_tests
    # Start coverage.
    if test.verbose:
        cov = coverage()
        cov.start()

    # Run tests.
    failures = run_tests((module,), verbosity=1, interactive=True)

    if test.verbose:
        # Stop and generate coverage report.
        cov.stop()
        log.info("\nCoverage Report:")
        cov.report(include=['%s*' % module,], omit=['*tests*',])
        cov.xml_report(include=['%s*' % module,], omit=['*tests*',])

        # Generate PEP8 report.
        pep_result = runpep8(module)
        if pep_result:
            log.info("\nPEP8 Report:")
            log.info(pep_result)

    sys.exit(failures)

<|endoftext|>


<commit_before>### $Id$
### $URL$

from sfa.util.faults import *
from sfa.util.method import Method
from sfa.util.parameter import Parameter, Mixed
from sfa.trust.auth import Auth
from sfa.util.record import GeniRecord
from sfa.util.genitable import GeniTable
from sfa.util.debug import log
from sfa.trust.credential import Credential
from sfa.server.registry import Registries

class remove(Method):
    """
    Remove an object from the registry. If the object represents a PLC object,
    then the PLC records will also be removed.

    @param cred credential string
    @param type record type
    @param hrn human readable name of record to remove

    @return 1 if successful, faults otherwise
    """

    interfaces = ['registry']

    accepts = [
        Parameter(str, "Credential string"),
        Parameter(str, "Record type"),
        Parameter(str, "Human readable name (hrn) of record to be removed"),
        Mixed(Parameter(str, "Request hash"),
              Parameter(None, "Request hash not specified"))
        ]

    returns = Parameter(int, "1 if successful")

    def call(self, cred, type, hrn, request_hash=None, origin_hrn=None):

        if origin_hrn==None:
            origin_hrn=Credential(string=cred).get_gid_caller().get_hrn()
        #log the call
        self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, hrn, self.name))
        # This cred will be an authority cred, not a user, so we cant use it to
        # authenticate the caller's request_hash. Let just get the caller's gid
        # from the cred and authenticate using that
        client_gid = Credential(string=cred).get_gid_caller()
        client_gid_str = client_gid.save_to_string(save_parents=True)
        self.api.auth.authenticateGid(client_gid_str, [cred, type, hrn], request_hash)
        self.api.auth.check(cred, "remove")
        self.api.auth.verify_object_permission(hrn)
        table = GeniTable()
        filter = {'hrn': hrn}
        if type not in ['all', '*']:
            filter['type'] = type
        records = table.find(filter)
        if not records:
            raise RecordNotFound(hrn)
        record = records[0]
        type = record['type']

        credential = self.api.getCredential()
       	registries = Registries(self.api)

        # Try to remove the object from the PLCDB of federated agg.
        # This is attempted before removing the object from the local agg's PLCDB and sfa table
        if hrn.startswith(self.api.hrn) and type in ['user', 'slice', 'authority']:
            for registry in registries:
                if registry not in [self.api.hrn]:
                    try:
			request_hash=None
                        result=registries[registry].remove_peer_object(credential, record, request_hash, origin_hrn)
                    except:
                        pass
        if type == "user":
            persons = self.api.plshell.GetPersons(self.api.plauth, record['pointer'])
            # only delete this person if he has site ids. if he doesnt, it probably means
            # he was just removed from a site, not actually deleted
            if persons and persons[0]['site_ids']:
                self.api.plshell.DeletePerson(self.api.plauth, record['pointer'])
        elif type == "slice":
            if self.api.plshell.GetSlices(self.api.plauth, record['pointer']):
                self.api.plshell.DeleteSlice(self.api.plauth, record['pointer'])
        elif type == "node":
            if self.api.plshell.GetNodes(self.api.plauth, record['pointer']):
                self.api.plshell.DeleteNode(self.api.plauth, record['pointer'])
        elif type == "authority":
            if self.api.plshell.GetSites(self.api.plauth, record['pointer']):
                self.api.plshell.DeleteSite(self.api.plauth, record['pointer'])
        else:
            raise UnknownGeniType(type)

        table.remove(record)

	# forward the call after replacing the root hrn

        return 1
<commit_msg>cosmetic<commit_after>### $Id$
### $URL$

from sfa.util.faults import *
from sfa.util.method import Method
from sfa.util.parameter import Parameter, Mixed
from sfa.trust.auth import Auth
from sfa.util.record import GeniRecord
from sfa.util.genitable import GeniTable
from sfa.util.debug import log
from sfa.trust.credential import Credential
from sfa.server.registry import Registries

class remove(Method):
    """
    Remove an object from the registry. If the object represents a PLC object,
    then the PLC records will also be removed.

    @param cred credential string
    @param type record type
    @param hrn human readable name of record to remove

    @return 1 if successful, faults otherwise
    """

    interfaces = ['registry']

    accepts = [
        Parameter(str, "Credential string"),
        Parameter(str, "Record type"),
        Parameter(str, "Human readable name (hrn) of record to be removed"),
        Mixed(Parameter(str, "Request hash"),
              Parameter(None, "Request hash not specified"))
        ]

    returns = Parameter(int, "1 if successful")

    def call(self, cred, type, hrn, request_hash=None, origin_hrn=None):
        # This cred will be an authority cred, not a user, so we cant use it to
        # authenticate the caller's request_hash. Let just get the caller's gid
        # from the cred and authenticate using that
        client_gid = Credential(string=cred).get_gid_caller()
        client_gid_str = client_gid.save_to_string(save_parents=True)
        self.api.auth.authenticateGid(client_gid_str, [cred, type, hrn], request_hash)
        self.api.auth.check(cred, "remove")
        self.api.auth.verify_object_permission(hrn)

        if origin_hrn==None:
            origin_hrn=Credential(string=cred).get_gid_caller().get_hrn()

        #log the call
        self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, hrn, self.name))
        table = GeniTable()
        filter = {'hrn': hrn}
        if type not in ['all', '*']:
            filter['type'] = type
        records = table.find(filter)
        if not records:
            raise RecordNotFound(hrn)
        record = records[0]
        type = record['type']

        credential = self.api.getCredential()
       	registries = Registries(self.api)

        # Try to remove the object from the PLCDB of federated agg.
        # This is attempted before removing the object from the local agg's PLCDB and sfa table
        if hrn.startswith(self.api.hrn) and type in ['user', 'slice', 'authority']:
            for registry in registries:
                if registry not in [self.api.hrn]:
                    try:
			request_hash=None
                        result=registries[registry].remove_peer_object(credential, record, request_hash, origin_hrn)
                    except:
                        pass
        if type == "user":
            persons = self.api.plshell.GetPersons(self.api.plauth, record['pointer'])
            # only delete this person if he has site ids. if he doesnt, it probably means
            # he was just removed from a site, not actually deleted
            if persons and persons[0]['site_ids']:
                self.api.plshell.DeletePerson(self.api.plauth, record['pointer'])
        elif type == "slice":
            if self.api.plshell.GetSlices(self.api.plauth, record['pointer']):
                self.api.plshell.DeleteSlice(self.api.plauth, record['pointer'])
        elif type == "node":
            if self.api.plshell.GetNodes(self.api.plauth, record['pointer']):
                self.api.plshell.DeleteNode(self.api.plauth, record['pointer'])
        elif type == "authority":
            if self.api.plshell.GetSites(self.api.plauth, record['pointer']):
                self.api.plshell.DeleteSite(self.api.plauth, record['pointer'])
        else:
            raise UnknownGeniType(type)

        table.remove(record)

	# forward the call after replacing the root hrn

        return 1
<|endoftext|>


<commit_before>#!/usr/bin/python
from twython import Twython
from random import randint
import json

frases = ["I love Penguins!", "Penguins are awesome!", "I like penguins : )", "Penguins are the best", "Penguins FTW", "Penguins <3", "Oh, Penguins!", "so you are thinking about penguins too ^^", "I love Penguins!"]
def pick():
    n = randint(0, len(frases))
    return frases[n]
def auth():
    with open("access.json", 'r') as f:
        db = json.load(f)
    return Twython(db["API_Key"], db["API_Secret"], db["Access_Token"],db["Access_Token_Secret"])

def load():
    with open("queue.json",'r') as f:
        queue = json.load(f)
    with open("info.json", 'r') as fi:
        info = json.load(fi)
    return queue, info

def respond(twitter, top_tweet):
    name = top_tweet["user"]["screen_name"]
    if name != "Penguin___Lover":
        #print "name",name
        twitter.update_status(status="@%s, " %(name)+pick(), in_reply_to_status_id=top_tweet["id"])

def dump(queue, info):
    with open("queue.json", 'w') as f:
        json.dump(queue, f)
    with open("info.json", 'w') as fi:
        json.dump(info, fi)

def main():
    twitter = auth()
    queue, info = load()
    tweets = twitter.search(q="penguins", result_type="recent", since_id=info["sinceid"], count='100')
    info["sinceid"] = tweets["search_metadata"]["max_id"]
    triggers = ("Penguins","penguins","Penguin","penguin")
    to_add = [tweet for tweet in tweets["statuses"] if not tweet["retweeted"] and not tweet.has_key("retweeted_status")]

    #to_add = [tweet for tweet in to_add if (tweet["text"].__contains__(triggers[0]) or tweet["text"].__contains__(triggers[1]) or tweet["text"].__contains__(triggers[2]) or tweet["text"].__contains__(triggers[3]))]

    #TOTEST

    #totaltext = tweet["text"]
    #print totaltext

    #tt = [x for x in totaltext.split() if x[0] != '@']
    #print tt
    #t = ' '.join(tt)
    #print t
    #    [y for y in x if y != 2]
    #to_add = [tweet for tweet in to_add if ( (triggers[0] in t) or (triggers[1] in t) or (triggers[2] in t) or (triggers[3] in t) )]

    filtered = []
    for tweet in to_add:
        totaltext = tweet["text"]
        #print totaltext

        tt = [x for x in totaltext.split() if x[0] != '@']
        #print tt
        t = ' '.join(tt)
        #print t

        if ( (triggers[0] in t) or (triggers[1] in t) or (triggers[2] in t) or (triggers[3] in t) ):
            filtered.append(tweet)
    #TOTEST

    queue = list(queue) + filtered
    mx = max(len(filtered), 20)
    if len(queue) > mx:
        queue = queue[-mx:]
    if len(queue) > 0:
        respond(twitter, queue.pop())
    dump(queue,info)

#it would have to run on cron every n minutes, it would be nice not to get banned for spam T.T
if __name__ == "__main__":
    main()
<commit_msg>This is it<commit_after>#!/usr/bin/python
from twython import Twython
from random import randint
import json

frases = ["I love Penguins!", "Penguins are awesome!", "I like penguins : )", "Penguins are the best", "Penguins FTW", "Penguins <3", "Oh, Penguins!", "so you are thinking about penguins too ^^", "I love Penguins!"]
def pick():
    n = randint(0, len(frases))
    return frases[n]
def auth():
    with open("access.json", 'r') as f:
        db = json.load(f)
    return Twython(db["API_Key"], db["API_Secret"], db["Access_Token"],db["Access_Token_Secret"])

def load():
    with open("queue.json",'r') as f:
        queue = json.load(f)
    with open("info.json", 'r') as fi:
        info = json.load(fi)
    return queue, info

def respond(twitter, top_tweet):
    name = top_tweet["user"]["screen_name"]
    if name != "Penguin___Lover":
        #print "name",name
        twitter.update_status(status="@%s, " %(name)+pick(), in_reply_to_status_id=top_tweet["id"])

def dump(queue, info):
    with open("queue.json", 'w') as f:
        json.dump(queue, f)
    with open("info.json", 'w') as fi:
        json.dump(info, fi)

def main():

    #think is ok
    r = random ()%4
    if ( r > 2 ):
        exit ()

    twitter = auth()
    queue, info = load()
    tweets = twitter.search(q="penguins", result_type="recent", since_id=info["sinceid"], count='100')
    info["sinceid"] = tweets["search_metadata"]["max_id"]
    triggers = ("Penguins","penguins","Penguin","penguin")
    to_add = [tweet for tweet in tweets["statuses"] if not tweet["retweeted"] and not tweet.has_key("retweeted_status")]

    #to_add = [tweet for tweet in to_add if (tweet["text"].__contains__(triggers[0]) or tweet["text"].__contains__(triggers[1]) or tweet["text"].__contains__(triggers[2]) or tweet["text"].__contains__(triggers[3]))]

    filtered = []
    for tweet in to_add:
        totaltext = tweet["text"]
        #print totaltext

        tt = [x for x in totaltext.split() if x[0] != '@']
        #print tt
        t = ' '.join(tt)
        #print t

        if ( (triggers[0] in t) or (triggers[1] in t) or (triggers[2] in t) or (triggers[3] in t) ):
            filtered.append(tweet)

    queue = list(queue) + filtered
    mx = max(len(filtered), 20)
    if len(queue) > mx:
        queue = queue[-mx:]
    if len(queue) > 0:
        respond(twitter, queue.pop())
    dump(queue,info)

#it would have to run on cron every n minutes, it would be nice not to get banned for spam T.T
if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import absolute_import

import traceback
import time
import sys
import subprocess
import platform
import tempfile
import os

from PyQt4 import QtCore, QtGui
Qt = QtCore.Qt

import shotgun_api3_registry

from . import tickets

class Dialog(QtGui.QDialog):

    def __init__(self, exceptions=None):
        super(Dialog, self).__init__()
        self._exception_list = list(exceptions or [])
        self._setup_ui()

    def _setup_ui(self):
        self.setWindowTitle('Open A Ticket')
        self.setMinimumSize(400, 300)
        self.setLayout(QtGui.QFormLayout())

        self._exception = QtGui.QComboBox()
        self._exception.addItem('None', None)
        for exc_type, exc_value, exc_traceback in self._exception_list:
            self._exception.addItem(
                '%s: %s [%s]' % (exc_type.__name__, exc_value, tickets.exception_uuid(exc_type, exc_value, exc_traceback)),
                (exc_type, exc_value, exc_traceback),
            )
        self._exception.setCurrentIndex(self._exception.count() - 1)
        self._exception.currentIndexChanged.connect(self._on_exception)
        self.layout().addRow("Exception", self._exception)

        # self._ticket = QtGui.QComboBox()
        # self._ticket.addItem("New")
        # self.layout().addRow("Ticket", self._ticket)

        self._title_label = QtGui.QLabel("Title")
        self._title = QtGui.QLineEdit('Bug Report')
        self.layout().addRow(self._title_label, self._title)

        self._description = QtGui.QTextEdit('Please describe the problem, and any steps to reproduce it and/or fix it.')
        self._description.focusInEvent = lambda *args: self._description.selectAll()
        self.layout().addRow("Description", self._description)

        self._screenshot_path = None
        self._screenshot = QtGui.QLabel()
        self._screenshot.setFixedSize(100, 100)
        self._screenshot.setFrameShadow(QtGui.QFrame.Sunken)
        self._screenshot.setFrameShape(QtGui.QFrame.Panel)
        self._screenshot.mouseReleaseEvent = self._on_screenshot
        self.layout().addRow("Screenshot", self._screenshot)

        buttons = QtGui.QHBoxLayout()
        self.layout().addRow("", buttons)

        # button = QtGui.QPushButton('Add Screenshot')
        # buttons.addWidget(button)

        buttons.addStretch()

        button = QtGui.QPushButton('Submit')
        button.clicked.connect(self._on_submit)
        buttons.addWidget(button)

        self._description.selectAll()
        self._on_exception(self._exception.currentIndex())

    def _on_exception(self, exc_index):
        exc_info = self._exception.itemData(exc_index).toPyObject()
        self._title_label.setVisible(not exc_info)
        self._title.setVisible(not exc_info)

    def _on_screenshot(self, *args):
        self.hide()
        path = tempfile.NamedTemporaryFile(suffix=".png", prefix="tanktmp", delete=False).name
        if platform.system() == "Darwin":
            # use built-in screenshot command on the mac
            proc = subprocess.Popen(['screencapture', '-mis', path])
        else:
            proc = subprocess.Popen(['import', path])
        proc.wait()
        self.show()

        self._screenshot_path = path
        pixmap = QtGui.QPixmap(path).scaledToHeight(100, Qt.SmoothTransformation)
        self._screenshot.setPixmap(pixmap)
        self._screenshot.setFixedSize(pixmap.size())

    def description(self):
        return str(self._description.toPlainText())

    def _get_reply_data(self):
        return [
            ('User Comment', str(self._description.toPlainText())),
            ('OS Environment', dict(os.environ)),
        ]

    def _on_submit(self, *args):

        exc_index = self._exception.currentIndex()
        exc_info = self._exception.itemData(exc_index).toPyObject()
        if exc_info:
            title = None
        else:
            exc_info = exc_info or (None, None, None)
            title = str(self._title.text())
        ticket_id = tickets.get_ticket_for_exception(*exc_info, title=title)
        reply_id = tickets.reply_to_ticket(ticket_id, self._get_reply_data(), user_id=tickets.guess_user_id())
        if self._screenshot_path:
            tickets.attach_to_ticket(ticket_id, self._screenshot_path)
        self.close()
        QtGui.QMessageBox.information(None,
            'Ticket Created',
            'Ticket #%d has been created on Shotgun' % ticket_id,
        )

__also_reload__ = [
    'sgactions.tickets',
]

def __before_reload__():
    # We have to manually clean this, since we aren't totally sure it will
    # always fall out of scope.
    global dialog
    if dialog:
        dialog.close()
        dialog.destroy()
        dialog = None

dialog = None

def run():
    global dialog
    if dialog:
        dialog.close()
    dialog = Dialog()
    dialog.show()
        <commit_msg>Include traceback<commit_after>from __future__ import absolute_import

import traceback
import time
import sys
import subprocess
import platform
import tempfile
import os

from PyQt4 import QtCore, QtGui
Qt = QtCore.Qt

import shotgun_api3_registry

from . import tickets

class Dialog(QtGui.QDialog):

    def __init__(self, exceptions=None):
        super(Dialog, self).__init__()
        self._exception_list = list(exceptions or [])
        self._setup_ui()

    def _setup_ui(self):
        self.setWindowTitle('Open A Ticket')
        self.setMinimumSize(400, 300)
        self.setLayout(QtGui.QFormLayout())

        self._exception = QtGui.QComboBox()
        self._exception.addItem('None', None)
        for exc_type, exc_value, exc_traceback in self._exception_list:
            self._exception.addItem(
                '%s: %s [%s]' % (exc_type.__name__, exc_value, tickets.exception_uuid(exc_type, exc_value, exc_traceback)),
                (exc_type, exc_value, exc_traceback),
            )
        self._exception.setCurrentIndex(self._exception.count() - 1)
        self._exception.currentIndexChanged.connect(self._on_exception)
        self.layout().addRow("Exception", self._exception)

        # self._ticket = QtGui.QComboBox()
        # self._ticket.addItem("New")
        # self.layout().addRow("Ticket", self._ticket)

        self._title_label = QtGui.QLabel("Title")
        self._title = QtGui.QLineEdit('Bug Report')
        self.layout().addRow(self._title_label, self._title)

        self._description = QtGui.QTextEdit('Please describe the problem, and any steps to reproduce it and/or fix it.')
        self._description.focusInEvent = lambda *args: self._description.selectAll()
        self.layout().addRow("Description", self._description)

        self._screenshot_path = None
        self._screenshot = QtGui.QLabel()
        self._screenshot.setFixedSize(100, 100)
        self._screenshot.setFrameShadow(QtGui.QFrame.Sunken)
        self._screenshot.setFrameShape(QtGui.QFrame.Panel)
        self._screenshot.mouseReleaseEvent = self._on_screenshot
        self.layout().addRow("Screenshot", self._screenshot)

        buttons = QtGui.QHBoxLayout()
        self.layout().addRow("", buttons)

        # button = QtGui.QPushButton('Add Screenshot')
        # buttons.addWidget(button)

        buttons.addStretch()

        button = QtGui.QPushButton('Submit')
        button.clicked.connect(self._on_submit)
        buttons.addWidget(button)

        self._description.selectAll()
        self._on_exception(self._exception.currentIndex())

    def _on_exception(self, exc_index):
        exc_info = self._exception.itemData(exc_index).toPyObject()
        self._title_label.setVisible(not exc_info)
        self._title.setVisible(not exc_info)

    def _on_screenshot(self, *args):
        self.hide()
        path = tempfile.NamedTemporaryFile(suffix=".png", prefix="tanktmp", delete=False).name
        if platform.system() == "Darwin":
            # use built-in screenshot command on the mac
            proc = subprocess.Popen(['screencapture', '-mis', path])
        else:
            proc = subprocess.Popen(['import', path])
        proc.wait()
        self.show()

        self._screenshot_path = path
        pixmap = QtGui.QPixmap(path).scaledToHeight(100, Qt.SmoothTransformation)
        self._screenshot.setPixmap(pixmap)
        self._screenshot.setFixedSize(pixmap.size())

    def description(self):
        return str(self._description.toPlainText())

    def _get_reply_data(self, exc_info):
        data = [('User Comment', str(self._description.toPlainText()))]
        if exc_info:
            data.append(('Traceback', exc_info))
        data.append(('OS Environment', dict(os.environ)))
        return data

    def _on_submit(self, *args):

        exc_index = self._exception.currentIndex()
        exc_info = self._exception.itemData(exc_index).toPyObject()
        data = self._get_reply_data(exc_info)
        if exc_info:
            title = None
        else:
            exc_info = exc_info or (None, None, None)
            title = str(self._title.text())
        ticket_id = tickets.get_ticket_for_exception(*exc_info, title=title)
        reply_id = tickets.reply_to_ticket(ticket_id, data, user_id=tickets.guess_user_id())
        if self._screenshot_path:
            tickets.attach_to_ticket(ticket_id, self._screenshot_path)
        self.close()
        QtGui.QMessageBox.information(None,
            'Ticket Created',
            'Ticket #%d has been created on Shotgun' % ticket_id,
        )

__also_reload__ = [
    'sgactions.tickets',
]

def __before_reload__():
    # We have to manually clean this, since we aren't totally sure it will
    # always fall out of scope.
    global dialog
    if dialog:
        dialog.close()
        dialog.destroy()
        dialog = None

dialog = None

def run():
    global dialog
    if dialog:
        dialog.close()
    dialog = Dialog()
    dialog.show()
        <|endoftext|>


<commit_before>from .session import Session
from .entity import Entity

__also_reload__ = ['.session', '.entity']
<commit_msg>Remove old __also_reload__<commit_after>from .session import Session
from .entity import Entity
<|endoftext|>


<commit_before>#!/usr/bin/python3

import bs4
import re

see_also_re = re.compile("\s+(?:\()?[Ss]ee also.*(  |$|\))")
nonletter_re = re.compile("[^A-Za-z]")

def trunc(s):    #these lengths are completely arbitrary
    s = s[:200]                        #chop at 200 characters
    x = s.rsplit(".  ", 1)
    if len(x[0]) > 160:                #if the last sentence break is more than 160 chars from the start
        return x[0] + "."
    elif len(s) == 200:                #if we ran right up to the end
        return s.rsplit(" ", 1)[0] + "..."
    else:
        return s

soup = bs4.BeautifulSoup(open("download/jargon.xml"))

for entry in soup.glossary.find_all("glossentry"):
    term = entry.glossterm.get_text()
    if entry.abbrev:
        pronounce = entry.abbrev.find("emphasis", role="pronunciation")
        grammar = entry.abbrev.find("emphasis", role="grammar")

        if pronounce:
            pronounce = pronounce.get_text()
        if pronounce == "//" or pronounce == None:
            pronounce = ""

        if grammar:
            grammar = grammar.get_text()
        if grammar == None:
            grammar = ""

    text = " ".join([x.get_text() for x in entry.find_all("glossdef")]) #concatenate all definitions
    text = " ".join([x.strip() for x in text.split("\n")])              #elide line breaks
    text = see_also_re.sub("", text)                                    #remove "see also", since we're including that in a separate field
    if grammar:
        text = "%s %s" % (grammar, text)                                #prepend grammar
    if pronounce:
        text = "%s %s" % (pronounce, text)                              #prepend part of speech
    text = trunc(text)                                                  #truncate

    see_also = "\\\\n".join(["[[%s]]" % x.get_text().replace("\n", "").replace("   ", " ") for x in entry.find_all("glossterm")][1:])    #the first one is the current term, so we skip that

    letter = term[0].upper()
    m = nonletter_re.match(letter)
    if m:
        letter = "0"
    url = "http://catb.org/jargon/html/%s/%s.html" % (letter, entry.get("id"))

    out = "\t".join([
        term,       #0
        "A",        #1
        "",         #2
        "",         #3
        "",         #4
        "",         #5
        see_also,   #6
        "",         #7
        "",         #8
        "",         #9
        "",         #10
        text,       #11
        url])       #12

    print(out)
<commit_msg>Jargon: Remove pronunciation.<commit_after>#!/usr/bin/python3

import bs4
import re

see_also_re = re.compile("\s+(?:\()?[Ss]ee also.*(  |$|\))")
nonletter_re = re.compile("[^A-Za-z]")

def trunc(s):    #these lengths are completely arbitrary
    s = s[:200]                        #chop at 200 characters
    x = s.rsplit(".  ", 1)
    if len(x[0]) > 160:                #if the last sentence break is more than 160 chars from the start
        return x[0] + "."
    elif len(s) == 200:                #if we ran right up to the end
        return s.rsplit(" ", 1)[0] + "..."
    else:
        return s

soup = bs4.BeautifulSoup(open("download/jargon.xml"))

for entry in soup.glossary.find_all("glossentry"):
    term = entry.glossterm.get_text()
    if entry.abbrev:
        grammar = entry.abbrev.find("emphasis", role="grammar")

        if grammar:
            grammar = grammar.get_text()
        if grammar == None:
            grammar = ""

    text = " ".join([x.get_text() for x in entry.find_all("glossdef")]) #concatenate all definitions
    text = " ".join([x.strip() for x in text.split("\n")])              #elide line breaks
    text = see_also_re.sub("", text)                                    #remove "see also", since we're including that in a separate field
    if grammar:
        text = "%s %s" % (grammar, text)                                #prepend grammar
    text = trunc(text)                                                  #truncate

    see_also = "\\\\n".join(["[[%s]]" % x.get_text().replace("\n", "").replace("   ", " ") for x in entry.find_all("glossterm")][1:])    #the first one is the current term, so we skip that

    letter = term[0].upper()
    m = nonletter_re.match(letter)
    if m:
        letter = "0"
    url = "http://catb.org/jargon/html/%s/%s.html" % (letter, entry.get("id"))

    out = "\t".join([
        term,       #0
        "A",        #1
        "",         #2
        "",         #3
        "",         #4
        "",         #5
        see_also,   #6
        "",         #7
        "",         #8
        "",         #9
        "",         #10
        text,       #11
        url])       #12

    print(out)
<|endoftext|>