<commit_before>#!/usr/bin/env python
#
# EC2 synthesis is designed for synthesis approach for EC2 Cloudl Application
# Basic purpose of this script is preparing kexec by making customized VM
# at block device.
#
# Copyright (C) 2011-2012 Carnegie Mellon University
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of version 2 of the GNU General Public License as published
# by the Free Software Foundation.  A copy of the GNU General Public License
# should have been distributed along with this program in the file
# LICENSE.GPL.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#

import os
import urllib2
from multiprocessing import Process, Queue, Pipe, JoinableQueue
from optparse import OptionParser
from datetime import datetime
import subprocess
import sys
import tempfile
from synthesis import network_worker, decomp_worker, delta_worker

CHUNK_SIZE = 1024*16

def piping_synthesis(overlay_url, base_path):
    prev = datetime.now()
    delta_processes = []
    tmp_dir = tempfile.mkdtemp()
    time_transfer = Queue()
    time_decomp = Queue()
    time_delta = Queue()

    download_queue = JoinableQueue()
    decomp_queue = JoinableQueue()
    (download_pipe_in, download_pipe_out) = Pipe()
    (decomp_pipe_in, decomp_pipe_out) = Pipe()
    recover_file = os.path.join(tmp_dir, overlay_url.split("/")[-1] + ".recover")

    # synthesis
    url = urllib2.urlopen(overlay_url)
    download_process = Process(target=network_worker, args=(url, download_queue, time_transfer, CHUNK_SIZE))
    decomp_process = Process(target=decomp_worker, args=(download_queue, decomp_queue, time_decomp))
    delta_process = Process(target=delta_worker, args=(decomp_queue, time_delta, base_path, recover_file))
    delta_processes.append(delta_process)
    download_process.start()
    decomp_process.start()
    delta_process.start()

    # wait until end
    delta_process.join()
    print "\n[Time] Total Time for synthesis(including download) : " + str(datetime.now()-prev)
    return recover_file

def process_command_line(argv):
    help_message = "\nEC2 synthesis is designed for rapid launching of customizing instance at Amazon EC2"

    parser = OptionParser(usage="usage: %prog -o [Overlay Download URL] -b [Base VM Path] -m [Instance Mount Path]" + help_message,
            version="EC2 Synthesys v0.1.1")
    parser.add_option(
            '-o', '--overlay', action='store', type='string', dest='overlay_download_url',
            help='Set overlay disk download URL.')
    parser.add_option(
            '-b', '--base', action='store', type='string', dest='base_path',
            help='Set Base disk path.')
    parser.add_option(
            '-m', '--mount', action='store', type='string', dest='output_mount',
            help='Set output Mount point. This Mount point is EC2 inital disk.')
    settings, args = parser.parse_args(argv)
    if not len(args) == 0:
        parser.error('program takes no command-line arguments; "%s" ignored.' % (args,))
    if settings.overlay_download_url == None or settings.base_path == None or settings.output_mount == None:
        parser.error('Read usage')

    return settings, args

def mount_launchVM(launch_disk_path):
    mount_dir = tempfile.mkdtemp()
    raw_vm = launch_disk_path + ".raw"

    # qemu-img convert
    start_time = datetime.now()
    cmd_convert = "qemu-img convert -f qcow2 %s -O raw %s" % (launch_disk_path, raw_vm)
    proc = subprocess.Popen(cmd_convert, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0 or os.path.exists(raw_vm) == False:
        print >> sys.strerr, "Error, Failed to QEMU-IMG Converting"
        sys.exit(2)
    convert_time = datetime.now()-start_time

    # mount
    start_time = datetime.now()
    cmd_mapping = "sudo kpartx -av %s" % (raw_vm)
    proc = subprocess.Popen(cmd_mapping, shell=True, stdin=sys.stdin, stdout=subprocess.PIPE)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to kpartx"
        sys.exit(2)
    output = proc.stdout.readline()
    output = output[output.find("loop"):]
    mapper_dev = output.split(" ")[0].strip()
    cmd_mount = "sudo mount /dev/mapper/%s %s" % (mapper_dev, mount_dir)
    proc = subprocess.Popen(cmd_mount, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to mount, ret code : " + str(proc.returncode)
        sys.exit(2)
    mount_time = datetime.now()-start_time

    print "[TIME] QCOW2 Converting time : %s" % (str(convert_time))
    print "[TIME] RAW Mouting time : %s" % (str(mount_time))
    return mount_dir, raw_vm

def rsync_overlayVM(vm_dir, instance_dir):
    instance_dir = os.path.join(instance_dir, ".")
    # TODO: check mount list and automatically umount residues
    subprocess.Popen("sudo umount %s" % (os.path.join(instance_dir, "run")) , shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # erase instance dir
    if os.path.exists(instance_dir):
        message = "Are you sure to delete all files at %s?(y/N) " % (instance_dir)
        ret = raw_input(message)
        if str(ret) != 'y':
            sys.exit(1)
    else:
        print >> sys.strerr, "Instance directory does not exists, " + str(instance_dir)
        sys.exit(1)

    cmd_erase = "sudo rm -rf %s" % os.path.join(instance_dir, "*")
    print "[INFO] erase instacnce dir: %s" % (cmd_erase)
    subprocess.Popen(cmd_erase, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # rsync
    print "[INFO] rsync from % to %" % (vm_dir, instance_dir)
    cmd_rsync = "sudo rsync -aHx %s/ %s/" % (vm_dir, instance_dir)
    subprocess.Popen(cmd_rsync, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # umount
    print "[INFO] umount instance dir, %s" % (instance_dir)
    subprocess.Popen("sudo umount %s" % (instance_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    print "[INFO] umount VM dir, %s" % (vm_dir)
    subprocess.Popen("sudo umount %s" % (vm_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

def clean_up(raw_vm, launch_vm):
    print "[INFO] clean up files : %s, %s" % (os.path.abspath(raw_vm), os.path.abspath(launch_vm))
    os.remove(raw_vm)
    os.remove(launch_vm)

def main(argv=None):
    settings, args = process_command_line(sys.argv[1:])

    # Synthesis overlay
    qcow_launch_image = piping_synthesis(settings.overlay_download_url, settings.base_path)

    # Mount VM File system
    launchVM_dir, VM_path =  mount_launchVM(qcow_launch_image)

    # rsync VM to origianl disk
    rsync_overlayVM(launchVM_dir, settings.output_mount)

    # clean up
    clean_up(VM_path, qcow_launch_image)

    return 0

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<commit_msg>syntax error<commit_after>#!/usr/bin/env python
#
# EC2 synthesis is designed for synthesis approach for EC2 Cloudl Application
# Basic purpose of this script is preparing kexec by making customized VM
# at block device.
#
# Copyright (C) 2011-2012 Carnegie Mellon University
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of version 2 of the GNU General Public License as published
# by the Free Software Foundation.  A copy of the GNU General Public License
# should have been distributed along with this program in the file
# LICENSE.GPL.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#

import os
import urllib2
from multiprocessing import Process, Queue, Pipe, JoinableQueue
from optparse import OptionParser
from datetime import datetime
import subprocess
import sys
import tempfile
from synthesis import network_worker, decomp_worker, delta_worker

CHUNK_SIZE = 1024*16

def piping_synthesis(overlay_url, base_path):
    prev = datetime.now()
    delta_processes = []
    tmp_dir = tempfile.mkdtemp()
    time_transfer = Queue()
    time_decomp = Queue()
    time_delta = Queue()

    download_queue = JoinableQueue()
    decomp_queue = JoinableQueue()
    (download_pipe_in, download_pipe_out) = Pipe()
    (decomp_pipe_in, decomp_pipe_out) = Pipe()
    recover_file = os.path.join(tmp_dir, overlay_url.split("/")[-1] + ".recover")

    # synthesis
    url = urllib2.urlopen(overlay_url)
    download_process = Process(target=network_worker, args=(url, download_queue, time_transfer, CHUNK_SIZE))
    decomp_process = Process(target=decomp_worker, args=(download_queue, decomp_queue, time_decomp))
    delta_process = Process(target=delta_worker, args=(decomp_queue, time_delta, base_path, recover_file))
    delta_processes.append(delta_process)
    download_process.start()
    decomp_process.start()
    delta_process.start()

    # wait until end
    delta_process.join()
    print "\n[Time] Total Time for synthesis(including download) : " + str(datetime.now()-prev)
    return recover_file

def process_command_line(argv):
    help_message = "\nEC2 synthesis is designed for rapid launching of customizing instance at Amazon EC2"

    parser = OptionParser(usage="usage: %prog -o [Overlay Download URL] -b [Base VM Path] -m [Instance Mount Path]" + help_message,
            version="EC2 Synthesys v0.1.1")
    parser.add_option(
            '-o', '--overlay', action='store', type='string', dest='overlay_download_url',
            help='Set overlay disk download URL.')
    parser.add_option(
            '-b', '--base', action='store', type='string', dest='base_path',
            help='Set Base disk path.')
    parser.add_option(
            '-m', '--mount', action='store', type='string', dest='output_mount',
            help='Set output Mount point. This Mount point is EC2 inital disk.')
    settings, args = parser.parse_args(argv)
    if not len(args) == 0:
        parser.error('program takes no command-line arguments; "%s" ignored.' % (args,))
    if settings.overlay_download_url == None or settings.base_path == None or settings.output_mount == None:
        parser.error('Read usage')

    return settings, args

def mount_launchVM(launch_disk_path):
    mount_dir = tempfile.mkdtemp()
    raw_vm = launch_disk_path + ".raw"

    # qemu-img convert
    start_time = datetime.now()
    cmd_convert = "qemu-img convert -f qcow2 %s -O raw %s" % (launch_disk_path, raw_vm)
    proc = subprocess.Popen(cmd_convert, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0 or os.path.exists(raw_vm) == False:
        print >> sys.stderr, "Error, Failed to QEMU-IMG Converting"
        print >> sys.stderr, "CMD: %s" % (cmd_convert)
        sys.exit(2)
    convert_time = datetime.now()-start_time

    # mount
    start_time = datetime.now()
    cmd_mapping = "sudo kpartx -av %s" % (raw_vm)
    proc = subprocess.Popen(cmd_mapping, shell=True, stdin=sys.stdin, stdout=subprocess.PIPE)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to kpartx"
        sys.exit(2)
    output = proc.stdout.readline()
    output = output[output.find("loop"):]
    mapper_dev = output.split(" ")[0].strip()
    cmd_mount = "sudo mount /dev/mapper/%s %s" % (mapper_dev, mount_dir)
    proc = subprocess.Popen(cmd_mount, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to mount, ret code : " + str(proc.returncode)
        sys.exit(2)
    mount_time = datetime.now()-start_time

    print "[TIME] QCOW2 Converting time : %s" % (str(convert_time))
    print "[TIME] RAW Mouting time : %s" % (str(mount_time))
    return mount_dir, raw_vm

def rsync_overlayVM(vm_dir, instance_dir):
    instance_dir = os.path.join(instance_dir, ".")
    # TODO: check mount list and automatically umount residues
    subprocess.Popen("sudo umount %s" % (os.path.join(instance_dir, "run")) , shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # erase instance dir
    if os.path.exists(instance_dir):
        message = "Are you sure to delete all files at %s?(y/N) " % (instance_dir)
        ret = raw_input(message)
        if str(ret) != 'y':
            sys.exit(1)
    else:
        print >> sys.strerr, "Instance directory does not exists, " + str(instance_dir)
        sys.exit(1)

    cmd_erase = "sudo rm -rf %s" % os.path.join(instance_dir, "*")
    print "[INFO] erase instacnce dir: %s" % (cmd_erase)
    subprocess.Popen(cmd_erase, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # rsync
    print "[INFO] rsync from % to %" % (vm_dir, instance_dir)
    cmd_rsync = "sudo rsync -aHx %s/ %s/" % (vm_dir, instance_dir)
    subprocess.Popen(cmd_rsync, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # umount
    print "[INFO] umount instance dir, %s" % (instance_dir)
    subprocess.Popen("sudo umount %s" % (instance_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    print "[INFO] umount VM dir, %s" % (vm_dir)
    subprocess.Popen("sudo umount %s" % (vm_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

def clean_up(raw_vm, launch_vm):
    print "[INFO] clean up files : %s, %s" % (os.path.abspath(raw_vm), os.path.abspath(launch_vm))
    os.remove(raw_vm)
    os.remove(launch_vm)

def main(argv=None):
    settings, args = process_command_line(sys.argv[1:])

    # Synthesis overlay
    qcow_launch_image = piping_synthesis(settings.overlay_download_url, settings.base_path)

    # Mount VM File system
    launchVM_dir, VM_path =  mount_launchVM(qcow_launch_image)

    # rsync VM to origianl disk
    rsync_overlayVM(launchVM_dir, settings.output_mount)

    # clean up
    clean_up(VM_path, qcow_launch_image)

    return 0

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<|endoftext|>


<commit_before>"""
*NOTE:* This version is slightly modified from the one in
    $CLAW/visclaw/src/python/visclaw/animation_tools.py

Some functions requires JSAnimation, either from Clawpack
or by installing it separately from
    https://github.com/jakevdp/JSAnimation

This animation_tools module contains tools to create animations in Python and
Jupyter notebooks.

Three types of animations are supported:
 - using the ipywidget interact to create a figure with a slider bar,
 - using JSAnimation to create Javascript code that loops over a set of
   images and adds controls to play as an animation.
 - creation of mp4 files using ffmpeg (provided this package is installed).

The set of images to combine in an animation can be specified as a
list of images, a list of `matplotlib` figures, or a directory of
`png` or other image files.

Utilities are provided to convert between these.

Functions are provided to create inline animations in Jupyter notebooks or
stand-alone files that can be viewed in other ways, including
 - An html file with the JSAnimation version,
 - A mp4 file,
 - A reStructured text file with the JSAnimation for inclusion in Sphinx docs.

The utility function make_anim_from_plotdir can be used to convert the png
files in a Clawpack _plots directory into standalone animations of the types
listed above.  See the file make_anim.py for an example of how this can be
invoked from an applications directory.

See also:
 https://ipywidgets.readthedocs.io/en/latest/#ipywidgets
 https://github.com/jakevdp/JSAnimation

More documentation of these functions is needed and they can probably be
improved.

"""

# use Python 3 style print function rather than Python 2 print statements:
from __future__ import print_function

from IPython.display import display
from matplotlib import image, animation
from matplotlib import pyplot as plt
from ipywidgets import interact, interact_manual
import ipywidgets
import io
from matplotlib import pyplot as plt

try:
    from JSAnimation import IPython_display
except:
    try:
        from clawpack.visclaw.JSAnimation import IPython_display
    except:
        print("*** Warning: JSAnimation not found")

def make_plotdir(plotdir='_plots', clobber=True):
    """
    Utility function to create a directory for storing a sequence of plot
    files, or if the directory already exists, clear out any old plots.
    If clobber==False then it will abort instead of deleting existing files.
    """

    import os
    if os.path.isdir(plotdir):
        if clobber:
            os.system("rm %s/*" % plotdir)
        else:
            raise IOError('*** Cannot clobber existing directory %s' % plotdir)
    else:
        os.system("mkdir %s" % plotdir)
    print("Figure files for each frame will be stored in ", plotdir)

def save_frame(frameno, plotdir='_plots', fname_base='frame', format='png',
               verbose=False, **kwargs):
    """
    After giving matplotlib commands to create the plot for a single frame
    of the desired animation, this can be called to save the figure with
    the appropriate file name such as _plots/frame00001.png.
    """

    plt.draw()
    filename = '%s/%s%s.%s' % (plotdir, fname_base, str(frameno).zfill(5), format)
    plt.savefig(filename, **kwargs)
    if verbose:
        print("Saved ",filename)

def make_anim(plotdir, fname_pattern='frame*.png', figsize=(10,6), dpi=None):
    """
    Assumes that a set of frames are available as png files in directory _plots,
    numbered consecutively, e.g. frame0000.png, frame0001.png, etc.

    Creates an animation based display each frame in turn, and returns anim.

    You can then display anim in an IPython notebook, or
    call make_html(anim) to create a stand-alone webpage.
    """

    import matplotlib

    if matplotlib.backends.backend in ['MacOSX']:
        print("*** animation.FuncAnimation doesn't work with backend %s" \
            % matplotlib.backends.backend)
        print("*** Suggest using 'Agg'")
        return

    import glob   # for finding all files matching a pattern

    # Find all frame files:
    filenames = glob.glob('%s/%s' % (plotdir, fname_pattern))

    # sort them into increasing order:
    filenames=sorted(filenames)

    fig = plt.figure(figsize=figsize, dpi=dpi)
    ax = fig.add_axes([0, 0, 1, 1])
    ax.axis('off')  # so there's not a second set of axes
    im = plt.imshow(image.imread(filenames[0]))

    def init():
        im.set_data(image.imread(filenames[0]))
        return im,

    def animate(i):
        image_i=image.imread(filenames[i])
        im.set_data(image_i)
        return im,

    anim = animation.FuncAnimation(fig, animate, init_func=init,
                          frames=len(filenames), interval=200, blit=True)

    return anim

def JSAnimate_images(images, figsize=(10,6), dpi=None):

    import matplotlib

    if matplotlib.backends.backend in ['MacOSX']:
        print("*** animation.FuncAnimation doesn't work with backend %s" \
            % matplotlib.backends.backend)
        print("*** Suggest using 'Agg'")
        return

    fig = plt.figure(figsize=figsize, dpi=None)
    ax = fig.add_axes([0, 0, 1, 1])
    ax.axis('off')  # so there's not a second set of axes

    im = plt.imshow(images[0])

    def init():
        im.set_data(images[0])
        return im,

    def animate(i):
        im.set_data(images[i])
        return im,

    anim = animation.FuncAnimation(fig, animate, init_func=init,
                          frames=len(images), interval=200, blit=True)

    plt.close(fig)
    return anim

def make_html(anim, file_name='anim.html', title=None, raw_html='', \
              fps=None, embed_frames=True, default_mode='once'):
    """
    Take an animation created by make_anim and convert it into a stand-alone
    html file.
    """

    from JSAnimation.IPython_display import anim_to_html

    html_body = anim_to_html(anim, fps=fps, embed_frames=embed_frames, \
                 default_mode=default_mode)

    html_file = open(file_name,'w')
    html_file.write("\n
%s
\n" % title)
    html_file.write(raw_html)
    html_file.write(html_body)
    html_file.close()
    print("Created %s" % file_name)

def make_rst(anim, file_name='anim.rst',
              fps=None, embed_frames=True, default_mode='once'):
    """
    Take an animation created by make_anim and convert it into an rst file
    (reStructuredText, for inclusion in Sphinx documentation, for example).
    """

    from JSAnimation.IPython_display import anim_to_html

    rst_body = anim_to_html(anim, fps=fps, embed_frames=embed_frames, \
                 default_mode=default_mode)

    rst_body = rst_body.split('\n')

    rst_file = open(file_name,'w')
    rst_file.write(".. raw:: html\n")
    for line in rst_body:
        rst_file.write("   %s\n" % line)
    rst_file.close()
    print("Created %s" % file_name)
    print("Imbed this in another rst file using:")
    print(".. include:: %s" % file_name)

def make_mp4(anim, file_name='anim.mp4',
              fps=None, embed_frames=True, default_mode='once'):
    """
    Take an animation and covert to mp4 file using ffmpeg, which must be
    installed.
    """
    import os

    if not animation.writers.is_available('ffmpeg'):
        print("** ffmpeg must be installed to create mp4 file")
        return

    if os.path.splitext(file_name)[1] != '.mp4':
        print("*** Might not work if file extension is not .mp4")
    if fps is None:
        fps = 3
    writer = animation.writers['ffmpeg'](fps=fps)
    anim.save(file_name, writer=writer)
    print("Created %s" % file_name)

def read_images(plotdir, fname_pattern='*.png'):

    import glob, os
    images = []
    files = glob.glob(os.path.join(plotdir, fname_pattern))
    for file in files:
        im = plt.imread(file)
        images.append(im)
    return images

def save_images(images, figsize=(8,6), plotdir='_plots', clobber=True, \
                fname_base='frame', format='png', verbose=False, **kwargs):

    make_plotdir(plotdir=plotdir, clobber=clobber)
    for frameno,image in enumerate(images):
        fig = imshow_noaxes(image, figsize)
        filename = '%s/%s%s.%s' % (plotdir, fname_base, str(frameno).zfill(5), format)
        plt.savefig(filename, format=format, **kwargs)
        plt.close(fig)
        if verbose:
            print("Saved ",filename)

def save_figs(figs, plotdir='_plots', clobber=True, \
                fname_base='frame', format='png', verbose=False, **kwargs):

    make_plotdir(plotdir=plotdir, clobber=clobber)
    for frameno,fig in enumerate(figs):
        filename = '%s/%s%s.%s' % (plotdir, fname_base, str(frameno).zfill(5), format)
        fig.savefig(filename, format=format, **kwargs)
        plt.close(fig)
        if verbose:
            print("Saved ",filename)

def make_image(fig, **kwargs):
    """
    Take a matplotlib figure *fig* and convert it to an image *im* that
    can be viewed with imshow.
    """

    import io
    png = io.BytesIO()
    fig.savefig(png,format='png', **kwargs)
    png.seek(0)
    im = plt.imread(png)
    return im

def make_images(figs, **kwargs):
    """
    Take a list of matplotlib figures *figs* and convert to list of images.
    """

    images = []
    for fig in figs:
        im = make_image(fig, **kwargs)
        images.append(im)
    return images

def imshow_noaxes(im, figsize=(8,6)):
    fig = plt.figure(figsize=figsize)
    ax = plt.axes()
    plt.imshow(im)
    ax.axis('off')
    return fig

def interact_animate_images(images, figsize=(10,6), manual=False, TextInput=False):

    def display_frame(frameno):
        imshow_noaxes(images[frameno], figsize=figsize)

    if TextInput:
        if TextInput:
            print("Valid frameno values: from %i to %i" % (0,len(images)-1))
        widget = ipywidgets.IntText(min=0,max=len(images)-1, value=0)
    else:
        widget = ipywidgets.IntSlider(min=0,max=len(images)-1, value=0)

    if manual:
        interact_manual(display_frame, frameno=widget)
    else:
        interact(display_frame, frameno=widget)

def interact_animate_figs(figs, manual=False, TextInput=False):

    def display_frame(frameno):
        display(figs[frameno])

    if TextInput:
        widget = ipywidgets.IntText(min=0,max=len(figs)-1, value=0)
    else:
        widget = ipywidgets.IntSlider(min=0,max=len(figs)-1, value=0)

    if manual:
        if TextInput:
            print("Valid frameno values: from %i to %i" % (0,len(figs)-1))
        interact_manual(display_frame, frameno=widget)
    else:
        interact(display_frame, frameno=widget)

def animate_figs(figs, style='ipywidgets', figsize=(10,6),  **kwargs):
    """
    Create an animation from a set of figures,
    style = 'ipywidgets' or 'JSAnimation'
    returns anim
    """

    images = make_images(figs)

    if style == 'ipywidgets':
        anim = interact_animate_images(images, figsize=figsize, **kwargs)
    elif style == 'JSAnimation':
        anim = JSAnimate_images(images, figsize=figsize, **kwargs)
    else:
        raise ValueError('** Unrecognized style = %s' % style)

    return anim

def make_anim_from_plotdir(plotdir='_plots', fignos='all',
        outputs=['mp4','html','rst'], file_name_prefix=None,
        figsize=(5,4), dpi=None, fps=5):

    """
    After running `make plots` using VisClaw, convert the png files in
    the plots directory into stand-alone files that can be embedded in
    webpages or Sphinx documentation.

    Call this from a script that starts with:
        import matplotlib
        matplotlib.use('Agg')
    """
    import glob, re

    if fignos == 'all':
        # determine what fignos are used in the plotdir
        movie_files = glob.glob(plotdir + '/movie*html')
        if len(movie_files) == 0:
            print('No movie files found in %s' % plotdir)
            return

        fignos = []
        regexp = re.compile(r"movie[^ ]*fig(?P<figno>[0-9]*)[.html]")
        for f in movie_files:
            result = regexp.search(f)
            fignos.append(result.group('figno'))

        print("Found these figures: %s" % fignos)

    for figno in fignos:

        fname_pattern = 'frame*fig%s.png' % figno
        anim = make_anim(plotdir, fname_pattern, figsize, dpi)

        if 'mp4' in outputs:
            file_name = file_name_prefix + 'fig%s.mp4' % figno
            make_mp4(anim, file_name, fps=fps, \
                embed_frames=True, default_mode='once')

        if 'html' in outputs:
            file_name = file_name_prefix + 'fig%s.html' % figno
            make_html(anim, file_name, fps=fps, \
                embed_frames=True, default_mode='once')

        if 'rst' in outputs:
            file_name = file_name_prefix + 'fig%s.rst' % figno
            make_rst(anim, file_name, fps=fps, \
                embed_frames=True, default_mode='once')

<commit_msg>Removed clawpack dependance on animation_tools too<commit_after>"""
*NOTE:* This version is slightly modified from the one in
    $CLAW/visclaw/src/python/visclaw/animation_tools.py

Some functions requires JSAnimation, either from Clawpack
or by installing it separately from
    https://github.com/jakevdp/JSAnimation

This animation_tools module contains tools to create animations in Python and
Jupyter notebooks.

Three types of animations are supported:
 - using the ipywidget interact to create a figure with a slider bar,
 - using JSAnimation to create Javascript code that loops over a set of
   images and adds controls to play as an animation.
 - creation of mp4 files using ffmpeg (provided this package is installed).

The set of images to combine in an animation can be specified as a
list of images, a list of `matplotlib` figures, or a directory of
`png` or other image files.

Utilities are provided to convert between these.

Functions are provided to create inline animations in Jupyter notebooks or
stand-alone files that can be viewed in other ways, including
 - An html file with the JSAnimation version,
 - A mp4 file,
 - A reStructured text file with the JSAnimation for inclusion in Sphinx docs.

The utility function make_anim_from_plotdir can be used to convert the png
files in a Clawpack _plots directory into standalone animations of the types
listed above.  See the file make_anim.py for an example of how this can be
invoked from an applications directory.

See also:
 https://ipywidgets.readthedocs.io/en/latest/#ipywidgets
 https://github.com/jakevdp/JSAnimation

More documentation of these functions is needed and they can probably be
improved.

"""

# use Python 3 style print function rather than Python 2 print statements:
from __future__ import print_function

from IPython.display import display
from matplotlib import image, animation
from matplotlib import pyplot as plt
from ipywidgets import interact, interact_manual
import ipywidgets
import io
from matplotlib import pyplot as plt

try:
    from JSAnimation import IPython_display
except:
    try:
        from clawpack.visclaw.JSAnimation import IPython_display
    except:
        print("*** Warning: JSAnimation not found")

def make_plotdir(plotdir='_plots', clobber=True):
    """
    Utility function to create a directory for storing a sequence of plot
    files, or if the directory already exists, clear out any old plots.
    If clobber==False then it will abort instead of deleting existing files.
    """

    import os
    if os.path.isdir(plotdir):
        if clobber:
            os.system("rm %s/*" % plotdir)
        else:
            raise IOError('*** Cannot clobber existing directory %s' % plotdir)
    else:
        os.system("mkdir %s" % plotdir)
    print("Figure files for each frame will be stored in ", plotdir)

def save_frame(frameno, plotdir='_plots', fname_base='frame', format='png',
               verbose=False, **kwargs):
    """
    After giving matplotlib commands to create the plot for a single frame
    of the desired animation, this can be called to save the figure with
    the appropriate file name such as _plots/frame00001.png.
    """

    plt.draw()
    filename = '%s/%s%s.%s' % (plotdir, fname_base, str(frameno).zfill(5), format)
    plt.savefig(filename, **kwargs)
    if verbose:
        print("Saved ",filename)

def make_anim(plotdir, fname_pattern='frame*.png', figsize=(10,6), dpi=None):
    """
    Assumes that a set of frames are available as png files in directory _plots,
    numbered consecutively, e.g. frame0000.png, frame0001.png, etc.

    Creates an animation based display each frame in turn, and returns anim.

    You can then display anim in an IPython notebook, or
    call make_html(anim) to create a stand-alone webpage.
    """

    import matplotlib

    if matplotlib.backends.backend in ['MacOSX']:
        print("*** animation.FuncAnimation doesn't work with backend %s" \
            % matplotlib.backends.backend)
        print("*** Suggest using 'Agg'")
        return

    import glob   # for finding all files matching a pattern

    # Find all frame files:
    filenames = glob.glob('%s/%s' % (plotdir, fname_pattern))

    # sort them into increasing order:
    filenames=sorted(filenames)

    fig = plt.figure(figsize=figsize, dpi=dpi)
    ax = fig.add_axes([0, 0, 1, 1])
    ax.axis('off')  # so there's not a second set of axes
    im = plt.imshow(image.imread(filenames[0]))

    def init():
        im.set_data(image.imread(filenames[0]))
        return im,

    def animate(i):
        image_i=image.imread(filenames[i])
        im.set_data(image_i)
        return im,

    anim = animation.FuncAnimation(fig, animate, init_func=init,
                          frames=len(filenames), interval=200, blit=True)

    return anim

def JSAnimate_images(images, figsize=(10,6), dpi=None):

    import matplotlib

    if matplotlib.backends.backend in ['MacOSX']:
        print("*** animation.FuncAnimation doesn't work with backend %s" \
            % matplotlib.backends.backend)
        print("*** Suggest using 'Agg'")
        return

    fig = plt.figure(figsize=figsize, dpi=None)
    ax = fig.add_axes([0, 0, 1, 1])
    ax.axis('off')  # so there's not a second set of axes

    im = plt.imshow(images[0])

    def init():
        im.set_data(images[0])
        return im,

    def animate(i):
        im.set_data(images[i])
        return im,

    anim = animation.FuncAnimation(fig, animate, init_func=init,
                          frames=len(images), interval=200, blit=True)

    plt.close(fig)
    return anim

def make_html(anim, file_name='anim.html', title=None, raw_html='', \
              fps=None, embed_frames=True, default_mode='once'):
    """
    Take an animation created by make_anim and convert it into a stand-alone
    html file.
    """
    try:
        from JSAnimation.IPython_display import anim_to_html
    except:
        try:
            from clawpack.visclaw.JSAnimation.IPython_display import anim_to_html
        except:
            print("*** Warning: JSAnimation not found, cannot import anim_to_html")

    html_body = anim_to_html(anim, fps=fps, embed_frames=embed_frames, \
                 default_mode=default_mode)

    html_file = open(file_name,'w')
    html_file.write("\n
%s
\n" % title)
    html_file.write(raw_html)
    html_file.write(html_body)
    html_file.close()
    print("Created %s" % file_name)

def make_rst(anim, file_name='anim.rst',
              fps=None, embed_frames=True, default_mode='once'):
    """
    Take an animation created by make_anim and convert it into an rst file
    (reStructuredText, for inclusion in Sphinx documentation, for example).
    """
    try:
        from JSAnimation.IPython_display import anim_to_html
    except:
        try:
            from clawpack.visclaw.JSAnimation.IPython_display import anim_to_html
        except:
            print("*** Warning: JSAnimation not found, cannot import anim_to_html")

    rst_body = anim_to_html(anim, fps=fps, embed_frames=embed_frames, \
                 default_mode=default_mode)

    rst_body = rst_body.split('\n')

    rst_file = open(file_name,'w')
    rst_file.write(".. raw:: html\n")
    for line in rst_body:
        rst_file.write("   %s\n" % line)
    rst_file.close()
    print("Created %s" % file_name)
    print("Imbed this in another rst file using:")
    print(".. include:: %s" % file_name)

def make_mp4(anim, file_name='anim.mp4',
              fps=None, embed_frames=True, default_mode='once'):
    """
    Take an animation and covert to mp4 file using ffmpeg, which must be
    installed.
    """
    import os

    if not animation.writers.is_available('ffmpeg'):
        print("** ffmpeg must be installed to create mp4 file")
        return

    if os.path.splitext(file_name)[1] != '.mp4':
        print("*** Might not work if file extension is not .mp4")
    if fps is None:
        fps = 3
    writer = animation.writers['ffmpeg'](fps=fps)
    anim.save(file_name, writer=writer)
    print("Created %s" % file_name)

def read_images(plotdir, fname_pattern='*.png'):

    import glob, os
    images = []
    files = glob.glob(os.path.join(plotdir, fname_pattern))
    for file in files:
        im = plt.imread(file)
        images.append(im)
    return images

def save_images(images, figsize=(8,6), plotdir='_plots', clobber=True, \
                fname_base='frame', format='png', verbose=False, **kwargs):

    make_plotdir(plotdir=plotdir, clobber=clobber)
    for frameno,image in enumerate(images):
        fig = imshow_noaxes(image, figsize)
        filename = '%s/%s%s.%s' % (plotdir, fname_base, str(frameno).zfill(5), format)
        plt.savefig(filename, format=format, **kwargs)
        plt.close(fig)
        if verbose:
            print("Saved ",filename)

def save_figs(figs, plotdir='_plots', clobber=True, \
                fname_base='frame', format='png', verbose=False, **kwargs):

    make_plotdir(plotdir=plotdir, clobber=clobber)
    for frameno,fig in enumerate(figs):
        filename = '%s/%s%s.%s' % (plotdir, fname_base, str(frameno).zfill(5), format)
        fig.savefig(filename, format=format, **kwargs)
        plt.close(fig)
        if verbose:
            print("Saved ",filename)

def make_image(fig, **kwargs):
    """
    Take a matplotlib figure *fig* and convert it to an image *im* that
    can be viewed with imshow.
    """

    import io
    png = io.BytesIO()
    fig.savefig(png,format='png', **kwargs)
    png.seek(0)
    im = plt.imread(png)
    return im

def make_images(figs, **kwargs):
    """
    Take a list of matplotlib figures *figs* and convert to list of images.
    """

    images = []
    for fig in figs:
        im = make_image(fig, **kwargs)
        images.append(im)
    return images

def imshow_noaxes(im, figsize=(8,6)):
    fig = plt.figure(figsize=figsize)
    ax = plt.axes()
    plt.imshow(im)
    ax.axis('off')
    return fig

def interact_animate_images(images, figsize=(10,6), manual=False, TextInput=False):

    def display_frame(frameno):
        imshow_noaxes(images[frameno], figsize=figsize)

    if TextInput:
        if TextInput:
            print("Valid frameno values: from %i to %i" % (0,len(images)-1))
        widget = ipywidgets.IntText(min=0,max=len(images)-1, value=0)
    else:
        widget = ipywidgets.IntSlider(min=0,max=len(images)-1, value=0)

    if manual:
        interact_manual(display_frame, frameno=widget)
    else:
        interact(display_frame, frameno=widget)

def interact_animate_figs(figs, manual=False, TextInput=False):

    def display_frame(frameno):
        display(figs[frameno])

    if TextInput:
        widget = ipywidgets.IntText(min=0,max=len(figs)-1, value=0)
    else:
        widget = ipywidgets.IntSlider(min=0,max=len(figs)-1, value=0)

    if manual:
        if TextInput:
            print("Valid frameno values: from %i to %i" % (0,len(figs)-1))
        interact_manual(display_frame, frameno=widget)
    else:
        interact(display_frame, frameno=widget)

def animate_figs(figs, style='ipywidgets', figsize=(10,6),  **kwargs):
    """
    Create an animation from a set of figures,
    style = 'ipywidgets' or 'JSAnimation'
    returns anim
    """

    images = make_images(figs)

    if style == 'ipywidgets':
        anim = interact_animate_images(images, figsize=figsize, **kwargs)
    elif style == 'JSAnimation':
        anim = JSAnimate_images(images, figsize=figsize, **kwargs)
    else:
        raise ValueError('** Unrecognized style = %s' % style)

    return anim

def make_anim_from_plotdir(plotdir='_plots', fignos='all',
        outputs=['mp4','html','rst'], file_name_prefix=None,
        figsize=(5,4), dpi=None, fps=5):

    """
    After running `make plots` using VisClaw, convert the png files in
    the plots directory into stand-alone files that can be embedded in
    webpages or Sphinx documentation.

    Call this from a script that starts with:
        import matplotlib
        matplotlib.use('Agg')
    """
    import glob, re

    if fignos == 'all':
        # determine what fignos are used in the plotdir
        movie_files = glob.glob(plotdir + '/movie*html')
        if len(movie_files) == 0:
            print('No movie files found in %s' % plotdir)
            return

        fignos = []
        regexp = re.compile(r"movie[^ ]*fig(?P<figno>[0-9]*)[.html]")
        for f in movie_files:
            result = regexp.search(f)
            fignos.append(result.group('figno'))

        print("Found these figures: %s" % fignos)

    for figno in fignos:

        fname_pattern = 'frame*fig%s.png' % figno
        anim = make_anim(plotdir, fname_pattern, figsize, dpi)

        if 'mp4' in outputs:
            file_name = file_name_prefix + 'fig%s.mp4' % figno
            make_mp4(anim, file_name, fps=fps, \
                embed_frames=True, default_mode='once')

        if 'html' in outputs:
            file_name = file_name_prefix + 'fig%s.html' % figno
            make_html(anim, file_name, fps=fps, \
                embed_frames=True, default_mode='once')

        if 'rst' in outputs:
            file_name = file_name_prefix + 'fig%s.rst' % figno
            make_rst(anim, file_name, fps=fps, \
                embed_frames=True, default_mode='once')

<|endoftext|>


<commit_before>__author__ = "jeanlouis.mbaka"

import csv
import sys

class Pareto():
	def __init__(self):
		pass

	def read_data(self, filename):
		"""
		"""
		results = read_csv(filename)
		self.header = results[0]
		self.data = results[1]

	def pareto(self):
		"""
		"""
		self.valorised_consumption()
		self.cumulative_valorised_consumption()
		self.abc_segmentation()

	def valorised_consumption(self):
		"""
		computes the valorised consumption.
		"""
		self.header.append("consommation valorisee")

		# compute the valorised consumption for each row
		num_of_data = len(self.data)
		for i in range(num_of_data):
			monthly_consumption = self.data[i][1]
			unit_price = self.data[i][2]
			self.data[i].append(unit_price * monthly_consumption)

	def cumulative_valorised_consumption(self):
		"""
		"""
		self.header.append("consomation cumulee")
		self.data = sorted(self.data, key=lambda row: row[-1], reverse=True) # sort by valorised consumption
		for index in range(len(self.data)):
			if index == 0:
				self.data[index].append(self.data[index][-1])
			else:
				self.data[index].append(self.data[index][-1] + self.data[index-1][-1])
		self.print_debug(heading="sorted cummulalive valorised consumption")

	def abc_segmentation(self):
		"""
		"""
		pass

	def print_debug(self, heading=""):
		print("<====={}=====>".format(heading))
		print(self.header)
		[print(row) for row in self.data]

def read_csv(filename):
	"""
	Read a csv file and print the content row by row
	:param filename: name/path to the csv file
	"""
	delimiter = ";"

	header = []
	data = []

	with open(filename, 'rt') as csv_file:
		reader = csv.reader(csv_file, dialect='excel-tab') # type CSV.READER
		i = 0
		for row in reader:
			row_content = row[0] # in excel-tab dialect, a row is an array containing one str
			row_content = str.split(row_content, delimiter) # split the string by ;

			if (i==0):
				header = row_content
				i += 1
				continue

			data_row = []
			for item in row_content:
				try:
					data_row.append(float(item.replace(",", ".")))
				except ValueError:
					data_row.append(item)
			data.append(data_row)
			i += 1
	return [header, data]

def parse_filename():
	"""
	:return: filename
	"""
	filename = ""
	if (len(sys.argv) > 1):
		filename = sys.argv[1]
	else:
		sys.exit(2)
	return filename

if __name__ == "__main__":
	my_pareto = Pareto()
	filename = parse_filename()
	my_pareto.read_data(filename)
	my_pareto.pareto()<commit_msg>cumulative % and ABC segmentation<commit_after>__author__ = "jeanlouis.mbaka"

import csv
import sys

class Pareto():
	def __init__(self):
		pass

	def read_data(self, filename):
		"""
		"""
		results = read_csv(filename)
		self.header = results[0]
		self.data = results[1]

	def pareto(self):
		"""
		"""
		self.valorised_consumption()
		self.cumulative_valorised_consumption()
		self.abc_segmentation()

	def valorised_consumption(self):
		"""
		computes the valorised consumption.
		"""
		self.header.append("consommation valorisee")

		num_of_data = len(self.data)
		for i in range(num_of_data):
			monthly_consumption = self.data[i][1]
			unit_price = self.data[i][2]
			self.data[i].append(unit_price * monthly_consumption)

	def cumulative_valorised_consumption(self):
		"""
		Sorted and cumululative consumption.
		"""
		self.header.append("consommation cumulee")
		self.data = sorted(self.data, key=lambda row: row[-1], reverse=True) # sort by valorised consumption
		for index in range(len(self.data)):
			if index == 0:
				self.data[index].append(self.data[index][-1])
			else:
				self.data[index].append(self.data[index][-1] + self.data[index-1][-1])

		self.print_debug(heading="sorted cumulalive valorised consumption")

	def abc_segmentation(self):
		"""
		ABC segmentation.
		"""
		self.header.append("PCT cumule")
		self.header.append("Categorie")

		for index in range(len(self.data)):
			cumulative_pct = self.data[index][-1] / self.data[-1][-1]
			self.data[index].append(cumulative_pct)
			if (cumulative_pct <= 0.8):
				self.data[index].append("A")
			elif (cumulative_pct > 0.8 and cumulative_pct <= 0.95):
				self.data[index].append("B")
			else:
				self.data[index].append("C")

		self.print_debug(heading="ABC Segmentation")

	def print_debug(self, heading=""):
		print("<====={}=====>".format(heading))
		print(self.header)
		[print(row) for row in self.data]

def read_csv(filename):
	"""
	Read a csv file and print the content row by row
	:param filename: name/path to the csv file
	"""
	delimiter = ";"

	header = []
	data = []

	with open(filename, 'rt') as csv_file:
		reader = csv.reader(csv_file, dialect='excel-tab') # type CSV.READER
		i = 0
		for row in reader:
			row_content = row[0] # in excel-tab dialect, a row is an array containing one str
			row_content = str.split(row_content, delimiter) # split the string by ;

			if (i==0):
				header = row_content
				i += 1
				continue

			data_row = []
			for item in row_content:
				try:
					data_row.append(float(item.replace(",", ".")))
				except ValueError:
					data_row.append(item)
			data.append(data_row)
			i += 1
	return [header, data]

def parse_filename():
	"""
	:return: filename
	"""
	filename = ""
	if (len(sys.argv) > 1):
		filename = sys.argv[1]
	else:
		sys.exit(2)
	return filename

if __name__ == "__main__":
	my_pareto = Pareto()
	filename = parse_filename()
	my_pareto.read_data(filename)
	my_pareto.pareto()<|endoftext|>


<commit_before>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""The ``spyne.model.binary`` package contains binary type markers."""

import os
import base64
import tempfile

from base64 import b64encode
from base64 import b64decode
from base64 import urlsafe_b64encode
from base64 import urlsafe_b64decode
from binascii import hexlify
from binascii import unhexlify

try:
    from cStringIO import StringIO
except ImportError: # Python 3
    from io import StringIO

from spyne.util import _bytes_join
from spyne.model import nillable_string
from spyne.model import ModelBase
from spyne.model import SimpleModel

class BINARY_ENCODING_HEX: pass
class BINARY_ENCODING_BASE64: pass
class BINARY_ENCODING_URLSAFE_BASE64: pass

class ByteArray(SimpleModel):
    """Canonical container for arbitrary data. Every protocol has a different
    way of encapsulating this type. E.g. xml-based protocols encode this as
    base64, while HttpRpc just hands it over.

    Its native python format is a sequence of ``str`` objects for Python 2.x
    and a sequence of ``bytes`` objects for Python 3.x.
    """

    __type_name__ = 'base64Binary'
    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    class Attributes(SimpleModel.Attributes):
        encoding = None
        """The binary encoding to use when the protocol does not enforce an
        encoding for binary data.

        One of (None, 'base64', 'hex')
        """

    def __new__(cls, **kwargs):
        tn = None
        if 'encoding' in kwargs:
            v = kwargs['encoding']

            if v in (None, 'base64', 'base64Binary', BINARY_ENCODING_BASE64):
                # This string is defined in the Xml Schema Standard
                tn = 'base64Binary'
                kwargs['encoding'] = BINARY_ENCODING_BASE64

            elif v in ('hex', 'hexBinary', BINARY_ENCODING_HEX):
                # This string is defined in the Xml Schema Standard
                tn = 'hexBinary'
                kwargs['encoding'] = BINARY_ENCODING_HEX

            else:
                raise ValueError("'encoding' must be one of: %r" % \
                                (tuple(ByteArray._encoding.handlers.values()),))

        retval = cls.customize(**kwargs)
        if tn is not None:
            retval.__type_name__ = tn
        return retval

    @staticmethod
    def is_default(cls):
        return True

    @classmethod
    @nillable_string
    def to_base64(cls, value):
        return b64encode(_bytes_join(value))

    @classmethod
    @nillable_string
    def from_base64(cls, value):
        return [b64decode(_bytes_join(value))]

    @classmethod
    @nillable_string
    def to_urlsafe_base64(cls, value):
        return urlsafe_b64encode(_bytes_join(value))

    @classmethod
    @nillable_string
    def from_urlsafe_base64(cls, value):
        return [urlsafe_b64decode(_bytes_join(value))]

    @classmethod
    @nillable_string
    def to_hex(cls, value):
        return hexlify(_bytes_join(value))

    @classmethod
    @nillable_string
    def from_hex(cls, value):
        return [unhexlify(_bytes_join(value))]

binary_encoding_handlers = {
    None: ''.join,
    BINARY_ENCODING_HEX: ByteArray.to_hex,
    BINARY_ENCODING_BASE64: ByteArray.to_base64,
    BINARY_ENCODING_URLSAFE_BASE64: urlsafe_b64encode,
}

binary_decoding_handlers = {
    None: lambda x: [x],
    BINARY_ENCODING_HEX: ByteArray.from_hex,
    BINARY_ENCODING_BASE64: ByteArray.from_base64,
    BINARY_ENCODING_URLSAFE_BASE64: ByteArray.from_urlsafe_base64,
}

class File(SimpleModel):
    """A compact way of dealing with incoming files for protocols with a
    standard way of encoding file metadata along with binary data. (E.g. Http)
    """

    __type_name__ = 'base64Binary'
    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    class Attributes(SimpleModel.Attributes):
        encoding = None
        """The binary encoding to use when the protocol does not enforce an
        encoding for binary data.

        One of (None, 'base64', 'hex')
        """

    class Value(object):
        """The class for values marked as ``File``.

        :param name: Original name of the file
        :param path: Current path to the file.
        :param type: The mime type of the file's contents.
        :param data: Optional sequence of ``str`` or ``bytes`` instances
            that contain the file's data.
        :param handle: :class:`file` object that contains the file's data.
            It is ignored unless the ``path`` argument is ``None``.
        """

        def __init__(self, name=None, path=None, type='application/octet-stream',
                                                            data=None, handle=None):
            self.name = name
            if self.name is not None:
                assert os.path.basename(self.name) == self.name

            self.path = path
            if self.path is not None:
                assert os.path.isabs(self.path)

            self.type = type
            self.data = data
            self.handle = handle

        def rollover(self):
            """This method normalizes the file object by making ``path``,
            ``name`` and ``handle`` properties consistent. It writes
            incoming data to the file object and points the ``data`` iterable
            to the contents of this file.
            """

            iter(self.data)

            if self.path is None:
                handle, self.path = tempfile.mkstemp()
                f = os.fdopen(handle, 'wb')
            else:
                assert os.path.isabs(self.path)
                f = open(self.path, 'wb')

            if self.name is None:
                self.name = os.path.basename(self.path)

            for data in self.data:
                f.write(data)

            f.close()

            self.data = None

    @classmethod
    @nillable_string
    def to_base64(cls, value):
        assert value.path, "You need to write data to persistent storage first " \
                           "if you want to read it back."
        f = open(value.path, 'rb')

        data = f.read(0x4000) # this needs to be a multiple of 4 (for base64)
        while len(data) > 0:
            yield base64.b64encode(data)
            data = f.read(0x4000)

        f.close()

    @classmethod
    @nillable_string
    def from_base64(cls, value):
        return File.Value(data=[base64.b64decode(value)])

    def __repr__(self):
        return "File(name=%r, path=%r, type=%r, data=%r)" % (self.name,
                                                self.path, self.type, self.data)

# **DEPRECATED!** Use ByteArray or File instead.
class Attachment(ModelBase):
    __type_name__ = 'base64Binary'
    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    def __init__(self, data=None, file_name=None):
        self.data = data
        self.file_name = file_name

    def save_to_file(self):
        '''This method writes the data to the specified file.  This method
        assumes that the file_name is the full path to the file to be written.
        This method also assumes that self.data is the base64 decoded data,
        and will do no additional transformations on it, simply write it to
        disk.
        '''

        if not self.data:
            raise Exception("No data to write")

        if not self.file_name:
            raise Exception("No file_name specified")

        f = open(self.file_name, 'wb')
        f.write(self.data)
        f.close()

    def load_from_file(self):
        '''This method loads the data from the specified file, and does
        no encoding/decoding of the data
        '''
        if not self.file_name:
            raise Exception("No file_name specified")
        f = open(self.file_name, 'rb')
        self.data = f.read()
        f.close()

    @classmethod
    @nillable_string
    def to_base64(cls, value):
        ostream = StringIO()
        if not (value.data is None):
            istream = StringIO(value.data)

        elif not (value.file_name is None):
            istream = open(value.file_name, 'rb')

        else:
            raise ValueError("Neither data nor a file_name has been specified")

        base64.encode(istream, ostream)
        ostream.seek(0)

        return ostream.read()

    @classmethod
    @nillable_string
    def from_base64(cls, value):
        istream = StringIO(value)
        ostream = StringIO()

        base64.decode(istream, ostream)
        ostream.seek(0)

        return Attachment(data=ostream.read())
<commit_msg>make ByteArray deserialization fail gracefully on input with invalid type.<commit_after>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""The ``spyne.model.binary`` package contains binary type markers."""

import os
import base64
import tempfile

from base64 import b64encode
from base64 import b64decode
from base64 import urlsafe_b64encode
from base64 import urlsafe_b64decode
from binascii import hexlify
from binascii import unhexlify

try:
    from cStringIO import StringIO
except ImportError: # Python 3
    from io import StringIO

from spyne.error import ValidationError
from spyne.util import _bytes_join
from spyne.model import nillable_string
from spyne.model import ModelBase
from spyne.model import SimpleModel

class BINARY_ENCODING_HEX: pass
class BINARY_ENCODING_BASE64: pass
class BINARY_ENCODING_URLSAFE_BASE64: pass

class ByteArray(SimpleModel):
    """Canonical container for arbitrary data. Every protocol has a different
    way of encapsulating this type. E.g. xml-based protocols encode this as
    base64, while HttpRpc just hands it over.

    Its native python format is a sequence of ``str`` objects for Python 2.x
    and a sequence of ``bytes`` objects for Python 3.x.
    """

    __type_name__ = 'base64Binary'
    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    class Attributes(SimpleModel.Attributes):
        encoding = None
        """The binary encoding to use when the protocol does not enforce an
        encoding for binary data.

        One of (None, 'base64', 'hex')
        """

    def __new__(cls, **kwargs):
        tn = None
        if 'encoding' in kwargs:
            v = kwargs['encoding']

            if v in (None, 'base64', 'base64Binary', BINARY_ENCODING_BASE64):
                # This string is defined in the Xml Schema Standard
                tn = 'base64Binary'
                kwargs['encoding'] = BINARY_ENCODING_BASE64

            elif v in ('hex', 'hexBinary', BINARY_ENCODING_HEX):
                # This string is defined in the Xml Schema Standard
                tn = 'hexBinary'
                kwargs['encoding'] = BINARY_ENCODING_HEX

            else:
                raise ValueError("'encoding' must be one of: %r" % \
                                (tuple(ByteArray._encoding.handlers.values()),))

        retval = cls.customize(**kwargs)
        if tn is not None:
            retval.__type_name__ = tn
        return retval

    @staticmethod
    def is_default(cls):
        return True

    @classmethod
    @nillable_string
    def to_base64(cls, value):
        return b64encode(_bytes_join(value))

    @classmethod
    @nillable_string
    def from_base64(cls, value):
        try:
            return [b64decode(_bytes_join(value))]
        except TypeError, e:
            raise ValidationError(value)

    @classmethod
    @nillable_string
    def to_urlsafe_base64(cls, value):
        return urlsafe_b64encode(_bytes_join(value))

    @classmethod
    @nillable_string
    def from_urlsafe_base64(cls, value):
        return [urlsafe_b64decode(_bytes_join(value))]

    @classmethod
    @nillable_string
    def to_hex(cls, value):
        return hexlify(_bytes_join(value))

    @classmethod
    @nillable_string
    def from_hex(cls, value):
        return [unhexlify(_bytes_join(value))]

binary_encoding_handlers = {
    None: ''.join,
    BINARY_ENCODING_HEX: ByteArray.to_hex,
    BINARY_ENCODING_BASE64: ByteArray.to_base64,
    BINARY_ENCODING_URLSAFE_BASE64: urlsafe_b64encode,
}

binary_decoding_handlers = {
    None: lambda x: [x],
    BINARY_ENCODING_HEX: ByteArray.from_hex,
    BINARY_ENCODING_BASE64: ByteArray.from_base64,
    BINARY_ENCODING_URLSAFE_BASE64: ByteArray.from_urlsafe_base64,
}

class File(SimpleModel):
    """A compact way of dealing with incoming files for protocols with a
    standard way of encoding file metadata along with binary data. (E.g. Http)
    """

    __type_name__ = 'base64Binary'
    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    class Attributes(SimpleModel.Attributes):
        encoding = None
        """The binary encoding to use when the protocol does not enforce an
        encoding for binary data.

        One of (None, 'base64', 'hex')
        """

    class Value(object):
        """The class for values marked as ``File``.

        :param name: Original name of the file
        :param path: Current path to the file.
        :param type: The mime type of the file's contents.
        :param data: Optional sequence of ``str`` or ``bytes`` instances
            that contain the file's data.
        :param handle: :class:`file` object that contains the file's data.
            It is ignored unless the ``path`` argument is ``None``.
        """

        def __init__(self, name=None, path=None, type='application/octet-stream',
                                                            data=None, handle=None):
            self.name = name
            if self.name is not None:
                assert os.path.basename(self.name) == self.name

            self.path = path
            if self.path is not None:
                assert os.path.isabs(self.path)

            self.type = type
            self.data = data
            self.handle = handle

        def rollover(self):
            """This method normalizes the file object by making ``path``,
            ``name`` and ``handle`` properties consistent. It writes
            incoming data to the file object and points the ``data`` iterable
            to the contents of this file.
            """

            iter(self.data)

            if self.path is None:
                handle, self.path = tempfile.mkstemp()
                f = os.fdopen(handle, 'wb')
            else:
                assert os.path.isabs(self.path)
                f = open(self.path, 'wb')

            if self.name is None:
                self.name = os.path.basename(self.path)

            for data in self.data:
                f.write(data)

            f.close()

            self.data = None

    @classmethod
    @nillable_string
    def to_base64(cls, value):
        assert value.path, "You need to write data to persistent storage first " \
                           "if you want to read it back."
        f = open(value.path, 'rb')

        data = f.read(0x4000) # this needs to be a multiple of 4 (for base64)
        while len(data) > 0:
            yield base64.b64encode(data)
            data = f.read(0x4000)

        f.close()

    @classmethod
    @nillable_string
    def from_base64(cls, value):
        return File.Value(data=[base64.b64decode(value)])

    def __repr__(self):
        return "File(name=%r, path=%r, type=%r, data=%r)" % (self.name,
                                                self.path, self.type, self.data)

# **DEPRECATED!** Use ByteArray or File instead.
class Attachment(ModelBase):
    __type_name__ = 'base64Binary'
    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    def __init__(self, data=None, file_name=None):
        self.data = data
        self.file_name = file_name

    def save_to_file(self):
        '''This method writes the data to the specified file.  This method
        assumes that the file_name is the full path to the file to be written.
        This method also assumes that self.data is the base64 decoded data,
        and will do no additional transformations on it, simply write it to
        disk.
        '''

        if not self.data:
            raise Exception("No data to write")

        if not self.file_name:
            raise Exception("No file_name specified")

        f = open(self.file_name, 'wb')
        f.write(self.data)
        f.close()

    def load_from_file(self):
        '''This method loads the data from the specified file, and does
        no encoding/decoding of the data
        '''
        if not self.file_name:
            raise Exception("No file_name specified")
        f = open(self.file_name, 'rb')
        self.data = f.read()
        f.close()

    @classmethod
    @nillable_string
    def to_base64(cls, value):
        ostream = StringIO()
        if not (value.data is None):
            istream = StringIO(value.data)

        elif not (value.file_name is None):
            istream = open(value.file_name, 'rb')

        else:
            raise ValueError("Neither data nor a file_name has been specified")

        base64.encode(istream, ostream)
        ostream.seek(0)

        return ostream.read()

    @classmethod
    @nillable_string
    def from_base64(cls, value):
        istream = StringIO(value)
        ostream = StringIO()

        base64.decode(istream, ostream)
        ostream.seek(0)

        return Attachment(data=ostream.read())
<|endoftext|>


<commit_before>#!/usr/bin/env python

import functools
import txtorcon
import perfconf

from pprint import pformat

from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from txsocksx.http import SOCKS5Agent

class BeginningPrinter(Protocol):
    def __init__(self, finished):
        self.finished = finished
        self.data = []

    def dataReceived(self, data):
        self.data.append(data)

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        print 'Body:', ''.join(self.data)
        self.finished.callback(None)

def cbRequest(response):
    print 'Response received'
    print 'Response headers:'
    print pformat(list(response.headers.getAllRawHeaders()))

    finished = Deferred()
    response.deliverBody(BeginningPrinter(finished))
    return finished

def cbShutdown(ignored):
    reactor.stop()

def do_request(state):
    torServerEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 9050)
    agent = SOCKS5Agent(reactor, proxyEndpoint=torServerEndpoint)
    d = agent.request('GET', 'http://www.google.com/')
    d.addCallback(cbRequest)
    d.addBoth(cbShutdown)

def setup_complete(proto):
    print "setup complete:", proto
    state = txtorcon.TorState(proto.tor_protocol)

    state.post_bootstrap.addCallback(do_request)
    state.post_bootstrap.addErrback(setup_failed)

def setup_failed(arg):
    print "Setup Failed", arg
    reactor.stop()

def updates(prog, tag, summary):
    print "%d%%: %s" % (prog, summary)

config = txtorcon.TorConfig()
config.OrPort = 1234
config.SocksPort = perfconf.tor_config['socks_port']

# Launch tor.
d = txtorcon.launch_tor(config, reactor, progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
reactor.run()
<commit_msg>remove unused imports<commit_after>#!/usr/bin/env python

import txtorcon
import perfconf

from pprint import pformat

from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from txsocksx.http import SOCKS5Agent

class BeginningPrinter(Protocol):
    def __init__(self, finished):
        self.finished = finished
        self.data = []

    def dataReceived(self, data):
        self.data.append(data)

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        print 'Body:', ''.join(self.data)
        self.finished.callback(None)

def cbRequest(response):
    print 'Response received'
    print 'Response headers:'
    print pformat(list(response.headers.getAllRawHeaders()))

    finished = Deferred()
    response.deliverBody(BeginningPrinter(finished))
    return finished

def cbShutdown(ignored):
    reactor.stop()

def do_request(state):
    torServerEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 9050)
    agent = SOCKS5Agent(reactor, proxyEndpoint=torServerEndpoint)
    d = agent.request('GET', 'http://www.google.com/')
    d.addCallback(cbRequest)
    d.addBoth(cbShutdown)

def setup_complete(proto):
    print "setup complete:", proto
    state = txtorcon.TorState(proto.tor_protocol)

    state.post_bootstrap.addCallback(do_request)
    state.post_bootstrap.addErrback(setup_failed)

def setup_failed(arg):
    print "Setup Failed", arg
    reactor.stop()

def updates(prog, tag, summary):
    print "%d%%: %s" % (prog, summary)

config = txtorcon.TorConfig()
config.OrPort = 1234
config.SocksPort = perfconf.tor_config['socks_port']

# Launch tor.
d = txtorcon.launch_tor(config, reactor, progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
reactor.run()
<|endoftext|>


<commit_before>#!/usr/bin/python
import time
import random
import sys
import alsaaudio
import wave
import struct
import json
import socket
import requests
import cStringIO
import os
from collections import OrderedDict
import math
from dotstar import Adafruit_DotStar

WHOAMI = socket.gethostname()
WHATAMI = os.path.basename(__file__).replace(".py", "")

#m = alsaaudio.Mixer('PCM')
#current_volume = m.getvolume() # Get the current Volume
#print("Cur Vol: %s " % current_volume)
#m.setvolume(100) # Set the volume to 70%.
#current_volume = m.getvolume() # Get the current Volume
#print("Cur Vol: %s " % current_volume)

numpixels = 120 # Number of LEDs in strip
# Here's how to control the strip from any two GPIO pins:
datapin   = 23
clockpin  = 24
defaultColor = 0x000099
defaultBright = 128
flashColor = 0x009999
flashBright = 255
lasthb = 0
hbinterval = 30
strip     = Adafruit_DotStar(numpixels, datapin, clockpin)
strip.setBrightness(defaultBright)
strip.begin()           # Initialize pins for output

hi_thres = 10
low_thres = 5
beat = False

def main():
    global strip
    global beat
    global lasthb
    global hbinterval

    logevent("startup", "startup", "Just started and ready to run")

    strip.setBrightness(defaultBright)
    setAllLEDS(strip, [defaultColor])
    strip.show()
    sounds = [0, 0, 0]

    channels = 2
    rate = 44100
    size = 1024
    out_stream = alsaaudio.PCM(alsaaudio.PCM_PLAYBACK, alsaaudio.PCM_NORMAL, 'default')
    out_stream.setformat(alsaaudio.PCM_FORMAT_S16_LE)
    out_stream.setchannels(channels)
    out_stream.setrate(rate)
    out_stream.setperiodsize(size)

    soundfiles = ['/home/pi/torture_audio.wav']
    memsound = {}
    print("Loading Sound files to memory")
    for sf in soundfiles:
        f = open(sf, "rb")
        sfdata = f.read()
        f.close()
        memsound[sf] = cStringIO.StringIO(sfdata)

    while True:
        curtime = int(time.time())
        if curtime - lasthb > hbinterval:
            logevent("heartbeat", "Working", "Standard HB")
            lasthb = curtime

        curfile = random.choice(soundfiles)
        memsound[curfile].seek(0)
        data = memsound[curfile].read(size)
        while data:
            out_stream.write(data)
            data = memsound[curfile].read(size)
            rmsval = rms(data)
            sounds.append(rmsval)
            ug = sounds.pop(0)
            try:
                sounds_avg = sum(sounds) / len(sounds)
            except:
                sounds_avg = 0
            if sounds_avg > hi_thres and beat == False:
                strip.setBrightness(flashBright)
                setAllLEDS(strip, [flashColor])
                beat = True
            if sounds_avg < low_thres and beat == True:
                strip.setBrightness(defaultBright)
                setAllLEDS(strip, [defaultColor])
                beat = False

    sys.exit(0)

def logevent(etype, edata, edesc):
    global WHOAMI
    global WHATAMI

    curtime = int(time.time())
    curts = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(curtime))
    outrec = OrderedDict()
    outrec['ts'] = curts
    outrec['host'] = WHOAMI
    outrec['script'] = WHATAMI
    outrec['event_type'] = etype
    outrec['event_data'] = edata
    outrec['event_desc'] = edesc
    sendlog(outrec, False)
    outrec = None

def setAllLEDS(strip, colorlist):
    for x in range(numpixels):
        strip.setPixelColor(x, colorlist[0])
    strip.show()

def rms(frame):
    SHORT_NORMALIZE = (1.0/32768.0)
    CHUNK = 1024
    swidth = 2

    count = len(frame)/swidth
    format = "%dh"%(count)
    shorts = struct.unpack( format, frame )

    sum_squares = 0.0
    for sample in shorts:
        n = sample * SHORT_NORMALIZE
        sum_squares += n*n
        rms = math.pow(sum_squares/count,0.5);
        return rms * 10000

def sendlog(log, debug):
    logurl = "http://hauntcontrol:5050/hauntlogs"
    try:
        r = requests.post(logurl, json=log)
        if debug:
            print("Posted to %s status code %s" % (logurl, r.status_code))
            print(json.dumps(log))
    except:
        if debug:
            print("Post to %s failed timed out?" % logurl)
            print(json.dumps(log))

if __name__ == "__main__":
    main()
<commit_msg>Update torture thresholds<commit_after>#!/usr/bin/python
import time
import random
import sys
import alsaaudio
import wave
import struct
import json
import socket
import requests
import cStringIO
import os
from collections import OrderedDict
import math
from dotstar import Adafruit_DotStar

WHOAMI = socket.gethostname()
WHATAMI = os.path.basename(__file__).replace(".py", "")

#m = alsaaudio.Mixer('PCM')
#current_volume = m.getvolume() # Get the current Volume
#print("Cur Vol: %s " % current_volume)
#m.setvolume(100) # Set the volume to 70%.
#current_volume = m.getvolume() # Get the current Volume
#print("Cur Vol: %s " % current_volume)

numpixels = 120 # Number of LEDs in strip
# Here's how to control the strip from any two GPIO pins:
datapin   = 23
clockpin  = 24
defaultColor = 0x000099
defaultBright = 128
flashColor = 0x009999
flashBright = 255
lasthb = 0
hbinterval = 30
strip     = Adafruit_DotStar(numpixels, datapin, clockpin)
strip.setBrightness(defaultBright)
strip.begin()           # Initialize pins for output

hi_thres = 50
low_thres = 40
beat = False

def main():
    global strip
    global beat
    global lasthb
    global hbinterval

    logevent("startup", "startup", "Just started and ready to run")

    strip.setBrightness(defaultBright)
    setAllLEDS(strip, [defaultColor])
    strip.show()
    sounds = [0, 0, 0]

    channels = 2
    rate = 44100
    size = 1024
    out_stream = alsaaudio.PCM(alsaaudio.PCM_PLAYBACK, alsaaudio.PCM_NORMAL, 'default')
    out_stream.setformat(alsaaudio.PCM_FORMAT_S16_LE)
    out_stream.setchannels(channels)
    out_stream.setrate(rate)
    out_stream.setperiodsize(size)

    soundfiles = ['/home/pi/torture_audio.wav']
    memsound = {}
    print("Loading Sound files to memory")
    for sf in soundfiles:
        f = open(sf, "rb")
        sfdata = f.read()
        f.close()
        memsound[sf] = cStringIO.StringIO(sfdata)

    while True:
        curtime = int(time.time())
        if curtime - lasthb > hbinterval:
            logevent("heartbeat", "Working", "Standard HB")
            lasthb = curtime

        curfile = random.choice(soundfiles)
        memsound[curfile].seek(0)
        data = memsound[curfile].read(size)
        while data:
            out_stream.write(data)
            data = memsound[curfile].read(size)
            rmsval = rms(data)
            sounds.append(rmsval)
            ug = sounds.pop(0)
            try:
                sounds_avg = sum(sounds) / len(sounds)
            except:
                sounds_avg = 0
            if sounds_avg > hi_thres and beat == False:
                strip.setBrightness(flashBright)
                setAllLEDS(strip, [flashColor])
                beat = True
            if sounds_avg < low_thres and beat == True:
                strip.setBrightness(defaultBright)
                setAllLEDS(strip, [defaultColor])
                beat = False

    sys.exit(0)

def logevent(etype, edata, edesc):
    global WHOAMI
    global WHATAMI

    curtime = int(time.time())
    curts = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(curtime))
    outrec = OrderedDict()
    outrec['ts'] = curts
    outrec['host'] = WHOAMI
    outrec['script'] = WHATAMI
    outrec['event_type'] = etype
    outrec['event_data'] = edata
    outrec['event_desc'] = edesc
    sendlog(outrec, False)
    outrec = None

def setAllLEDS(strip, colorlist):
    for x in range(numpixels):
        strip.setPixelColor(x, colorlist[0])
    strip.show()

def rms(frame):
    SHORT_NORMALIZE = (1.0/32768.0)
    CHUNK = 1024
    swidth = 2

    count = len(frame)/swidth
    format = "%dh"%(count)
    shorts = struct.unpack( format, frame )

    sum_squares = 0.0
    for sample in shorts:
        n = sample * SHORT_NORMALIZE
        sum_squares += n*n
        rms = math.pow(sum_squares/count,0.5);
        return rms * 10000

def sendlog(log, debug):
    logurl = "http://hauntcontrol:5050/hauntlogs"
    try:
        r = requests.post(logurl, json=log)
        if debug:
            print("Posted to %s status code %s" % (logurl, r.status_code))
            print(json.dumps(log))
    except:
        if debug:
            print("Post to %s failed timed out?" % logurl)
            print(json.dumps(log))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Test cases for tournament.py

import tournament
import names
import datetime
import random
import time

def testDeleteMatches():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    print "1. Old matches can be deleted."

def testDelete():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    print "2. Player records can be deleted."

def testCount():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c == '0':
        raise TypeError(
            "countPlayers() should return numeric zero, not string '0'.")
    if c != 0:
        raise ValueError("After deleting, countPlayers should return zero.")
    print "3. After deleting, countPlayers() returns zero."

def testRegister():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    print ("register player")
    swiss_trnmnt.registerPlayer("Chandra Nalaar")
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c != 1:
        raise ValueError(
            "After one player registers, countPlayers() should be 1.")
    print "4. After registering a player, countPlayers() returns 1."

def testRegisterCountDelete():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Markov Chaney")
    swiss_trnmnt.registerPlayer("Joe Malik")
    swiss_trnmnt.registerPlayer("Mao Tsu-hsi")
    swiss_trnmnt.registerPlayer("Atlanta Hope")
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c != 4:
        raise ValueError(
            "After registering four players, countPlayers should be 4.")
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c != 0:
        raise ValueError("After deleting, countPlayers should return zero.")
    print "5. Players can be registered and deleted."

def testStandingsBeforeMatches():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Melpomene Murray")
    swiss_trnmnt.registerPlayer("Randy Schwartz")
    standings = swiss_trnmnt.playerStandings()
    if len(standings) < 2:
        raise ValueError("Players should appear in playerStandings even before "
                         "they have played any matches.")
    elif len(standings) > 2:
        raise ValueError("Only registered players should appear in standings.")
    if len(standings[0]) != 4:
        raise ValueError("Each playerStandings row should have four columns.")
    [(id1, name1, wins1, matches1), (id2, name2, wins2, matches2)] = standings
    if matches1 != 0 or matches2 != 0 or wins1 != 0 or wins2 != 0:
        raise ValueError(
            "Newly registered players should have no matches or wins.")
    if set([name1, name2]) != set(["Melpomene Murray", "Randy Schwartz"]):
        raise ValueError("Registered players' names should appear in standings, "
                         "even if they have no matches played.")
    print "6. Newly registered players appear in the standings with no matches."

def reportMatches():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Bruno Walton")
    swiss_trnmnt.registerPlayer("Boots O'Neal")
    swiss_trnmnt.registerPlayer("Cathy Burton")
    swiss_trnmnt.registerPlayer("Diane Grant")
    standings = swiss_trnmnt.playerStandings()
    [id1, id2, id3, id4] = [row[0] for row in standings]
    swiss_trnmnt.reportMatch(id1, id2)
    swiss_trnmnt.reportMatch(id3, id4)
    standings = swiss_trnmnt.playerStandings()
    for (i, n, w, m) in standings:
        if m != 1:
            raise ValueError("Each player should have one match recorded.")
        if i in (id1, id3) and w != 1:
            raise ValueError("Each match winner should have one win recorded.")
        elif i in (id2, id4) and w != 0:
            raise ValueError("Each match lenoser should have zero wins recorded.")
    print "7. After a match, players have updated standings."

def testPairings():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Twilight Sparkle")
    swiss_trnmnt.registerPlayer("Fluttershy")
    swiss_trnmnt.registerPlayer("Applejack")
    swiss_trnmnt.registerPlayer("Pinkie Pie")
    standings = swiss_trnmnt.playerStandings()
    [id1, id2, id3, id4] = [row[0] for row in standings]
    swiss_trnmnt.reportMatch(id1, id2)
    swiss_trnmnt.reportMatch(id3, id4)
    pairings = swiss_trnmnt.swissPairings()
    if len(pairings) != 2:
        raise ValueError(
            "For four players, swissPairings should return two pairs.")
    [(pid1, pname1, pid2, pname2), (pid3, pname3, pid4, pname4)] = pairings
    correct_pairs = set([frozenset([id1, id3]), frozenset([id2, id4])])
    actual_pairs = set([frozenset([pid1, pid2]), frozenset([pid3, pid4])])
    if correct_pairs != actual_pairs:
        raise ValueError(
            "After one match, players with one win should be paired.")
    print "8. After one match, players with one win are paired."

def testTournamentMultiPlayers():
    id_round=1
    start_time = time.time()
    final_players= list()
    print "Number of competitors [",swiss_trnmnt._competitors,"]"
    for i in range(swiss_trnmnt._competitors):
        # register players' name randomly using third-package names
        try:
            swiss_trnmnt.registerPlayer(names.get_full_name())
        except Exception as ex:
            swiss_trnmnt.rollback()
            if ex.pgcode == '23505': # avoit duplicated player in same tournamnt
                swiss_trnmnt.registerPlayer(names.get_full_name())
            else:
                raise Exception('Unexpected error registering players',str(ex))
    swiss_trnmnt.rankingInit()
    while id_round <=swiss_trnmnt._total_rounds:
        print "Round=[",id_round,"]"
        # result of pairings for next match [(id1,name1,id2,name2),...]
        swiss= swiss_trnmnt.swissPairings()
        print '\n'
        print "Next match=",swiss,'\n'
        testMatchesRandomly(swiss,id_round) # establish winner of match randomly
        id_round +=1
    print '--- SWISS TOURNAMENT FINISHED ---\n'
    print '   Final Standings'
    print ('ID\tNAME\t\tWINS')
    print ('---------------------------')
    final_stands=swiss_trnmnt.playerStandings()
    for i in final_stands:
        print i[0],'\t',i[1][:13],'\t',i[2] # it prints ID | NAME | WINS
    final_players=swiss_trnmnt.topEightPlayers()
    print (''' \nThe final TOP players using opponents-points for tie-break''')
    print ('ID\tNAME\t    POSITION')
    print ('---------------------------')
    x=1
    for top8 in final_players:
        for rows in top8:
            print rows[0],'\t',rows[1][:10],'\t',x
            x+=1
    print '\n---Starting single-elimination tournament---'
    while id_round <=swiss_trnmnt._total_rounds+swiss_trnmnt._rounds_single: # round's number for single is 3
        print "Round final=[",id_round,"]"
        single= swiss_trnmnt.siglePairingElimination()
        print "Next match=",single, '\n'
        testMatchesRandomly(single,id_round) # establish winner of match randomly
        id_round +=1
    swiss_trnmnt.setWinnerTournament()
    swiss_trnmnt.closeTournament()
    print("Total execution --- %s seconds ---" % (time.time() - start_time))

def testMatchesRandomly(pairings,id_round):
    current_date = datetime.datetime.now()
    current_date= current_date.replace(microsecond=0)
    for z in range(len(pairings)):
        winner=random.randint(0,1)
        if winner==0:
            losser=2
        else:
            losser=0
            winner=2
        swiss_trnmnt.reportMatch(pairings[z][winner],pairings[z][losser],
            id_round,current_date)
    swiss_trnmnt.commitTournament()

if __name__ == '__main__':
    print " -----   STARTING SWISS-TOURNAMENT   -----"
    #Define settings of the swiss_trnmnt, required for testTournamentMultiPlayers
    #[tournmt_id,tournmt_name,location,date_start,date_end,number_competitors]
    data=[1234,'swiss_trnmnt swissPairings','Mexico city','2015-12-01',
          '205-12-30',19]
    swiss_trnmnt=tournament.Swiss()
    swiss_trnmnt.setTournamentInfo(data)

    # Udacity test nanodegree program
    '''testDeleteMatches()
    testDelete()
    testCount()
    testRegister()
    testRegisterCountDelete()
    testStandingsBeforeMatches()
    reportMatches()
    testPairings()
    '''
    # To test extended implementation (multi-players and multi-swiss_trnmnts)
    # "comment methods for testing nanodegree program"

    testTournamentMultiPlayers()

    print "Success!  All tests pass!"

<commit_msg>changes of project<commit_after>#!/usr/bin/env python
#
# Test cases for tournament.py

import tournament
import names
import datetime
import random
import time

def testDeleteMatches():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    print "1. Old matches can be deleted."

def testDelete():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    print "2. Player records can be deleted."

def testCount():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c == '0':
        raise TypeError(
            "countPlayers() should return numeric zero, not string '0'.")
    if c != 0:
        raise ValueError("After deleting, countPlayers should return zero.")
    print "3. After deleting, countPlayers() returns zero."

def testRegister():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    print ("register player")
    swiss_trnmnt.registerPlayer("Chandra Nalaar")
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c != 1:
        raise ValueError(
            "After one player registers, countPlayers() should be 1.")
    print "4. After registering a player, countPlayers() returns 1."

def testRegisterCountDelete():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Markov Chaney")
    swiss_trnmnt.registerPlayer("Joe Malik")
    swiss_trnmnt.registerPlayer("Mao Tsu-hsi")
    swiss_trnmnt.registerPlayer("Atlanta Hope")
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c != 4:
        raise ValueError(
            "After registering four players, countPlayers should be 4.")
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    c = swiss_trnmnt.conn_trnmt_db.countPlayers()
    if c != 0:
        raise ValueError("After deleting, countPlayers should return zero.")
    print "5. Players can be registered and deleted."

def testStandingsBeforeMatches():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Melpomene Murray")
    swiss_trnmnt.registerPlayer("Randy Schwartz")
    standings = swiss_trnmnt.playerStandings()
    if len(standings) < 2:
        raise ValueError("Players should appear in playerStandings even before "
                         "they have played any matches.")
    elif len(standings) > 2:
        raise ValueError("Only registered players should appear in standings.")
    if len(standings[0]) != 4:
        raise ValueError("Each playerStandings row should have four columns.")
    [(id1, name1, wins1, matches1), (id2, name2, wins2, matches2)] = standings
    if matches1 != 0 or matches2 != 0 or wins1 != 0 or wins2 != 0:
        raise ValueError(
            "Newly registered players should have no matches or wins.")
    if set([name1, name2]) != set(["Melpomene Murray", "Randy Schwartz"]):
        raise ValueError("Registered players' names should appear in standings, "
                         "even if they have no matches played.")
    print "6. Newly registered players appear in the standings with no matches."

def reportMatches():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Bruno Walton")
    swiss_trnmnt.registerPlayer("Boots O'Neal")
    swiss_trnmnt.registerPlayer("Cathy Burton")
    swiss_trnmnt.registerPlayer("Diane Grant")
    standings = swiss_trnmnt.playerStandings()
    [id1, id2, id3, id4] = [row[0] for row in standings]
    swiss_trnmnt.reportMatch(id1, id2)
    swiss_trnmnt.reportMatch(id3, id4)
    standings = swiss_trnmnt.playerStandings()
    for (i, n, w, m) in standings:
        if m != 1:
            raise ValueError("Each player should have one match recorded.")
        if i in (id1, id3) and w != 1:
            raise ValueError("Each match winner should have one win recorded.")
        elif i in (id2, id4) and w != 0:
            raise ValueError("Each match lenoser should have zero wins recorded.")
    print "7. After a match, players have updated standings."

def testPairings():
    swiss_trnmnt.conn_trnmt_db.deleteMatches()
    swiss_trnmnt.conn_trnmt_db.deletePlayers()
    swiss_trnmnt.registerPlayer("Twilight Sparkle")
    swiss_trnmnt.registerPlayer("Fluttershy")
    swiss_trnmnt.registerPlayer("Applejack")
    swiss_trnmnt.registerPlayer("Pinkie Pie")
    standings = swiss_trnmnt.playerStandings()
    [id1, id2, id3, id4] = [row[0] for row in standings]
    swiss_trnmnt.reportMatch(id1, id2)
    swiss_trnmnt.reportMatch(id3, id4)
    pairings = swiss_trnmnt.swissPairings()
    if len(pairings) != 2:
        raise ValueError(
            "For four players, swissPairings should return two pairs.")
    [(pid1, pname1, pid2, pname2), (pid3, pname3, pid4, pname4)] = pairings
    correct_pairs = set([frozenset([id1, id3]), frozenset([id2, id4])])
    actual_pairs = set([frozenset([pid1, pid2]), frozenset([pid3, pid4])])
    if correct_pairs != actual_pairs:
        raise ValueError(
            "After one match, players with one win should be paired.")
    print "8. After one match, players with one win are paired."

def testTournamentMultiPlayers():
    id_round=1
    start_time = time.time()
    final_players= list()
    print "Number of competitors [",swiss_trnmnt._competitors,"]"
    for i in range(swiss_trnmnt._competitors):
        # register players' name randomly using third-package names
        try:
            swiss_trnmnt.registerPlayer(names.get_full_name())
        except Exception as ex:
            swiss_trnmnt.rollback()
            if ex.pgcode == '23505': # avoit duplicated player in same tournamnt
                swiss_trnmnt.registerPlayer(names.get_full_name())
            else:
                raise Exception('Unexpected error registering players',str(ex))
    swiss_trnmnt.rankingInit()
    while id_round <=swiss_trnmnt._total_rounds:
        print "Round=[",id_round,"]"
        # result of pairings for next match [(id1,name1,id2,name2),...]
        swiss= swiss_trnmnt.swissPairings()
        print '\n'
        print "Next match=",swiss,'\n'
        testMatchesRandomly(swiss,id_round) # establish winner of match randomly
        id_round +=1
    print '--- SWISS TOURNAMENT FINISHED ---\n'
    print '   Final Standings'
    print ('ID\tNAME\t\tWINS')
    print ('---------------------------')
    final_stands=swiss_trnmnt.playerStandings()
    for i in final_stands:
        print i[0],'\t',i[1][:13],'\t',i[2] # it prints ID | NAME | WINS
    final_players=swiss_trnmnt.topEightPlayers()
    print (''' \nThe final TOP players using opponents-points for tie-break''')
    print ('ID\tNAME\t    POSITION')
    print ('---------------------------')
    x=1
    for top8 in final_players:
        for rows in top8:
            print rows[0],'\t',rows[1][:10],'\t',x
            x+=1
    print '\n---Starting single-elimination tournament---'
    while id_round <=swiss_trnmnt._total_rounds+swiss_trnmnt._rounds_single: # round's number for single is 3
        print "Round final=[",id_round,"]"
        single= swiss_trnmnt.siglePairingElimination()
        print "Next match=",single, '\n'
        testMatchesRandomly(single,id_round) # establish winner of match randomly
        id_round +=1
    swiss_trnmnt.setWinnerTournament()
    swiss_trnmnt.closeTournament()
    print("Total execution --- %s seconds ---" % (time.time() - start_time))

def testMatchesRandomly(pairings,id_round):
    current_date = datetime.datetime.now()
    current_date= current_date.replace(microsecond=0)
    for z in range(len(pairings)):
        winner=random.randint(0,1)
        if winner==0:
            losser=2
        else:
            losser=0
            winner=2
        swiss_trnmnt.reportMatch(pairings[z][winner],pairings[z][losser],
            id_round,current_date)
    swiss_trnmnt.commitTournament()

if __name__ == '__main__':
    print " -----   STARTING SWISS-TOURNAMENT   -----"
    #Define settings of the tournament, required for testTournamentMultiPlayers
    #[tournmt_id,tournmt_name,location,date_start,date_end,number_competitors]
    data=[1234,'swiss_trnmnt swissPairings','Mexico city','2015-12-01',
          '205-12-30',19]
    swiss_trnmnt=tournament.Swiss()
    swiss_trnmnt.setTournamentInfo(data)

    # Udacity test nanodegree program
    '''testDeleteMatches()
    testDelete()
    testCount()
    testRegister()
    testRegisterCountDelete()
    testStandingsBeforeMatches()
    reportMatches()
    testPairings()
    '''
    # To test extended implementation (multi-players and multi-tournaments)
    # "comment methods for testing nanodegree program"

    testTournamentMultiPlayers()

    print "Success!  All tests pass!"

<|endoftext|>