<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyClick(PythonPackage):
    """Python composable command line interface toolkit."""

    homepage = "https://click.palletsprojects.com"
    pypi = "click/click-7.1.2.tar.gz"

    version('8.0.1', sha256='8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a')
    version('7.1.2', sha256='d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a')
    version('7.0', sha256='5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7',
            url='https://pypi.io/packages/source/c/click/Click-7.0.tar.gz')
    version('6.6', sha256='cc6a19da8ebff6e7074f731447ef7e112bd23adf3de5c597cf9989f2fd8defe9')

    depends_on('python@3.6:', type=('build', 'run'), when='@8.0.0:')
    depends_on('python@2.7:2.8,3.5:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')

    depends_on('py-importlib-metadata', type=('build', 'run'), when='@8.0.0: ^python@:3.7')
    # depends_on('py-colorama', type=('build', 'run'), when='platform=windows')
<commit_msg>py-click: add 8.0.3 (#27549)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyClick(PythonPackage):
    """Python composable command line interface toolkit."""

    homepage = "https://click.palletsprojects.com"
    pypi = "click/click-7.1.2.tar.gz"

    version('8.0.3', sha256='410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b')
    version('8.0.1', sha256='8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a')
    version('7.1.2', sha256='d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a')
    version('7.0', sha256='5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7',
            url='https://pypi.io/packages/source/c/click/Click-7.0.tar.gz')
    version('6.6', sha256='cc6a19da8ebff6e7074f731447ef7e112bd23adf3de5c597cf9989f2fd8defe9')

    depends_on('python@3.6:', type=('build', 'run'), when='@8.0.0:')
    depends_on('python@2.7:2.8,3.5:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')

    depends_on('py-importlib-metadata', type=('build', 'run'), when='@8.0.0: ^python@:3.7')
    # depends_on('py-colorama', type=('build', 'run'), when='platform=windows')
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyFiona(PythonPackage):
    """Fiona reads and writes spatial data files."""

    homepage = "https://github.com/Toblerity/Fiona"
    pypi = "Fiona/Fiona-1.8.18.tar.gz"
    git      = "https://github.com/Toblerity/Fiona.git"

    maintainers = ['adamjstewart']

    version('master', branch='master')
    version('1.8.18', sha256='b732ece0ff8886a29c439723a3e1fc382718804bb057519d537a81308854967a')
    version('1.8.6',  sha256='fa31dfe8855b9cd0b128b47a4df558f1b8eda90d2181bff1dd9854e5556efb3e')
    version('1.7.12', sha256='8b54eb8422d7c502bb7776b184018186bede1a489cf438a7a47f992ade6a0e51')

    depends_on('python@3.6:', type=('build', 'link', 'run'), when='@1.9:')
    depends_on('python@2.6:', type=('build', 'link', 'run'))
    depends_on('gdal@1.11:', type=('build', 'link', 'run'), when='@1.9:')
    depends_on('gdal@1.8:', type=('build', 'link', 'run'))

    depends_on('py-setuptools', type=('build', 'run'))
    depends_on('py-cython', type='build', when='@master')
    depends_on('py-attrs@17:', type=('build', 'run'))
    depends_on('py-certifi', type=('build', 'run'), when='@1.8.18')
    depends_on('py-click@4:7', type=('build', 'run'))
    depends_on('py-cligj@0.5:', type=('build', 'run'))
    depends_on('py-click-plugins@1:', type=('build', 'run'))
    depends_on('py-six@1.7:', type=('build', 'run'))
    depends_on('py-munch', type=('build', 'run'))
    depends_on('py-argparse', type=('build', 'run'), when='^python@:2.6')
    depends_on('py-ordereddict', type=('build', 'run'), when='^python@:2.6')
    depends_on('py-enum34', type=('build', 'run'), when='^python@:3.3')
<commit_msg>py-fiona: add v1.8.20 (#25196)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyFiona(PythonPackage):
    """Fiona reads and writes spatial data files."""

    homepage = "https://github.com/Toblerity/Fiona"
    pypi = "Fiona/Fiona-1.8.18.tar.gz"
    git      = "https://github.com/Toblerity/Fiona.git"

    maintainers = ['adamjstewart']

    version('master', branch='master')
    version('1.8.20', sha256='a70502d2857b82f749c09cb0dea3726787747933a2a1599b5ab787d74e3c143b')
    version('1.8.18', sha256='b732ece0ff8886a29c439723a3e1fc382718804bb057519d537a81308854967a')
    version('1.8.6',  sha256='fa31dfe8855b9cd0b128b47a4df558f1b8eda90d2181bff1dd9854e5556efb3e')
    version('1.7.12', sha256='8b54eb8422d7c502bb7776b184018186bede1a489cf438a7a47f992ade6a0e51')

    depends_on('python@3.6:', type=('build', 'link', 'run'), when='@1.9:')
    depends_on('python@2.6:', type=('build', 'link', 'run'))
    depends_on('gdal@1.11:', type=('build', 'link', 'run'), when='@1.9:')
    depends_on('gdal@1.8:', type=('build', 'link', 'run'))

    depends_on('py-cython', type='build', when='@master')
    depends_on('py-attrs@17:', type=('build', 'run'))
    depends_on('py-certifi', type=('build', 'run'), when='@1.8.18:')
    depends_on('py-click@4:', type=('build', 'run'))
    depends_on('py-cligj@0.5:', type=('build', 'run'))
    depends_on('py-click-plugins@1:', type=('build', 'run'))
    depends_on('py-six@1.7:', type=('build', 'run'))
    depends_on('py-munch', type=('build', 'run'))
    depends_on('py-setuptools', type=('build', 'run'))
    depends_on('py-argparse', type=('build', 'run'), when='^python@:2.6')
    depends_on('py-ordereddict', type=('build', 'run'), when='^python@:2.6')
    depends_on('py-enum34', type=('build', 'run'), when='^python@:3.3')
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class PyMongo(PythonPackage):
    """Python driver for MongoDB <http://www.mongodb.org>"""

    homepage = "http://github.com/mongodb/mongo-python-driver"
    url      = "https://pypi.io/packages/source/p/pymongo/pymongo-3.3.0.tar.gz"

    version('3.3.0', '42cd12a5014fb7d3e1987ca04f5c651f')

    depends_on('python@2.6:2.8,3.3:')

    depends_on('setuptools', type='build')
<commit_msg>Fix pymongo package (#6832)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class PyMongo(PythonPackage):
    """Python driver for MongoDB <http://www.mongodb.org>"""

    homepage = "http://github.com/mongodb/mongo-python-driver"
    url      = "https://pypi.io/packages/source/p/pymongo/pymongo-3.6.0.tar.gz"

    version('3.6.0', '2f64fa7691c77535b72050704cc12afb')
    version('3.3.0', '42cd12a5014fb7d3e1987ca04f5c651f')

    depends_on('python@2.6:2.8,3.3:')

    depends_on('py-setuptools', type='build')
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RTictoc(RPackage):
    """tictoc: Functions for timing R scripts, as well as implementations of
    Stack and List structures

    This package provides the timing functions 'tic' and 'toc'
    that can be nested. One can record all timings while a
    complex script is running, and examine the values later. It
    is also possible to instrument the timing calls with custom
    callbacks. In addition, this package provides class
    'Stack', implemented as a vector, and class 'List',
    implemented as a list, both of which support operations
    'push', 'pop', 'first', 'last' and 'clear'."""

    homepage = "https://collectivemedia.github.io/tictoc/"
    url      = "https://cloud.r-project.org/src/contrib/tictoc_1.0.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/tictoc"

    version('1.0', sha256='47da097c1822caa2d8e262381987cfa556ad901131eb96109752742526b2e2fe')

    depends_on('r@3.0.3:', type=('build', 'run'))
<commit_msg>r-tictoc: add 1.0.1 (#27715)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RTictoc(RPackage):
    """Functions for timing R scripts, as well as implementations of Stack and
    List structures.

    This package provides the timing functions 'tic' and 'toc'
    that can be nested. One can record all timings while a
    complex script is running, and examine the values later. It
    is also possible to instrument the timing calls with custom
    callbacks. In addition, this package provides class
    'Stack', implemented as a vector, and class 'List',
    implemented as a list, both of which support operations
    'push', 'pop', 'first', 'last' and 'clear'."""

    homepage = "https://collectivemedia.github.io/tictoc/"
    cran = "tictoc"

    version('1.0.1', sha256='a09a1535c417ddf6637bbbda5fca6edab6c7f7b252a64e57e99d4d0748712705')
    version('1.0', sha256='47da097c1822caa2d8e262381987cfa556ad901131eb96109752742526b2e2fe')

    depends_on('r@3.0.3:', type=('build', 'run'), when='@1.0.1:')
    depends_on('r@3.0.3:4.0', type=('build', 'run'), when='@1.0')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
# author 92ez.com

from flask import Flask, render_template, request, abort, send_from_directory
import requests
import sys
import os
import re

app = Flask(__name__)

@app.route("/search", methods=['POST'])
def search():
    nickname = request.form.get('nickname')
    source_type = request.form.get('type')
    user_id = request.form.get('uid')
    if user_id != "userid123":
        return "can not use!"
    else:
        if len(nickname) > 0:
            get_source(nickname, 1, source_type, user_id)
            return "complete!"
        else:
            return "nickname is empty"

@app.route("/history", methods=['GET'])
def history():
    user_id = request.args.get('uid')
    if len(user_id) < 1:
        return "user id is empty !"
    else:
        try:
            list_data = os.listdir(sys.path[0] + '/download/' + user_id)
            return render_template("history.html", uid=user_id, data=list_data)
        except Exception as e:
            return "user is not exist!"

@app.route("/type", methods=['GET'])
def gettype():
    user_id = request.args.get('uid')
    blog = request.args.get('blog')
    if len(user_id) < 1:
        return "user id is empty !"
    else:
        try:
            list_data = os.listdir(sys.path[0] + '/download/' + user_id + '/' + blog)
            return render_template("type.html", blog=blog, uid=user_id, data=list_data)
        except Exception as e:
            return "user is not exist!"

@app.route("/list", methods=['GET'])
def getlist():
    user_id = request.args.get('uid')
    blog = request.args.get('blog')
    dir = request.args.get('dir')
    if len(user_id) < 1:
        return "user id is empty !"
    else:
        try:
            list_data = os.listdir(sys.path[0] + '/download/' + user_id + '/' + blog + '/' + dir)
            return render_template("list.html", blog=blog, type=dir, uid=user_id, data=list_data)
        except Exception as e:
            return "user is not exist!"

@app.route("/download", methods=['GET'])
def download():
    filepath = sys.path[0] + '/download/' + request.args.get('path')
    filename = request.args.get('name')
    if os.path.isfile(filepath + filename):
        print(filepath + filename)
        return send_from_directory(filepath, filename, as_attachment=True)
    else:
        abort(404)

@app.route("/")
def index():
    return render_template("index.html")

def get_source(nickname, page_index, source_type, user_id):
    aim_url = "https://%s.tumblr.com/page/%d" % (nickname, page_index)
    print('[o] Get source from blog %s ...' % aim_url)
    try:
        response_string = requests.get(url=aim_url, timeout=50).content.decode('utf8').replace('\n', '')
        if "posts-no-posts content" not in response_string:
            if source_type == "images":
                source_elements = re.findall(r'<img(.+?)>', response_string)
                source_iframes = re.findall(r'id="photoset_iframe(.+?)>', response_string)  # for special images iframe
                if len(source_iframes) > 0:
                    for iframe_item in source_iframes:
                        this_iframe_url = re.findall(r'src="(.+?)"', iframe_item)[0]
                        image_iframe_url = "https://%s.tumblr.com" % nickname + this_iframe_url
                        images_iframe_response = requests.get(url=image_iframe_url, timeout=50).content.decode('utf8').replace('\n', '')
                        iframe_images = re.findall(r'<img(.+?)>', images_iframe_response)
                        source_elements += iframe_images
            else:
                source_elements = re.findall(r'<iframe(.+?)>', response_string)

            if len(source_elements) > 0:
                dir_path = sys.path[0] + '/download/' + user_id + '/' + aim_url.split('//')[1].split('.')[0] + '/'
                if not os.path.exists(dir_path):
                    os.makedirs(dir_path)

                for this_source in source_elements:
                    if source_type == "images":
                        img_dir = dir_path + 'image/'
                        if not os.path.exists(img_dir):
                            os.makedirs(img_dir)
                        image_url = re.findall(r'src="(.+?)"', this_source)[0]
                        image_name = image_url.split('/')[-1]
                        if "tumblr_" in image_name:
                            write_file(image_url, img_dir, image_name)
                    else:
                        if "/video/" in this_source:
                            video_dir = dir_path + 'video/'
                            if not os.path.exists(video_dir):
                                os.makedirs(video_dir)
                            video_url = re.findall(r"src='(.+?)'", this_source)[0]
                            video_response = requests.get(url=video_url, timeout=50).content.decode('utf8').replace('\n', '')
                            video_source = re.findall(r'<source src="(.+?)"', video_response)[0]
                            video_name = video_source.split('/')[-1] + '.mp4'
                            write_file(video_source, video_dir, video_name)

            else:
                print('[x] Can not get any source.')
            page_index += 1
            get_source(nickname, page_index, source_type)
        else:
            print('[!] Get source complete!')
    except Exception as e:
        print(e)
        get_source(nickname, page_index, source_type)

def write_file(source_url, dir_path, file_name):
    if not os.path.exists(dir_path + file_name):
        print('[*] Source %s is downloading...' % file_name)
        file_download = requests.get(url=source_url, timeout=50)
        if file_download.status_code == 200:
            open(dir_path + file_name, 'wb').write(file_download.content)
    else:
        print('[*] Source %s has been downloaded.' % file_name)

if __name__ == "__main__":
    app.run(debug=True)<commit_msg>flask编写简易的在线版本<commit_after>#!/usr/bin/env python
# coding=utf-8
# author 92ez.com

from flask import Flask, render_template, request, abort, send_from_directory
import requests
import sys
import os
import re

app = Flask(__name__)

@app.route("/search", methods=['POST'])
def search():
    nickname = request.form.get('nickname')
    source_type = request.form.get('type')
    user_id = request.form.get('uid')
    if user_id == "user0001":
        if len(nickname) > 0:
            get_source(nickname, 1, source_type, user_id)
            return "complete!"
        else:
            return "nickname is empty"
    else:
        return "can not use!!"

@app.route("/history", methods=['GET'])
def history():
    user_id = request.args.get('uid')
    if len(user_id) < 1:
        return "user id is empty !"
    else:
        try:
            list_data = os.listdir(sys.path[0] + '/download/' + user_id)
            return render_template("history.html", uid=user_id, data=list_data)
        except Exception as e:
            return "user is not exist!"

@app.route("/type", methods=['GET'])
def gettype():
    user_id = request.args.get('uid')
    blog = request.args.get('blog')
    if len(user_id) < 1:
        return "user id is empty !"
    else:
        try:
            list_data = os.listdir(sys.path[0] + '/download/' + user_id + '/' + blog)
            return render_template("type.html", blog=blog, uid=user_id, data=list_data)
        except Exception as e:
            return "user is not exist!"

@app.route("/list", methods=['GET'])
def getlist():
    user_id = request.args.get('uid')
    blog = request.args.get('blog')
    dir = request.args.get('dir')
    if len(user_id) < 1:
        return "user id is empty !"
    else:
        try:
            list_data = os.listdir(sys.path[0] + '/download/' + user_id + '/' + blog + '/' + dir)
            return render_template("list.html", blog=blog, type=dir, uid=user_id, data=list_data)
        except Exception as e:
            return "user is not exist!"

@app.route("/download", methods=['GET'])
def download():
    filepath = sys.path[0] + '/download/' + request.args.get('path')
    filename = request.args.get('name')
    if os.path.isfile(filepath + filename):
        print(filepath + filename)
        return send_from_directory(filepath, filename, as_attachment=True)
    else:
        abort(404)

@app.route("/")
def index():
    return render_template("index.html")

def get_source(nickname, page_index, source_type, user_id):
    aim_url = "https://%s.tumblr.com/page/%d" % (nickname, page_index)
    print('[o] Get source from blog %s ...' % aim_url)
    try:
        response_string = requests.get(url=aim_url, timeout=50).content.decode('utf8').replace('\n', '')
        if "posts-no-posts content" not in response_string:
            if source_type == "images":
                source_elements = re.findall(r'<img(.+?)>', response_string)
                source_iframes = re.findall(r'id="photoset_iframe(.+?)>', response_string)  # for special images iframe
                if len(source_iframes) > 0:
                    for iframe_item in source_iframes:
                        this_iframe_url = re.findall(r'src="(.+?)"', iframe_item)[0]
                        image_iframe_url = "https://%s.tumblr.com" % nickname + this_iframe_url
                        images_iframe_response = requests.get(url=image_iframe_url, timeout=50).content.decode('utf8').replace('\n', '')
                        iframe_images = re.findall(r'<img(.+?)>', images_iframe_response)
                        source_elements += iframe_images
            else:
                source_elements = re.findall(r'<iframe(.+?)>', response_string)

            if len(source_elements) > 0:
                dir_path = sys.path[0] + '/download/' + user_id + '/' + aim_url.split('//')[1].split('.')[0] + '/'
                if not os.path.exists(dir_path):
                    os.makedirs(dir_path)

                for this_source in source_elements:
                    if source_type == "images":
                        img_dir = dir_path + 'image/'
                        if not os.path.exists(img_dir):
                            os.makedirs(img_dir)
                        image_url = re.findall(r'src="(.+?)"', this_source)[0]
                        image_name = image_url.split('/')[-1]
                        if "tumblr_" in image_name:
                            write_file(image_url, img_dir, image_name)
                    else:
                        if "/video/" in this_source:
                            video_dir = dir_path + 'video/'
                            if not os.path.exists(video_dir):
                                os.makedirs(video_dir)
                            video_url = re.findall(r"src='(.+?)'", this_source)[0]
                            video_response = requests.get(url=video_url, timeout=50).content.decode('utf8').replace('\n', '')
                            video_source = re.findall(r'<source src="(.+?)"', video_response)[0]
                            video_name = video_source.split('/')[-1] + '.mp4'
                            write_file(video_source, video_dir, video_name)

            else:
                print('[x] Can not get any source.')
            page_index += 1
            get_source(nickname, page_index, source_type)
        else:
            print('[!] Get source complete!')
    except Exception as e:
        print(e)
        get_source(nickname, page_index, source_type)

def write_file(source_url, dir_path, file_name):
    if not os.path.exists(dir_path + file_name):
        print('[*] Source %s is downloading...' % file_name)
        file_download = requests.get(url=source_url, timeout=50)
        if file_download.status_code == 200:
            open(dir_path + file_name, 'wb').write(file_download.content)
    else:
        print('[*] Source %s has been downloaded.' % file_name)

if __name__ == "__main__":
    app.run(debug=True)<|endoftext|>


<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from builtins import *  # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order
import ctypes
import platform
from contextlib import contextmanager
from os.path import dirname

import gi

gi.require_version("Gst", "1.0")
from gi.repository import Gst  # pylint:disable=wrong-import-order

# Here we are using ctypes to call `gst_buffer_map` and `gst_buffer_unmap`
# because PyGObject does not properly expose struct GstMapInfo (see
# [bz #678663]).  Apparently this is fixed upstream but we are still awaiting
# an upstream release (Mar 2014).  Hopefully this can be removed in the future.

_GST_PADDING = 4  # From gstconfig.h

# From struct GstMapInfo in gstreamer/gst/gstmemory.h:
class _GstMapInfo(ctypes.Structure):
    _fields_ = [("memory", ctypes.c_void_p),   # GstMemory *memory
                ("flags", ctypes.c_int),       # GstMapFlags flags
                ("data", ctypes.POINTER(ctypes.c_byte)),    # guint8 *data
                ("size", ctypes.c_size_t),     # gsize size
                ("maxsize", ctypes.c_size_t),  # gsize maxsize
                ("user_data", ctypes.c_void_p * 4),     # gpointer user_data[4]
                # gpointer _gst_reserved[GST_PADDING]:
                ("_gst_reserved", ctypes.c_void_p * _GST_PADDING)]

_GstMapInfo_p = ctypes.POINTER(_GstMapInfo)

if platform.system() == "Darwin":
    _libgst = ctypes.CDLL(dirname(Gst.__path__) + "/../libgstreamer-1.0.dylib")
else:
    _libgst = ctypes.CDLL("libgstreamer-1.0.so.0")
_libgst.gst_buffer_map.argtypes = [ctypes.c_void_p, _GstMapInfo_p, ctypes.c_int]
_libgst.gst_buffer_map.restype = ctypes.c_int

_libgst.gst_buffer_unmap.argtypes = [ctypes.c_void_p, _GstMapInfo_p]
_libgst.gst_buffer_unmap.restype = None

_libgst.gst_buffer_get_size.argtypes = [ctypes.c_void_p]
_libgst.gst_buffer_get_size.restype = ctypes.c_size_t

_libgst.gst_sample_get_buffer.argtypes = [ctypes.c_void_p]
_libgst.gst_sample_get_buffer.restype = ctypes.c_void_p

_libgst.gst_mini_object_is_writable.argtypes = [ctypes.c_void_p]
_libgst.gst_mini_object_is_writable.restype = ctypes.c_int

@contextmanager
def _map_gst_buffer(pbuffer, flags):
    if pbuffer is None:
        raise TypeError("Cannot pass NULL to _map_gst_buffer")
    if flags & Gst.MapFlags.WRITE \
            and _libgst.gst_mini_object_is_writable(pbuffer) == 0:
        raise ValueError(
            "Writable array requested but buffer is not writeable")

    mapping = _GstMapInfo()
    success = _libgst.gst_buffer_map(pbuffer, mapping, flags)
    if not success:
        raise RuntimeError("Couldn't map buffer")
    try:
        yield ctypes.cast(
            mapping.data, ctypes.POINTER(ctypes.c_byte * mapping.size)).contents
    finally:
        _libgst.gst_buffer_unmap(pbuffer, mapping)

@contextmanager
def map_gst_sample(sample, flags):
    with _sample_borrow_buffer(sample) as pbuffer:
        with _map_gst_buffer(pbuffer, flags) as x:
            yield x

@contextmanager
def _sample_borrow_buffer(sample):
    """
    This function exists because of a change in behaviour of pygobject 3.13 that
    broke our ability to get a writable buffer from a GstSample.

    See https://bugzilla.gnome.org/show_bug.cgi?id=736896 for more information.
    """
    if not isinstance(sample, Gst.Sample):
        raise TypeError("sample_borrow_buffer must take a Gst.Sample.  "
                        "Received a %s instead" % str(type(sample)))

    # hashing a GObject actually gives the address (pointer) of the C struct
    # that backs it!:
    pbuffer = _libgst.gst_sample_get_buffer(hash(sample))
    if pbuffer is None:
        raise ValueError(
            "sample_borrow_buffer: Provided GstSample doesn't contain a buffer")
    yield pbuffer

def sample_get_size(sample):
    with _sample_borrow_buffer(sample) as pbuffer:
        return _libgst.gst_buffer_get_size(pbuffer)

def test_map_sample_reading_data():
    Gst.init([])

    s = Gst.Sample.new(Gst.Buffer.new_wrapped(b"hello"), None, None, None)
    with map_gst_sample(s, Gst.MapFlags.READ) as a:
        assert b'hello' == b''.join(chr(x) for x in a)

def test_map_sample_modifying_data():
    Gst.init([])

    s = Gst.Sample.new(Gst.Buffer.new_wrapped(b"hello"), None, None, None)
    with map_gst_sample(s, Gst.MapFlags.WRITE | Gst.MapFlags.READ) as a:
        a[2] = 1

    assert s.get_buffer().extract_dup(0, 5) == b"he\x01lo"

def test_map_sample_without_buffer():
    Gst.init([])

    sample = Gst.Sample.new(None, None, None, None)
    try:
        # pylint:disable=no-value-for-parameter
        with map_gst_sample(sample, Gst.MapFlags.READ):
            assert False
    except ValueError:
        pass
<commit_msg>Python3: Fix test_map_sample_reading_data<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from builtins import *  # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order
import ctypes
import platform
from contextlib import contextmanager
from os.path import dirname

import gi

gi.require_version("Gst", "1.0")
from gi.repository import Gst  # pylint:disable=wrong-import-order

# Here we are using ctypes to call `gst_buffer_map` and `gst_buffer_unmap`
# because PyGObject does not properly expose struct GstMapInfo (see
# [bz #678663]).  Apparently this is fixed upstream but we are still awaiting
# an upstream release (Mar 2014).  Hopefully this can be removed in the future.

_GST_PADDING = 4  # From gstconfig.h

# From struct GstMapInfo in gstreamer/gst/gstmemory.h:
class _GstMapInfo(ctypes.Structure):
    _fields_ = [("memory", ctypes.c_void_p),   # GstMemory *memory
                ("flags", ctypes.c_int),       # GstMapFlags flags
                ("data", ctypes.POINTER(ctypes.c_byte)),    # guint8 *data
                ("size", ctypes.c_size_t),     # gsize size
                ("maxsize", ctypes.c_size_t),  # gsize maxsize
                ("user_data", ctypes.c_void_p * 4),     # gpointer user_data[4]
                # gpointer _gst_reserved[GST_PADDING]:
                ("_gst_reserved", ctypes.c_void_p * _GST_PADDING)]

_GstMapInfo_p = ctypes.POINTER(_GstMapInfo)

if platform.system() == "Darwin":
    _libgst = ctypes.CDLL(dirname(Gst.__path__) + "/../libgstreamer-1.0.dylib")
else:
    _libgst = ctypes.CDLL("libgstreamer-1.0.so.0")
_libgst.gst_buffer_map.argtypes = [ctypes.c_void_p, _GstMapInfo_p, ctypes.c_int]
_libgst.gst_buffer_map.restype = ctypes.c_int

_libgst.gst_buffer_unmap.argtypes = [ctypes.c_void_p, _GstMapInfo_p]
_libgst.gst_buffer_unmap.restype = None

_libgst.gst_buffer_get_size.argtypes = [ctypes.c_void_p]
_libgst.gst_buffer_get_size.restype = ctypes.c_size_t

_libgst.gst_sample_get_buffer.argtypes = [ctypes.c_void_p]
_libgst.gst_sample_get_buffer.restype = ctypes.c_void_p

_libgst.gst_mini_object_is_writable.argtypes = [ctypes.c_void_p]
_libgst.gst_mini_object_is_writable.restype = ctypes.c_int

@contextmanager
def _map_gst_buffer(pbuffer, flags):
    if pbuffer is None:
        raise TypeError("Cannot pass NULL to _map_gst_buffer")
    if flags & Gst.MapFlags.WRITE \
            and _libgst.gst_mini_object_is_writable(pbuffer) == 0:
        raise ValueError(
            "Writable array requested but buffer is not writeable")

    mapping = _GstMapInfo()
    success = _libgst.gst_buffer_map(pbuffer, mapping, flags)
    if not success:
        raise RuntimeError("Couldn't map buffer")
    try:
        yield ctypes.cast(
            mapping.data, ctypes.POINTER(ctypes.c_byte * mapping.size)).contents
    finally:
        _libgst.gst_buffer_unmap(pbuffer, mapping)

@contextmanager
def map_gst_sample(sample, flags):
    with _sample_borrow_buffer(sample) as pbuffer:
        with _map_gst_buffer(pbuffer, flags) as x:
            yield x

@contextmanager
def _sample_borrow_buffer(sample):
    """
    This function exists because of a change in behaviour of pygobject 3.13 that
    broke our ability to get a writable buffer from a GstSample.

    See https://bugzilla.gnome.org/show_bug.cgi?id=736896 for more information.
    """
    if not isinstance(sample, Gst.Sample):
        raise TypeError("sample_borrow_buffer must take a Gst.Sample.  "
                        "Received a %s instead" % str(type(sample)))

    # hashing a GObject actually gives the address (pointer) of the C struct
    # that backs it!:
    pbuffer = _libgst.gst_sample_get_buffer(hash(sample))
    if pbuffer is None:
        raise ValueError(
            "sample_borrow_buffer: Provided GstSample doesn't contain a buffer")
    yield pbuffer

def sample_get_size(sample):
    with _sample_borrow_buffer(sample) as pbuffer:
        return _libgst.gst_buffer_get_size(pbuffer)

def test_map_sample_reading_data():
    Gst.init([])

    s = Gst.Sample.new(Gst.Buffer.new_wrapped(b"hello"), None, None, None)
    with map_gst_sample(s, Gst.MapFlags.READ) as a:
        assert b'hello' == ''.join(chr(x) for x in a).encode("ascii")

def test_map_sample_modifying_data():
    Gst.init([])

    s = Gst.Sample.new(Gst.Buffer.new_wrapped(b"hello"), None, None, None)
    with map_gst_sample(s, Gst.MapFlags.WRITE | Gst.MapFlags.READ) as a:
        a[2] = 1

    assert s.get_buffer().extract_dup(0, 5) == b"he\x01lo"

def test_map_sample_without_buffer():
    Gst.init([])

    sample = Gst.Sample.new(None, None, None, None)
    try:
        # pylint:disable=no-value-for-parameter
        with map_gst_sample(sample, Gst.MapFlags.READ):
            assert False
    except ValueError:
        pass
<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE

import numpy as np
import os
import matplotlib as mpl
import matplotlib.pyplot as p
from pandas import read_csv

def comparison_plot(path, num_fids=5, verbose=False, obs_to_des=False,
                    obs_to_fid=False, obs_to_fid_data=None,
                    statistics=["Wavelet", "MVC", "PSpec", "Bispectrum",
                                "DeltaVariance", "Genus", "VCS",
                                "VCS_Density", "VCS_Velocity", "VCA",
                                "Tsallis", "PCA", "SCF", "Cramer",
                                "Skewness", "Kurtosis", "Dendrogram_Hist",
                                "Dendrogram_Num", "PDF"],
                    comparisons=["0_0", "0_1", "0_2", "1_0", "1_1", "1_2",
                                 "2_0", "2_1", "2_2"]):
    '''
    Requires results converted into csv form!!

    This function plots a comparison of the distances between the different
    simulations and the fiducial runs. All face combinations are checked for
    in the given path. The plots adjust to the available amount of data.

    Parameters
    ----------
    path : str
           Path to folder containing the HDF5 files with the distance results.
    analysis_fcn : function, optional
        Function to apply to the time-step data.
    verbose : bool, optional
        Enables plotting.
    cross_compare : bool, optional
        Include comparisons between faces.
    statistics : list, optional
        Statistics to plot. Default is all.
    comparisons : list, optional
        The face comparisons to include in the plots. The order is set here as
        well.
    '''

    if path[-1] != "/":
        path += "/"

    # Set the order by the given comparison list
    order = comparisons

    # All possible face combinations
    data_files = {"0_0": ["Face 0 to 0", None, None],
                  "1_1": ["Face 1 to 1", None, None],
                  "2_2": ["Face 2 to 2", None, None],
                  "0_1": ["Face 0 to 1", None, None],
                  "0_2": ["Face 0 to 2", None, None],
                  "1_2": ["Face 1 to 2", None, None],
                  "2_1": ["Face 2 to 1", None, None],
                  "2_0": ["Face 2 to 0", None, None],
                  "1_0": ["Face 1 to 0", None, None]}

    # Remove comparisons that aren't requested
    for key in data_files.keys():
        if not key in comparisons:
            del data_files[key]

    if obs_to_des:
        data_files["0_obs"] = ["Face 0 to Obs"],
        data_files["1_obs"] = ["Face 1 to Obs"],
        data_files["2_obs"] = ["Face 2 to Obs"],

    if obs_to_fid:
        obs_to_fid_data = {0: None,
                           1: None,
                           2: None}

    # Read in the data and match it to one of the face combinations.
    for x in os.listdir(path):
        if not os.path.isfile(os.path.join(path, x)):
            continue
        if not x[-3:] == "csv":
            continue

        # Separate out the observational csv files.
        if obs_to_fid and 'complete' in x:
            for key in obs_to_fid_data.keys():
                if "face_"+str(key):
                    obs_to_fid_data[key] = read_csv(os.path.join(path, x),
                                                    index_col=0)
                    break
        else:
            for key in data_files.keys():
                if key in x:
                    data = read_csv(os.path.join(path, x))
                    if "fiducial" in x:
                        data_files[key][1] = data
                    elif "distances" in x:
                        data_files[key][2] = data
                    else:
                        pass
                    break

    # Now delete the keys with no data
    for key in data_files.keys():
        if len(data_files[key]) == 1:
            del data_files[key]
            order.remove(key)

    if data_files.keys() == []:
        print "No csv files found in %s" % (path)
        return

    # Set the colour cycle
    colour_cycle = mpl.rcParams['axes.color_cycle']
    if len(colour_cycle) < num_fids:
        # Need to define our own in this case.
        mpl.rcParams['axes.color_cycle'] = \
            [mpl.cmap.jet(i) for i in np.linspace(0.0, 1.0, num_fids)]
    else:
        mpl.rcParams['axes.color_cycle'] = colour_cycle[:num_fids]

    for stat in statistics:
        # Divide by 2 b/c there should be 2 files for each comparison b/w faces
        (fig, ax) = _plot_size(len(data_files.keys()))
        if len(data_files.keys()) == 1:
            shape = (1, )
            ax = np.array([ax])
        else:
            shape = ax.shape

        if len(shape) == 1:
            ax = ax[:, np.newaxis]
            shape = ax.shape
        for k, (key, axis) in enumerate(zip(order, ax.flatten())):
            bottom = False
            if k >= len(ax.flatten()) - shape[1]:
                bottom = True
            if k / float(shape[0]) in [0, 1, 2]:
                left = True
            _plotter(axis, data_files[key][2][stat], data_files[key][1][stat],
                     num_fids, data_files[key][0], stat, bottom, left)
            if obs_to_fid and k <= k / float(shape[1]):
                obs_key = int(key[0])
                _horiz_obs_plot(axis, obs_to_fid_data[obs_key][stat],
                                num_fids)

        if verbose:
            p.autoscale(True)
            fig.show()
        else:
            p.autoscale(True)
            fig.savefig("distance_comparisons_" + stat + ".pdf")
            fig.clf()

def _plot_size(num):
    if num <= 3:
        return p.subplots(num, sharex=True)
    elif num > 3 and num <= 8:
        rows = num / 2 + num % 2
        return p.subplots(nrows=rows, ncols=2, figsize=(14, 14), dpi=100, sharex=True)
    elif num == 9:
        return p.subplots(nrows=3, ncols=3, figsize=(14, 14), dpi=100, sharex=True)
    else:
        print "There should be a maximum of 9 comparisons."
        return

def _plotter(ax, data, fid_data, num_fids, title, stat, bottom, left):

    num_design = (max(data.shape) / num_fids)
    x_vals = np.arange(0, num_design)
    xtick_labels = [str(i) for i in x_vals]
    fid_labels = [str(i) for i in range(num_fids-1)]
    # Plot designs
    for i in range(num_fids):
        y_vals = data.ix[int(i * num_design):int(((i + 1) * num_design)-1)]
        ax.plot(x_vals, y_vals, "-o", label="Fiducial " + str(i), alpha=0.6)
    # Set title in upper left hand corner
    ax.annotate(title, xy=(0, 1), xytext=(12, -6), va='top',
                xycoords='axes fraction', textcoords='offset points',
                fontsize=12, alpha=0.75)
    if left:
        # Set the ylabel using the stat name. Replace underscores
        ax.set_ylabel(stat.replace("_", " ")+"\nDistance", fontsize=10,
                      multialignment='center')
    else:
        ax.set_ylabel("")

    # If the plot is on the bottom of a column, add labels
    if bottom:
        # Put two 'labels' for the x axis
        ax.annotate("Designs", xy=(0.4, -0.25), xytext=(0.4, -0.25),
                    va='top', xycoords='axes fraction',
                    textcoords='offset points',
                    fontsize=12)
        ax.annotate("Fiducials", xy=(0.9, -0.25),
                    xytext=(0.9, -0.25),
                    va='top', xycoords='axes fraction',
                    textcoords='offset points',
                    fontsize=12)

    #Plot fiducials
    # fid_comps = (num_fids**2 + num_fids) / 2
    x_fid_vals = np.arange(num_design, num_design + num_fids)
    prev = 0
    for i, posn in enumerate(np.arange(num_fids - 1, 0, -1)):
        ax.plot(x_fid_vals[:len(x_fid_vals)-i-1],
                fid_data[prev:posn+prev], "ko", alpha=0.6)
        prev += posn
    # Make the legend
    ax.legend(loc="upper right", prop={'size': 10})
    ax.set_xlim([-1, num_design + num_fids + 8])
    ax.set_xticks(np.append(x_vals, x_fid_vals))
    ax.set_xticklabels(xtick_labels+fid_labels, rotation=90, size=12)

def _horiz_obs_plot(ax, data, num_fids):
    '''
    Plot a horizontal line with surrounding shading across
    the plot to signify the distance of the observational data.
    '''

    # This eventually needs to be generalized
    labels_dict = {"ophA.13co.fits": "OphA",
                   "ngc1333.13co.fits": "NGC-1333",
                   "ic348.13co.fits": "IC-348"}

    x_vals = ax.axis()[:2]

    num_obs = data.shape[0] / num_fids

    obs_names = data.index[:num_obs]

    for i, obs in enumerate(obs_names):
        for j in range(num_fids):
            y_vals = 2*[data.ix[int(j * num_obs)+i]]
            ax.plot(x_vals, y_vals, "-", label="Fiducial " + str(j), alpha=0.4,
                    linewidth=3)

        y_vals = np.asarray(data.ix[i::num_obs])

        yposn = np.nanmean(y_vals)
        ymax = np.nanmax(y_vals)
        ymin = np.nanmin(y_vals)

        trans = ax.get_yaxis_transform()
        ax.annotate(labels_dict[obs], xy=(1.0, ymax), xytext=(1.03, yposn),
                    fontsize=15, xycoords=trans,
                    arrowprops=dict(facecolor='k',
                                    width=0.05, alpha=1.0, headwidth=0.1),
                    horizontalalignment='left',
                    verticalalignment='center')
        ax.annotate(labels_dict[obs], xy=(1.0, ymin), xytext=(1.03, yposn),
                    fontsize=15, xycoords=trans,
                    arrowprops=dict(facecolor='k',
                                    width=0.05, alpha=1.0, headwidth=0.1),
                    horizontalalignment='left',
                    verticalalignment='center')

def timestep_comparisons(path, verbose=False):
    '''
    Use pandas built-in plotting to look at the variation across time-steps.

    Parameters
    ----------
    path : str
        Path to files.
    verbose : bool, optional
        Enables plotting.
    '''
    data_files = [os.path.join(path, x) for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))
                  and x[-2:] == "h5"]
    if len(data_files) == 0:
        print "The inputed path contains no HDF5 files."
        return None

    data = [HDFStore(filename) for filename in data_files]

    for key in data[0].keys():
        for i, dataset in enumerate(data):
            # p.subplot(3,3,i)
            df = dataset[key].sort(axis=0).sort(axis=1)
            df.T.plot(style="D--")
            p.legend(prop={'size': 8}, loc="best")
            p.title(str(key)[1:])
            locs, xlabels = p.xticks(size=8)
            p.setp(xlabels, rotation=70)

            if verbose:
                p.show()
            else:
                p.savefig(
                    "timestep_comparisons_" + str(data_files[i][38:-19]) + "_" + str(key[1:]) + ".pdf")
                p.close()

def num_fiducials(N):
    '''

    Return the number of fiducials based on the number of lines in the
    comparison file.

    Parameters
    ----------
    N : int
        Number of rows in the data frame.
    '''

    n = 1

    while n < N:
        if n * (n - 1) == 2 * N:
            return n - 1
        else:
            n += 1

    return "Doesn't factor into an integer value."
<commit_msg>Overplot observational distances for each subplot<commit_after># Licensed under an MIT open source license - see LICENSE

import numpy as np
import os
import matplotlib as mpl
import matplotlib.pyplot as p
from pandas import read_csv

def comparison_plot(path, num_fids=5, verbose=False, obs_to_des=False,
                    obs_to_fid=False, obs_to_fid_data=None,
                    statistics=["Wavelet", "MVC", "PSpec", "Bispectrum",
                                "DeltaVariance", "Genus", "VCS",
                                "VCS_Density", "VCS_Velocity", "VCA",
                                "Tsallis", "PCA", "SCF", "Cramer",
                                "Skewness", "Kurtosis", "Dendrogram_Hist",
                                "Dendrogram_Num", "PDF"],
                    comparisons=["0_0", "0_1", "0_2", "1_0", "1_1", "1_2",
                                 "2_0", "2_1", "2_2"]):
    '''
    Requires results converted into csv form!!

    This function plots a comparison of the distances between the different
    simulations and the fiducial runs. All face combinations are checked for
    in the given path. The plots adjust to the available amount of data.

    Parameters
    ----------
    path : str
           Path to folder containing the HDF5 files with the distance results.
    analysis_fcn : function, optional
        Function to apply to the time-step data.
    verbose : bool, optional
        Enables plotting.
    cross_compare : bool, optional
        Include comparisons between faces.
    statistics : list, optional
        Statistics to plot. Default is all.
    comparisons : list, optional
        The face comparisons to include in the plots. The order is set here as
        well.
    '''

    if path[-1] != "/":
        path += "/"

    # Set the order by the given comparison list
    order = comparisons

    # All possible face combinations
    data_files = {"0_0": ["Face 0 to 0", None, None],
                  "1_1": ["Face 1 to 1", None, None],
                  "2_2": ["Face 2 to 2", None, None],
                  "0_1": ["Face 0 to 1", None, None],
                  "0_2": ["Face 0 to 2", None, None],
                  "1_2": ["Face 1 to 2", None, None],
                  "2_1": ["Face 2 to 1", None, None],
                  "2_0": ["Face 2 to 0", None, None],
                  "1_0": ["Face 1 to 0", None, None]}

    # Remove comparisons that aren't requested
    for key in data_files.keys():
        if not key in comparisons:
            del data_files[key]

    if obs_to_des:
        data_files["0_obs"] = ["Face 0 to Obs"],
        data_files["1_obs"] = ["Face 1 to Obs"],
        data_files["2_obs"] = ["Face 2 to Obs"],

    if obs_to_fid:
        obs_to_fid_data = {0: None,
                           1: None,
                           2: None}

    # Read in the data and match it to one of the face combinations.
    for x in os.listdir(path):
        if not os.path.isfile(os.path.join(path, x)):
            continue
        if not x[-3:] == "csv":
            continue

        # Separate out the observational csv files.
        if obs_to_fid and 'complete' in x:
            for key in obs_to_fid_data.keys():
                if "face_"+str(key):
                    obs_to_fid_data[key] = read_csv(os.path.join(path, x),
                                                    index_col=0)
                    break
        else:
            for key in data_files.keys():
                if key in x:
                    data = read_csv(os.path.join(path, x))
                    if "fiducial" in x:
                        data_files[key][1] = data
                    elif "distances" in x:
                        data_files[key][2] = data
                    else:
                        pass
                    break

    # Now delete the keys with no data
    for key in data_files.keys():
        if len(data_files[key]) == 1:
            del data_files[key]
            order.remove(key)

    if data_files.keys() == []:
        print "No csv files found in %s" % (path)
        return

    # Set the colour cycle
    colour_cycle = mpl.rcParams['axes.color_cycle']
    if len(colour_cycle) < num_fids:
        # Need to define our own in this case.
        mpl.rcParams['axes.color_cycle'] = \
            [mpl.cmap.jet(i) for i in np.linspace(0.0, 1.0, num_fids)]
    else:
        mpl.rcParams['axes.color_cycle'] = colour_cycle[:num_fids]

    for stat in statistics:
        # Divide by 2 b/c there should be 2 files for each comparison b/w faces
        (fig, ax) = _plot_size(len(data_files.keys()))
        if len(data_files.keys()) == 1:
            shape = (1, )
            ax = np.array([ax])
        else:
            shape = ax.shape

        if len(shape) == 1:
            ax = ax[:, np.newaxis]
            shape = ax.shape
        for k, (key, axis) in enumerate(zip(order, ax.flatten())):
            bottom = False
            if k >= len(ax.flatten()) - shape[1]:
                bottom = True
            if k / float(shape[0]) in [0, 1, 2]:
                left = True
            _plotter(axis, data_files[key][2][stat], data_files[key][1][stat],
                     num_fids, data_files[key][0], stat, bottom, left)
            if obs_to_fid:
                obs_key = int(key[0])
                _horiz_obs_plot(axis, obs_to_fid_data[obs_key][stat],
                                num_fids)

        if verbose:
            p.autoscale(True)
            fig.show()
        else:
            p.autoscale(True)
            fig.savefig("distance_comparisons_" + stat + ".pdf")
            fig.clf()

def _plot_size(num):
    if num <= 3:
        return p.subplots(num, sharex=True)
    elif num > 3 and num <= 8:
        rows = num / 2 + num % 2
        return p.subplots(nrows=rows, ncols=2, figsize=(14, 14), dpi=100, sharex=True)
    elif num == 9:
        return p.subplots(nrows=3, ncols=3, figsize=(14, 14), dpi=100, sharex=True)
    else:
        print "There should be a maximum of 9 comparisons."
        return

def _plotter(ax, data, fid_data, num_fids, title, stat, bottom, left):

    num_design = (max(data.shape) / num_fids)
    x_vals = np.arange(0, num_design)
    xtick_labels = [str(i) for i in x_vals]
    fid_labels = [str(i) for i in range(num_fids-1)]
    # Plot designs
    for i in range(num_fids):
        y_vals = data.ix[int(i * num_design):int(((i + 1) * num_design)-1)]
        ax.plot(x_vals, y_vals, "-o", label="Fiducial " + str(i), alpha=0.6)
    # Set title in upper left hand corner
    ax.annotate(title, xy=(0, 1), xytext=(12, -6), va='top',
                xycoords='axes fraction', textcoords='offset points',
                fontsize=12, alpha=0.75)
    if left:
        # Set the ylabel using the stat name. Replace underscores
        ax.set_ylabel(stat.replace("_", " ")+"\nDistance", fontsize=10,
                      multialignment='center')
    else:
        ax.set_ylabel("")

    # If the plot is on the bottom of a column, add labels
    if bottom:
        # Put two 'labels' for the x axis
        ax.annotate("Designs", xy=(0.4, -0.25), xytext=(0.4, -0.25),
                    va='top', xycoords='axes fraction',
                    textcoords='offset points',
                    fontsize=12)
        ax.annotate("Fiducials", xy=(0.9, -0.25),
                    xytext=(0.9, -0.25),
                    va='top', xycoords='axes fraction',
                    textcoords='offset points',
                    fontsize=12)

    #Plot fiducials
    # fid_comps = (num_fids**2 + num_fids) / 2
    x_fid_vals = np.arange(num_design, num_design + num_fids)
    prev = 0
    for i, posn in enumerate(np.arange(num_fids - 1, 0, -1)):
        ax.plot(x_fid_vals[:len(x_fid_vals)-i-1],
                fid_data[prev:posn+prev], "ko", alpha=0.6)
        prev += posn
    # Make the legend
    ax.legend(loc="upper right", prop={'size': 10})
    ax.set_xlim([-1, num_design + num_fids + 8])
    ax.set_xticks(np.append(x_vals, x_fid_vals))
    ax.set_xticklabels(xtick_labels+fid_labels, rotation=90, size=12)

def _horiz_obs_plot(ax, data, num_fids):
    '''
    Plot a horizontal line with surrounding shading across
    the plot to signify the distance of the observational data.
    '''

    # This eventually needs to be generalized
    labels_dict = {"ophA.13co.fits": "OphA",
                   "ngc1333.13co.fits": "NGC-1333",
                   "ic348.13co.fits": "IC-348"}

    x_vals = ax.axis()[:2]

    num_obs = data.shape[0] / num_fids

    obs_names = data.index[:num_obs]

    for i, obs in enumerate(obs_names):
        for j in range(num_fids):
            y_vals = 2*[data.ix[int(j * num_obs)+i]]
            ax.plot(x_vals, y_vals, "-", label="Fiducial " + str(j), alpha=0.4,
                    linewidth=3)

        y_vals = np.asarray(data.ix[i::num_obs])

        yposn = np.nanmean(y_vals)
        ymax = np.nanmax(y_vals)
        ymin = np.nanmin(y_vals)

        trans = ax.get_yaxis_transform()
        ax.annotate(labels_dict[obs], xy=(1.0, ymax), xytext=(1.03, yposn),
                    fontsize=15, xycoords=trans,
                    arrowprops=dict(facecolor='k',
                                    width=0.05, alpha=1.0, headwidth=0.1),
                    horizontalalignment='left',
                    verticalalignment='center')
        ax.annotate(labels_dict[obs], xy=(1.0, ymin), xytext=(1.03, yposn),
                    fontsize=15, xycoords=trans,
                    arrowprops=dict(facecolor='k',
                                    width=0.05, alpha=1.0, headwidth=0.1),
                    horizontalalignment='left',
                    verticalalignment='center')

def timestep_comparisons(path, verbose=False):
    '''
    Use pandas built-in plotting to look at the variation across time-steps.

    Parameters
    ----------
    path : str
        Path to files.
    verbose : bool, optional
        Enables plotting.
    '''
    data_files = [os.path.join(path, x) for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))
                  and x[-2:] == "h5"]
    if len(data_files) == 0:
        print "The inputed path contains no HDF5 files."
        return None

    data = [HDFStore(filename) for filename in data_files]

    for key in data[0].keys():
        for i, dataset in enumerate(data):
            # p.subplot(3,3,i)
            df = dataset[key].sort(axis=0).sort(axis=1)
            df.T.plot(style="D--")
            p.legend(prop={'size': 8}, loc="best")
            p.title(str(key)[1:])
            locs, xlabels = p.xticks(size=8)
            p.setp(xlabels, rotation=70)

            if verbose:
                p.show()
            else:
                p.savefig(
                    "timestep_comparisons_" + str(data_files[i][38:-19]) + "_" + str(key[1:]) + ".pdf")
                p.close()

def num_fiducials(N):
    '''

    Return the number of fiducials based on the number of lines in the
    comparison file.

    Parameters
    ----------
    N : int
        Number of rows in the data frame.
    '''

    n = 1

    while n < N:
        if n * (n - 1) == 2 * N:
            return n - 1
        else:
            n += 1

    return "Doesn't factor into an integer value."
<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE

import numpy as np
import os
import matplotlib as mpl
import matplotlib.pyplot as p
from pandas import read_csv, DataFrame

def comparison_plot(path, num_fids=5, verbose=False, obs_to_des=False,
                    obs_to_fid=False, obs_to_fid_shade=True, legend=True,
                    statistics=["Wavelet", "MVC", "PSpec", "Bispectrum",
                                "DeltaVariance", "Genus", "VCS",
                                "VCS_Density", "VCS_Velocity", "VCA",
                                "Tsallis", "PCA", "SCF", "Cramer",
                                "Skewness", "Kurtosis", "Dendrogram_Hist",
                                "Dendrogram_Num", "PDF"],
                    comparisons=["0_0", "0_1", "0_2", "1_0", "1_1", "1_2",
                                 "2_0", "2_1", "2_2"],
                    out_path=None, design_matrix=None):
    '''
    Requires results converted into csv form!!

    This function plots a comparison of the distances between the different
    simulations and the fiducial runs. All face combinations are checked for
    in the given path. The plots adjust to the available amount of data.

    Parameters
    ----------
    path : str
           Path to folder containing the HDF5 files with the distance results.
    analysis_fcn : function, optional
        Function to apply to the time-step data.
    verbose : bool, optional
        Enables plotting.
    obs_to_des : bool, optional
        Add in subplots where observational cubes are treayed as the fiducial.
    obs_to_fid : bool, optional
        Include observational to fiducial distances in the distance subplots.
    obs_to_fid_shade : bool, optional
        Plots the observational distances as a single band instead of by
        fiducial.
    statistics : list, optional
        Statistics to plot. Default is all.
    comparisons : list, optional
        The face comparisons to include in the plots. The order is set here as
        well.
    design_matrix : str or pandas.DataFrame, optional
        The experimental design, which is used for the distances. If a string,
        is given, it is assumed that it's the path to the saved csv file. When
        given, the labels of the plots will be coded in 'binary' according to
        the levels in the design.
    '''

    if path[-1] != "/":
        path += "/"

    # Set the order by the given comparison list
    order = comparisons

    # All possible face combinations
    data_files = {"0_0": ["Face 0 to 0", None, None],
                  "1_1": ["Face 1 to 1", None, None],
                  "2_2": ["Face 2 to 2", None, None],
                  "0_1": ["Face 0 to 1", None, None],
                  "0_2": ["Face 0 to 2", None, None],
                  "1_2": ["Face 1 to 2", None, None],
                  "2_1": ["Face 2 to 1", None, None],
                  "2_0": ["Face 2 to 0", None, None],
                  "1_0": ["Face 1 to 0", None, None]}

    # Remove comparisons that aren't requested
    for key in data_files.keys():
        if not key in comparisons:
            del data_files[key]

    if obs_to_des:
        data_files["0_obs"] = ["Face 0 to Obs"],
        data_files["1_obs"] = ["Face 1 to Obs"],
        data_files["2_obs"] = ["Face 2 to Obs"],

    if obs_to_fid:
        obs_to_fid_data = {0: None,
                           1: None,
                           2: None}

    # Read in the data and match it to one of the face combinations.
    for x in os.listdir(path):
        if not os.path.isfile(os.path.join(path, x)):
            continue
        if not x[-3:] == "csv":
            continue

        # Separate out the observational csv files.
        if obs_to_fid and 'complete' in x:
            for key in obs_to_fid_data.keys():
                if "face_"+str(key) in x:
                    obs_to_fid_data[key] = read_csv(os.path.join(path, x),
                                                    index_col=0)
                    break
        else:
            for key in data_files.keys():
                if key in x:
                    data = read_csv(os.path.join(path, x))
                    if "fiducial" in x:
                        data_files[key][1] = data
                    elif "distances" in x:
                        data_files[key][2] = data
                    else:
                        pass
                    break

    # Now delete the keys with no data
    for key in data_files.keys():
        if len(data_files[key]) == 1:
            del data_files[key]
            order.remove(key)

    if data_files.keys() == []:
        print "No csv files found in %s" % (path)
        return

    # Check for a design
    if design_matrix is not None:
        if isinstance(design_matrix, DataFrame):
            pass
        elif isinstance(design_matrix, unicode) or isinstance(design_matrix, str):
            design_matrix = read_csv(design_matrix, index_col=0)
        else:
            print type(design_matrix)
            raise TypeError("design_matrix must be a pandas.DataFrame or "
                            "a path to a csv file.")

        # Set -1 to 0 for cleanliness
        design_matrix[design_matrix == -1] = 0.0

        design_labels = []

        for ind in design_matrix.index():

            label = "".join([str(int(val)) for val in design_matrix.ix[ind]])

            design_labels.append(label)

        else:
            design_labels = None

    # Set the colour cycle
    colour_cycle = mpl.rcParams['axes.color_cycle']
    if len(colour_cycle) < num_fids:
        # Need to define our own in this case.
        mpl.rcParams['axes.color_cycle'] = \
            [mpl.cm.jet(i) for i in np.linspace(0.0, 1.0, num_fids)]
    else:
        mpl.rcParams['axes.color_cycle'] = colour_cycle[:num_fids]

    for stat in statistics:
        # Divide by 2 b/c there should be 2 files for each comparison b/w faces
        (fig, ax) = _plot_size(len(data_files.keys()))
        if len(data_files.keys()) == 1:
            shape = (1, )
            ax = np.array([ax])
        else:
            shape = ax.shape

        if len(shape) == 1:
            ax = ax[:, np.newaxis]
            shape = ax.shape
        for k, (key, axis) in enumerate(zip(order, ax.flatten())):
            bottom = False
            if k >= len(ax.flatten()) - shape[1]:
                bottom = True
            if k / float(shape[0]) in [0, 1, 2]:
                left = True
            _plotter(axis, data_files[key][2][stat], data_files[key][1][stat],
                     num_fids, data_files[key][0], stat, bottom, left,
                     legend=legend, labels=design_labels)
            if obs_to_fid:
                obs_key = int(key[0])
                _horiz_obs_plot(axis, obs_to_fid_data[obs_key][stat],
                                num_fids, shading=obs_to_fid_shade)

        if verbose:
            p.autoscale(True)
            fig.show()
        else:
            p.autoscale(True)
            save_name = "distance_comparisons_" + stat + ".pdf"
            if out_path is not None:
                if out_path[-1] != "/":
                    out_path += "/"
                save_name = out_path + save_name
            fig.savefig(save_name)
            fig.clf()

def _plot_size(num):
    if num <= 3:
        return p.subplots(num, sharex=True)
    elif num > 3 and num <= 8:
        rows = num / 2 + num % 2
        return p.subplots(nrows=rows, ncols=2, figsize=(14, 14), dpi=100, sharex=True)
    elif num == 9:
        return p.subplots(nrows=3, ncols=3, figsize=(14, 14), dpi=100, sharex=True)
    else:
        print "There should be a maximum of 9 comparisons."
        return

def _plotter(ax, data, fid_data, num_fids, title, stat, bottom, left,
             legend=True, labels=None):

    num_design = (max(data.shape) / num_fids)

    if labels is not None:
        if len(labels) != num_design:
            raise Warning("Design matrix contains different number of designs "
                          "than the data. Double check the inputted "
                          "design_matrix.")

    x_vals = np.arange(0, num_design)
    xtick_labels = [str(i) for i in x_vals]
    fid_labels = [str(i) for i in range(num_fids-1)]
    # Plot designs
    for i in range(num_fids):
        y_vals = data.ix[int(i * num_design):int(((i + 1) * num_design)-1)]
        ax.plot(x_vals, y_vals, "-o", label="Fiducial " + str(i), alpha=0.6)
    # Set title in upper left hand corner
    ax.annotate(title, xy=(0, 1), xytext=(12, -6), va='top',
                xycoords='axes fraction', textcoords='offset points',
                fontsize=12, alpha=0.75)
    if left:
        # Set the ylabel using the stat name. Replace underscores
        ax.set_ylabel(stat.replace("_", " ")+"\nDistance", fontsize=10,
                      multialignment='center')
    else:
        ax.set_ylabel("")

    # If the plot is on the bottom of a column, add labels
    if bottom:
        trans = ax.get_xaxis_transform()

        # Put two 'labels' for the x axis
        ax.annotate("Designs", xy=(num_design/2 - 9, -0.15),
                    xytext=(num_design/2 - 1, -0.15),
                    va='top', xycoords=trans,
                    fontsize=12)
        fid_x = num_design + num_fids/2 - 3
        ax.annotate("Fiducials", xy=(fid_x, -0.15),
                    xytext=(fid_x, -0.15),
                    va='top', xycoords=trans,
                    fontsize=12)

    #Plot fiducials
    # fid_comps = (num_fids**2 + num_fids) / 2
    x_fid_vals = np.arange(num_design, num_design + num_fids)
    prev = 0
    for i, posn in enumerate(np.arange(num_fids - 1, 0, -1)):
        ax.plot(x_fid_vals[:len(x_fid_vals)-i-1],
                fid_data[prev:posn+prev], "ko", alpha=0.6)
        prev += posn
    # Make the legend
    if legend:
        ax.legend(loc="upper right", prop={'size': 10})
    ax.set_xlim([-1, num_design + num_fids + 8])
    ax.set_xticks(np.append(x_vals, x_fid_vals))
    if labels is None:
        ax.set_xticklabels(xtick_labels+fid_labels, rotation=90, size=12)
    else:
        ax.set_xticklabels(labels, rotation=90, size=12)

def _horiz_obs_plot(ax, data, num_fids, shading=False):
    '''
    Plot a horizontal line with surrounding shading across
    the plot to signify the distance of the observational data.
    '''

    # This eventually needs to be generalized
    labels_dict = {"ophA.13co.fits": "OphA",
                   "ngc1333.13co.fits": "NGC 1333",
                   "ic348.13co.fits": "IC 348"}

    # Also needs to be generalized
    colors = ["r", "g", "b"]

    x_vals = ax.axis()[:2]

    num_obs = data.shape[0] / num_fids

    obs_names = data.index[:num_obs]

    for i, obs in enumerate(obs_names):

        y_vals = np.asarray(data.ix[i::num_obs])

        yposn = np.nanmean(y_vals)
        ymax = np.nanmax(y_vals)
        ymin = np.nanmin(y_vals)

        if shading:

            # Want to plot a single line at the mean, then shade to show
            # variance.

            ax.fill_between(x_vals, ymax, ymin, facecolor=colors[i],
                            interpolate=True, alpha=0.4,
                            edgecolor=colors[i])

            middle = (ymax + ymin) / 2

            trans = ax.get_yaxis_transform()
            ax.annotate(labels_dict[obs], xy=(0.9, middle), xytext=(0.9, middle),
                        fontsize=12, xycoords=trans,
                        verticalalignment='center',
                        horizontalalignment='center')

        else:

            for j in range(num_fids):
                y_vals = 2*[data.ix[int(j * num_obs)+i]]
                ax.plot(x_vals, y_vals, "-", label="Fiducial " + str(j),
                        alpha=0.4, linewidth=3)

            trans = ax.get_yaxis_transform()
            ax.annotate(labels_dict[obs], xy=(1.0, ymax), xytext=(1.03, yposn),
                        fontsize=15, xycoords=trans,
                        arrowprops=dict(facecolor='k',
                                        width=0.05, alpha=1.0, headwidth=0.1),
                        horizontalalignment='left',
                        verticalalignment='center')
            ax.annotate(labels_dict[obs], xy=(1.0, ymin), xytext=(1.03, yposn),
                        fontsize=15, xycoords=trans,
                        arrowprops=dict(facecolor='k',
                                        width=0.05, alpha=1.0, headwidth=0.1),
                        horizontalalignment='left',
                        verticalalignment='center')

def timestep_comparisons(path, verbose=False):
    '''
    Use pandas built-in plotting to look at the variation across time-steps.

    Parameters
    ----------
    path : str
        Path to files.
    verbose : bool, optional
        Enables plotting.
    '''
    data_files = [os.path.join(path, x) for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))
                  and x[-2:] == "h5"]
    if len(data_files) == 0:
        print "The inputed path contains no HDF5 files."
        return None

    data = [HDFStore(filename) for filename in data_files]

    for key in data[0].keys():
        for i, dataset in enumerate(data):
            # p.subplot(3,3,i)
            df = dataset[key].sort(axis=0).sort(axis=1)
            df.T.plot(style="D--")
            p.legend(prop={'size': 8}, loc="best")
            p.title(str(key)[1:])
            locs, xlabels = p.xticks(size=8)
            p.setp(xlabels, rotation=70)

            if verbose:
                p.show()
            else:
                p.savefig(
                    "timestep_comparisons_" + str(data_files[i][38:-19]) + "_" + str(key[1:]) + ".pdf")
                p.close()

def num_fiducials(N):
    '''

    Return the number of fiducials based on the number of lines in the
    comparison file.

    Parameters
    ----------
    N : int
        Number of rows in the data frame.
    '''

    n = 1

    while n < N:
        if n * (n - 1) == 2 * N:
            return n - 1
        else:
            n += 1

    return "Doesn't factor into an integer value."
<commit_msg>not callable<commit_after># Licensed under an MIT open source license - see LICENSE

import numpy as np
import os
import matplotlib as mpl
import matplotlib.pyplot as p
from pandas import read_csv, DataFrame

def comparison_plot(path, num_fids=5, verbose=False, obs_to_des=False,
                    obs_to_fid=False, obs_to_fid_shade=True, legend=True,
                    statistics=["Wavelet", "MVC", "PSpec", "Bispectrum",
                                "DeltaVariance", "Genus", "VCS",
                                "VCS_Density", "VCS_Velocity", "VCA",
                                "Tsallis", "PCA", "SCF", "Cramer",
                                "Skewness", "Kurtosis", "Dendrogram_Hist",
                                "Dendrogram_Num", "PDF"],
                    comparisons=["0_0", "0_1", "0_2", "1_0", "1_1", "1_2",
                                 "2_0", "2_1", "2_2"],
                    out_path=None, design_matrix=None):
    '''
    Requires results converted into csv form!!

    This function plots a comparison of the distances between the different
    simulations and the fiducial runs. All face combinations are checked for
    in the given path. The plots adjust to the available amount of data.

    Parameters
    ----------
    path : str
           Path to folder containing the HDF5 files with the distance results.
    analysis_fcn : function, optional
        Function to apply to the time-step data.
    verbose : bool, optional
        Enables plotting.
    obs_to_des : bool, optional
        Add in subplots where observational cubes are treayed as the fiducial.
    obs_to_fid : bool, optional
        Include observational to fiducial distances in the distance subplots.
    obs_to_fid_shade : bool, optional
        Plots the observational distances as a single band instead of by
        fiducial.
    statistics : list, optional
        Statistics to plot. Default is all.
    comparisons : list, optional
        The face comparisons to include in the plots. The order is set here as
        well.
    design_matrix : str or pandas.DataFrame, optional
        The experimental design, which is used for the distances. If a string,
        is given, it is assumed that it's the path to the saved csv file. When
        given, the labels of the plots will be coded in 'binary' according to
        the levels in the design.
    '''

    if path[-1] != "/":
        path += "/"

    # Set the order by the given comparison list
    order = comparisons

    # All possible face combinations
    data_files = {"0_0": ["Face 0 to 0", None, None],
                  "1_1": ["Face 1 to 1", None, None],
                  "2_2": ["Face 2 to 2", None, None],
                  "0_1": ["Face 0 to 1", None, None],
                  "0_2": ["Face 0 to 2", None, None],
                  "1_2": ["Face 1 to 2", None, None],
                  "2_1": ["Face 2 to 1", None, None],
                  "2_0": ["Face 2 to 0", None, None],
                  "1_0": ["Face 1 to 0", None, None]}

    # Remove comparisons that aren't requested
    for key in data_files.keys():
        if not key in comparisons:
            del data_files[key]

    if obs_to_des:
        data_files["0_obs"] = ["Face 0 to Obs"],
        data_files["1_obs"] = ["Face 1 to Obs"],
        data_files["2_obs"] = ["Face 2 to Obs"],

    if obs_to_fid:
        obs_to_fid_data = {0: None,
                           1: None,
                           2: None}

    # Read in the data and match it to one of the face combinations.
    for x in os.listdir(path):
        if not os.path.isfile(os.path.join(path, x)):
            continue
        if not x[-3:] == "csv":
            continue

        # Separate out the observational csv files.
        if obs_to_fid and 'complete' in x:
            for key in obs_to_fid_data.keys():
                if "face_"+str(key) in x:
                    obs_to_fid_data[key] = read_csv(os.path.join(path, x),
                                                    index_col=0)
                    break
        else:
            for key in data_files.keys():
                if key in x:
                    data = read_csv(os.path.join(path, x))
                    if "fiducial" in x:
                        data_files[key][1] = data
                    elif "distances" in x:
                        data_files[key][2] = data
                    else:
                        pass
                    break

    # Now delete the keys with no data
    for key in data_files.keys():
        if len(data_files[key]) == 1:
            del data_files[key]
            order.remove(key)

    if data_files.keys() == []:
        print "No csv files found in %s" % (path)
        return

    # Check for a design
    if design_matrix is not None:
        if isinstance(design_matrix, DataFrame):
            pass
        elif isinstance(design_matrix, unicode) or isinstance(design_matrix, str):
            design_matrix = read_csv(design_matrix, index_col=0)
        else:
            print type(design_matrix)
            raise TypeError("design_matrix must be a pandas.DataFrame or "
                            "a path to a csv file.")

        # Set -1 to 0 for cleanliness
        design_matrix[design_matrix == -1] = 0.0

        design_labels = []

        for ind in design_matrix.index:

            label = "".join([str(int(val)) for val in design_matrix.ix[ind]])

            design_labels.append(label)

        else:
            design_labels = None

    # Set the colour cycle
    colour_cycle = mpl.rcParams['axes.color_cycle']
    if len(colour_cycle) < num_fids:
        # Need to define our own in this case.
        mpl.rcParams['axes.color_cycle'] = \
            [mpl.cm.jet(i) for i in np.linspace(0.0, 1.0, num_fids)]
    else:
        mpl.rcParams['axes.color_cycle'] = colour_cycle[:num_fids]

    for stat in statistics:
        # Divide by 2 b/c there should be 2 files for each comparison b/w faces
        (fig, ax) = _plot_size(len(data_files.keys()))
        if len(data_files.keys()) == 1:
            shape = (1, )
            ax = np.array([ax])
        else:
            shape = ax.shape

        if len(shape) == 1:
            ax = ax[:, np.newaxis]
            shape = ax.shape
        for k, (key, axis) in enumerate(zip(order, ax.flatten())):
            bottom = False
            if k >= len(ax.flatten()) - shape[1]:
                bottom = True
            if k / float(shape[0]) in [0, 1, 2]:
                left = True
            _plotter(axis, data_files[key][2][stat], data_files[key][1][stat],
                     num_fids, data_files[key][0], stat, bottom, left,
                     legend=legend, labels=design_labels)
            if obs_to_fid:
                obs_key = int(key[0])
                _horiz_obs_plot(axis, obs_to_fid_data[obs_key][stat],
                                num_fids, shading=obs_to_fid_shade)

        if verbose:
            p.autoscale(True)
            fig.show()
        else:
            p.autoscale(True)
            save_name = "distance_comparisons_" + stat + ".pdf"
            if out_path is not None:
                if out_path[-1] != "/":
                    out_path += "/"
                save_name = out_path + save_name
            fig.savefig(save_name)
            fig.clf()

def _plot_size(num):
    if num <= 3:
        return p.subplots(num, sharex=True)
    elif num > 3 and num <= 8:
        rows = num / 2 + num % 2
        return p.subplots(nrows=rows, ncols=2, figsize=(14, 14), dpi=100, sharex=True)
    elif num == 9:
        return p.subplots(nrows=3, ncols=3, figsize=(14, 14), dpi=100, sharex=True)
    else:
        print "There should be a maximum of 9 comparisons."
        return

def _plotter(ax, data, fid_data, num_fids, title, stat, bottom, left,
             legend=True, labels=None):

    num_design = (max(data.shape) / num_fids)

    if labels is not None:
        if len(labels) != num_design:
            raise Warning("Design matrix contains different number of designs "
                          "than the data. Double check the inputted "
                          "design_matrix.")

    x_vals = np.arange(0, num_design)
    xtick_labels = [str(i) for i in x_vals]
    fid_labels = [str(i) for i in range(num_fids-1)]
    # Plot designs
    for i in range(num_fids):
        y_vals = data.ix[int(i * num_design):int(((i + 1) * num_design)-1)]
        ax.plot(x_vals, y_vals, "-o", label="Fiducial " + str(i), alpha=0.6)
    # Set title in upper left hand corner
    ax.annotate(title, xy=(0, 1), xytext=(12, -6), va='top',
                xycoords='axes fraction', textcoords='offset points',
                fontsize=12, alpha=0.75)
    if left:
        # Set the ylabel using the stat name. Replace underscores
        ax.set_ylabel(stat.replace("_", " ")+"\nDistance", fontsize=10,
                      multialignment='center')
    else:
        ax.set_ylabel("")

    # If the plot is on the bottom of a column, add labels
    if bottom:
        trans = ax.get_xaxis_transform()

        # Put two 'labels' for the x axis
        ax.annotate("Designs", xy=(num_design/2 - 9, -0.15),
                    xytext=(num_design/2 - 1, -0.15),
                    va='top', xycoords=trans,
                    fontsize=12)
        fid_x = num_design + num_fids/2 - 3
        ax.annotate("Fiducials", xy=(fid_x, -0.15),
                    xytext=(fid_x, -0.15),
                    va='top', xycoords=trans,
                    fontsize=12)

    #Plot fiducials
    # fid_comps = (num_fids**2 + num_fids) / 2
    x_fid_vals = np.arange(num_design, num_design + num_fids)
    prev = 0
    for i, posn in enumerate(np.arange(num_fids - 1, 0, -1)):
        ax.plot(x_fid_vals[:len(x_fid_vals)-i-1],
                fid_data[prev:posn+prev], "ko", alpha=0.6)
        prev += posn
    # Make the legend
    if legend:
        ax.legend(loc="upper right", prop={'size': 10})
    ax.set_xlim([-1, num_design + num_fids + 8])
    ax.set_xticks(np.append(x_vals, x_fid_vals))
    if labels is None:
        ax.set_xticklabels(xtick_labels+fid_labels, rotation=90, size=12)
    else:
        ax.set_xticklabels(labels, rotation=90, size=12)

def _horiz_obs_plot(ax, data, num_fids, shading=False):
    '''
    Plot a horizontal line with surrounding shading across
    the plot to signify the distance of the observational data.
    '''

    # This eventually needs to be generalized
    labels_dict = {"ophA.13co.fits": "OphA",
                   "ngc1333.13co.fits": "NGC 1333",
                   "ic348.13co.fits": "IC 348"}

    # Also needs to be generalized
    colors = ["r", "g", "b"]

    x_vals = ax.axis()[:2]

    num_obs = data.shape[0] / num_fids

    obs_names = data.index[:num_obs]

    for i, obs in enumerate(obs_names):

        y_vals = np.asarray(data.ix[i::num_obs])

        yposn = np.nanmean(y_vals)
        ymax = np.nanmax(y_vals)
        ymin = np.nanmin(y_vals)

        if shading:

            # Want to plot a single line at the mean, then shade to show
            # variance.

            ax.fill_between(x_vals, ymax, ymin, facecolor=colors[i],
                            interpolate=True, alpha=0.4,
                            edgecolor=colors[i])

            middle = (ymax + ymin) / 2

            trans = ax.get_yaxis_transform()
            ax.annotate(labels_dict[obs], xy=(0.9, middle), xytext=(0.9, middle),
                        fontsize=12, xycoords=trans,
                        verticalalignment='center',
                        horizontalalignment='center')

        else:

            for j in range(num_fids):
                y_vals = 2*[data.ix[int(j * num_obs)+i]]
                ax.plot(x_vals, y_vals, "-", label="Fiducial " + str(j),
                        alpha=0.4, linewidth=3)

            trans = ax.get_yaxis_transform()
            ax.annotate(labels_dict[obs], xy=(1.0, ymax), xytext=(1.03, yposn),
                        fontsize=15, xycoords=trans,
                        arrowprops=dict(facecolor='k',
                                        width=0.05, alpha=1.0, headwidth=0.1),
                        horizontalalignment='left',
                        verticalalignment='center')
            ax.annotate(labels_dict[obs], xy=(1.0, ymin), xytext=(1.03, yposn),
                        fontsize=15, xycoords=trans,
                        arrowprops=dict(facecolor='k',
                                        width=0.05, alpha=1.0, headwidth=0.1),
                        horizontalalignment='left',
                        verticalalignment='center')

def timestep_comparisons(path, verbose=False):
    '''
    Use pandas built-in plotting to look at the variation across time-steps.

    Parameters
    ----------
    path : str
        Path to files.
    verbose : bool, optional
        Enables plotting.
    '''
    data_files = [os.path.join(path, x) for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))
                  and x[-2:] == "h5"]
    if len(data_files) == 0:
        print "The inputed path contains no HDF5 files."
        return None

    data = [HDFStore(filename) for filename in data_files]

    for key in data[0].keys():
        for i, dataset in enumerate(data):
            # p.subplot(3,3,i)
            df = dataset[key].sort(axis=0).sort(axis=1)
            df.T.plot(style="D--")
            p.legend(prop={'size': 8}, loc="best")
            p.title(str(key)[1:])
            locs, xlabels = p.xticks(size=8)
            p.setp(xlabels, rotation=70)

            if verbose:
                p.show()
            else:
                p.savefig(
                    "timestep_comparisons_" + str(data_files[i][38:-19]) + "_" + str(key[1:]) + ".pdf")
                p.close()

def num_fiducials(N):
    '''

    Return the number of fiducials based on the number of lines in the
    comparison file.

    Parameters
    ----------
    N : int
        Number of rows in the data frame.
    '''

    n = 1

    while n < N:
        if n * (n - 1) == 2 * N:
            return n - 1
        else:
            n += 1

    return "Doesn't factor into an integer value."
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os
import io
import datetime
import json
import pytz
from werkzeug.datastructures import FileStorage
from flask import current_app as app, url_for
from flask.ext.restful import reqparse
from tabular_validator import pipeline
from tabular_validator import exceptions
from tvweb import compat

REMOTE_SCHEMES = ('http', 'https', 'ftp', 'ftps')
JSON_DATE_FORMAT = '%Y-%m-%d'
JSON_TIME_FORMAT = '%H:%M:%S'
JSON_DATETIME_FORMAT = '{0}T{1}'.format(JSON_DATE_FORMAT, JSON_TIME_FORMAT)

def get_datetime(date_only=False):
    """Simple wrapper to get a UTC aware `now` or `date`."""
    now = datetime.datetime.now(pytz.utc)
    if date_only:
        return now.date()
    return now

def get_pipeline(runargs=None):
    """Get a pipeline as per config."""
    config = app.config['TVWEB_PIPELINE_DEFAULT_CONFIG']
    if runargs:
        config['options']['schema']['schema'] = runargs.pop('schema')
        config['options']['structure']['ignore_empty_rows'] = runargs.pop('ignore_empty_rows')
        config.update(runargs)
    try:
        rv = pipeline.Pipeline(**config)
    except exceptions.PipelineBuildError as e:
        rv = None
    return rv

def get_runargs():
    """Get valid args from request."""
    parser = reqparse.RequestParser()
    location = ['json', 'values', 'form', 'args', 'files']
    parser.add_argument('data', location=location)
    parser.add_argument('schema', location=location)
    parser.add_argument('data_url', location=location)
    parser.add_argument('schema_url', location=location)
    parser.add_argument('data_file', type=FileStorage, location=location)
    parser.add_argument('schema_file', type=FileStorage, location=location)
    parser.add_argument('report_limit', type=int, default=1000)
    parser.add_argument('row_limit', type=int, default=20000)
    parser.add_argument('fail_fast', type=bool)
    parser.add_argument('ignore_empty_rows', type=bool)
    parser.add_argument('format')
    parser.add_argument('encoding')
    return parser.parse_args()

def clean_payload(payload):
    payload['data'] = resolve_payload_item('data', payload)
    payload['schema'] = resolve_payload_item('schema', payload)
    del payload['data_url']
    del payload['data_file']
    del payload['schema_url']
    del payload['schema_file']
    return payload

def resolve_payload_item(key, payload):
    """Resolve data or schema from request."""
    as_file = '{0}_file'.format(key)
    as_url = '{0}_url'.format(key)
    if payload.get(key):
        return payload[key]
    if payload.get(as_url):
        return payload[as_url]
    elif payload.get(as_file):
        if key.startswith('data'):
            return payload[as_file].stream
        else:
            return payload[as_file].read().decode('utf-8')
    else:
        return None

def get_report_permalinks(payload):

    if not isinstance(payload['data'], (io.IOBase, FileStorage)) and \
           compat.parse.urlparse(payload['data']).scheme in REMOTE_SCHEMES:
        if not hasattr(payload.get('schema'), 'filename'):
            domain = app.config['TVWEB_URL']
            api_fragment = url_for('api.run')
            ui_fragment = url_for('pages.reports')
            params = compat.urlencode({'data': payload['data'], 'schema': payload['schema'] or ''})
            permalinks = {
                'html': '{0}{1}?{2}'.format(domain, ui_fragment, params),
                'json': '{0}{1}?{2}'.format(domain, api_fragment, params)
            }

            return permalinks

    return {}
<commit_msg>Pass all params for get URL<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os
import io
import datetime
import json
import pytz
from werkzeug.datastructures import FileStorage
from flask import current_app as app, url_for
from flask.ext.restful import reqparse
from tabular_validator import pipeline
from tabular_validator import exceptions
from tvweb import compat

REMOTE_SCHEMES = ('http', 'https', 'ftp', 'ftps')
JSON_DATE_FORMAT = '%Y-%m-%d'
JSON_TIME_FORMAT = '%H:%M:%S'
JSON_DATETIME_FORMAT = '{0}T{1}'.format(JSON_DATE_FORMAT, JSON_TIME_FORMAT)

def get_datetime(date_only=False):
    """Simple wrapper to get a UTC aware `now` or `date`."""
    now = datetime.datetime.now(pytz.utc)
    if date_only:
        return now.date()
    return now

def get_pipeline(runargs=None):
    """Get a pipeline as per config."""
    config = app.config['TVWEB_PIPELINE_DEFAULT_CONFIG']
    if runargs:
        config['options']['schema']['schema'] = runargs.pop('schema')
        config['options']['structure']['ignore_empty_rows'] = runargs.pop('ignore_empty_rows')
        config.update(runargs)
    try:
        rv = pipeline.Pipeline(**config)
    except exceptions.PipelineBuildError as e:
        rv = None
    return rv

def get_runargs():
    """Get valid args from request."""
    parser = reqparse.RequestParser()
    location = ['json', 'values', 'form', 'args', 'files']
    parser.add_argument('data', location=location)
    parser.add_argument('schema', location=location)
    parser.add_argument('data_url', location=location)
    parser.add_argument('schema_url', location=location)
    parser.add_argument('data_file', type=FileStorage, location=location)
    parser.add_argument('schema_file', type=FileStorage, location=location)
    parser.add_argument('report_limit', type=int, default=1000)
    parser.add_argument('row_limit', type=int, default=20000)
    parser.add_argument('fail_fast', type=bool)
    parser.add_argument('ignore_empty_rows', type=bool)
    parser.add_argument('format')
    parser.add_argument('encoding')
    return parser.parse_args()

def clean_payload(payload):
    payload['data'] = resolve_payload_item('data', payload)
    payload['schema'] = resolve_payload_item('schema', payload)
    del payload['data_url']
    del payload['data_file']
    del payload['schema_url']
    del payload['schema_file']
    return payload

def resolve_payload_item(key, payload):
    """Resolve data or schema from request."""
    as_file = '{0}_file'.format(key)
    as_url = '{0}_url'.format(key)
    if payload.get(key):
        return payload[key]
    if payload.get(as_url):
        return payload[as_url]
    elif payload.get(as_file):
        if key.startswith('data'):
            return payload[as_file].stream
        else:
            return payload[as_file].read().decode('utf-8')
    else:
        return None

def get_report_permalinks(payload):

    if not isinstance(payload['data'], (io.IOBase, FileStorage)) and \
           compat.parse.urlparse(payload['data']).scheme in REMOTE_SCHEMES:
        if not hasattr(payload.get('schema'), 'filename'):
            domain = app.config['TVWEB_URL']
            api_fragment = url_for('api.run')
            ui_fragment = url_for('pages.reports')
            params = compat.urlencode({k: v for k, v in payload.items() if v})
            permalinks = {
                'html': '{0}{1}?{2}'.format(domain, ui_fragment, params),
                'json': '{0}{1}?{2}'.format(domain, api_fragment, params)
            }

            return permalinks

    return {}
<|endoftext|>


<commit_before>from twisted.internet import defer
from zope.interface import classProvides, implements
from automatron.command import IAutomatronCommandHandler
from automatron.plugin import IAutomatronPluginFactory, STOP

class AutomatronControlPlugin(object):
    classProvides(IAutomatronPluginFactory)
    implements(IAutomatronCommandHandler)

    name = 'notify_control'
    priority = 100

    def __init__(self, controller):
        self.controller = controller

    command_map = {
        #command: (help, min_args, max_args, permission)
        'join': ('<channel> [key]', 1, 2, 'channel'),
        'leave': ('<channel> [reason]', 1, 2, 'channel'),
        'say': ('<channel> <message>', 2, 2, 'say'),
        'identify': ('[channel]', 0, 1, None)
    }

    def on_command(self, client, user, command, args):
        if command in self.command_map:
            self._on_command(client, user, command, args)
            return STOP

    @defer.inlineCallbacks
    def _on_command(self, client, user, command, args):
        nickname = client.parse_user(user)[0]
        config = self.command_map[command]

        if config[3] is not None:
            if not (yield self.controller.config.has_permission(client.server, None, user, config[3])):
                client.msg(nickname, 'You\'re not authorized to do that.')
                return

        if not config[1] <= len(args) <= config[2]:
            client.msg(nickname, 'Invalid syntax. Use: %s %s' % (command, config[0]))
            return

        getattr(self, '_on_command_%s' % command)(client, user, *args)

    def _on_command_join(self, client, user, channel, key=None):
        if key is not None:
            self.controller.config.update_value('channel', client.server, channel, 'key', key)
            client.join(channel, key)
        else:
            d = self.controller.config.get_value('channel', client.server, channel, 'key')
            d.addCallback(lambda channel_key, _: client.join(channel, channel_key))

    def _on_command_leave(self, client, user, channel, reason=None):
        client.leave(channel, reason if reason is not None else 'Leaving...')

    def _on_command_say(self, client, user, channel, message):
        client.msg(channel, message)

    @defer.inlineCallbacks
    def _on_command_identify(self, client, user, channel=None):
        nickname = client.parse_user(user)[0]
        username, username_relevance = yield self.controller.config.get_username_by_hostmask(client.server, user)
        if username is not None:
            if username_relevance == 0:
                identity = 'You are globally known as %s' % username
            else:
                identity = 'You are known as %s' % username

            role, role_relevance = yield self.controller.config.get_role_by_username(client.server, channel, username)
            if role_relevance is not None and role_relevance < username_relevance:
                role = role_relevance = None

            if role_relevance is None:
                client.msg(nickname, identity)
            elif role_relevance in (2, 3):
                client.msg(nickname, '%s and your role in %s is %s' % (identity, channel, role))
            else:
                client.msg(nickname, '%s and your role is %s' % (identity, role))
        else:
            client.msg(nickname, 'I don\'t know you...')
<commit_msg>Small cleanup in leave command.<commit_after>from twisted.internet import defer
from zope.interface import classProvides, implements
from automatron.command import IAutomatronCommandHandler
from automatron.plugin import IAutomatronPluginFactory, STOP

class AutomatronControlPlugin(object):
    classProvides(IAutomatronPluginFactory)
    implements(IAutomatronCommandHandler)

    name = 'notify_control'
    priority = 100

    def __init__(self, controller):
        self.controller = controller

    command_map = {
        #command: (help, min_args, max_args, permission)
        'join': ('<channel> [key]', 1, 2, 'channel'),
        'leave': ('<channel> [reason]', 1, 2, 'channel'),
        'say': ('<channel> <message>', 2, 2, 'say'),
        'identify': ('[channel]', 0, 1, None)
    }

    def on_command(self, client, user, command, args):
        if command in self.command_map:
            self._on_command(client, user, command, args)
            return STOP

    @defer.inlineCallbacks
    def _on_command(self, client, user, command, args):
        nickname = client.parse_user(user)[0]
        config = self.command_map[command]

        if config[3] is not None:
            if not (yield self.controller.config.has_permission(client.server, None, user, config[3])):
                client.msg(nickname, 'You\'re not authorized to do that.')
                return

        if not config[1] <= len(args) <= config[2]:
            client.msg(nickname, 'Invalid syntax. Use: %s %s' % (command, config[0]))
            return

        getattr(self, '_on_command_%s' % command)(client, user, *args)

    def _on_command_join(self, client, user, channel, key=None):
        if key is not None:
            self.controller.config.update_value('channel', client.server, channel, 'key', key)
            client.join(channel, key)
        else:
            d = self.controller.config.get_value('channel', client.server, channel, 'key')
            d.addCallback(lambda channel_key, _: client.join(channel, channel_key))

    def _on_command_leave(self, client, user, channel, reason='Leaving...'):
        client.leave(channel, reason)

    def _on_command_say(self, client, user, channel, message):
        client.msg(channel, message)

    @defer.inlineCallbacks
    def _on_command_identify(self, client, user, channel=None):
        nickname = client.parse_user(user)[0]
        username, username_relevance = yield self.controller.config.get_username_by_hostmask(client.server, user)
        if username is not None:
            if username_relevance == 0:
                identity = 'You are globally known as %s' % username
            else:
                identity = 'You are known as %s' % username

            role, role_relevance = yield self.controller.config.get_role_by_username(client.server, channel, username)
            if role_relevance is not None and role_relevance < username_relevance:
                role = role_relevance = None

            if role_relevance is None:
                client.msg(nickname, identity)
            elif role_relevance in (2, 3):
                client.msg(nickname, '%s and your role in %s is %s' % (identity, channel, role))
            else:
                client.msg(nickname, '%s and your role is %s' % (identity, role))
        else:
            client.msg(nickname, 'I don\'t know you...')
<|endoftext|>