<commit_before>from plugin_manager import BasePlugin
from utils import Command, respond, process_args
from random import choice
from asyncio import get_event_loop

class MusicPlayer(BasePlugin):
    name = "music_player"
    default_config = {
        'music_channel': "CHANNEL ID HERE",
        'force_music_channel': False,
        'no_permission_lines': [
            "**NEGATIVE. Insufficient permissions for funky beats in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for rocking you like a hurricane in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for putting hands in the air in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for wanting to rock in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for dropping the beat in channel: {}.**"
        ],
        'max_video_length': 10 * 60,
        'max_queue_length': 30,
        'default_volume': 100,
        'ytdl_options': {
            'format': 'bestaudio/best',
            'extractaudio': True,
            'audioformat': 'mp3',
            'outtmpl': '%(extractor)s-%(id)s-%(title)s.%(ext)s',
            'restrictfilenames': True,
            'noplaylist': True,
            'nocheckcertificate': True,
            'ignoreerrors': False,
            'logtostderr': False,
            'quiet': True,
            'no_warnings': False,
            'default_search': 'auto',
            'source_address': '0.0.0.0'
        }
    }

    def activate(self):
        c = self.plugin_config
        self.vc = False
        self.player = False
        self.queue = []
        # TODO move no_perm_lines to config
        self.no_perm_lines = c.no_permission_lines
        self.ytdl_options = c.ytdl_options
        self.volume = c.default_volume
        self.max_length = c.max_video_length
        self.max_queue = c.max_queue_length
        self.m_channel = c.music_channel if c.force_music_channel else False

    @Command("joinvc",
             category="voice",
             syntax="",
             doc="Joins same voice channel as user.")
    async def _joinvc(self, data):
        """
        Joins the same voice chat as the caller.
        Checks the permissions before joining - must be able to join, speak and not use ptt.
        """
        # leave if there's already a vc client in self.
        if self.vc:
            self.vc.disconnect()
        for server in self.client.servers:
            # doublecheck, just in case bot crashed earlier and discord is being weird
            if self.client.is_voice_connected(server):
                self.client.voice_client_in(server).disconnect()
            a_voice = self.m_channel
            if not self.m_channel:
                a_voice = data.author.voice.voice_channel
            perms = server.me.permissions_in(a_voice)
            if perms.connect and perms.speak and perms.use_voice_activation:
                self.vc = await self.client.join_voice_channel(a_voice)
                await respond(self.client, data, f"**AFFIRMATIVE. Connected to: {a_voice}.**")
            else:
                await respond(self.client, data, choice(self.no_perm_lines).format(a_voice))

    @Command("playvc",
             category="voice",
             syntax="[URL or search query]",
             doc="Plays presented youtube video or searches for one.\nNO PLAYLISTS ALLOWED.")
    async def _playvc(self, data):
        """
        Decorates the input to make sure ytdl can eat it and filters out playlists before pushing the video in the
        queue.
        """
        if not self.vc:
            await respond(self.client, data, "**WARNING: Can not play music while not connected.**")
            return
        args = data.content.split(' ', 1)
        if len(args) > 1:
            if not (args[1].startswith("http://") or args[1].startswith("https://")):
                args[1] = "ytsearch:" + args[1]
            if args[1].find("list=") > -1:
                raise SyntaxWarning("No playlists allowed!")
            await self.play_video(args[1], data)
        else:
            raise SyntaxError("Expected URL or search query.")

    async def play_video(self, vid, data):
        """
        Processes provided video request, either starting to play it instantly or adding it to queue.
        :param vid: URL or ytsearch: query to process
        :param data: message data for responses
        """
        if self.player and not self.player.is_done():
            if len(self.queue)<self.max_queue:
                self.queue.append(vid)
                t_string = "\n".join(self.queue)
                await respond(self.client, data, f"**AFFIRMATIVE. ADDING \"{vid}\" to queue.\n"
                                                 f"Current queue:**\n```{t_string}```")
            else:
                await respond(self.client, data, f"**NEGATIVE. ANALYSIS: Queue full. Dropping \"{vid}\".\n"
                                                 f"Current queue:**\n```{t_string}```")
        else:
            t_loop = get_event_loop()
            # creates a player with a callback to play next video
            self.player = await self.vc.create_ytdl_player(vid, ytdl_options=self.ytdl_options,
                                                           after=lambda: t_loop.create_task(self.play_next(
                                                                   data)))
            if self.player.duration <= self.max_length:
                self.player.volume = self.volume / 100
                self.player.start()
                await respond(self.client, data, f"**CURRENTLY PLAYING: \"{vid}\"**")
            else:
                self.player.stop()
                await respond(self.client, data, f"**WARNING: \"{vid}\" is too long, skipping.**")

    async def play_next(self, data):
        """
        Attempts to play next video in queue
        :param data: message data for responses
        """
        if len(self.queue) > 0:
            await self.play_video(self.queue.pop(0), data)

    @Command("skipvc",
             category="voice",
             syntax="",
             doc="Skips current video.")
    async def _skipvc(self, data):
        if self.player:
            self.player.stop()

    @Command("volvc",
             category="voice",
             syntax="[volume from 0 to 200]",
             doc="Adjusts volume, from 0 to 200%.")
    async def _volvc(self, data):
        """
        Checks that the user didn't put in something stupid and adjusts volume.
        """
        args = process_args(data.content.split())
        if len(args) > 1:
            try:
                vol = int(args[1])
            except ValueError:
                raise SyntaxError("Expected integer value between 0 and 200!")
            if vol < 0:
                vol = 0
            if vol > 200:
                vol = 200
            self.volume = vol
            if self.player:
                self.player.volume = vol / 100
        else:
            raise SyntaxError("Expected integer value between 0 and 200!")

    @Command("stopvc",
             perms={"mute_members"},
             category="voice",
             syntax="",
             doc="Stops the music. All of it.")
    async def _stopvc(self, data):
        if len(self.queue) > 0:
            self.queue = []
        if self.player:
            self.player.stop()
        await respond(self.client, data, "**AFFIRMATIVE. Ceasing the rhythmical noise.**")

    @Command("queuevc",
             category="voice",
             syntax="",
             doc="Writes out the current queue.")
    async def _queuevc(self, data):
        if len(self.queue) > 0:
            t_string = "\n".join(self.queue)
            await respond(self.client, data, f"**ANALYSIS: Current queue:**\n```{t_string}```")
        else:
            await respond(self.client, data, "**ANALYSIS: queie empty.**")
<commit_msg>Skipping is now democracy. Unless you just do it.<commit_after>from plugin_manager import BasePlugin
from utils import Command, respond, process_args
from random import choice
from asyncio import get_event_loop

class MusicPlayer(BasePlugin):
    name = "music_player"
    default_config = {
        'music_channel': "CHANNEL ID HERE",
        'force_music_channel': False,
        'no_permission_lines': [
            "**NEGATIVE. Insufficient permissions for funky beats in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for rocking you like a hurricane in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for putting hands in the air in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for wanting to rock in channel: {}.**",
            "**NEGATIVE. Insufficient permissions for dropping the beat in channel: {}.**"
        ],
        'max_video_length': 10 * 60,
        'max_queue_length': 30,
        'default_volume': 100,
        'ytdl_options': {
            'format': 'bestaudio/best',
            'extractaudio': True,
            'audioformat': 'mp3',
            'outtmpl': '%(extractor)s-%(id)s-%(title)s.%(ext)s',
            'restrictfilenames': True,
            'noplaylist': True,
            'nocheckcertificate': True,
            'ignoreerrors': False,
            'logtostderr': False,
            'quiet': True,
            'no_warnings': False,
            'default_search': 'auto',
            'source_address': '0.0.0.0'
        }
    }

    def activate(self):
        c = self.plugin_config
        self.vc = False
        self.player = False
        self.queue = []
        self.vote_set = set()
        # stuff from config
        self.no_perm_lines = c.no_permission_lines
        self.ytdl_options = c.ytdl_options
        self.volume = c.default_volume
        self.max_length = c.max_video_length
        self.max_queue = c.max_queue_length
        self.m_channel = c.music_channel if c.force_music_channel else False

    @Command("joinvc",
             category="voice",
             syntax="",
             doc="Joins same voice channel as user.")
    async def _joinvc(self, data):
        """
        Joins the same voice chat as the caller.
        Checks the permissions before joining - must be able to join, speak and not use ptt.
        """
        # leave if there's already a vc client in self.
        if self.vc:
            self.vc.disconnect()
        for server in self.client.servers:
            # doublecheck, just in case bot crashed earlier and discord is being weird
            if self.client.is_voice_connected(server):
                self.client.voice_client_in(server).disconnect()
            a_voice = self.m_channel
            if not self.m_channel:
                a_voice = data.author.voice.voice_channel
            perms = server.me.permissions_in(a_voice)
            if perms.connect and perms.speak and perms.use_voice_activation:
                self.vc = await self.client.join_voice_channel(a_voice)
                await respond(self.client, data, f"**AFFIRMATIVE. Connected to: {a_voice}.**")
            else:
                await respond(self.client, data, choice(self.no_perm_lines).format(a_voice))

    @Command("playvc",
             category="voice",
             syntax="[URL or search query]",
             doc="Plays presented youtube video or searches for one.\nNO PLAYLISTS ALLOWED.")
    async def _playvc(self, data):
        """
        Decorates the input to make sure ytdl can eat it and filters out playlists before pushing the video in the
        queue.
        """
        if not self.vc:
            await respond(self.client, data, "**WARNING: Can not play music while not connected.**")
            return
        args = data.content.split(' ', 1)
        if len(args) > 1:
            if not (args[1].startswith("http://") or args[1].startswith("https://")):
                args[1] = "ytsearch:" + args[1]
            if args[1].find("list=") > -1:
                raise SyntaxWarning("No playlists allowed!")
            await self.play_video(args[1], data)
        else:
            raise SyntaxError("Expected URL or search query.")

    async def play_video(self, vid, data):
        """
        Processes provided video request, either starting to play it instantly or adding it to queue.
        :param vid: URL or ytsearch: query to process
        :param data: message data for responses
        """
        if self.player and not self.player.is_done():
            if len(self.queue)<self.max_queue:
                self.queue.append(vid)
                t_string = "\n".join(self.queue)
                await respond(self.client, data, f"**AFFIRMATIVE. ADDING \"{vid}\" to queue.\n"
                                                 f"Current queue:**\n```{t_string}```")
            else:
                await respond(self.client, data, f"**NEGATIVE. ANALYSIS: Queue full. Dropping \"{vid}\".\n"
                                                 f"Current queue:**\n```{t_string}```")
        else:
            t_loop = get_event_loop()
            self.vote_set = set()
            # creates a player with a callback to play next video
            self.player = await self.vc.create_ytdl_player(vid, ytdl_options=self.ytdl_options,
                                                           after=lambda: t_loop.create_task(self.play_next(
                                                                   data)))
            if self.player.duration <= self.max_length:
                self.player.volume = self.volume / 100
                self.player.start()
                await respond(self.client, data, f"**CURRENTLY PLAYING: \"{vid}\"**")
            else:
                self.player.stop()
                await respond(self.client, data, f"**WARNING: \"{vid}\" is too long, skipping.**")

    async def play_next(self, data):
        """
        Attempts to play next video in queue
        :param data: message data for responses
        """
        if len(self.queue) > 0:
            await self.play_video(self.queue.pop(0), data)

    @Command("skipvc",
             category="voice",
             syntax="",
             doc="Skips current video.")
    async def _skipvc(self, data):
        """
        Collects votes for skipping current song or skips if you got mute_members permission
        """
        self.vote_set.add(data.author.id)
        override = data.author.permissions_in(self.vc.channel).mute_members
        votes = len(self.vote_set)
        m_votes = len(self.vc.channel.voice_members)/2
        if votes >= m_votes or override:
            if self.player:
                self.player.stop()
                await respond(self.client, data, "**AFFIRMATIVE. Skipping current song.**"
                              if not override else "**AFFIRMATIVE. Override accepted. Skipping current song.**")
        else:
            await respond(self.client, data, f"**Skip vote: ACCEPTED. {votes} out of required {}**")

    @Command("volvc",
             category="voice",
             syntax="[volume from 0 to 200]",
             doc="Adjusts volume, from 0 to 200%.")
    async def _volvc(self, data):
        """
        Checks that the user didn't put in something stupid and adjusts volume.
        """
        args = process_args(data.content.split())
        if len(args) > 1:
            try:
                vol = int(args[1])
            except ValueError:
                raise SyntaxError("Expected integer value between 0 and 200!")
            if vol < 0:
                vol = 0
            if vol > 200:
                vol = 200
            self.volume = vol
            if self.player:
                self.player.volume = vol / 100
        else:
            raise SyntaxError("Expected integer value between 0 and 200!")

    @Command("stopvc",
             perms={"mute_members"},
             category="voice",
             syntax="",
             doc="Stops the music. All of it.")
    async def _stopvc(self, data):
        if len(self.queue) > 0:
            self.queue = []
        if self.player:
            self.player.stop()
        await respond(self.client, data, "**AFFIRMATIVE. Ceasing the rhythmical noise.**")

    @Command("queuevc",
             category="voice",
             syntax="",
             doc="Writes out the current queue.")
    async def _queuevc(self, data):
        if len(self.queue) > 0:
            t_string = "\n".join(self.queue)
            await respond(self.client, data, f"**ANALYSIS: Current queue:**\n```{t_string}```")
        else:
            await respond(self.client, data, "**ANALYSIS: queie empty.**")
<|endoftext|>


<commit_before>#def apptest(environ, start_response):
#    status = '200 OK'
#    output = 'Hello World from within the hiv app, after all that!'
#
#    response_headers = [('Content-type', 'text/plain'),
#                        ('Content-Length', str(len(output)))]
#
#    start_response(status, response_headers)
#
#    return [output]
from flask import Flask

# TODO: fix static folder, this requires adapting HTML templates
hiv = Flask(__name__)
hiv.config.from_object('config')

# welcome page is the index
from .blueprints.welcome import welcome
hiv.register_blueprint(welcome, url_prefix='')

from .blueprints.tutorial import tutorial
hiv.register_blueprint(tutorial, url_prefix='/tutorial')

from .blueprints.patient import patient
hiv.register_blueprint(patient)

from .blueprints.region import region
hiv.register_blueprint(region)

from .blueprints.data import data
hiv.register_blueprint(data)

from .blueprints.method import method
hiv.register_blueprint(method)

from .blueprints.static import static
hiv.register_blueprint(static)

from .blueprints.download import download
hiv.register_blueprint(download)

# RESTful API
from .blueprints.api import api_bp
hiv.register_blueprint(api_bp)
<commit_msg>tiny docs<commit_after># vim: fdm=indent
'''
author:     Fabio Zanini
date:       13/06/15
content:    Hivwholeweb: a web application for longitudinal NGS data of HIV-1.
'''
#def apptest(environ, start_response):
#    status = '200 OK'
#    output = 'Hello World from within the hiv app, after all that!'
#
#    response_headers = [('Content-type', 'text/plain'),
#                        ('Content-Length', str(len(output)))]
#
#    start_response(status, response_headers)
#
#    return [output]
from flask import Flask

# TODO: fix static folder, this requires adapting HTML templates
hiv = Flask(__name__)
hiv.config.from_object('config')

# welcome page is the index
from .blueprints.welcome import welcome
hiv.register_blueprint(welcome, url_prefix='')

from .blueprints.tutorial import tutorial
hiv.register_blueprint(tutorial, url_prefix='/tutorial')

from .blueprints.patient import patient
hiv.register_blueprint(patient)

from .blueprints.region import region
hiv.register_blueprint(region)

from .blueprints.data import data
hiv.register_blueprint(data)

from .blueprints.method import method
hiv.register_blueprint(method)

from .blueprints.static import static
hiv.register_blueprint(static)

from .blueprints.download import download
hiv.register_blueprint(download)

# RESTful API
from .blueprints.api import api_bp
hiv.register_blueprint(api_bp)
<|endoftext|>


<commit_before>import unittest
from cStringIO import StringIO
from pytactoe.presenter import Presenter
from pytactoe.io import InputOutput

class PresenterTests(unittest.TestCase):

    def setUp(self):
        self.chosen_output = StringIO()
        self.io = InputOutput(None, self.chosen_output)
        self.presenter = Presenter(self.io)

    def test_presenting_the_welcome_message(self):
        expected_welcome = "Welcome to Tic Tac Toe\n\n"
        self.presenter.welcome_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_welcome)

    def test_presenting_the_move_prompt(self):
        expected_prompt = "X, please select a move:\n"
        self.presenter.move_prompt("X")
        received_output = self.chosen_output.getvalue()

    def test_presenting_the_player_type_prompt(self):
        expected_prompt = "The first player is a... type 'human' or 'computer'\n"
        self.presenter.player_type_prompt("first")
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_prompt)

    def test_presenting_the_player_type_prompt(self):
        expected_prompt = "Would you like to play again? type 'yes' or 'no'\n"
        self.presenter.play_again_prompt()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_prompt)

    def test_presenting_the_winner_message(self):
        expected_winner_message = "O has won the game!\n"
        self.presenter.winner_message("O")
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_winner_message)

    def test_presenting_invalid_move_message(self):
        expected_invalid_move_message = "Invalid move. Please try again\n"
        self.presenter.invalid_move_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_invalid_move_message)

    def test_presenting_stalemate_message(self):
        expected_stalemate_message = "The game ended in a stalemate\n"
        self.presenter.stalemate_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_stalemate_message)

    def test_presenting_stalemate_message(self):
        expected_computer_move_message = "Computer is thinking...\n\n"
        self.presenter.computer_move_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_computer_move_message)

    def test_presenting_available_moves(self):
        expected_available_moves_message = "Available moves: [1, 2, 3]\n\n\n"
        self.presenter.available_moves_message([1, 2, 3])
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_available_moves_message)
<commit_msg>separates presenter prompts from messages<commit_after>import unittest
from cStringIO import StringIO
from pytactoe.presenter import Presenter
from pytactoe.io import InputOutput

class PresenterTests(unittest.TestCase):

    def setUp(self):
        self.chosen_output = StringIO()
        self.io = InputOutput(None, self.chosen_output)
        self.presenter = Presenter(self.io)

    def test_presenting_the_welcome_message(self):
        expected_welcome = "Welcome to Tic Tac Toe\n\n"
        self.presenter.welcome_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_welcome)

    def test_presenting_the_winner_message(self):
        expected_winner_message = "O has won the game!\n"
        self.presenter.winner_message("O")
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_winner_message)

    def test_presenting_invalid_move_message(self):
        expected_invalid_move_message = "Invalid move. Please try again\n"
        self.presenter.invalid_move_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_invalid_move_message)

    def test_presenting_stalemate_message(self):
        expected_stalemate_message = "The game ended in a stalemate\n"
        self.presenter.stalemate_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_stalemate_message)

    def test_presenting_stalemate_message(self):
        expected_computer_move_message = "Computer is thinking...\n\n"
        self.presenter.computer_move_message()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_computer_move_message)

    def test_presenting_available_moves(self):
        expected_available_moves_message = "Available moves: [1, 2, 3]\n\n\n"
        self.presenter.available_moves_message([1, 2, 3])
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_available_moves_message)

    def test_presenting_the_move_prompt(self):
        expected_prompt = "X, please select a move:\n"
        self.presenter.move_prompt("X")
        received_output = self.chosen_output.getvalue()

    def test_presenting_the_player_type_prompt(self):
        expected_prompt = "The first player is a... type 'human' or 'computer'\n"
        self.presenter.player_type_prompt("first")
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_prompt)

    def test_presenting_the_player_type_prompt(self):
        expected_prompt = "Would you like to play again? type 'yes' or 'no'\n"
        self.presenter.play_again_prompt()
        received_output = self.chosen_output.getvalue()

        self.assertEqual(received_output, expected_prompt)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import csv
import collections
import sys
import re
import os
import binascii
import itertools
import traceback

# enable local imports; redirect config calls to general config
import add_install_path
import config
settings = config.settings()

try:
    import arcpy

except:
    """
    The config import above thows a warning in this case,
    just swallow it here and let this script import, since most of
    these utils don't depend on arcpy.
    """
    pass

class Loci(object):
    """ A basic class to store the Loci attributes we commonly use."""
    def __init__(self, input_features):
        self.fields = self.loci_fields(input_features)
        self.count = self.loci_count()
        self.columns = self.loci_columns()
        self.names = self.loci_names()

    def loci_fields(self, input_features):
        """ Actual loci field names, e.g. L_Ev23_1."""
        # map loci fields to values
        loci = collections.OrderedDict()
        # optional: use this to also filter if the genetic columns are up to date
        genetic_columns = settings.genetic_columns.split(";")
        loci_expr = '^l_(.*)_[0-9]+'
        for field in [f.name for f in arcpy.ListFields(input_features)]:
            match = re.match(loci_expr, field, re.IGNORECASE)
            if match:
                name = match.groups()[0]
                if loci.has_key(name):
                    loci[name].append(field)
                else:
                    loci[name] = [field]
        return loci

    def defined(self):
        """ Are any loci fields defined?"""
        defined = False
        if self.column is not None:
            defined = True
        return defined

    def loci_count(self):
        """ Number of distinct loci containing fields."""
        return len(self.fields.keys())

    def loci_columns(self):
        """ All column names containing loci."""
        loci = self.fields
        return list(itertools.chain(*loci.values()))

    def loci_names(self):
        """ Loci names, e.g. Ev23."""
        loci = self.fields
        return loci.keys()

class Haplotype(object):
    def __init__(self, input_features):
        self.column = self.haplotype_column(input_features)
        self.defined = self.defined()
        self.counter = self.haplotype_data(input_features)
        self.names = self.haplotype_names()
        self.indexed = self.haplotype_indexed()

    def haplotype_column(self, input_features):
        haplo_col = None
        haplo_expr = '^haplotype|dlphap$'
        for field in [f.name for f in arcpy.ListFields(input_features)]:
            match = re.match(haplo_expr, field, re.IGNORECASE)
            if match:
                haplo_col = field
        return haplo_col

    def defined(self):
        defined = False
        if self.column is not None:
            defined = True
        return defined

    def haplotype_data(self, input_features):
        # TODO: this currently just looks up all the data,
        # better yet we should generate a table on import and keep it updated
        # when edits are made to the input table.
        haplo_data = {}
        if self.defined:
            res = [r[0] for r in arcpy.da.SearchCursor(input_features, [self.column])]
            haplo_data = collections.Counter(filter(None, res))
        return haplo_data

    def haplotype_names(self):
        return self.counter.keys()

    def haplotype_indexed(self):
        # map sorted haplotypes to integers; equivalent to Shepherd's approach
        sorted_cols = sorted(self.names)
        # 1: A+, 2: E1, ...
        return itertools.izip(itertools.count(1), sorted_cols)

class MissingCSVHeader(Exception):
    def __init__(self, csv):
        self.csv = csv
        Exception.__init__(self, "CSV file is missing a header: {}".format(csv))

def parameters_from_args(defaults_tuple=None, sys_args=None):
    """Provided a set of tuples for default values, return a list of mapped
       variables."""
    defaults = collections.OrderedDict(defaults_tuple)
    if defaults_tuple is not None:
        args = len(sys_args) - 1
        for i, key in enumerate(defaults.keys()):
            idx = i + 1
            if idx <= args:
                defaults[key] = sys_args[idx]
    return defaults

def msg(output_msg, mtype='message', exception=None):
    if mtype == 'error':
        arcpy_messages = arcpy.GetMessages()
        try:
            tb = sys.exc_info()[2]
            tbinfo = traceback.format_tb(tb)[0]
        except:
            tbinfo = "No traceback reported."

        if settings.mode == 'script':
            if exception:
                # print the raw exception
                print exception
            # Arcpy and Python stuff, hopefully also helpful
            err_msg = "ArcPy Error: {msg_text}\nPython Error: ${tbinfo}".format(
                msg_text=arcpy_messages, tbinfo=tbinfo)
        else:
            arcpy.AddMessage(output_msg)
            if exception:
                arcpy.AddError(exception)
            arcpy.AddError(arcpy_messages)
            arcpy.AddMessage("Python Error: ${tbinfo}".format(tbinfo=tbinfo))
    elif settings.mode == 'script':
        print output_msg
    else:
        if mtype == 'message':
            arcpy.AddMessage(output_msg)
        elif mtype == 'warning':
            arcpy.AddWarning(output_msg)

def file_type(filename):
    """ Map of 'known' extensions to filetypes."""
    expected_type = None
    known_types = {
        '.csv' : 'Text',
        '.txt' : 'Text',
        '.asc' : 'Text',
        '.tab' : 'Text',
        '.xls' : 'Excel',
        '.xlsx': 'Excel',
    }
    ext = os.path.splitext(filename)[1].lower()
    if known_types.has_key(ext):
        expected_type = known_types[ext]
    else:
        raise UnknownType
    return expected_type

def parse_table(input_file):
    """ Parse a text table (usually CSV) determine its type,
        and validate."""
    # TODO: handle UTF-8 encodings robustly
    header = None
    data = None
    dialect = None
    try:
        with open(input_file, 'rb') as input_table:
            # sample the first 4k of the file
            sample = input_table.read(4096)
            sniffer = csv.Sniffer()

            if not sniffer.has_header(sample):
                # require the input to have a valid header
                raise MissingCSVHeader
            else:
                dialect = sniffer.sniff(sample)
                # reset reading
                input_table.seek(0)
                # for reading, rely on dialect parsing to correctly dermine
                # the input file's traits (e.g. proper quoting).
                table = csv.reader(input_table, dialect=dialect)

                # pull off the first line of the CSV
                header = table.next()
                data = []
                for row in table:
                    data.append(row)
    except Exception as e:
        raise e
    return (header, data, dialect)

# TODO: ADD TEST CASES FOR:
#  - quoted strings inside CSV fields
#  - our various validations in validate_column_label.

def validate_table(input_file):
    (header, data, dialect) = parse_table(input_file)

    # Handle multiple columns with the same name
    validated_header = []
    # Generate a list of columns which have duplicate names.
    duplicate_cols = set([i for i in header if header.count(i) > 1])
    duplicate_positions = collections.Counter(duplicate_cols)
    for col in header:
        label = validate_column_label(col)
        # handle the duplicate columns, labeling each item uniquely
        if col in duplicate_cols:
            # get current dupe position
            label = label + "_" + str(duplicate_positions[col])
            duplicate_positions[col] += 1
        validated_header.append(label)

    # set up output file name
    temp_dir = os.path.dirname(input_file)
    (label, ext) = os.path.splitext(os.path.basename(input_file))

    # generate a random name, but include the original file suffix for Arc
    temp_name = binascii.b2a_hex(os.urandom(15))
    tmp_fn = "".join([temp_name, ext])
    temp_csv = os.path.join(temp_dir, tmp_fn)

    with open(temp_csv, 'wb') as output_file:
        writer = csv.writer(output_file, dialect=dialect, quotechar='"', quoting=csv.QUOTE_ALL)
        writer.writerow(validated_header)
        for row in data:
            writer.writerow(row)
    return temp_csv

def validated_table_results(input_file):
    validated_table = validate_table(input_file)
    table_parts = parse_table(validated_table)
    # delete the temporary table generated; only want the components in this case.
    os.unlink(validated_table)
    return table_parts

def validate_column_label(column):
    """
    Modify the column labels to reflect ArcGIS restrictions:
     - no spaces, replaced here with underscores.
     - no special characters: `~@#$%^&*()-+=|\,<>?/{}.![]:;
     - Field names need to start with a letter
     - 64 character field names
     - can't use reserved words [http://support.microsoft.com/kb/286335]

    Alternatively, can use ValidateFieldName for a similar effect,
    without the same fine-grained control:
      http://resources.arcgis.com/en/help/main/10.1/index.html#//018v00000060000000
    """
    INVALID_CHARS = '`~@#$%^&*()+=|\,<>?/{}.![]:;'

    # replace a few common characters with underscores
    column = column.replace(" ", "_").replace("-", "_")
    for c in INVALID_CHARS: column = column.replace(c, '')
    # if the column starts with non-text, prefix the string with 'genegis'
    if not re.match('^[A-z]', column):
        column = 'genegis_' + column

    # 'Field names are limited to 64 characters for both file and
    # personal geodatabases.'
    if len(column) > 64:
        column = column[:63]

    return column

def protect_columns(input_table_name=None, protected_columns={}):
    """
    Force data typing on columns using schema.ini to store the necessary
    configuration. This prevents ArcGIS from trying to auto-detect the
    data type, which doesn't work for some of our input data, such as
    haplotypes, which it wants to treat as coordinate values.
    """

    # input table directory
    input_dir = os.path.dirname(input_table_name)

    # just the table name
    table_name = os.path.basename(input_table_name)

    # open the relevant schema.ini file
    schema_path = os.path.join(input_dir, 'schema.ini')

    # If it exists already, we want append. Don't use binary mode, let
    # Python auto-conver the newlines.
    if os.path.exists(schema_path):
        mode = 'a+'
    else:
        mode = 'w+'
    with open(schema_path, mode) as schema_file:
        header = "[{table_name}]\n".format(table_name=table_name)
        schema_file.write(header)
        if protected_columns:
            for (value, res_dict) in protected_columns.items():
                (idx, data_type) = res_dict
                col_label = "Col{idx}={value} {data_type}\n".format(
                    idx=idx, value=value, data_type=data_type)
                schema_file.write(col_label)

    return schema_path

def set_file_extension(param, ext):
    # make sure the output file name has the correct extension.
    param_input = param.valueAsText
    if param_input is not None:
        param_output = add_file_extension(param_input, ext)
    else:
        param_output = param_input
    return param_output

def add_file_extension(input_name, expected_ext):
    name_with_ext = input_name
    ext = expected_ext.lower()
    (label, input_ext) = os.path.splitext(os.path.basename(input_name))
    if input_ext.lower() != ext:
        name_with_ext = "{label}.{ext}".format(label = label, ext=ext)
    return os.path.join(os.path.dirname(input_name), name_with_ext)

def xstr(s):
    """ Replace None values with empty strings."""
    if s is None:
        return ''
    return str(s)

def zstr(s):
    """ Replace None values with zeros."""
    if s is None:
        return '0'
    return str(s)

# FIXME: duplicated from Install\utils.py
def currentLayers():
    # find layers in current map document
    layers = []
    # inspect the layer list, find the first point layer
    mxd = arcpy.mapping.MapDocument("current")
    # get a list of all layers, store it
    config.all_layers = arcpy.mapping.ListLayers(mxd)

    # iterate over our layers, find those which are candidates for analysis
    if config.all_layers is not None:
        for layer in config.all_layers:
            try:
                # FIXME: check performance on this. if expensive, do something cheaper
                desc = arcpy.Describe(layer)
                if desc.datasetType in config.allowed_formats and \
                    desc.shapeType in config.allowed_types:
                    layers.append(layer)
            except:
                # silently skip layers which don't support describe (e.g. AGOL).
                continue
    return layers
<commit_msg>Add docstrings to Haplotype class as well.<commit_after># -*- coding: utf-8 -*-
import csv
import collections
import sys
import re
import os
import binascii
import itertools
import traceback

# enable local imports; redirect config calls to general config
import add_install_path
import config
settings = config.settings()

try:
    import arcpy

except:
    """
    The config import above thows a warning in this case,
    just swallow it here and let this script import, since most of
    these utils don't depend on arcpy.
    """
    pass

class Loci(object):
    """ A basic class to store the Loci attributes we commonly use."""
    def __init__(self, input_features):
        self.fields = self.loci_fields(input_features)
        self.count = self.loci_count()
        self.columns = self.loci_columns()
        self.names = self.loci_names()

    def loci_fields(self, input_features):
        """ Actual loci field names, e.g. L_Ev23_1."""
        # map loci fields to values
        loci = collections.OrderedDict()
        # optional: use this to also filter if the genetic columns are up to date
        genetic_columns = settings.genetic_columns.split(";")
        loci_expr = '^l_(.*)_[0-9]+'
        for field in [f.name for f in arcpy.ListFields(input_features)]:
            match = re.match(loci_expr, field, re.IGNORECASE)
            if match:
                name = match.groups()[0]
                if loci.has_key(name):
                    loci[name].append(field)
                else:
                    loci[name] = [field]
        return loci

    def defined(self):
        """ Are any loci fields defined?"""
        defined = False
        if self.column is not None:
            defined = True
        return defined

    def loci_count(self):
        """ Number of distinct loci containing fields."""
        return len(self.fields.keys())

    def loci_columns(self):
        """ All column names containing loci."""
        loci = self.fields
        return list(itertools.chain(*loci.values()))

    def loci_names(self):
        """ Loci names, e.g. Ev23."""
        loci = self.fields
        return loci.keys()

class Haplotype(object):
    """ Track our Haplotype column, and provide a few useful summarizations
        of the data contained within."""

    def __init__(self, input_features):
        self.column = self.haplotype_column(input_features)
        self.defined = self.defined()
        self.counter = self.haplotype_data(input_features)
        self.names = self.haplotype_names()
        self.indexed = self.haplotype_indexed()

    def haplotype_column(self, input_features):
        """ The column name containin the haplotype data."""
        haplo_col = None
        haplo_expr = '^haplotype|dlphap$'
        for field in [f.name for f in arcpy.ListFields(input_features)]:
            match = re.match(haplo_expr, field, re.IGNORECASE)
            if match:
                haplo_col = field
        return haplo_col

    def defined(self):
        """ Is a haplotype column defined?"""
        defined = False
        if self.column is not None:
            defined = True
        return defined

    def haplotype_data(self, input_features):
        """ Counts of each haplotype type."""
        # TODO: this currently just looks up all the data,
        # better yet we should generate a table on import and keep it updated
        # when edits are made to the input table.
        haplo_data = {}
        if self.defined:
            res = [r[0] for r in arcpy.da.SearchCursor(input_features, [self.column])]
            haplo_data = collections.Counter(filter(None, res))
        return haplo_data

    def haplotype_names(self):
        """ Distinct haplotype names found."""
        return self.counter.keys()

    def haplotype_indexed(self):
        """ A mapping of haplotype name to integers for interchange."""
        # map sorted haplotypes to integers; equivalent to Shepherd's approach
        sorted_cols = sorted(self.names)
        # 1: A+, 2: E1, ...
        return itertools.izip(itertools.count(1), sorted_cols)

class MissingCSVHeader(Exception):
    def __init__(self, csv):
        self.csv = csv
        Exception.__init__(self, "CSV file is missing a header: {}".format(csv))

def parameters_from_args(defaults_tuple=None, sys_args=None):
    """Provided a set of tuples for default values, return a list of mapped
       variables."""
    defaults = collections.OrderedDict(defaults_tuple)
    if defaults_tuple is not None:
        args = len(sys_args) - 1
        for i, key in enumerate(defaults.keys()):
            idx = i + 1
            if idx <= args:
                defaults[key] = sys_args[idx]
    return defaults

def msg(output_msg, mtype='message', exception=None):
    if mtype == 'error':
        arcpy_messages = arcpy.GetMessages()
        try:
            tb = sys.exc_info()[2]
            tbinfo = traceback.format_tb(tb)[0]
        except:
            tbinfo = "No traceback reported."

        if settings.mode == 'script':
            if exception:
                # print the raw exception
                print exception
            # Arcpy and Python stuff, hopefully also helpful
            err_msg = "ArcPy Error: {msg_text}\nPython Error: ${tbinfo}".format(
                msg_text=arcpy_messages, tbinfo=tbinfo)
        else:
            arcpy.AddMessage(output_msg)
            if exception:
                arcpy.AddError(exception)
            arcpy.AddError(arcpy_messages)
            arcpy.AddMessage("Python Error: ${tbinfo}".format(tbinfo=tbinfo))
    elif settings.mode == 'script':
        print output_msg
    else:
        if mtype == 'message':
            arcpy.AddMessage(output_msg)
        elif mtype == 'warning':
            arcpy.AddWarning(output_msg)

def file_type(filename):
    """ Map of 'known' extensions to filetypes."""
    expected_type = None
    known_types = {
        '.csv' : 'Text',
        '.txt' : 'Text',
        '.asc' : 'Text',
        '.tab' : 'Text',
        '.xls' : 'Excel',
        '.xlsx': 'Excel',
    }
    ext = os.path.splitext(filename)[1].lower()
    if known_types.has_key(ext):
        expected_type = known_types[ext]
    else:
        raise UnknownType
    return expected_type

def parse_table(input_file):
    """ Parse a text table (usually CSV) determine its type,
        and validate."""
    # TODO: handle UTF-8 encodings robustly
    header = None
    data = None
    dialect = None
    try:
        with open(input_file, 'rb') as input_table:
            # sample the first 4k of the file
            sample = input_table.read(4096)
            sniffer = csv.Sniffer()

            if not sniffer.has_header(sample):
                # require the input to have a valid header
                raise MissingCSVHeader
            else:
                dialect = sniffer.sniff(sample)
                # reset reading
                input_table.seek(0)
                # for reading, rely on dialect parsing to correctly dermine
                # the input file's traits (e.g. proper quoting).
                table = csv.reader(input_table, dialect=dialect)

                # pull off the first line of the CSV
                header = table.next()
                data = []
                for row in table:
                    data.append(row)
    except Exception as e:
        raise e
    return (header, data, dialect)

# TODO: ADD TEST CASES FOR:
#  - quoted strings inside CSV fields
#  - our various validations in validate_column_label.

def validate_table(input_file):
    (header, data, dialect) = parse_table(input_file)

    # Handle multiple columns with the same name
    validated_header = []
    # Generate a list of columns which have duplicate names.
    duplicate_cols = set([i for i in header if header.count(i) > 1])
    duplicate_positions = collections.Counter(duplicate_cols)
    for col in header:
        label = validate_column_label(col)
        # handle the duplicate columns, labeling each item uniquely
        if col in duplicate_cols:
            # get current dupe position
            label = label + "_" + str(duplicate_positions[col])
            duplicate_positions[col] += 1
        validated_header.append(label)

    # set up output file name
    temp_dir = os.path.dirname(input_file)
    (label, ext) = os.path.splitext(os.path.basename(input_file))

    # generate a random name, but include the original file suffix for Arc
    temp_name = binascii.b2a_hex(os.urandom(15))
    tmp_fn = "".join([temp_name, ext])
    temp_csv = os.path.join(temp_dir, tmp_fn)

    with open(temp_csv, 'wb') as output_file:
        writer = csv.writer(output_file, dialect=dialect, quotechar='"', quoting=csv.QUOTE_ALL)
        writer.writerow(validated_header)
        for row in data:
            writer.writerow(row)
    return temp_csv

def validated_table_results(input_file):
    validated_table = validate_table(input_file)
    table_parts = parse_table(validated_table)
    # delete the temporary table generated; only want the components in this case.
    os.unlink(validated_table)
    return table_parts

def validate_column_label(column):
    """
    Modify the column labels to reflect ArcGIS restrictions:
     - no spaces, replaced here with underscores.
     - no special characters: `~@#$%^&*()-+=|\,<>?/{}.![]:;
     - Field names need to start with a letter
     - 64 character field names
     - can't use reserved words [http://support.microsoft.com/kb/286335]

    Alternatively, can use ValidateFieldName for a similar effect,
    without the same fine-grained control:
      http://resources.arcgis.com/en/help/main/10.1/index.html#//018v00000060000000
    """
    INVALID_CHARS = '`~@#$%^&*()+=|\,<>?/{}.![]:;'

    # replace a few common characters with underscores
    column = column.replace(" ", "_").replace("-", "_")
    for c in INVALID_CHARS: column = column.replace(c, '')
    # if the column starts with non-text, prefix the string with 'genegis'
    if not re.match('^[A-z]', column):
        column = 'genegis_' + column

    # 'Field names are limited to 64 characters for both file and
    # personal geodatabases.'
    if len(column) > 64:
        column = column[:63]

    return column

def protect_columns(input_table_name=None, protected_columns={}):
    """
    Force data typing on columns using schema.ini to store the necessary
    configuration. This prevents ArcGIS from trying to auto-detect the
    data type, which doesn't work for some of our input data, such as
    haplotypes, which it wants to treat as coordinate values.
    """

    # input table directory
    input_dir = os.path.dirname(input_table_name)

    # just the table name
    table_name = os.path.basename(input_table_name)

    # open the relevant schema.ini file
    schema_path = os.path.join(input_dir, 'schema.ini')

    # If it exists already, we want append. Don't use binary mode, let
    # Python auto-conver the newlines.
    if os.path.exists(schema_path):
        mode = 'a+'
    else:
        mode = 'w+'
    with open(schema_path, mode) as schema_file:
        header = "[{table_name}]\n".format(table_name=table_name)
        schema_file.write(header)
        if protected_columns:
            for (value, res_dict) in protected_columns.items():
                (idx, data_type) = res_dict
                col_label = "Col{idx}={value} {data_type}\n".format(
                    idx=idx, value=value, data_type=data_type)
                schema_file.write(col_label)

    return schema_path

def set_file_extension(param, ext):
    # make sure the output file name has the correct extension.
    param_input = param.valueAsText
    if param_input is not None:
        param_output = add_file_extension(param_input, ext)
    else:
        param_output = param_input
    return param_output

def add_file_extension(input_name, expected_ext):
    name_with_ext = input_name
    ext = expected_ext.lower()
    (label, input_ext) = os.path.splitext(os.path.basename(input_name))
    if input_ext.lower() != ext:
        name_with_ext = "{label}.{ext}".format(label = label, ext=ext)
    return os.path.join(os.path.dirname(input_name), name_with_ext)

def xstr(s):
    """ Replace None values with empty strings."""
    if s is None:
        return ''
    return str(s)

def zstr(s):
    """ Replace None values with zeros."""
    if s is None:
        return '0'
    return str(s)

# FIXME: duplicated from Install\utils.py
def currentLayers():
    # find layers in current map document
    layers = []
    # inspect the layer list, find the first point layer
    mxd = arcpy.mapping.MapDocument("current")
    # get a list of all layers, store it
    config.all_layers = arcpy.mapping.ListLayers(mxd)

    # iterate over our layers, find those which are candidates for analysis
    if config.all_layers is not None:
        for layer in config.all_layers:
            try:
                # FIXME: check performance on this. if expensive, do something cheaper
                desc = arcpy.Describe(layer)
                if desc.datasetType in config.allowed_formats and \
                    desc.shapeType in config.allowed_types:
                    layers.append(layer)
            except:
                # silently skip layers which don't support describe (e.g. AGOL).
                continue
    return layers
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import Gaffer
import GafferUI

import GafferScene
import GafferSceneUI

Gaffer.Metadata.registerNodeDescription(

GafferScene.Prune,

"""A node for removing whole branches from the scene hierarchy.""",

"filter",
"""The branches to prune. The specified locations and all locations below them will be removed from the scene.""",

"adjustBounds",
"""Computes new tightened bounding boxes taking into account the removed locations. This can be an expensive operation - turn on with care.""",

)
<commit_msg>PruneUI : Prefer Metadata.registerNode() to registerNodeDescription().<commit_after>##########################################################################
#
#  Copyright (c) 2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import Gaffer
import GafferScene

Gaffer.Metadata.registerNode(

	GafferScene.Prune,

	"description",
	"""
	A node for removing whole branches from the scene hierarchy.
	""",

	plugs = {

		"filter" : [

			"description",
			"""
			Filter to specify the branches to prune. The specified
			locations and all locations below them will be removed from
			the scene.
			""",

		],

		"adjustBounds" : [

			"description",
			"""
			Computes new tightened bounding boxes taking into account
			the removed locations. This can be an expensive operation -
			turn on with care.
			""",

		],

	}

)
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
This module contains helper functions for handling metadata.
"""

from copy import deepcopy

class MergeConflictError(TypeError):
    pass

class MergeConflictWarning(Warning):
    pass

def take_left(left, right):
    return left

def take_right(left, right):
    return left

def concat(left, right):
    """
    Concatenate the ``left`` and ``right`` objects

    Currently only lists and tuples are allowed, but this could change.
    """
    if type(left) != type(right):
        raise MergeConflictError()
    if any(isinstance(left, cls) for cls in (tuple, list)):
        return left + right
    else:
        raise MergeConflictError()

def raise_error(left, right):
    """
    Raise a MergeConflictError if left and right sequences
    are being merged.
    """
    raise MergeConflictError()

def _both_isinstance(left, right, cls):
    return isinstance(left, cls) and isinstance(right, cls)

def merge(left, right, merge_func=concat):
    """
    Merge the ``left`` and ``right`` metadata objects.

    This is a simplistic and limited implemenation at this point.
    """
    if not _both_isinstance(left, right, dict):
        raise MergeConflictError('Can only merge two dict-based objects')

    out = left.__class__()

    for key, val in left.items():
        out[key] = deepcopy(val)

    for key, val in right.items():
        # If no conflict then insert val into out dict and continue
        if key not in out:
            out[key] = deepcopy(val)
            continue

        # There is a conflict that must be resolved
        if _both_isinstance(left[key], right[key], dict):
            out[key] = merge(left[key], right[key], merge_func)

        else:
            try:
                out[key] = merge_func(left[key], right[key])
            except MergeConflictError:
                raise MergeConflictError('Cannot merge meta key {0!r} types {1!r} and {2!r}'
                                         .format(key, type(left[key]), type(right[key])))

    return out
<commit_msg>Use single deepcopy instead of once per key in meta merge<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
This module contains helper functions for handling metadata.
"""

from copy import deepcopy

class MergeConflictError(TypeError):
    pass

class MergeConflictWarning(Warning):
    pass

def take_left(left, right):
    return left

def take_right(left, right):
    return left

def concat(left, right):
    """
    Concatenate the ``left`` and ``right`` objects

    Currently only lists and tuples are allowed, but this could change.
    """
    if type(left) != type(right):
        raise MergeConflictError()
    if any(isinstance(left, cls) for cls in (tuple, list)):
        return left + right
    else:
        raise MergeConflictError()

def raise_error(left, right):
    """
    Raise a MergeConflictError if left and right sequences
    are being merged.
    """
    raise MergeConflictError()

def _both_isinstance(left, right, cls):
    return isinstance(left, cls) and isinstance(right, cls)

def merge(left, right, merge_func=concat):
    """
    Merge the ``left`` and ``right`` metadata objects.

    This is a simplistic and limited implemenation at this point.
    """
    if not _both_isinstance(left, right, dict):
        raise MergeConflictError('Can only merge two dict-based objects')

    out = deepcopy(left)

    for key, val in right.items():
        # If no conflict then insert val into out dict and continue
        if key not in out:
            out[key] = deepcopy(val)
            continue

        # There is a conflict that must be resolved
        if _both_isinstance(left[key], right[key], dict):
            out[key] = merge(left[key], right[key], merge_func)

        else:
            try:
                out[key] = merge_func(left[key], right[key])
            except MergeConflictError:
                raise MergeConflictError('Cannot merge meta key {0!r} types {1!r} and {2!r}'
                                         .format(key, type(left[key]), type(right[key])))

    return out
<|endoftext|>


<commit_before>import os, sys, traceback
import xml.etree.ElementTree as xml

##
# Stores error information needed for printing diff messages
class XMLError(object):

  ##
  # Constructor.
  # @param err A one line error message
  # @param msg A detailed message describing the error
  def __init__(self, err, msg):
    self.error = err
    self.message = msg

##
# A class for finding difference between XML documents
class XMLDiffer(object):

  ##
  # Constructor.
  # @param file1 The master (gold) file to check against
  # @param file2 The file to compare to the master file
  #
  # Optional Arguments:
  #   abz_zero Any value less than this is assumed zero (default: 1e-11)
  #   rel_tol Relative tolerance to check numeric values against (default: 5.5e-6)
  #   max_values The maximum number of values to test
  def __init__(self, file1, file2, **kwargs):

    # Store the file names
    self._file = [file1, file2]

    # Extract the optional arguments
    self._abs_zero = float(kwargs.pop('abs_zero', 1e-11))
    self._rtol = float(kwargs.pop('rel_tol', 5.5e-6))

    # Storage for XMLError objects
    self._errors = []

    # Extract the XML tree from the files
    self._root1 = self._extractXML(file1)
    self._root2 = self._extractXML(file2)

    # Perform the comparision
    self._compare()

  ##
  # Check the comparision status (public)
  # Returns True if the comparision fails
  def fail(self):
    return len(self._errors) > 0

  ##
  # Print the error message(s) (public)
  # @return The output as a single string
  def message(self, **kwargs):

    # Header
    output = []
    output.append('Running XMLDiffer.py')
    output.append( '         File 1: ' + self._file[0])
    output.append( '         File 2: ' + self._file[1])
    output.append( '        rel_tol: ' + str(self._rtol))
    output.append( '       abz_zero: ' + str(self._abs_zero))
    output.append( '  No. of errors: ' + str(len(self._errors)))

    # Errors
    cnt = 0
    for e in self._errors:
      cnt += 1
      output.append('\n')
      output.append('ERROR ' + str(cnt) + ':')
      output.append('  ' + e.error)
      for m in e.message:
        output.append('    ' + m)

    # Print the output
    if kwargs.pop('output', False):
      print '\n'.join(output)

    # Return the text, as a single string
    return '\n'.join(output)

  ##
  # Add an XMLError object to the storage vector (private)
  # @param err A string containing the error message or an XMLError object
  # @param msg A detailed message for the error (ignored if XMLError is passed to err)
  def _addError(self, err, msg=[]):

    # Add object directly
    if isinstance(err, XMLError):
      self._errors.append(err)

    # Create and add the object
    else:
      obj = XMLError(err, msg)
      self._errors.append(obj)

  ##
  # Reads the XML file (private)
  # @param filename The name of the XML file to read
  # @return An xml.etree.ElementTree root object
  def _extractXML(self, filename):

    # Check for file existence
    if not os.path.isfile(filename):
      self._addError('Could not open ' + filename + ', the file does not exist.')
      return None

    # Get the root object of the XML tree
    try:
      root = xml.parse(filename).getroot()

    # Catch parser errors
    except xml.ParseError:
      err = 'An XML parser error occured attemptting to read XML tree from ' + filename + '.'
      msg = traceback.format_exc().splitlines()
      self._addError(err, msg)
      root = None

    # Return the object
    return root

  ##
  # Perform the block by block comparision (private)
  def _compare(self):

    # Define local variables
    root = [self._root1, self._root2]
    name = ['file 1', 'file 2']

    # Do nothing if the objects do not exist
    if root[0] == None or root[1] == None:
      return

    # Loop through each tree object in the master file
    for elem0 in root[0].iter():

      # Intialize the result and error storage
      results = []
      errors  = []

      # Loop through all blocks in the second file with the current tag
      for elem1 in root[1].iter(elem0.tag):

        # Perform the comparision
        r, e = self._compareBlock(elem0, elem1)

        # Append the test results
        results.append(r)
        errors.append(e)

      # If all results are False, there was no match
      if not any(results):

        # Fiter out errors (elem.text failure)
        errors = filter(None, errors)

        # If no erros exist there was no block or block with identical attributes located
        if len(errors) == 0:
          msg = self._getAttrib(elem0)
          if len(msg) == 0:
            err = 'Unable to locate an XML Block with the tag "' + elem0.tag + '" in file 2.'
            self._addError(err)
          else:
            err = 'Unable to locate an XML Block with the tag "' + elem0.tag + '" and the following attributes in file 2.'
            self._addError(err, msg)

        # Had a text error within similar blocks
        else:
          for e in errors:
            self._addError(e)

  ##
  # Compares XML blocks (private)
  # This function first compares the XML block attributes, if those match
  # then the XML text is also compared.
  # @param elem0 The master XML element object
  # @param elem1 The XML element object to compare the master against
  # @return A pair containg the test result (True or False) and an error indicator,
  #         this 'indicator' is None if the result of the match is True or if the
  #         attributes fail to match. When the text fails to match then it contains
  #         the XMLError object.
  def _compareBlock(self, elem0, elem1):

    # Perform attribute comparision
    test_attrib = self._compareAttributes(elem0, elem1)

    # If the attributes match, compare the text and return those results
    if test_attrib:
      test_text, err = self._compareText(elem0, elem1)
      return test_text, err

    # Otherwise the attributes do match
    else:
      return False, None

  ##
  # Perform attribute comparison (private)
  # @param elem0 The master XML element object
  # @param elem1 The XML element object to compare the master against
  def _compareAttributes(self, elem0, elem1):

    # Initialize the output (assume match)
    result = True

    # Loop through each attribute of the master object
    for key0, value0 in elem0.attrib.iteritems():

      # Attribute is missing from the slave object, match fails
      if not elem1.attrib.has_key(key0):
        return  False

      # If the slave object has the same attribute, perform a comparison
      elif elem1.attrib.has_key(key0):
        value1 = elem1.attrib[key0]

        # Attempt to perform a numeric comparison
        try:
          tvalue, rel_diff = self._isClose(value0, value1)
          if not tvalue:
            return False

        except:
          if value0 != value1:
            return False

    # Return the results
    return result

  ## Perform comparison of text for two XML blocks (private)
  # @param elem0 The master XML element object
  # @param elem1 The XML element object to compare the master against
  # @return A pair of items, either True, None or False, XMLError
  def _compareText(self, elem0, elem1):

    # Initialize the output
    result = True
    err = None

    # Return if no text exists
    if elem0.text == None and elem1.text == None:
      return (result, err)
    elif elem0.text == None or elem1.text == None:
      return (False, err)

    # Convert the text to a list of strings
    text0 = elem0.text.replace('\n', '').strip().split(' ')
    text1 = elem1.text.replace('\n', '').strip().split(' ')
    text0 = filter(None, text0)
    text1 = filter(None, text1)

    # Check that the lengths are the same
    if len(text0) != len(text1):
      result = False
      err = 'An XML block with the tag "' + elem0.tag + '" and the following attributes exists in both files, but the blocks have a diffenent number of values.'
      msg = self._getAttrib(elem0)
      msg.append('No. items file 1: ' + '%d' % len(text0))
      msg.append('No. items file 2: ' + '%d' % len(text1))
      err = XMLError(err, msg)
      return (False, err)

    for i in xrange(len(text0)):
      value, rel_diff = self._isClose(text0[i], text1[i])

      if not value:
        err = 'An XML block with the tag "' + elem0.tag + '" and the following attributes has differing values on file 2.'
        msg = self._getAttrib(elem0)
        msg.append('Index ' + str(i) + ' : ' + text0[i] + ' ~ ' + text1[i] + ', rel diff: ' + '%e' % rel_diff)
        err = XMLError(err, msg)
        return (False, err)

    return result, err

  ##
  # Perform relative tolerance check between two numbers (private)
  # @param value0 A string or list of strings containing the first number
  # @param value1 A string or list of strings containing the second number
  def _isClose(self, value0, value1):

    # Return values
    result = True
    rel_diff = 0

    # Convert the strings to floats
    value0 = float(value0)
    value1 = float(value1)

    # Apply the absolute zeros
    if abs(value0) < self._abs_zero:
      value0 = 0
    if abs(value1) < self._abs_zero:
      value1 = 0

    # Check for zero
    if value0 == 0 and value1 == 0:
      result = True

    # Check the relative error
    else:
      rel_diff = abs( ( value0 - value1 ) / max( abs(value0), abs(value1) ) )
      if rel_diff > self._rtol:
        result = False

    # Return the comparision
    return result, rel_diff

  ##
  # Get the attributes (dict) as a string (private)
  # @return Attribute message string
  def _getAttrib(self, elem):
    msg = []
    for k, v in elem.attrib.iteritems():
      msg.append('  ' + k + ' = ' + v)
    return msg

if __name__ == '__main__':

  # 'gold' file
  file0 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test0.vtu')

  # Has invalid tags
  file1 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test1.vtu')

  # Has blocks with different attributes and values (text)
  file2 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test2.vtu')

  # Has blocks with tolerance differences
  file3 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test3.vtu')

  # Lines for making things look pretty
  lines = '-'*129

  # Test invalid filenames (1 error)
  print lines
  print 'Test 1: Invalid filename (1 error)'
  print lines
  d = XMLDiffer(file0, 'adfadfas.vtk')
  if d.fail():
    d.message(output=True)

  # Parser error (i.e., opening/closing tags do not match)
  print '\n' + lines
  print 'Test 2: Parser Error (1 error)'
  print lines
  d = XMLDiffer(file0, file2)
  if d.fail():
    d.message(output=True)

  # Test missing block and different attribute
  #  (1) Unable locate block (Piece as different attributes in 2)
  #  (2) Differing values in Points/DataArray
  #  (3) No CellData block in 2
  #  (4) Different no. of values in Cells/DataArray (Name = types)
  print '\n' + lines
  print 'Test 3: Block Errors (4 errors)'
  print lines
  d = XMLDiffer(file0, file1)
  if d.fail():
    d.message(output=True)

  # Test abs_zero
  # The first call should not error, the second should diff
  print '\n' + lines
  print 'Test 4: Absolute zero (1 error)'
  print lines
  # This should not error
  d = XMLDiffer(file0, file3, abs_zero=1e-10)
  if not d.fail():
    print 'Successfully set value with lower abs_zero flag\n'

  d = XMLDiffer(file0, file3, abs_zero=1e-12)
  if d.fail():
    d.message(output=True)

  # Test rel_tol
  # The first call should not error, the second should diff
  print '\n' + lines
  print 'Test 5: Relative Tolerance (1 error)'
  print lines
  d = XMLDiffer(file0, file3)
  if not d.fail():
    print 'Successfully checked value within default tolerance\n'

  d = XMLDiffer(file0, file3, rel_tol=1e-10)
  if d.fail():
    d.message(output=True)

  # Test no-error
  print '\n' + lines
  print 'Test 6: Same File (0 error)'
  print lines
  d = XMLDiffer(file0, file0)
  if not d.fail():
    print 'Successfully checked the same file\n'
<commit_msg>Fix Python2.6 compatibility: Note: We'll need to update the recipes on moosebuild after this is merged. refs #4163<commit_after>import os, sys, traceback
import xml.etree.ElementTree as xml

##
# Stores error information needed for printing diff messages
class XMLError(object):

  ##
  # Constructor.
  # @param err A one line error message
  # @param msg A detailed message describing the error
  def __init__(self, err, msg):
    self.error = err
    self.message = msg

##
# A class for finding difference between XML documents
class XMLDiffer(object):

  ##
  # Constructor.
  # @param file1 The master (gold) file to check against
  # @param file2 The file to compare to the master file
  #
  # Optional Arguments:
  #   abz_zero Any value less than this is assumed zero (default: 1e-11)
  #   rel_tol Relative tolerance to check numeric values against (default: 5.5e-6)
  #   max_values The maximum number of values to test
  def __init__(self, file1, file2, **kwargs):

    # Store the file names
    self._file = [file1, file2]

    # Extract the optional arguments
    self._abs_zero = float(kwargs.pop('abs_zero', 1e-11))
    self._rtol = float(kwargs.pop('rel_tol', 5.5e-6))

    # Storage for XMLError objects
    self._errors = []

    # Extract the XML tree from the files
    self._root1 = self._extractXML(file1)
    self._root2 = self._extractXML(file2)

    # Perform the comparision
    self._compare()

  ##
  # Check the comparision status (public)
  # Returns True if the comparision fails
  def fail(self):
    return len(self._errors) > 0

  ##
  # Print the error message(s) (public)
  # @return The output as a single string
  def message(self, **kwargs):

    # Header
    output = []
    output.append('Running XMLDiffer.py')
    output.append( '         File 1: ' + self._file[0])
    output.append( '         File 2: ' + self._file[1])
    output.append( '        rel_tol: ' + str(self._rtol))
    output.append( '       abz_zero: ' + str(self._abs_zero))
    output.append( '  No. of errors: ' + str(len(self._errors)))

    # Errors
    cnt = 0
    for e in self._errors:
      cnt += 1
      output.append('\n')
      output.append('ERROR ' + str(cnt) + ':')
      output.append('  ' + e.error)
      for m in e.message:
        output.append('    ' + m)

    # Print the output
    if kwargs.pop('output', False):
      print '\n'.join(output)

    # Return the text, as a single string
    return '\n'.join(output)

  ##
  # Add an XMLError object to the storage vector (private)
  # @param err A string containing the error message or an XMLError object
  # @param msg A detailed message for the error (ignored if XMLError is passed to err)
  def _addError(self, err, msg=[]):

    # Add object directly
    if isinstance(err, XMLError):
      self._errors.append(err)

    # Create and add the object
    else:
      obj = XMLError(err, msg)
      self._errors.append(obj)

  ##
  # Reads the XML file (private)
  # @param filename The name of the XML file to read
  # @return An xml.etree.ElementTree root object
  def _extractXML(self, filename):

    # Check for file existence
    if not os.path.isfile(filename):
      self._addError('Could not open ' + filename + ', the file does not exist.')
      return None

    # Get the root object of the XML tree
    try:
      root = xml.parse(filename).getroot()

    # Catch parser errors
    except xml.ParseError:
      err = 'An XML parser error occured attemptting to read XML tree from ' + filename + '.'
      msg = traceback.format_exc().splitlines()
      self._addError(err, msg)
      root = None

    # Return the object
    return root

  ##
  # Perform the block by block comparision (private)
  def _compare(self):

    # Define local variables
    root = [self._root1, self._root2]
    name = ['file 1', 'file 2']

    # Do nothing if the objects do not exist
    if root[0] == None or root[1] == None:
      return

    # Loop through each tree object in the master file
    for elem0 in root[0].getiterator():

      # Intialize the result and error storage
      results = []
      errors  = []

      # Loop through all blocks in the second file with the current tag
      for elem1 in root[1].getiterator(elem0.tag):

        # Perform the comparision
        r, e = self._compareBlock(elem0, elem1)

        # Append the test results
        results.append(r)
        errors.append(e)

      # If all results are False, there was no match
      if not any(results):

        # Fiter out errors (elem.text failure)
        errors = filter(None, errors)

        # If no erros exist there was no block or block with identical attributes located
        if len(errors) == 0:
          msg = self._getAttrib(elem0)
          if len(msg) == 0:
            err = 'Unable to locate an XML Block with the tag "' + elem0.tag + '" in file 2.'
            self._addError(err)
          else:
            err = 'Unable to locate an XML Block with the tag "' + elem0.tag + '" and the following attributes in file 2.'
            self._addError(err, msg)

        # Had a text error within similar blocks
        else:
          for e in errors:
            self._addError(e)

  ##
  # Compares XML blocks (private)
  # This function first compares the XML block attributes, if those match
  # then the XML text is also compared.
  # @param elem0 The master XML element object
  # @param elem1 The XML element object to compare the master against
  # @return A pair containg the test result (True or False) and an error indicator,
  #         this 'indicator' is None if the result of the match is True or if the
  #         attributes fail to match. When the text fails to match then it contains
  #         the XMLError object.
  def _compareBlock(self, elem0, elem1):

    # Perform attribute comparision
    test_attrib = self._compareAttributes(elem0, elem1)

    # If the attributes match, compare the text and return those results
    if test_attrib:
      test_text, err = self._compareText(elem0, elem1)
      return test_text, err

    # Otherwise the attributes do match
    else:
      return False, None

  ##
  # Perform attribute comparison (private)
  # @param elem0 The master XML element object
  # @param elem1 The XML element object to compare the master against
  def _compareAttributes(self, elem0, elem1):

    # Initialize the output (assume match)
    result = True

    # Loop through each attribute of the master object
    for key0, value0 in elem0.attrib.iteritems():

      # Attribute is missing from the slave object, match fails
      if not elem1.attrib.has_key(key0):
        return  False

      # If the slave object has the same attribute, perform a comparison
      elif elem1.attrib.has_key(key0):
        value1 = elem1.attrib[key0]

        # Attempt to perform a numeric comparison
        try:
          tvalue, rel_diff = self._isClose(value0, value1)
          if not tvalue:
            return False

        except:
          if value0 != value1:
            return False

    # Return the results
    return result

  ## Perform comparison of text for two XML blocks (private)
  # @param elem0 The master XML element object
  # @param elem1 The XML element object to compare the master against
  # @return A pair of items, either True, None or False, XMLError
  def _compareText(self, elem0, elem1):

    # Initialize the output
    result = True
    err = None

    # Return if no text exists
    if elem0.text == None and elem1.text == None:
      return (result, err)
    elif elem0.text == None or elem1.text == None:
      return (False, err)

    # Convert the text to a list of strings
    text0 = elem0.text.replace('\n', '').strip().split(' ')
    text1 = elem1.text.replace('\n', '').strip().split(' ')
    text0 = filter(None, text0)
    text1 = filter(None, text1)

    # Check that the lengths are the same
    if len(text0) != len(text1):
      result = False
      err = 'An XML block with the tag "' + elem0.tag + '" and the following attributes exists in both files, but the blocks have a diffenent number of values.'
      msg = self._getAttrib(elem0)
      msg.append('No. items file 1: ' + '%d' % len(text0))
      msg.append('No. items file 2: ' + '%d' % len(text1))
      err = XMLError(err, msg)
      return (False, err)

    for i in xrange(len(text0)):
      value, rel_diff = self._isClose(text0[i], text1[i])

      if not value:
        err = 'An XML block with the tag "' + elem0.tag + '" and the following attributes has differing values on file 2.'
        msg = self._getAttrib(elem0)
        msg.append('Index ' + str(i) + ' : ' + text0[i] + ' ~ ' + text1[i] + ', rel diff: ' + '%e' % rel_diff)
        err = XMLError(err, msg)
        return (False, err)

    return result, err

  ##
  # Perform relative tolerance check between two numbers (private)
  # @param value0 A string or list of strings containing the first number
  # @param value1 A string or list of strings containing the second number
  def _isClose(self, value0, value1):

    # Return values
    result = True
    rel_diff = 0

    # Convert the strings to floats
    value0 = float(value0)
    value1 = float(value1)

    # Apply the absolute zeros
    if abs(value0) < self._abs_zero:
      value0 = 0
    if abs(value1) < self._abs_zero:
      value1 = 0

    # Check for zero
    if value0 == 0 and value1 == 0:
      result = True

    # Check the relative error
    else:
      rel_diff = abs( ( value0 - value1 ) / max( abs(value0), abs(value1) ) )
      if rel_diff > self._rtol:
        result = False

    # Return the comparision
    return result, rel_diff

  ##
  # Get the attributes (dict) as a string (private)
  # @return Attribute message string
  def _getAttrib(self, elem):
    msg = []
    for k, v in elem.attrib.iteritems():
      msg.append('  ' + k + ' = ' + v)
    return msg

if __name__ == '__main__':

  # 'gold' file
  file0 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test0.vtu')

  # Has invalid tags
  file1 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test1.vtu')

  # Has blocks with different attributes and values (text)
  file2 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test2.vtu')

  # Has blocks with tolerance differences
  file3 = os.path.join(os.getenv('MOOSE_DIR'), 'framework', 'scripts', 'TestHarness', 'test_files', 'xml_differ_test3.vtu')

  # Lines for making things look pretty
  lines = '-'*129

  # Test invalid filenames (1 error)
  print lines
  print 'Test 1: Invalid filename (1 error)'
  print lines
  d = XMLDiffer(file0, 'adfadfas.vtk')
  if d.fail():
    d.message(output=True)

  # Parser error (i.e., opening/closing tags do not match)
  print '\n' + lines
  print 'Test 2: Parser Error (1 error)'
  print lines
  d = XMLDiffer(file0, file2)
  if d.fail():
    d.message(output=True)

  # Test missing block and different attribute
  #  (1) Unable locate block (Piece as different attributes in 2)
  #  (2) Differing values in Points/DataArray
  #  (3) No CellData block in 2
  #  (4) Different no. of values in Cells/DataArray (Name = types)
  print '\n' + lines
  print 'Test 3: Block Errors (4 errors)'
  print lines
  d = XMLDiffer(file0, file1)
  if d.fail():
    d.message(output=True)

  # Test abs_zero
  # The first call should not error, the second should diff
  print '\n' + lines
  print 'Test 4: Absolute zero (1 error)'
  print lines
  # This should not error
  d = XMLDiffer(file0, file3, abs_zero=1e-10)
  if not d.fail():
    print 'Successfully set value with lower abs_zero flag\n'

  d = XMLDiffer(file0, file3, abs_zero=1e-12)
  if d.fail():
    d.message(output=True)

  # Test rel_tol
  # The first call should not error, the second should diff
  print '\n' + lines
  print 'Test 5: Relative Tolerance (1 error)'
  print lines
  d = XMLDiffer(file0, file3)
  if not d.fail():
    print 'Successfully checked value within default tolerance\n'

  d = XMLDiffer(file0, file3, rel_tol=1e-10)
  if d.fail():
    d.message(output=True)

  # Test no-error
  print '\n' + lines
  print 'Test 6: Same File (0 error)'
  print lines
  d = XMLDiffer(file0, file0)
  if not d.fail():
    print 'Successfully checked the same file\n'
<|endoftext|>


<commit_before>from scrapy import cmdline
import os

class Crawler:
    def execute(self, module):
        command = ("scrapy crawl %s" % module)
        cmdline.execute(command.split())

def main():
    utils = Crawler()

    # utils.execute("haraj")
    # utils.execute('harajwatch')

    # ===============
    # test
    # ===============
    # utils.execute("opensooq_debug")
    # ===============
    # watch
    # ===============
    utils.execute('opensooqwatch_debug')
    # utils.execute("mstamlwatch_debug")

if __name__ == '__main__':
    main()
<commit_msg>cwharaj.<commit_after>from scrapy import cmdline
import os

class Crawler:
    def execute(self, module):
        command = ("scrapy crawl %s" % module)
        cmdline.execute(command.split())

def main():
    utils = Crawler()

    # utils.execute("haraj")
    # utils.execute('harajwatch')

    # ===============
    # test
    # ===============
    # utils.execute("opensooq_debug")
    # ===============
    # watch
    # ===============
    # utils.execute('opensooqwatch_debug')
    utils.execute("mstamlwatch_debug")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># RUN: %PYTHON %s 2>&1 | FileCheck %s

# This file contains small benchmarks with reasonably-sized problem/tiling sizes
# and codegen options.

from ..core.experts import *
from ..core.harness import *
from ..core.transforms import *

from ..contraction.definitions import EinsumProblem

################################################################################
### Compilation strategies.
################################################################################

all_experts = [
    e.print_ir(after_all=False, at_begin=False, llvm=False) for e in [
    SingleTilingExpert('matmul_on_tensors',
                       'linalg.generic',
                       tile_sizes=[12, 32, 8],
                       tile_interchange=[0, 1, 2],
                       pad=True,
                       pack_paddings=[1, 1, 0],
                       hoist_paddings=[2, 3, 0]),
    DoubleTileAndDecompose('matmul_on_tensors',
                            'linalg.generic',
                            tile_sizes1=[288, 128, 512],
                            tile_interchange1=[0, 2, 1],
                            tile_sizes2=[9, 32, 16],
                            tile_interchange2=[0, 1, 2],
                            pad2=True,
                            pack_paddings2=[1, 1, 0],
                            hoist_paddings2=[5, 6, 0]).then(\
      Vectorize('matmul_on_tensors',
                'linalg.generic')).then(\
      LoweringOnlyExpert('matmul_on_tensors',
                         'linalg.generic',
                         transpose_lowering='eltwise')),
    DoubleTileAndDecompose('matmul_on_tensors',
                            'linalg.generic',
                            tile_sizes1=[128, 384, 512],
                            tile_interchange1=[0, 1, 2],
                            tile_sizes2=[12, 32, 1],
                            tile_interchange2=[1, 0, 2],
                            pad2=True,
                            pack_paddings2=[1, 1, 0],
                            hoist_paddings2=[3, 2, 0]).then(\
      Vectorize('matmul_on_tensors',
                'linalg.generic')).then(\
      LoweringOnlyExpert('matmul_on_tensors',
                         'linalg.generic',
                         transpose_lowering='eltwise'))
    ]]

################################################################################
### Problem instantiations.
################################################################################

keys = ['m', 'n', 'k']

def make_size_list(sizes: Sequence):
  return {k: v for k, v in zip(keys, sizes)}

# CHECK-NOT: FAILURE
def main():
  n_iters = 100
  problem_size_list = [
      [1, 384, 384],
      [128, 384, 384],
      [128, 1536, 384],
      [128, 384, 1536],
      [192, 128, 256],
      [260, 280, 300],
      [1000, 1000, 1000],
      [1020, 1020, 1020],
      [1020, 1021, 1022],
      [1024, 1024, 1024],
      [2048, 2048, 347],
  ]

  def numpy_kernel(args, sizes, types):
    A, B, C = args
    C.fill(0.)
    np.dot(A, B, out=C)

  def pytorch_kernel(args, sizes, types):
    A, B, C = args
    C.fill_(0.)
    torch.mm(A, B, out=C)

  for runtime_only in [
      [],  # case 1: static at compile time
      ['m', 'k'],  # case 2: partially dynamic at compile time
      keys # case 3: fully dynamic at compile time
  ]:
    #            fastest           slowest
    # specs: C +=  A^T.B      A.B    A.B^T
    for spec in ['km,kn', 'mk,kn', 'mk,nk' ]:
      test_harness(lambda s, t: EinsumProblem(spec), [[np.float32] * 3],
                  map(make_size_list, problem_size_list),
                  all_experts,
                  n_iters=n_iters,
                  runtime_only_sizes=set(runtime_only),
                  function_name='matmul_on_tensors',
                  dump_ir_to_file='/tmp/abc.mlir',
                  dump_obj_to_file='/tmp/abc.o',
                  numpy_benchmark=numpy_kernel,
                  pytorch_benchmark=pytorch_kernel)

def benchmark():
  n_iters = 10
  problem_size_list = [
      [200, 200, 200],
      [1000, 1000, 1000],
      [2000, 2000, 2000],
  ]
  test_harness(lambda s, t: EinsumProblem('mk,kn'), [[np.float32] * 3],
               map(make_size_list, problem_size_list),
               all_experts,
               n_iters=n_iters,
               function_name='matmul_on_tensors')

if __name__ == '__main__':
  main()
<commit_msg>Add missing import for PyTorch<commit_after># RUN: %PYTHON %s 2>&1 | FileCheck %s

# This file contains small benchmarks with reasonably-sized problem/tiling sizes
# and codegen options.

from ..core.experts import *
from ..core.harness import *
from ..core.transforms import *

from ..contraction.definitions import EinsumProblem

################################################################################
### Compilation strategies.
################################################################################

all_experts = [
    e.print_ir(after_all=False, at_begin=False, llvm=False) for e in [
    SingleTilingExpert('matmul_on_tensors',
                       'linalg.generic',
                       tile_sizes=[12, 32, 8],
                       tile_interchange=[0, 1, 2],
                       pad=True,
                       pack_paddings=[1, 1, 0],
                       hoist_paddings=[2, 3, 0]),
    DoubleTileAndDecompose('matmul_on_tensors',
                            'linalg.generic',
                            tile_sizes1=[288, 128, 512],
                            tile_interchange1=[0, 2, 1],
                            tile_sizes2=[9, 32, 16],
                            tile_interchange2=[0, 1, 2],
                            pad2=True,
                            pack_paddings2=[1, 1, 0],
                            hoist_paddings2=[5, 6, 0]).then(\
      Vectorize('matmul_on_tensors',
                'linalg.generic')).then(\
      LoweringOnlyExpert('matmul_on_tensors',
                         'linalg.generic',
                         transpose_lowering='eltwise')),
    DoubleTileAndDecompose('matmul_on_tensors',
                            'linalg.generic',
                            tile_sizes1=[128, 384, 512],
                            tile_interchange1=[0, 1, 2],
                            tile_sizes2=[12, 32, 1],
                            tile_interchange2=[1, 0, 2],
                            pad2=True,
                            pack_paddings2=[1, 1, 0],
                            hoist_paddings2=[3, 2, 0]).then(\
      Vectorize('matmul_on_tensors',
                'linalg.generic')).then(\
      LoweringOnlyExpert('matmul_on_tensors',
                         'linalg.generic',
                         transpose_lowering='eltwise'))
    ]]

################################################################################
### Problem instantiations.
################################################################################

keys = ['m', 'n', 'k']

def make_size_list(sizes: Sequence):
  return {k: v for k, v in zip(keys, sizes)}

# CHECK-NOT: FAILURE
def main():
  n_iters = 100
  problem_size_list = [
      [1, 384, 384],
      [128, 384, 384],
      [128, 1536, 384],
      [128, 384, 1536],
      [192, 128, 256],
      [260, 280, 300],
      [1000, 1000, 1000],
      [1020, 1020, 1020],
      [1020, 1021, 1022],
      [1024, 1024, 1024],
      [2048, 2048, 347],
  ]

  def numpy_kernel(args, sizes, types):
    A, B, C = args
    C.fill(0.)
    np.dot(A, B, out=C)

  def pytorch_kernel(args, sizes, types):
    import torch
    A, B, C = args
    C.fill_(0.)
    torch.mm(A, B, out=C)

  for runtime_only in [
      [],  # case 1: static at compile time
      ['m', 'k'],  # case 2: partially dynamic at compile time
      keys # case 3: fully dynamic at compile time
  ]:
    #            fastest           slowest
    # specs: C +=  A^T.B      A.B    A.B^T
    for spec in ['km,kn', 'mk,kn', 'mk,nk' ]:
      test_harness(lambda s, t: EinsumProblem(spec), [[np.float32] * 3],
                  map(make_size_list, problem_size_list),
                  all_experts,
                  n_iters=n_iters,
                  runtime_only_sizes=set(runtime_only),
                  function_name='matmul_on_tensors',
                  dump_ir_to_file='/tmp/abc.mlir',
                  dump_obj_to_file='/tmp/abc.o',
                  numpy_benchmark=numpy_kernel,
                  pytorch_benchmark=pytorch_kernel)

def benchmark():
  n_iters = 10
  problem_size_list = [
      [200, 200, 200],
      [1000, 1000, 1000],
      [2000, 2000, 2000],
  ]
  test_harness(lambda s, t: EinsumProblem('mk,kn'), [[np.float32] * 3],
               map(make_size_list, problem_size_list),
               all_experts,
               n_iters=n_iters,
               function_name='matmul_on_tensors')

if __name__ == '__main__':
  main()
<|endoftext|>