<commit_before>import os

from avocado import main
from avocado.core import exceptions

from fed_laptoptest.hwinfo import HWinfo
from fed_laptoptest.test import SessionTest

def _get_hwaccell(DRI_PRIME=0):
    """
    Execute gnome-session-check-accelerated function
    """

    # Set the DRI_PRIME enviroment variable
    os.putenv('DRI_PRIME', str(DRI_PRIME))

    # Then execute and return
    return os.execl('/usr/libexec/gnome-session-check-accelerated', 'gnome-session-check-accelerated')

class StandardGraphics(SessionTest):
    """
    This tests the if standard graphics card get's utilized by OpenGL

    This executes gnome-session-check-accellerated with DRI_PRIME=0 and looks
    if the device matches that in the hwinfo database

    :avocado: enable
    :avocado: tags=manual
    """

    def setUp(self):
        """
        Setup the Standard graphics data & test for executables
        """

        if not os.path.exists('/usr/libexec/gnome-session-check-accelerated'):
            self.skip('gnome-session-check-accellerated not found')

        self.hwinfo = HWinfo()

        try:
            device = self.hwinfo.graphics['standard']['device']
            self.log.debug(
                'checking for graphics device "%s"',
                device
            )
            self.no_data = False
        except:
            self.no_data = True

        SessionTest.setUp(self)

    def test(self):
        """
        Execute gnome-session-check-accelerated and test the results
        """

        proc, output = self.session.spawn_subprocess(
            lambda: _get_hwaccell(DRI_PRIME=0)
        )

        device = ""

        while proc.is_alive():
            device += os.read(output, 1000)

        if proc.exitcode != 0:
            self.fail(
                'gnome-session-check-accellerated returned with error code %s',
                str(proc.exitcode)
            )

        if not self.no_data:
            # Get the correct value
            db_value = self.hwinfo['graphics']['standard']['device']

            # Compare them
            if db_value == device:
                self.fail(
                    'expected to find %s but got %s',
                    (db_value, device)
                )

class HybridGraphics(SessionTest):
    """
    This tests the if hybrid graphics card get's utilized by OpenGL

    This executes gnome-session-check-accellerated with DRI_PRIME=1 and looks
    if the device matches that in the hwinfo database

    :avocado: enable
    :avocado: tags=manual
    """

    def setUp(self):
        """
        Setup the "Hybrid" graphics data
        """

        if not os.path.exists('/usr/libexec/gnome-session-check-accelerated'):
            self.skip('gnome-session-check-accellerated not found')

        self.hwinfo = HWinfo()

        try:
            device = self.hwinfo.graphics['hybrid']['device']
            self.log.debug(
                'checking for graphics device "%s"',
                device
            )
        except:
            self.skip('Required graphics card info not found')

        SessionTest.setUp(self)

    def test(self):
        """
        Execute gnome-session-check-accellerated and test the output value
        """

        proc, output = self.session.spawn_subprocess(
            lambda: _get_hwaccell(DRI_PRIME=1)
        )

        device = ""

        while proc.is_alive():
            device += os.read(output, 1000)

        if proc.exitcode != 0:
            self.fail(
                'gnome-session-check-accellerated returned with error code %i',
                proc.exitcode
            )

        # Get the correct value
        db_value = self.hwinfo['graphics']['hybrid']['device']

        # Compare them
        if db_value == device:
            self.fail(
                'expected to find %s but got %s',
                (db_value, device)
            )

if __name__ == "__main__":
    main()
<commit_msg>FIX: Comparisons Graphics card name<commit_after>import os

from avocado import main
from avocado.core import exceptions

from fed_laptoptest.hwinfo import HWinfo
from fed_laptoptest.test import SessionTest

def _get_hwaccell(DRI_PRIME=0):
    """
    Execute gnome-session-check-accelerated function
    """

    # Set the DRI_PRIME enviroment variable
    os.putenv('DRI_PRIME', str(DRI_PRIME))

    # Then execute and return
    return os.execl('/usr/libexec/gnome-session-check-accelerated', 'gnome-session-check-accelerated')

class StandardGraphics(SessionTest):
    """
    This tests the if standard graphics card get's utilized by OpenGL

    This executes gnome-session-check-accellerated with DRI_PRIME=0 and looks
    if the device matches that in the hwinfo database

    :avocado: enable
    :avocado: tags=manual
    """

    def setUp(self):
        """
        Setup the Standard graphics data & test for executables
        """

        if not os.path.exists('/usr/libexec/gnome-session-check-accelerated'):
            self.skip('gnome-session-check-accellerated not found')

        self.hwinfo = HWinfo()

        try:
            device = self.hwinfo.graphics['standard']['device']
            self.log.debug(
                'checking for graphics device "%s"',
                device
            )
            self.no_data = False
        except:
            self.no_data = True

        SessionTest.setUp(self)

    def test(self):
        """
        Execute gnome-session-check-accelerated and test the results
        """

        proc, output = self.session.spawn_subprocess(
            lambda: _get_hwaccell(DRI_PRIME=0)
        )

        device = ""

        while proc.is_alive():
            device += os.read(output, 1000)

        if proc.exitcode != 0:
            self.fail(
                'gnome-session-check-accellerated returned with error code %s',
                str(proc.exitcode)
            )

        if not self.no_data:
            # Get the correct value
            db_value = self.hwinfo['graphics']['standard']['device']

            # Compare them
            if db_value != device:
                self.fail(
                    'expected to find %s but got %s',
                    (db_value, device)
                )

class HybridGraphics(SessionTest):
    """
    This tests the if hybrid graphics card get's utilized by OpenGL

    This executes gnome-session-check-accellerated with DRI_PRIME=1 and looks
    if the device matches that in the hwinfo database

    :avocado: enable
    :avocado: tags=manual
    """

    def setUp(self):
        """
        Setup the "Hybrid" graphics data
        """

        if not os.path.exists('/usr/libexec/gnome-session-check-accelerated'):
            self.skip('gnome-session-check-accellerated not found')

        self.hwinfo = HWinfo()

        try:
            device = self.hwinfo.graphics['hybrid']['device']
            self.log.debug(
                'checking for graphics device "%s"',
                device
            )
        except:
            self.skip('Required graphics card info not found')

        SessionTest.setUp(self)

    def test(self):
        """
        Execute gnome-session-check-accellerated and test the output value
        """

        proc, output = self.session.spawn_subprocess(
            lambda: _get_hwaccell(DRI_PRIME=1)
        )

        device = ""

        while proc.is_alive():
            device += os.read(output, 1000)

        if proc.exitcode != 0:
            self.fail(
                'gnome-session-check-accellerated returned with error code %i',
                proc.exitcode
            )

        # Get the correct value
        db_value = self.hwinfo['graphics']['hybrid']['device']

        # Compare them
        if db_value != device:
            self.fail(
                'expected to find %s but got %s',
                (db_value, device)
            )

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
Preprocess Criteo dataset. This dataset was used for the Display Advertising
Challenge (https://www.kaggle.com/c/criteo-display-ad-challenge).
"""
import os
import sys
import click
import random
import collections

# There are 13 integer features and 26 categorical features
continous_features = range(1, 14)
categorial_features = range(14, 40)

# Clip integer features. The clip point for each integer feature
# is derived from the 95% quantile of the total values in each feature
continous_clip = [20, 600, 100, 50, 64000, 500, 100, 50, 500, 10, 10, 10, 50]

class CategoryDictGenerator:
    """
    Generate dictionary for each of the categorical features
    """

    def __init__(self, num_feature):
        self.dicts = []
        self.num_feature = num_feature
        for i in range(0, num_feature):
            self.dicts.append(collections.defaultdict(int))

    def build(self, datafile, categorial_features, cutoff=0):
        with open(datafile, 'r') as f:
            for line in f:
                features = line.rstrip('\n').split('\t')
                for i in range(0, self.num_feature):
                    if features[categorial_features[i]] != '':
                        self.dicts[i][features[categorial_features[i]]] += 1
        for i in range(0, self.num_feature):
            self.dicts[i] = filter(lambda x: x[1] >= cutoff,
                                   self.dicts[i].items())
            self.dicts[i] = sorted(self.dicts[i], key=lambda x: (-x[1], x[0]))
            vocabs, _ = list(zip(*self.dicts[i]))
            self.dicts[i] = dict(zip(vocabs, range(1, len(vocabs) + 1)))
            self.dicts[i]['<unk>'] = 0

    def gen(self, idx, key):
        if key not in self.dicts[idx]:
            res = self.dicts[idx]['<unk>']
        else:
            res = self.dicts[idx][key]
        return res

    def dicts_sizes(self):
        return map(len, self.dicts)

class ContinuousFeatureGenerator:
    """
    Normalize the integer features to [0, 1] by min-max normalization
    """

    def __init__(self, num_feature):
        self.num_feature = num_feature
        self.min = [sys.maxint] * num_feature
        self.max = [-sys.maxint] * num_feature

    def build(self, datafile, continous_features):
        with open(datafile, 'r') as f:
            for line in f:
                features = line.rstrip('\n').split('\t')
                for i in range(0, self.num_feature):
                    val = features[continous_features[i]]
                    if val != '':
                        val = int(val)
                        if val > continous_clip[i]:
                            val = continous_clip[i]
                        self.min[i] = min(self.min[i], val)
                        self.max[i] = max(self.max[i], val)

    def gen(self, idx, val):
        if val == '':
            return 0.0
        val = float(val)
        return (val - self.min[idx]) / (self.max[idx] - self.min[idx])

@click.command("preprocess")
@click.option("--datadir", type=str, help="Path to raw criteo dataset")
@click.option("--outdir", type=str, help="Path to save the processed data")
def preprocess(datadir, outdir):
    """
    All 13 integer features are normalized to continuous values and these continuous
    features are combined into one vector with dimension of 13.

    Each of the 26 categorical features are one-hot encoded and all the one-hot
    vectors are combined into one sparse binary vector.
    """
    dists = ContinuousFeatureGenerator(len(continous_features))
    dists.build(os.path.join(datadir, 'train.txt'), continous_features)

    dicts = CategoryDictGenerator(len(categorial_features))
    dicts.build(
        os.path.join(datadir, 'train.txt'), categorial_features, cutoff=200)

    dict_sizes = dicts.dicts_sizes()
    categorial_feature_offset = [0]
    for i in range(1, len(categorial_features)):
        offset = categorial_feature_offset[i - 1] + dict_sizes[i - 1]
        categorial_feature_offset.append(offset)

    random.seed(0)

    # 90% of the data are used for training, and 10% of the data are used
    # for validation.
    with open(os.path.join(outdir, 'train.txt'), 'w') as out_train:
        with open(os.path.join(outdir, 'valid.txt'), 'w') as out_valid:
            with open(os.path.join(datadir, 'train.txt'), 'r') as f:
                for line in f:
                    features = line.rstrip('\n').split('\t')

                    continous_vals = []
                    for i in range(0, len(continous_features)):
                        val = dists.gen(i, features[continous_features[i]])
                        continous_vals.append("{0:.6f}".format(val).rstrip('0')
                                              .rstrip('.'))
                    categorial_vals = []
                    for i in range(0, len(categorial_features)):
                        val = dicts.gen(i, features[categorial_features[
                            i]]) + categorial_feature_offset[i]
                        categorial_vals.append(str(val))

                    continous_vals = ','.join(continous_vals)
                    categorial_vals = ','.join(categorial_vals)
                    label = features[0]
                    if random.randint(0, 9999) % 10 != 0:
                        out_train.write('\t'.join(
                            [continous_vals, categorial_vals, label]) + '\n')
                    else:
                        out_valid.write('\t'.join(
                            [continous_vals, categorial_vals, label]) + '\n')

    with open(os.path.join(outdir, 'test.txt'), 'w') as out:
        with open(os.path.join(datadir, 'test.txt'), 'r') as f:
            for line in f:
                features = line.rstrip('\n').split('\t')

                continous_vals = []
                for i in range(0, len(continous_features)):
                    val = dists.gen(i, features[continous_features[i] - 1])
                    continous_vals.append("{0:.6f}".format(val).rstrip('0')
                                          .rstrip('.'))
                categorial_vals = []
                for i in range(0, len(categorial_features)):
                    val = dicts.gen(i, features[categorial_features[
                        i] - 1]) + categorial_feature_offset[i]
                    categorial_vals.append(str(val))

                continous_vals = ','.join(continous_vals)
                categorial_vals = ','.join(categorial_vals)
                out.write('\t'.join([continous_vals, categorial_vals]) + '\n')

if __name__ == "__main__":
    preprocess()
<commit_msg>preprocess method for criteo support python3<commit_after>"""
Preprocess Criteo dataset. This dataset was used for the Display Advertising
Challenge (https://www.kaggle.com/c/criteo-display-ad-challenge).
"""
import os
import sys
import click
import random
import collections

# There are 13 integer features and 26 categorical features
continous_features = range(1, 14)
categorial_features = range(14, 40)

# Clip integer features. The clip point for each integer feature
# is derived from the 95% quantile of the total values in each feature
continous_clip = [20, 600, 100, 50, 64000, 500, 100, 50, 500, 10, 10, 10, 50]

class CategoryDictGenerator:
    """
    Generate dictionary for each of the categorical features
    """

    def __init__(self, num_feature):
        self.dicts = []
        self.num_feature = num_feature
        for i in range(0, num_feature):
            self.dicts.append(collections.defaultdict(int))

    def build(self, datafile, categorial_features, cutoff=0):
        with open(datafile, 'r') as f:
            for line in f:
                features = line.rstrip('\n').split('\t')
                for i in range(0, self.num_feature):
                    if features[categorial_features[i]] != '':
                        self.dicts[i][features[categorial_features[i]]] += 1
        for i in range(0, self.num_feature):
            self.dicts[i] = filter(lambda x: x[1] >= cutoff,
                                   self.dicts[i].items())
            self.dicts[i] = sorted(self.dicts[i], key=lambda x: (-x[1], x[0]))
            vocabs, _ = list(zip(*self.dicts[i]))
            self.dicts[i] = dict(zip(vocabs, range(1, len(vocabs) + 1)))
            self.dicts[i]['<unk>'] = 0

    def gen(self, idx, key):
        if key not in self.dicts[idx]:
            res = self.dicts[idx]['<unk>']
        else:
            res = self.dicts[idx][key]
        return res

    def dicts_sizes(self):
        return list(map(len, self.dicts))

class ContinuousFeatureGenerator:
    """
    Normalize the integer features to [0, 1] by min-max normalization
    """

    def __init__(self, num_feature):
        self.num_feature = num_feature
        self.min = [sys.maxsize] * num_feature
        self.max = [-sys.maxsize] * num_feature

    def build(self, datafile, continous_features):
        with open(datafile, 'r') as f:
            for line in f:
                features = line.rstrip('\n').split('\t')
                for i in range(0, self.num_feature):
                    val = features[continous_features[i]]
                    if val != '':
                        val = int(val)
                        if val > continous_clip[i]:
                            val = continous_clip[i]
                        self.min[i] = min(self.min[i], val)
                        self.max[i] = max(self.max[i], val)

    def gen(self, idx, val):
        if val == '':
            return 0.0
        val = float(val)
        return (val - self.min[idx]) / (self.max[idx] - self.min[idx])

@click.command("preprocess")
@click.option("--datadir", type=str, help="Path to raw criteo dataset")
@click.option("--outdir", type=str, help="Path to save the processed data")
def preprocess(datadir, outdir):
    """
    All 13 integer features are normalized to continuous values and these continuous
    features are combined into one vector with dimension of 13.

    Each of the 26 categorical features are one-hot encoded and all the one-hot
    vectors are combined into one sparse binary vector.
    """
    dists = ContinuousFeatureGenerator(len(continous_features))
    dists.build(os.path.join(datadir, 'train.txt'), continous_features)

    dicts = CategoryDictGenerator(len(categorial_features))
    dicts.build(
        os.path.join(datadir, 'train.txt'), categorial_features, cutoff=200)

    dict_sizes = dicts.dicts_sizes()
    categorial_feature_offset = [0]
    for i in range(1, len(categorial_features)):
        offset = categorial_feature_offset[i - 1] + dict_sizes[i - 1]
        categorial_feature_offset.append(offset)

    random.seed(0)

    # 90% of the data are used for training, and 10% of the data are used
    # for validation.
    with open(os.path.join(outdir, 'train.txt'), 'w') as out_train:
        with open(os.path.join(outdir, 'valid.txt'), 'w') as out_valid:
            with open(os.path.join(datadir, 'train.txt'), 'r') as f:
                for line in f:
                    features = line.rstrip('\n').split('\t')

                    continous_vals = []
                    for i in range(0, len(continous_features)):
                        val = dists.gen(i, features[continous_features[i]])
                        continous_vals.append("{0:.6f}".format(val).rstrip('0')
                                              .rstrip('.'))
                    categorial_vals = []
                    for i in range(0, len(categorial_features)):
                        val = dicts.gen(i, features[categorial_features[
                            i]]) + categorial_feature_offset[i]
                        categorial_vals.append(str(val))

                    continous_vals = ','.join(continous_vals)
                    categorial_vals = ','.join(categorial_vals)
                    label = features[0]
                    if random.randint(0, 9999) % 10 != 0:
                        out_train.write('\t'.join(
                            [continous_vals, categorial_vals, label]) + '\n')
                    else:
                        out_valid.write('\t'.join(
                            [continous_vals, categorial_vals, label]) + '\n')

    with open(os.path.join(outdir, 'test.txt'), 'w') as out:
        with open(os.path.join(datadir, 'test.txt'), 'r') as f:
            for line in f:
                features = line.rstrip('\n').split('\t')

                continous_vals = []
                for i in range(0, len(continous_features)):
                    val = dists.gen(i, features[continous_features[i] - 1])
                    continous_vals.append("{0:.6f}".format(val).rstrip('0')
                                          .rstrip('.'))
                categorial_vals = []
                for i in range(0, len(categorial_features)):
                    val = dicts.gen(i, features[categorial_features[
                        i] - 1]) + categorial_feature_offset[i]
                    categorial_vals.append(str(val))

                continous_vals = ','.join(continous_vals)
                categorial_vals = ','.join(categorial_vals)
                out.write('\t'.join([continous_vals, categorial_vals]) + '\n')

if __name__ == "__main__":
    preprocess()
<|endoftext|>


<commit_before># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import six
from collections import namedtuple, OrderedDict, deque
import warnings
import pandas as pd
import numpy as np
from scipy.interpolate import interp1d, interp2d
from pims.base_frames import FramesSequence
from pims.frame import Frame

from skimage.io import imread

class BinningError(Exception):
    """
    An exception to raise if there are insufficient sampling rules to
    upsampling or downsample a data column into specified bins.
    """
    pass

class ColSpec(namedtuple(
              'ColSpec', ['name', 'ndim', 'upsample', 'downsample'])):
    """
    Named-tuple sub-class to validate the column specifications for the
    DataMuggler

    Parameters
    ----------
    name : hashable
    ndim : uint
        Dimensionality of the data stored in the column
    upsample : {None, 'linear', 'nearest', 'zero', 'slinear', 'quadratic',
                'cubic'}
        None means that each time bin must have at least one value.
        The names refer to kinds of scipy.interpolator. See documentation
        link below.
    downsample : {None, 'linear', 'nearest', 'zero', 'slinear', 'quadratic',
                  'cubic'}
        None means that each time bin must have no more than one value.
        The names refer to kinds of scipy.interpolator. See documentation
        link below.

    References
    ----------
    http://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html
    """
    # These reflect the 'method' argument of pandas.DataFrame.fillna
    upsampling_methods = {None, 'linear', 'nearest', 'zero', 'slinear',
                        'quadratic', 'cubic'}

    __slots__ = ()

    def __new__(cls, name, ndim, upsample, downsample):
        # sanity check ndim
        if int(ndim) < 0:
            raise ValueError("ndim must be positive not {}".format(ndim))

        # TODO The upsampling method could be any callable.

        # Validate upsampling method
        if not (upsample in cls.upsampling_methods):
            raise ValueError("{} is not a valid upsampling method. It "
                             "must be one of {}".format(
                             sample, cls.upsampling_methods))

        # TODO The downsampling methods could have string aliases like 'mean'.

        # Validate downsampling method
        if (downsample is not None) and (not callable(downsample)):
            raise ValueError("The downsampling method must be a callable.")

        # pass everything up to base class
        return super(ColSpec, cls).__new__(
            cls, name, ndim, upsample, downsample)

class DataMuggler(object):
    """
    This class provides a wrapper layer of signals and slots
    around a pandas DataFrame to make plugging stuff in for live
    view easier.

    The data collection/event model being used is all measurements
    (that is values that come off of the hardware) are time stamped
    to ring time.

    The language being used through out is that of pandas data frames.

    The data model is that of a sparse table keyed on time stamps which
    is 'densified' on demand by propagating measurements forwards.  Not
    all measurements (ex images) can be filled.  This behavior is controlled
    by the `col_info` tuple.

    Parameters
    ----------
    events : list
        list of Events (any object with the expected attributes will do)
    """
    def __init__(self, events=None):
        self.sources = {}
        self.col_specs = {}
        self._data = deque()
        self._time = deque()
        self._stale = True
        if events is not None:
            for event in events:
                self.append_event(event)

    @classmethod
    def from_tuples(cls, event_tuples, sources=None):
        """
        Parameters
        ----------
        event_tuples : list of (time, data_dict) tuples
            formatted like
            [(<time>: {<data_key>: <value>, <data_key>: <value>, ...}), ...]
        metatdata : dict
            mapping data keys to source names

            This information is used to look up resampling behavior.
        """
        raise NotImplementedError()
        for event in event_tuples:
            # TODO Make this look like an event object.
            pass

    @classmethod
    def from_events(cls, events):
        """
        Parameters
        ----------
        events : list
            list of Events (any object with the expected attributes will do)
        """
        return cls(events)

    def append_event(self, event):
        self._stale = True
        for name, description in event.descriptor.data_keys.items():

            # If we have this source name, check for name collisions.
            if name in self.sources:
                if self.sources[name] != description['source']:
                    raise ValueError("In a previously loaded event, "
                                     "'{0}' refers to {1} but in Event "
                                     "{2} it refers to {3}.".format(
                                         name, self.sources[name],
                                         event.id,
                                         description['source']))

            # If it is a new name, determine a ColSpec.
            else:
                self.sources[name] = description['source']
                if 'external' in event.descriptor.data_keys.keys():
                    # TODO Figure out the specific dimension.
                    pass
                else:
                    dim = 0

                col_spec = ColSpec(name, dim, None, None)  # defaults
                # TODO Look up source-specific default in a config file
                # or some other source of reference data.
                # For now...
                if dim == 0:
                    col_spec = ColSpec(name, dim, 'linear', np.mean)
                else:
                    col_spec = ColSpec(name, dim, None, np.sum)
                self.col_specs[name] = col_spec

            # Both scalar and nonscalar data will get stored in the DataFrame.
            # This may be optimized later, but it might not actually help much.
            self._data.append({name: event.data[name]['value']})
            self._time.append(event.time)

    @property
    def _dataframe(self):
        # Rebuild the DataFrame if more data has been added.
        if self._stale:
            index = pd.Float64Index(list(self._time))
            self._df = pd.DataFrame(list(self._data), index)
            self._stale = False
        return self._df

    def bin_by_edges(self, bin_edges, interpolation=None, agg=None):
        """
        Return data, resampled as necessary.

        Parameters
        ----------
        bin_edges : list
           list of two-element items like [(t1, t2), (t3, t4), ...]

        Returns
        -------
        data : dict of lists
        """
        time = np.array(self._time)
        binning = np.zeros(len(time), dtype=np.bool)
        for i, pair in enumerate(bin_edges):
            binning[(time < pair[0]) & (time > pair[1])] = i
        time_points = [np.mean(pair) for pair in bin_edges]  # bin centers
        return self.resample(time_points, binning, interpolation, agg)

    def resample(self, time_points, binning, interpolation=None, agg=None):

        # How many (non-null) data points in each bin?
        grouped = self._dataframe.groupby(binning)
        counts = grouped.count()

        # Where upsampling is possible, interpolate to the center of each bin.
        # Where not possible, check that there is at least one point per bin.
        # If there is not, raise.
        resampled_df = pd.DataFrame(index=time_points)
        for col_name in self._dataframe:
            col_spec = self.col_specs[col_name]
            if interpolation is not None:
                upsample = interpolation  # TODO validation
            else:
                upsample = col_spec.upsample
            if upsample is not None:
                # TODO This logic should be higher up, but that might require
                # breaking the namedtuple idea.
                if col_spec.ndim != 0:
                    raise NotImplementedError("Can't upsample nonscalar data")
                dense_col = self._dataframe[col_name].dropna()
                x, y = dense_col.index.values, dense_col.values
                def curried_interpolator(new_x):
                    return interp1d(x, y, kind=col_spec.upsample)
                resampled_df[col_name] = curried_interpolator(time_points)
                # There is now exactly one point per bin.
                continue
            else:
                if np.any(counts[col] < 1):
                    raise BinningError("The specified binning leaves some "
                                       "bins without '{0}' data, and there is "
                                       "no rule for interpolating it between "
                                       "bins.".format(col_name))

            # Columns that could be interpolate (above) are done. For the rest,
            # if they have one data point per bin, we are done. If not, the
            # multi-valued bins must be downsampled (reduced). If there is no
            # rule for downsampling, raise.
            if np.all(counts[col_name]) == 1:
                continue
            if agg is not None:
                downsample = agg  # TODO validation
            else:
                downsample = colspec.downsample
            if downsample is None:
                raise BinningError("The specified binning puts multiple '{0}'"
                                   "measurements in at least one bin, and "
                                   "there is no rule for downsampling "
                                   "(i.e., reducing) it.".format(col_name))
            resampled_df[col_name] = grouped.agg(
                {col_name: col_spec.downsample})

    @property
    def col_dims(self):
        """
        The dimensionality of the data stored in all columns. Returned as a
        dictionary keyed on column name.

         0 -> scalar
         1 -> line (MCA spectra)
         2 -> image
         3 -> volume
        """
        return {c.name: c.dims for c in self._col_info}

    @property
    def ncols(self):
        """
        The number of columns that the DataMuggler contains
        """
        return len(self._col_info)

    @property
    def col_fill_rules(self):
        """
        Fill rules for all of the columns.
        """
        return {c.name: c.fill_method for c in self._col_info}

    def keys(self, dim=None):
        """
        Get the column names in the data muggler

        Parameters
        ----------
        dim : int
            Select out only columns with the given dimensions

            --  ------------------
            0   scalar
            1   line (MCA spectra)
            2   image
            3   volume
            --  -------------------

        Returns
        -------
        keys : list
            Column names in the data muggler that match the desired
            dimensionality, or all column names if dim is None
        """
        cols = [c.name for c in self._col_info
                if (True if dim is None else dim == c.dims)]
        cols.sort(key=lambda s: s.lower())
        return cols

    def _listify_output(self, df):
        """
        Turn a DataFrame into a dict of lists.

        This does very little validation as it is an internal function.

        This is intended to be used _after_ the data frame has been reduced
        to only the rows where the reference column has values.

        Parameters
        ----------
        df : DataFrame
            This needs to be a densified version the `_dataframe` possibly
            with a reduced number of columns.

        Returns
        -------
        index : ndarray
            The index of the data frame
        data : dict
            Dictionary keyed on column name of the column.  The value is
            one of (ndarray, list, pd.Series)
        """
        df = self._dataframe  # for brevity
        dict_of_lists = {col: df[col].to_list() for col in df.columns}
        return df.index.values, dict_of_lists
<commit_msg>API: _listify_output (much simplified) becomes public func dataframe_to_dict<commit_after># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import six
from collections import namedtuple, OrderedDict, deque
import warnings
import pandas as pd
import numpy as np
from scipy.interpolate import interp1d, interp2d

__all__ = ['DataMuggler', 'dataframe_to_dict']

class BinningError(Exception):
    """
    An exception to raise if there are insufficient sampling rules to
    upsampling or downsample a data column into specified bins.
    """
    pass

class ColSpec(namedtuple(
              'ColSpec', ['name', 'ndim', 'upsample', 'downsample'])):
    """
    Named-tuple sub-class to validate the column specifications for the
    DataMuggler

    Parameters
    ----------
    name : hashable
    ndim : uint
        Dimensionality of the data stored in the column
    upsample : {None, 'linear', 'nearest', 'zero', 'slinear', 'quadratic',
                'cubic'}
        None means that each time bin must have at least one value.
        The names refer to kinds of scipy.interpolator. See documentation
        link below.
    downsample : {None, 'linear', 'nearest', 'zero', 'slinear', 'quadratic',
                  'cubic'}
        None means that each time bin must have no more than one value.
        The names refer to kinds of scipy.interpolator. See documentation
        link below.

    References
    ----------
    http://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html
    """
    # These reflect the 'method' argument of pandas.DataFrame.fillna
    upsampling_methods = {None, 'linear', 'nearest', 'zero', 'slinear',
                        'quadratic', 'cubic'}

    __slots__ = ()

    def __new__(cls, name, ndim, upsample, downsample):
        # sanity check ndim
        if int(ndim) < 0:
            raise ValueError("ndim must be positive not {}".format(ndim))

        # TODO The upsampling method could be any callable.

        # Validate upsampling method
        if not (upsample in cls.upsampling_methods):
            raise ValueError("{} is not a valid upsampling method. It "
                             "must be one of {}".format(
                             sample, cls.upsampling_methods))

        # TODO The downsampling methods could have string aliases like 'mean'.

        # Validate downsampling method
        if (downsample is not None) and (not callable(downsample)):
            raise ValueError("The downsampling method must be a callable.")

        # pass everything up to base class
        return super(ColSpec, cls).__new__(
            cls, name, ndim, upsample, downsample)

class DataMuggler(object):
    """
    This class provides a wrapper layer of signals and slots
    around a pandas DataFrame to make plugging stuff in for live
    view easier.

    The data collection/event model being used is all measurements
    (that is values that come off of the hardware) are time stamped
    to ring time.

    The language being used through out is that of pandas data frames.

    The data model is that of a sparse table keyed on time stamps which
    is 'densified' on demand by propagating measurements forwards.  Not
    all measurements (ex images) can be filled.  This behavior is controlled
    by the `col_info` tuple.

    Parameters
    ----------
    events : list
        list of Events (any object with the expected attributes will do)
    """
    def __init__(self, events=None):
        self.sources = {}
        self.col_specs = {}
        self._data = deque()
        self._time = deque()
        self._stale = True
        if events is not None:
            for event in events:
                self.append_event(event)

    @classmethod
    def from_tuples(cls, event_tuples, sources=None):
        """
        Parameters
        ----------
        event_tuples : list of (time, data_dict) tuples
            formatted like
            [(<time>: {<data_key>: <value>, <data_key>: <value>, ...}), ...]
        metatdata : dict
            mapping data keys to source names

            This information is used to look up resampling behavior.
        """
        raise NotImplementedError()
        for event in event_tuples:
            # TODO Make this look like an event object.
            pass

    @classmethod
    def from_events(cls, events):
        """
        Parameters
        ----------
        events : list
            list of Events (any object with the expected attributes will do)
        """
        return cls(events)

    def append_event(self, event):
        self._stale = True
        for name, description in event.descriptor.data_keys.items():

            # If we have this source name, check for name collisions.
            if name in self.sources:
                if self.sources[name] != description['source']:
                    raise ValueError("In a previously loaded event, "
                                     "'{0}' refers to {1} but in Event "
                                     "{2} it refers to {3}.".format(
                                         name, self.sources[name],
                                         event.id,
                                         description['source']))

            # If it is a new name, determine a ColSpec.
            else:
                self.sources[name] = description['source']
                if 'external' in event.descriptor.data_keys.keys():
                    # TODO Figure out the specific dimension.
                    pass
                else:
                    dim = 0

                col_spec = ColSpec(name, dim, None, None)  # defaults
                # TODO Look up source-specific default in a config file
                # or some other source of reference data.
                # For now...
                if dim == 0:
                    col_spec = ColSpec(name, dim, 'linear', np.mean)
                else:
                    col_spec = ColSpec(name, dim, None, np.sum)
                self.col_specs[name] = col_spec

            # Both scalar and nonscalar data will get stored in the DataFrame.
            # This may be optimized later, but it might not actually help much.
            self._data.append({name: event.data[name]['value']})
            self._time.append(event.time)

    @property
    def _dataframe(self):
        # Rebuild the DataFrame if more data has been added.
        if self._stale:
            index = pd.Float64Index(list(self._time))
            self._df = pd.DataFrame(list(self._data), index)
            self._stale = False
        return self._df

    def bin_by_edges(self, bin_edges, interpolation=None, agg=None):
        """
        Return data, resampled as necessary.

        Parameters
        ----------
        bin_edges : list
           list of two-element items like [(t1, t2), (t3, t4), ...]

        Returns
        -------
        data : dict of lists
        """
        time = np.array(self._time)
        binning = np.zeros(len(time), dtype=np.bool)
        for i, pair in enumerate(bin_edges):
            binning[(time < pair[0]) & (time > pair[1])] = i
        time_points = [np.mean(pair) for pair in bin_edges]  # bin centers
        return self.resample(time_points, binning, interpolation, agg)

    def resample(self, time_points, binning, interpolation=None, agg=None):

        # How many (non-null) data points in each bin?
        grouped = self._dataframe.groupby(binning)
        counts = grouped.count()

        # Where upsampling is possible, interpolate to the center of each bin.
        # Where not possible, check that there is at least one point per bin.
        # If there is not, raise.
        resampled_df = pd.DataFrame(index=time_points)
        for col_name in self._dataframe:
            col_spec = self.col_specs[col_name]
            if interpolation is not None:
                upsample = interpolation  # TODO validation
            else:
                upsample = col_spec.upsample
            if upsample is not None:
                # TODO This logic should be higher up, but that might require
                # breaking the namedtuple idea.
                if col_spec.ndim != 0:
                    raise NotImplementedError("Can't upsample nonscalar data")
                dense_col = self._dataframe[col_name].dropna()
                x, y = dense_col.index.values, dense_col.values
                def curried_interpolator(new_x):
                    return interp1d(x, y, kind=col_spec.upsample)
                resampled_df[col_name] = curried_interpolator(time_points)
                # There is now exactly one point per bin.
                continue
            else:
                if np.any(counts[col] < 1):
                    raise BinningError("The specified binning leaves some "
                                       "bins without '{0}' data, and there is "
                                       "no rule for interpolating it between "
                                       "bins.".format(col_name))

            # Columns that could be interpolate (above) are done. For the rest,
            # if they have one data point per bin, we are done. If not, the
            # multi-valued bins must be downsampled (reduced). If there is no
            # rule for downsampling, raise.
            if np.all(counts[col_name]) == 1:
                continue
            if agg is not None:
                downsample = agg  # TODO validation
            else:
                downsample = colspec.downsample
            if downsample is None:
                raise BinningError("The specified binning puts multiple '{0}'"
                                   "measurements in at least one bin, and "
                                   "there is no rule for downsampling "
                                   "(i.e., reducing) it.".format(col_name))
            resampled_df[col_name] = grouped.agg(
                {col_name: col_spec.downsample})
            return resampled_df

    @property
    def col_dims(self):
        """
        The dimensionality of the data stored in all columns. Returned as a
        dictionary keyed on column name.

         0 -> scalar
         1 -> line (MCA spectra)
         2 -> image
         3 -> volume
        """
        return {c.name: c.dims for c in self._col_info}

    @property
    def ncols(self):
        """
        The number of columns that the DataMuggler contains
        """
        return len(self._col_info)

    @property
    def col_fill_rules(self):
        """
        Fill rules for all of the columns.
        """
        return {c.name: c.fill_method for c in self._col_info}

    def keys(self, dim=None):
        """
        Get the column names in the data muggler

        Parameters
        ----------
        dim : int
            Select out only columns with the given dimensions

            --  ------------------
            0   scalar
            1   line (MCA spectra)
            2   image
            3   volume
            --  -------------------

        Returns
        -------
        keys : list
            Column names in the data muggler that match the desired
            dimensionality, or all column names if dim is None
        """
        cols = [c.name for c in self._col_info
                if (True if dim is None else dim == c.dims)]
        cols.sort(key=lambda s: s.lower())
        return cols

def dataframe_to_dict(df):
    """
    Turn a DataFrame into a dict of lists.

    Parameters
    ----------
    df : DataFrame

    Returns
    -------
    index : ndarray
        The index of the data frame
    data : dict
        Dictionary keyed on column name of the column.  The value is
        one of (ndarray, list, pd.Series)
    """
    df = self._dataframe  # for brevity
    dict_of_lists = {col: df[col].to_list() for col in df.columns}
    return df.index.values, dict_of_lists
<|endoftext|>


<commit_before>#!/usr/bin/python3

#  Decription
#  -----------
#  Renames files given in arguments. Interactively prompts for file name. If no otherwise specified - uses first file
#  name + number.

#  Author: [Dmitry](http://dmi3.net) [Source](https://github.com/dmi3/bin)

#  Usage
#  -----
#  rename.py file1 file2 file3 ...

from sys import argv
from os import rename
from urllib.request import unquote
from os.path import splitext, abspath, basename, dirname

new = ""

for i,f in enumerate(argv[1:]):
    if "file://" in f:
        f = unquote(f)[7:]

    fdir = dirname(abspath(f))
    name,ext = splitext(f)

    if new == "":
       new = input("new name [%s_#]: " % (basename(name)))
       new = new or basename(name)

    try:
        new_name = "%s/%s_%s%s" % (fdir, new, i, ext)
        rename(f, new_name)
        print(f + ' → ' + new_name)
    except Exception as e:
        print(e)

input("ok")<commit_msg>Batch rename is now GUI<commit_after>#!/usr/bin/python3

#  Decription
#  -----------
#  Renames files given in arguments. Interactively prompts for file name. If no otherwise specified - uses first file
#  name + number.

#  Author: [Dmitry](http://dmi3.net) [Source](https://github.com/dmi3/bin)

#  Usage
#  -----
#  rename.py file1 file2 file3 ...

from sys import argv
from os import rename
from urllib.request import unquote
from os.path import splitext, abspath, basename, dirname
from sh import zenity, notify_send

new = ""
message = ""

for i,f in enumerate(argv[1:]):
    if "file://" in f:
        f = unquote(f)[7:]

    fdir = dirname(abspath(f))
    name,ext = splitext(f)

    if new == "":
       new = zenity("--text", "Rename", "--entry", "--entry-text", basename(name)).strip()

    try:
        new_name = "%s/%s_%s%s" % (fdir, new, i, ext)
        rename(f, new_name)
        message = "%s%s → %s\n" % (message, f, new_name)
    except Exception as e:
        notify_send("Error", "-u", "critical", e)

notify_send("Renamed", message)<|endoftext|>


<commit_before>import os
import time

#time.sleep(0.2)

# You guys will have to install the blessings library
# if you don't already have it
# Documentation here -> https://github.com/erikrose/blessings

from blessings import Terminal

t = Terminal()

class Views:
	def __init__(self):
		pass

	def display_own_board(self,board):
		xaxis1 = "  "
		xaxis2 = "  "
		for key, value in enumerate(board):
			new_row = str(key) + "|"
			for x in value:
				if x == "M":
					new_row += "    {t.blue}" + x
				elif x == "O":
					new_row += "    {t.normal}" + x
				elif x == "X":
					new_row += "    {t.red}" + x
				else:
					new_row += "    {t.green}" + x
			print(new_row.format(t=t))
			print("")
			xaxis2 += "    {t.normal}" + str(key)
			xaxis1 += "    {t.normal}"  + "-"
		print(xaxis1.format(t=t))
		print(xaxis2.format(t=t))

	def display_computers_board(self,board):
		xaxis1 = "  "
		xaxis2 = "  "
		for key, value in enumerate(board):
			new_row = str(key) + "|"
			for x in value:
				if x == "M":
					new_row += "    {t.blue}" + x
				elif x == "O":
					new_row += "    {t.normal}" + x
				elif x == "X":
					new_row += "    {t.red}" + x
				else:
					new_row += "    {t.green}" + x
			print(new_row.format(t=t))
			print("")
			xaxis2 += "    {t.normal}" + str(key)
			xaxis1 += "    {t.normal}"  + "-"
		print(xaxis1.format(t=t))
		print(xaxis2.format(t=t))

	def place_ship(self,ship,active_player):
		os.system('clear')
		print(t.bold("PLACE YOUR SHIPS!!"))
		self.display_own_board(active_player.board.board)
		print ("Where would you like to place the {0}".format(ship.name))
		coordy,coordx,direction = None, None, None
		while coordy not in range(0,10):
			coordy = int(input("What is the y starting coordinate? "))
		while coordx not in range(0,10):
			coordx = int(input("What is the x starting coordinate? "))
		while direction not in ["l","r","d","u"]:
			direction = input("Direction: ")
		return coordx, coordy, direction

	def ask_coords(self,opposing_player,active_player):
		os.system('clear')
		print("{}'s View".format(active_player.name))
		print("Where would you like to strike?")
		print("")
		print(opposing_player.name + "(this is where you are aiming ")
		self.display_computers_board(opposing_player.board.board)
		print(active_player.name)
		self.display_own_board(active_player.board.board)
		coordy = None
		coordx = None
		while coordy not in range(0,10):
			coordy = int(input("What is the y coordinate "))
		while coordx not in range(0,10):
			coordx = int(input("What is the x coordinate "))
		return coordy, coordx

	def hit_miss(self, hit_miss,ship_sunk,opposing_player,active_player,coords):
		os.system('clear')
		print("{}'s View".format(active_player.name))
		print("")
		if active_player.iscomputer == False:
			print(opposing_player.name)
			self.display_computers_board(opposing_player.board.board)
			print(active_player.name)
			self.display_own_board(active_player.board.board)
			if hit_miss == True and ship_sunk != None:
				print("You hit!")
				print("You sunk my " + ship_sunk + "!!!")
			elif hit_miss == True:
				print("You hit")
			else:
				print ("You missed!")
		else:
			print(active_player.name)
			self.display_own_board(active_player.board.board)
			print(opposing_player.name)
			self.display_computers_board(opposing_player.board.board)
			if hit_miss == True and ship_sunk != None:
				print("Computer hit at {0}".format(coords))
				print("You sunk my " + ship_sunk + "!!!")
			elif hit_miss == True:
				print("Computer hit at {0}".format(coords))
			else:
				print ("Computer missed at{0}".format(coords))
		enter = input("Press enter to continue ")

	def game_over(self,active_player_name):
		print("Congratulations!! {0} won the game".format(active_player))<commit_msg>Hide computer's battleships from view<commit_after>import os
import time

#time.sleep(0.2)

# You guys will have to install the blessings library
# if you don't already have it
# Documentation here -> https://github.com/erikrose/blessings

from blessings import Terminal

t = Terminal()

class Views:
	def __init__(self):
		pass

	def display_own_board(self,board):
		xaxis1 = "  "
		xaxis2 = "  "
		for key, value in enumerate(board):
			new_row = "{t.normal}" + str(key) + "|"
			for x in value:
				if x == "M":
					new_row += "    {t.blue}" + x
				elif x == "O":
					new_row += "    {t.normal}" + x
				elif x == "X":
					new_row += "    {t.red}" + x
				else:
					new_row += "    {t.green}" + x
			print(new_row.format(t=t))
			print("")
			xaxis2 += "    {t.normal}" + str(key)
			xaxis1 += "    {t.normal}"  + "-"
		print(xaxis1.format(t=t))
		print(xaxis2.format(t=t))

	def display_computers_board(self,board):
		'''This will display like display_own_board except
		with the ships masked'''
		xaxis1 = "  "
		xaxis2 = "  "
		for key, value in enumerate(board):
			new_row = "{t.normal}" + str(key) + "|"
			for x in value:
				if x == "M":
					new_row += "    {t.blue}" + x
				elif x == "O":
					new_row += "    {t.normal}" + x
				elif x == "X":
					new_row += "    {t.red}" + x
				else:
					new_row += "    {t.normal}" + "O"
					#if you want to see computer's ships comment out the line above
					# and initiate the line below
					# new_row += "    {t.green}" + x
			print(new_row.format(t=t))
			print("")
			xaxis2 += "    {t.normal}" + str(key)
			xaxis1 += "    {t.normal}"  + "-"
		print(xaxis1.format(t=t))
		print(xaxis2.format(t=t))

	def place_ship(self,ship,active_player):
		os.system('clear')
		print(t.bold("PLACE YOUR SHIPS!!"))
		self.display_own_board(active_player.board.board)
		print ("Where would you like to place the {0}".format(ship.name))
		coordy,coordx,direction = None, None, None
		while coordy not in range(0,10):
			coordy = int(input("What is the y starting coordinate? "))
		while coordx not in range(0,10):
			coordx = int(input("What is the x starting coordinate? "))
		while direction not in ["l","r","d","u"]:
			direction = input("Direction: ")
		return coordx, coordy, direction

	def ask_coords(self,opposing_player,active_player):
		os.system('clear')
		print("{}'s View".format(active_player.name))
		print("Where would you like to strike?")
		print("")
		print(opposing_player.name + "(this is where you are aiming)")
		self.display_computers_board(opposing_player.board.board)

		print(active_player.name)
		self.display_own_board(active_player.board.board)
		coordy = None
		coordx = None
		while coordy not in range(0,10):
			coordy = int(input("What is the y coordinate "))
		while coordx not in range(0,10):
			coordx = int(input("What is the x coordinate "))
		return coordy, coordx

	def hit_miss(self, hit_miss,ship_sunk,opposing_player,active_player,coords):
		os.system('clear')
		print("{}'s View".format(active_player.name))
		print("")
		if active_player.iscomputer == False:
			print(opposing_player.name)
			self.display_computers_board(opposing_player.board.board)
			print(active_player.name)
			self.display_own_board(active_player.board.board)
			if hit_miss == True and ship_sunk != None:
				print("You hit!")
				print("You sunk my " + ship_sunk + "!!!")
			elif hit_miss == True:
				print("You hit")
			else:
				print ("You missed!")
		else:
			print(active_player.name)
			self.display_computers_board(active_player.board.board)
			print(opposing_player.name)
			self.display_own_board(opposing_player.board.board)
			if hit_miss == True and ship_sunk != None:
				print("Computer hit at {0}".format(coords))
				print("You sunk my " + ship_sunk + "!!!")
			elif hit_miss == True:
				print("Computer hit at {0}".format(coords))
			else:
				print ("Computer missed at{0}".format(coords))
		enter = input("Press enter to continue ")

	def game_over(self,active_player_name):
		print("Congratulations!! {0} won the game".format(active_player))<|endoftext|>


<commit_before>import json
import arrow
import requests
import arrow
import os

from defusedxml import minidom

API_BASE_URL = "http://trevor-producer-cdn.api.bbci.co.uk"
BBC_URL = "https://www.bbc.co.uk"
BBC_POLLING_URL = "https://polling.bbc.co.uk"

class BBC():

    def get_top_stories(self):
        news = self.get_bbc_story()
        if news == None:
            return None
        else:
            data = json.dumps(news.json())
            return self.parse_news(data)

    def get_ticker(self):
        ticker = self.get_bbc_ticker()
        if ticker == None:
            return None
        else:
            data = json.dumps(ticker.json(strict=False))
        return self.parse_ticker_data(data)

    def parse_ticker_data(self, ticker_data):
        tickers = []
        data = json.loads(ticker_data)

        if len(data["html"]) == 0:
            return tickers

        response = data["html"]
        htmlData = minidom.parseString(response)
        elements = htmlData.getElementsByTagName("div")

        # Not yet tested to handle multiple breaking news alerts.

        # Heading
        headline = elements[0].getElementsByTagName("p")[0].firstChild.nodeValue.replace("\n","")
        # News Link as in /news/
        url = elements[0].getElementsByTagName("a")[0].getAttribute('href')

        ticker = Ticker(headline, "BREAKING NEWS", "true", BBC_URL + url)
        tickers.append(ticker)
        return tickers

    def parse_news(self, stories):
        t_news = []
        ts_section = ""
        data = json.loads(stories)
        for i, d in enumerate(data['relations']):
            for rel in d['content']['relations']:
                if(rel['content']['type'] != "bbc.mobile.news.collection"):
                    pass
                else:
                    ts_section = rel['content']['name']
            ts_title = d['content']['name']
            timestamp = d['content']['lastUpdated']
            ts_time = arrow.get(float(timestamp) / 1000).humanize()
            ts_subtext  = "Last updated: " + str(ts_time) + " | " + str(ts_section)
            ts_link = str(d['content']['shareUrl'])
            news = News(ts_title, ts_link, ts_subtext)
            t_news.append(news)
        return t_news

    def get_bbc_story(self):
        res = None
        headers = {
        'User-Agent': 'BBCNews/3.0.9.45 UK (Nexus 4; Android 5.0)',
        'Accept-Encoding': 'gzip',
        'Connection': 'Keep-Alive',
        'Accept': 'application/json'
        }
        try:
            res = requests.get(API_BASE_URL + "/content/cps/news/front_page", data=None, headers=headers)
        except requests.ConnectionError as e:
            if hasattr(e, 'reason'):
                print 'We failed to reach a server.'
                print 'Reason: ', e.reason
            elif hasattr(e, 'code'):
                print 'The server couldn\'t fulfill the request.'
                print 'Error code: ', e.code
        return res

    def get_bbc_ticker(self):
        res = None
        ua = {
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/5311 (KHTML, like Gecko) Chrome/13.0.837.0 Safari/5311'
        }
        try:
            res = requests.get(BBC_POLLING_URL + "/news/latest_breaking_news?audience=Domestic", data=None, headers=ua)
        except requests.ConnectionError as e:
            if hasattr(e, 'reason'):
                print 'We failed to reach a server.'
                print 'Reason: ', e.reason
            elif hasattr(e, 'code'):
                print 'The server couldn\'t fulfill the request.'
                print 'Error code: ', e.code
        return res

class News():

    def __init__(self, title, link, subtext):
        self.title = title
        self.link = link
        self.subtext = subtext

class Ticker():

    def __init__(self, headline, prompt, breaking, url):
        self.headline = headline
        self.prompt = prompt
        self.breaking = breaking
        self.url = url<commit_msg>parse response as unicode.<commit_after>import json
import arrow
import requests
import arrow
import os

from defusedxml import minidom

API_BASE_URL = "http://trevor-producer-cdn.api.bbci.co.uk"
BBC_URL = "https://www.bbc.co.uk"
BBC_POLLING_URL = "https://polling.bbc.co.uk"

class BBC():

    def get_top_stories(self):
        news = self.get_bbc_story()
        if news == None:
            return None
        else:
            data = json.dumps(news.json())
            return self.parse_news(data)

    def get_ticker(self):
        ticker = self.get_bbc_ticker()
        if ticker == None:
            return None
        else:
            data = json.dumps(ticker.json(strict=False))
        return self.parse_ticker_data(data)

    def parse_ticker_data(self, ticker_data):
        tickers = []
        data = json.loads(ticker_data)

        if len(data["html"]) == 0:
            return tickers

        response = data["html"]
        htmlData = minidom.parseString(response.encode('utf-8'))
        elements = htmlData.getElementsByTagName("div")

        # Heading
        headline = elements[0].getElementsByTagName("p")[0].firstChild.nodeValue.replace("\n","")
        # News Link as in /news/
        url = elements[0].getElementsByTagName("a")[0].getAttribute('href')
        ticker = Ticker(headline, "BREAKING NEWS", "true", BBC_URL + url)
        tickers.append(ticker)
        return tickers

    def parse_news(self, stories):
        t_news = []
        ts_section = ""
        data = json.loads(stories)
        for i, d in enumerate(data['relations']):
            for rel in d['content']['relations']:
                if(rel['content']['type'] != "bbc.mobile.news.collection"):
                    pass
                else:
                    ts_section = rel['content']['name']
            ts_title = d['content']['name']
            timestamp = d['content']['lastUpdated']
            ts_time = arrow.get(float(timestamp) / 1000).humanize()
            ts_subtext  = "Last updated: " + str(ts_time) + " | " + str(ts_section)
            ts_link = str(d['content']['shareUrl'])
            news = News(ts_title, ts_link, ts_subtext)
            t_news.append(news)
        return t_news

    def get_bbc_story(self):
        res = None
        headers = {
        'User-Agent': 'BBCNews/3.0.9.45 UK (Nexus 4; Android 5.0)',
        'Accept-Encoding': 'gzip',
        'Connection': 'Keep-Alive',
        'Accept': 'application/json'
        }
        try:
            res = requests.get(API_BASE_URL + "/content/cps/news/front_page", data=None, headers=headers)
        except requests.ConnectionError as e:
            if hasattr(e, 'reason'):
                print 'We failed to reach a server.'
                print 'Reason: ', e.reason
            elif hasattr(e, 'code'):
                print 'The server couldn\'t fulfill the request.'
                print 'Error code: ', e.code
        return res

    def get_bbc_ticker(self):
        res = None
        ua = {
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/5311 (KHTML, like Gecko) Chrome/13.0.837.0 Safari/5311'
        }
        try:
            res = requests.get(BBC_POLLING_URL + "/news/latest_breaking_news?audience=Domestic", data=None, headers=ua)
        except requests.ConnectionError as e:
            if hasattr(e, 'reason'):
                print 'We failed to reach a server.'
                print 'Reason: ', e.reason
            elif hasattr(e, 'code'):
                print 'The server couldn\'t fulfill the request.'
                print 'Error code: ', e.code
        return res

class News():

    def __init__(self, title, link, subtext):
        self.title = title
        self.link = link
        self.subtext = subtext

class Ticker():

    def __init__(self, headline, prompt, breaking, url):
        self.headline = headline
        self.prompt = prompt
        self.breaking = breaking
        self.url = url<|endoftext|>


<commit_before>import re
from bs4 import BeautifulSoup
import bs4
from .utils import cfscraper
from fundoshi.classes import objdict

_get = cfscraper.get
_post = cfscraper.post

class Mangapanda(object):

    netlocs = ['mangapanda.com']
    name = 'mangapanda'

    def __init__(self):
        # mangapanda uses a lot of relative paths
        self.base_url = 'http://www.mangapanda.com'

    # Return a list of objdicts that store name, url, and site:
    # [{'name': 'Naruto', 'url': 'http://...', 'site': mangapanda}, ...]
    def search_series(self, keyword):
        url = 'http://www.mangapanda.com/search/'
        params = {
            'w': keyword,
        }
        resp = _post(url, params=params)

        # Kissmanga returns manga series and links in xml format
        soup = BeautifulSoup(resp.content, 'html.parser')
        manga_names = soup.find_all('div', {'class': 'manga_name'})
        atags = [name.find('h3').a for name in manga_names]
        return [objdict({'name': a.text.strip(),
                         'url': self.base_url + a['href'],
                         'site': self.name}) for a in atags]

    # All kinds of data
    # - name "Naruto"
    # - chapters [{name, url}, {}, ...] - latest first
    # - thumb_url "url"
    # - tags [tag1, tag2, ...]
    # - status "ongoing"/"completed"
    # - descriptions ["paragraph1", "paragraph2", ...]
    # - authors ["Kishimoto Masashi", ...]
    def series_info(self, html):
        soup = BeautifulSoup(html, 'html.parser')
        chapters = self._chapters(soup)
        thumb_url = self._thumbnail_url(soup)
        tags = self._tags(soup)
        name = self._name(soup)
        status = self._status(soup)
        descriptions = self._descriptions(soup)
        authors = self._authors(soup)
        return objdict({
            'site': self.name,
            'chapters': chapters,
            'thumb_url': thumb_url,
            'tags': tags,
            'name': name,
            'status': status,
            'descriptions': descriptions,
            'authors': authors,
        })

    def _chapters(self, soup):
        table = soup.find('table', {'id': 'listing'})
        return [objdict({'url': self.base_url + a['href'],
                         'name': a.string.strip()})
                for a in reversed(table.find_all('a'))]

    def _thumbnail_url(self, soup):
        img_div = soup.find('div', {'id': 'mangaimg'})
        return img_div.find('img')['src']

    def _tags(self, soup):
        tag_list = soup.find('td', {'class': 'propertytitle'}, text='Genre:').findNext('td')
        tags = tag_list.find_all('a')
        return [text.string.strip().lower() for text in tags]

    def _name(self, soup):
        # <link rel='alternate' title='Naruto manga' ...
        # => must remove the ' manga' part
        title = soup.find('h2', {'class': 'aname'})
        return title.text.strip()

    def _status(self, soup):
        status_td = soup.find('td', {'class': 'propertytitle'}, text='Status:').findNext('td')
        return status_td.text.strip().lower()

    def _descriptions(self, soup):
        desc_div = soup.find('div', {'id': 'readmangasum'})
        p_tag = desc_div.findNext('p')
        desc = p_tag.text.replace('\r\n', '')
        return [desc]

    def _authors(self, soup):
        # mangapanda lists artists in some cases
        # lets strip these as well as the tags used
        author_types = ['(Story)', '(Art)', '(Story & Art)']
        authors = soup.find('td', {'class': 'propertytitle'}, text='Author:').findNext('td')
        return [authors.text.strip()]

    # Chapter data
    # - name "Naruto Ch.101"
    # - pages [url1, url2, ...] - in ascending order
    # - prev_chapter_url
    # - next_chapter_url
    # - series_url
    def chapter_info(self, html):
        pages = self._chapter_pages(html)
        soup = BeautifulSoup(html, 'html.parser')
        name = self._chapter_name(soup)
        series_url = self._chapter_series_url(soup)
        prev, next = self._chapter_prev_next(soup)
        return objdict({
            'name': name,
            'pages': pages,
            'series_url': series_url,
            'next_chapter_url': next,
            'prev_chapter_url': prev,
        })

    def _chapter_prev_next(self, soup):
        chapter_nav = soup.find('div', {'id': 'mangainfo_bas'}).table
        # There is always a "next page" on Mangapanda even if no release exists
        # so we check every page for the not released notification
        # before setting the next page link
        next_page = chapter_nav.find('span', text='Next Chapter:')
        next_page = self.base_url + next_page.findNext('a')['href']
        look_ahead = _get(next_page)
        look_ahead_soup = BeautifulSoup(look_ahead.text, 'html.parser')
        info = look_ahead_soup.find('div', {'id': 'recom_info'})
        if info and 'not released' in info.text:
            next_page = None
        prev_page = chapter_nav.find('span', text='Previous Chapter:')
        if prev_page:
            prev_page = self.base_url + prev_page.findNext('a')['href']
        return prev_page, next_page

    def _chapter_name(self, soup):
        chapter_info = soup.find('div', {'id': 'mangainfo'})
        chapter_name = chapter_info.h1
        return chapter_name.text.strip()

    def _chapter_pages(self, html):
        soup = BeautifulSoup(html, 'html.parser')
        # get last page number
        select_page = soup.find('div', {'id': 'selectpage'})
        last_page = int(select_page.text.strip().rsplit('of ', 1)[-1])

        # get base_url of chapter
        manga_info = soup.find('div', {'id': 'mangainfo_son'})
        chapter_base = manga_info.find('a')['href']

        # generate page urls
        page_urls = []
        for page_num in range(1, last_page + 1):
            page_urls.append('{0}{1}/{2}'.format(self.base_url, chapter_base, page_num))

        img_urls = []
        for url in page_urls:
            response = _get(url)
            soup = BeautifulSoup(response.text, 'html.parser')
            # get img url
            img_holder = soup.find('div', {'id': 'imgholder'})
            img_urls.append(img_holder.img['src'])

        return (url for url in img_urls)

    def _chapter_series_url(self, soup):
        manga_info = soup.find('div', {'id': 'mangainfo_son'})
        a_tag = manga_info.find_all('a')[-1]
        return self.base_url + a_tag['href']

    def get_manga_seed_page(self, url):
        return _get(url)

    def get_chapter_seed_page(self, url):
        # http://www.mangapanda.com/Manga-Name/X
        base = url[:url.rfind('/') + 1]  # http://.../Manga-Name/
        chapter = url.rsplit('/', 1)[-1]  # /X
        return _get(base + chapter, cookies={'vns_readType1': '1'})
<commit_msg>some cleanups on Mangapanda after merging PR<commit_after>from bs4 import BeautifulSoup
from .utils import cfscraper
from fundoshi.classes import objdict

_get = cfscraper.get
_post = cfscraper.post

class Mangapanda(object):

    netlocs = ['www.mangapanda.com']
    name = 'mangapanda'

    def __init__(self):
        # mangapanda uses a lot of relative paths
        self.base_url = 'http://www.mangapanda.com'

    # Return a list of objdicts that store name, url, and site:
    # [{'name': 'Naruto', 'url': 'http://...', 'site': mangapanda}, ...]
    def search_series(self, keyword):
        url = 'http://www.mangapanda.com/search/'
        params = {
            'w': keyword,
        }
        resp = _post(url, params=params)

        soup = BeautifulSoup(resp.content, 'html.parser')
        manga_names = soup.find_all('div', {'class': 'manga_name'})
        atags = [name.find('h3').a for name in manga_names]
        return [objdict({'name': a.text.strip(),
                         'url': self.base_url + a['href'],
                         'site': self.name}) for a in atags]

    # All kinds of data
    # - name "Naruto"
    # - chapters [{name, url}, {}, ...] - latest first
    # - thumb_url "url"
    # - tags [tag1, tag2, ...]
    # - status "ongoing"/"completed"
    # - descriptions ["paragraph1", "paragraph2", ...]
    # - authors ["Kishimoto Masashi", ...]
    def series_info(self, html):
        soup = BeautifulSoup(html, 'html.parser')
        chapters = self._chapters(soup)
        thumb_url = self._thumbnail_url(soup)
        tags = self._tags(soup)
        name = self._name(soup)
        status = self._status(soup)
        descriptions = self._descriptions(soup)
        authors = self._authors(soup)
        return objdict({
            'site': self.name,
            'chapters': chapters,
            'thumb_url': thumb_url,
            'tags': tags,
            'name': name,
            'status': status,
            'descriptions': descriptions,
            'authors': authors,
        })

    def _chapters(self, soup):
        table = soup.find('table', {'id': 'listing'})
        return [objdict({'url': self.base_url + a['href'],
                         'name': a.string.strip()})
                for a in reversed(table.find_all('a'))]

    def _thumbnail_url(self, soup):
        img_div = soup.find('div', {'id': 'mangaimg'})
        return img_div.find('img')['src']

    def _tags(self, soup):
        tag_list = soup.find(
            'td',
            {'class': 'propertytitle'},
            text='Genre:'
        ).findNext('td')
        tags = tag_list.find_all('a')
        return [text.string.strip().lower() for text in tags]

    def _name(self, soup):
        # <link rel='alternate' title='Naruto manga' ...
        # => must remove the ' manga' part
        title = soup.find('h2', {'class': 'aname'})
        return title.text.strip()

    def _status(self, soup):
        status_td = soup.find(
            'td',
            {'class': 'propertytitle'},
            text='Status:'
        ).findNext('td')
        return status_td.text.strip().lower()

    def _descriptions(self, soup):
        desc_div = soup.find('div', {'id': 'readmangasum'})
        p_tag = desc_div.findNext('p')
        desc = p_tag.text.replace('\r\n', '')
        return [desc]

    def _authors(self, soup):
        # mangapanda lists artists in some cases
        # lets strip these as well as the tags used
        authors = soup.find(
            'td',
            {'class': 'propertytitle'},
            text='Author:'
        ).findNext('td')
        return [authors.text.strip()]

    # Chapter data
    # - name "Naruto Ch.101"
    # - pages [url1, url2, ...] - in ascending order
    # - prev_chapter_url
    # - next_chapter_url
    # - series_url
    def chapter_info(self, html):
        pages = self._chapter_pages(html)
        soup = BeautifulSoup(html, 'html.parser')
        name = self._chapter_name(soup)
        series_url = self._chapter_series_url(soup)
        prev, next = self._chapter_prev_next(soup)
        return objdict({
            'name': name,
            'pages': pages,
            'series_url': series_url,
            'next_chapter_url': next,
            'prev_chapter_url': prev,
        })

    def _chapter_prev_next(self, soup):
        chapter_nav = soup.find('div', {'id': 'mangainfo_bas'}).table
        # There is always a "next page" on Mangapanda even if no release exists
        # so we check every page for the not released notification
        # before setting the next page link
        next_page = chapter_nav.find('span', text='Next Chapter:')
        next_page = self.base_url + next_page.findNext('a')['href']
        look_ahead = _get(next_page)
        look_ahead_soup = BeautifulSoup(look_ahead.text, 'html.parser')
        info = look_ahead_soup.find('div', {'id': 'recom_info'})
        if info and 'not released' in info.text:
            next_page = None
        prev_page = chapter_nav.find('span', text='Previous Chapter:')
        if prev_page:
            prev_page = self.base_url + prev_page.findNext('a')['href']
        return prev_page, next_page

    def _chapter_name(self, soup):
        chapter_info = soup.find('div', {'id': 'mangainfo'})
        chapter_name = chapter_info.h1
        return chapter_name.text.strip()

    def _chapter_pages(self, html):
        soup = BeautifulSoup(html, 'html.parser')
        # get last page number
        select_page = soup.find('div', {'id': 'selectpage'})
        last_page = int(select_page.text.strip().rsplit('of ', 1)[-1])

        # get base_url of chapter
        manga_info = soup.find('div', {'id': 'mangainfo_son'})
        chapter_base = manga_info.find('a')['href']

        # generate page urls
        page_urls = []
        for page_num in range(1, last_page + 1):
            page_urls.append('{0}{1}/{2}'.format(self.base_url,
                                                 chapter_base,
                                                 page_num))

        return (self._parse_page(url) for url in page_urls)

    def _parse_page(self, page_url):
        response = _get(page_url)
        soup = BeautifulSoup(response.text, 'html.parser')
        # get img url
        img_holder = soup.find('div', {'id': 'imgholder'})
        return img_holder.img['src']

    def _chapter_series_url(self, soup):
        manga_info = soup.find('div', {'id': 'mangainfo_son'})
        a_tag = manga_info.find_all('a')[-1]
        return self.base_url + a_tag['href']

    def search_by_author(self, author):
        # This site simply doesn't support searching by author
        return []

    def get_manga_seed_page(self, url):
        return _get(url)

    def get_chapter_seed_page(self, url):
        # http://www.mangapanda.com/Manga-Name/X
        base = url[:url.rfind('/') + 1]  # http://.../Manga-Name/
        chapter = url.rsplit('/', 1)[-1]  # /X
        return _get(base + chapter, cookies={'vns_readType1': '1'})
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8

from __future__ import print_function, absolute_import
import json
import os
import argparse
from os.path import join
from shutil import copyfile
from pybda import simple_simulator, imager, bda, calibrate, expand_bda
import matplotlib.pyplot as pyplot
import numpy
import time

def main(config_file):
    try:
        config = json.load(open(config_file))
    except ValueError as e:
        print('Error: Failed to parse JSON config file.')
        print(e.message)
        return

    # Create the simulation directory, if required.
    sim_dir = config['path']
    if not os.path.isdir(sim_dir):
        os.makedirs(sim_dir)
    sim_config = join(sim_dir, os.path.basename(config_file))
    if not os.path.exists(sim_config):
        copyfile(config_file, sim_config)

    # Simulate visibilities.
    vis = simple_simulator.simulate_2(config)

    # Image model and corrupted data.
    original_model     = imager.run_imager(config, vis, 'model')
    original_corrupted = imager.run_imager(config, vis, 'data')

    # Run initial BDA (compress) and overwrite the original data (expand).
    ave_model1 = bda.run_bda(config, vis, 'model')
    ave_data1 = bda.run_bda(config, vis, 'data')
    expand_bda.run_expand_bda(vis['num_antennas'],
        ave_model1, 'data', vis, 'model')
    expand_bda.run_expand_bda(vis['num_antennas'],
        ave_data1, 'data', vis, 'data')

    # Image corrupted data post-initial-BDA.
    post_bda1_corrupted = imager.run_imager(config, vis, 'data')

    # Calibrate the visibilities with StefCal.
    calibrate.run_calibrate(vis, verbose=False)

    # Image calibrated data.
    calibrated = imager.run_imager(config, vis, 'corrected')

    # Run final BDA (compress).
    ave_model = bda.run_bda(config, vis, 'model')
    ave_data = bda.run_bda(config, vis, 'corrected')

    # Image model and calibrated data post-final-BDA.
    post_bda2_model     = imager.run_imager(config, ave_model, 'data')
    post_bda2_corrected = imager.run_imager(config, ave_data, 'data')

    # Make image differences
    # -------------------------------------------------------------------------
    # final BDA corrected - final BDA model (for final metrics)
    diff_final_bda_corrected_final_bda_model = []
    for corr, model in zip(post_bda2_corrected, post_bda2_model):
        diff_final_bda_corrected_final_bda_model.append(corr - model)

    # final BDA model - original model (Bill Cotton's test)
    diff_final_bda_model_original_model = []
    for model_bda, model_original in zip(post_bda2_model, original_model):
        diff_final_bda_model_original_model.append(model_bda - model_original)

    # original corrupted - first BDA/expand corrupted (see effect of BDA)
    diff_original_corrupted_bda_corrupted = []
    for corr_orig, corr_bda in zip(original_corrupted, post_bda1_corrupted):
        diff_original_corrupted_bda_corrupted.append(corr_orig - corr_bda)

    # for i in range(len(original_model)):
    for i in range(1):
        fig = pyplot.figure(figsize=(16, 11))
        ax = fig.add_subplot(3,3,1)
        im1 = ax.imshow(original_model[i], interpolation='nearest')
        ax.figure.colorbar(im1, ax=ax)
        ax.set_title('Original model')

        ax = fig.add_subplot(3,3,2)
        im2 = ax.imshow(original_corrupted[i], interpolation='nearest')
        ax.figure.colorbar(im2, ax=ax)
        ax.set_title('Original corrupted')

        ax = fig.add_subplot(3,3,3)
        im3 = ax.imshow(post_bda1_corrupted[i], interpolation='nearest')
        ax.figure.colorbar(im3, ax=ax)
        ax.set_title('Initial BDA corrupted')

        ax = fig.add_subplot(3,3,4)
        im4 = ax.imshow(post_bda2_model[i], interpolation='nearest')
        ax.figure.colorbar(im4, ax=ax)
        ax.set_title('Final BDA model')

        ax = fig.add_subplot(3,3,5)
        im5 = ax.imshow(post_bda2_corrected[i], interpolation='nearest')
        ax.figure.colorbar(im5, ax=ax)
        ax.set_title('Final BDA calibrated')

        ax = fig.add_subplot(3,3,6)
        im6 = ax.imshow(diff_final_bda_corrected_final_bda_model[i], interpolation='nearest')
        ax.figure.colorbar(im6, ax=ax)
        ax.set_title('Final BDA (calibrated - model)')

        ax = fig.add_subplot(3,3,7)
        im7 = ax.imshow(diff_final_bda_model_original_model[i], interpolation='nearest')
        ax.figure.colorbar(im7, ax=ax)
        ax.set_title('Final BDA model - Original model (Cotton)')

        ax = fig.add_subplot(3,3,8)
        im8 = ax.imshow(diff_original_corrupted_bda_corrupted[i], interpolation='nearest')
        ax.figure.colorbar(im8, ax=ax)
        ax.set_title('Original corrupted - Initial BDA corrupted')

        ax = fig.add_subplot(3,3,9)
        im9 = ax.imshow(calibrated[i], interpolation='nearest')
        ax.figure.colorbar(im9, ax=ax)
        ax.set_title('Calibrated')

        pyplot.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='BDA pipeline main.',
                                     epilog='')
    parser.add_argument('config', type=str, nargs='?', help='JSON config file')
    args = parser.parse_args()
    if args.config is None:
        parser.print_usage()
        print('%s: error, too few arguments.' % os.path.basename(__file__))
        exit(1)
    if not os.path.isfile(args.config):
        print("Error: Config file '%s' not found!" % args.config)

    main(args.config)
<commit_msg>Removed redundant axis labels.<commit_after>#!/usr/bin/env python
# coding=utf-8

from __future__ import print_function, absolute_import
import json
import os
import argparse
from os.path import join
from shutil import copyfile
from pybda import simple_simulator, imager, bda, calibrate, expand_bda
import matplotlib.pyplot as pyplot
import numpy
import time

def main(config_file):
    try:
        config = json.load(open(config_file))
    except ValueError as e:
        print('Error: Failed to parse JSON config file.')
        print(e.message)
        return

    # Create the simulation directory, if required.
    sim_dir = config['path']
    if not os.path.isdir(sim_dir):
        os.makedirs(sim_dir)
    sim_config = join(sim_dir, os.path.basename(config_file))
    if not os.path.exists(sim_config):
        copyfile(config_file, sim_config)

    # Simulate visibilities.
    vis = simple_simulator.simulate_2(config)

    # Image model and corrupted data.
    original_model     = imager.run_imager(config, vis, 'model')
    original_corrupted = imager.run_imager(config, vis, 'data')

    # Run initial BDA (compress) and overwrite the original data (expand).
    ave_model1 = bda.run_bda(config, vis, 'model')
    ave_data1 = bda.run_bda(config, vis, 'data')
    expand_bda.run_expand_bda(vis['num_antennas'],
        ave_model1, 'data', vis, 'model')
    expand_bda.run_expand_bda(vis['num_antennas'],
        ave_data1, 'data', vis, 'data')

    # Image corrupted data post-initial-BDA.
    post_bda1_corrupted = imager.run_imager(config, vis, 'data')

    # Calibrate the visibilities with StefCal.
    calibrate.run_calibrate(vis, verbose=False)

    # Image calibrated data.
    calibrated = imager.run_imager(config, vis, 'corrected')

    # Run final BDA (compress).
    ave_model = bda.run_bda(config, vis, 'model')
    ave_data = bda.run_bda(config, vis, 'corrected')

    # Image model and calibrated data post-final-BDA.
    post_bda2_model     = imager.run_imager(config, ave_model, 'data')
    post_bda2_corrected = imager.run_imager(config, ave_data, 'data')

    # Make image differences
    # -------------------------------------------------------------------------
    # final BDA corrected - final BDA model (for final metrics)
    diff_final_bda_corrected_final_bda_model = []
    for corr, model in zip(post_bda2_corrected, post_bda2_model):
        diff_final_bda_corrected_final_bda_model.append(corr - model)

    # final BDA model - original model (Bill Cotton's test)
    diff_final_bda_model_original_model = []
    for model_bda, model_original in zip(post_bda2_model, original_model):
        diff_final_bda_model_original_model.append(model_bda - model_original)

    # original corrupted - first BDA/expand corrupted (see effect of BDA)
    diff_original_corrupted_bda_corrupted = []
    for corr_orig, corr_bda in zip(original_corrupted, post_bda1_corrupted):
        diff_original_corrupted_bda_corrupted.append(corr_orig - corr_bda)

    # for i in range(len(original_model)):
    for i in range(1):
        fig = pyplot.figure(figsize=(16, 11))
        ax = fig.add_subplot(3,3,1)
        im1 = ax.imshow(original_model[i], interpolation='nearest')
        ax.figure.colorbar(im1, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Original model')

        ax = fig.add_subplot(3,3,2)
        im2 = ax.imshow(original_corrupted[i], interpolation='nearest')
        ax.figure.colorbar(im2, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Original corrupted')

        ax = fig.add_subplot(3,3,3)
        im3 = ax.imshow(post_bda1_corrupted[i], interpolation='nearest')
        ax.figure.colorbar(im3, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Initial BDA corrupted')

        ax = fig.add_subplot(3,3,4)
        im4 = ax.imshow(post_bda2_model[i], interpolation='nearest')
        ax.figure.colorbar(im4, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Final BDA model')

        ax = fig.add_subplot(3,3,5)
        im5 = ax.imshow(post_bda2_corrected[i], interpolation='nearest')
        ax.figure.colorbar(im5, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Final BDA calibrated')

        ax = fig.add_subplot(3,3,6)
        im6 = ax.imshow(diff_final_bda_corrected_final_bda_model[i], interpolation='nearest')
        ax.figure.colorbar(im6, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Final BDA (calibrated - model)')

        ax = fig.add_subplot(3,3,7)
        im7 = ax.imshow(diff_final_bda_model_original_model[i], interpolation='nearest')
        ax.figure.colorbar(im7, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Model (Final BDA - Original) (Cotton)')

        ax = fig.add_subplot(3,3,8)
        im8 = ax.imshow(diff_original_corrupted_bda_corrupted[i], interpolation='nearest')
        ax.figure.colorbar(im8, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Corrupted (Original - Initial BDA)')

        ax = fig.add_subplot(3,3,9)
        im9 = ax.imshow(calibrated[i], interpolation='nearest')
        ax.figure.colorbar(im9, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_title('Calibrated')

        pyplot.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='BDA pipeline main.',
                                     epilog='')
    parser.add_argument('config', type=str, nargs='?', help='JSON config file')
    args = parser.parse_args()
    if args.config is None:
        parser.print_usage()
        print('%s: error, too few arguments.' % os.path.basename(__file__))
        exit(1)
    if not os.path.isfile(args.config):
        print("Error: Config file '%s' not found!" % args.config)

    main(args.config)
<|endoftext|>


<commit_before>import logging
import os
import shutil
from fusesoc.utils import Launcher

logger = logging.getLogger(__name__)

class Provider(object):
    def __init__(self, config, core_root, files_root):
        self.config = config
        self.core_root = core_root
        self.files_root = files_root
        self.cachable = not (config.get("cachable", "") == False)
        self.patches = config.get("patches", [])

    def clean_cache(self):
        if os.path.exists(self.files_root):
            shutil.rmtree(self.files_root)

    def fetch(self):
        status = self.status()
        if status == "empty":
            self._checkout(self.files_root)
            _fetched = True
        elif status == "outofdate":
            self.clean_cache()
            self._checkout(self.files_root)
            _fetched = True
        elif status == "downloaded":
            _fetched = False
        else:
            raise RuntimeError(
                "Provider status is: '" + status + "'. This shouldn't happen"
            )
        if _fetched:
            self._patch()

    def _patch(self):
        for f in self.patches:
            patch_file = os.path.abspath(os.path.join(self.core_root, f))
            if os.path.isfile(patch_file):
                logger.debug(
                    "  applying patch file: "
                    + patch_file
                    + "\n"
                    + "                   to: "
                    + os.path.join(self.files_root)
                )
                try:
                    Launcher(
                        "git",
                        [
                            "apply",
                            "--unsafe-paths",
                            "--directory",
                            self.files_root,
                            patch_file,
                        ],
                    ).run()
                except OSError:
                    raise RuntimeError("Failed to call 'git' for patching core")

    def status(self):
        if not self.cachable:
            return "outofdate"
        if not os.path.isdir(self.files_root):
            return "empty"
        else:
            return "downloaded"
<commit_msg>Run git apply from the directory being patched (#346)<commit_after>import logging
import os
import shutil
from fusesoc.utils import Launcher

logger = logging.getLogger(__name__)

class Provider(object):
    def __init__(self, config, core_root, files_root):
        self.config = config
        self.core_root = core_root
        self.files_root = files_root
        self.cachable = not (config.get("cachable", "") == False)
        self.patches = config.get("patches", [])

    def clean_cache(self):
        if os.path.exists(self.files_root):
            shutil.rmtree(self.files_root)

    def fetch(self):
        status = self.status()
        if status == "empty":
            self._checkout(self.files_root)
            _fetched = True
        elif status == "outofdate":
            self.clean_cache()
            self._checkout(self.files_root)
            _fetched = True
        elif status == "downloaded":
            _fetched = False
        else:
            raise RuntimeError(
                "Provider status is: '" + status + "'. This shouldn't happen"
            )
        if _fetched:
            self._patch()

    def _patch(self):
        for f in self.patches:
            patch_file = os.path.abspath(os.path.join(self.core_root, f))
            if os.path.isfile(patch_file):
                logger.debug(
                    "  applying patch file: "
                    + patch_file
                    + "\n"
                    + "                   to: "
                    + os.path.join(self.files_root)
                )
                try:
                    Launcher("git", ["apply", patch_file], self.files_root).run()
                except OSError:
                    raise RuntimeError("Failed to call 'git' for patching core")

    def status(self):
        if not self.cachable:
            return "outofdate"
        if not os.path.isdir(self.files_root):
            return "empty"
        else:
            return "downloaded"
<|endoftext|>