<commit_before>#    Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
import subprocess
from cStringIO import StringIO

from wlauto import Command, ExtensionLoader, settings
from wlauto.utils.doc import (get_summary, get_description, get_type_name, format_column, format_body,
                              format_paragraph, indent, strip_inlined_text)
from wlauto.utils.misc import get_pager
from wlauto.utils.terminalsize import get_terminal_size

class ShowCommand(Command):

    name = 'show'

    description = """
    Display documentation for the specified extension (workload, instrument, etc.).
    """

    def initialize(self):
        self.parser.add_argument('name', metavar='EXTENSION',
                                 help='''The name of the extension for which information will
                                         be shown.''')

    def execute(self, args):
        # pylint: disable=unpacking-non-sequence
        ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths)
        extension = ext_loader.get_extension_class(args.name)
        out = StringIO()
        term_width, term_height = get_terminal_size()
        format_extension(extension, out, term_width)
        text = out.getvalue()
        pager = get_pager()
        if len(text.split('\n')) > term_height and pager:
            try:
                sp = subprocess.Popen(pager, stdin=subprocess.PIPE)
                sp.communicate(text)
            except OSError:
                self.logger.warning('Could not use PAGER "{}"'.format(pager))
                sys.stdout.write(text)
        else:
            sys.stdout.write(text)

def format_extension(extension, out, width):
    format_extension_name(extension, out)
    out.write('\n')
    format_extension_summary(extension, out, width)
    out.write('\n')
    if extension.parameters:
        format_extension_parameters(extension, out, width)
        out.write('\n')
    format_extension_description(extension, out, width)

def format_extension_name(extension, out):
    out.write('\n{}\n'.format(extension.name))

def format_extension_summary(extension, out, width):
    out.write('{}\n'.format(format_body(strip_inlined_text(get_summary(extension)), width)))

def format_extension_description(extension, out, width):
    # skip the initial paragraph of multi-paragraph description, as already
    # listed above.
    description = get_description(extension).split('\n\n', 1)[-1]
    out.write('{}\n'.format(format_body(strip_inlined_text(description), width)))

def format_extension_parameters(extension, out, width, shift=4):
    out.write('parameters:\n\n')
    param_texts = []
    for param in extension.parameters:
        description = format_paragraph(strip_inlined_text(param.description or ''), width - shift)
        param_text = '{}'.format(param.name)
        if param.mandatory:
            param_text += " (MANDATORY)"
        param_text += '\n{}\n'.format(description)
        param_text += indent('type: {}\n'.format(get_type_name(param.kind)))
        if param.allowed_values:
            param_text += indent('allowed values: {}\n'.format(', '.join(map(str, param.allowed_values))))
        elif param.constraint:
            param_text += indent('constraint: {}\n'.format(get_type_name(param.constraint)))
        if param.default is not None:
            param_text += indent('default: {}\n'.format(param.default))
        param_texts.append(indent(param_text, shift))

    out.write(format_column('\n'.join(param_texts), width))

<commit_msg>show command: adding supported platforms<commit_after>#    Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
import subprocess
from cStringIO import StringIO

from wlauto import Command, ExtensionLoader, settings
from wlauto.utils.doc import (get_summary, get_description, get_type_name, format_column, format_body,
                              format_paragraph, indent, strip_inlined_text)
from wlauto.utils.misc import get_pager
from wlauto.utils.terminalsize import get_terminal_size

class ShowCommand(Command):

    name = 'show'

    description = """
    Display documentation for the specified extension (workload, instrument, etc.).
    """

    def initialize(self):
        self.parser.add_argument('name', metavar='EXTENSION',
                                 help='''The name of the extension for which information will
                                         be shown.''')

    def execute(self, args):
        # pylint: disable=unpacking-non-sequence
        ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths)
        extension = ext_loader.get_extension_class(args.name)
        out = StringIO()
        term_width, term_height = get_terminal_size()
        format_extension(extension, out, term_width)
        text = out.getvalue()
        pager = get_pager()
        if len(text.split('\n')) > term_height and pager:
            try:
                sp = subprocess.Popen(pager, stdin=subprocess.PIPE)
                sp.communicate(text)
            except OSError:
                self.logger.warning('Could not use PAGER "{}"'.format(pager))
                sys.stdout.write(text)
        else:
            sys.stdout.write(text)

def format_extension(extension, out, width):
    format_extension_name(extension, out)
    out.write('\n')
    format_extension_summary(extension, out, width)
    out.write('\n')
    if hasattr(extension, 'supported_platforms'):
        format_supported_platforms(extension, out, width)
        out.write('\n')
    if extension.parameters:
        format_extension_parameters(extension, out, width)
        out.write('\n')
    format_extension_description(extension, out, width)

def format_extension_name(extension, out):
    out.write('\n{}\n'.format(extension.name))

def format_extension_summary(extension, out, width):
    out.write('{}\n'.format(format_body(strip_inlined_text(get_summary(extension)), width)))

def format_supported_platforms(extension, out, width):
    text = 'supported on: {}'.format(', '.join(extension.supported_platforms))
    out.write('{}\n'.format(format_body(text, width)))

def format_extension_description(extension, out, width):
    # skip the initial paragraph of multi-paragraph description, as already
    # listed above.
    description = get_description(extension).split('\n\n', 1)[-1]
    out.write('{}\n'.format(format_body(strip_inlined_text(description), width)))

def format_extension_parameters(extension, out, width, shift=4):
    out.write('parameters:\n\n')
    param_texts = []
    for param in extension.parameters:
        description = format_paragraph(strip_inlined_text(param.description or ''), width - shift)
        param_text = '{}'.format(param.name)
        if param.mandatory:
            param_text += " (MANDATORY)"
        param_text += '\n{}\n'.format(description)
        param_text += indent('type: {}\n'.format(get_type_name(param.kind)))
        if param.allowed_values:
            param_text += indent('allowed values: {}\n'.format(', '.join(map(str, param.allowed_values))))
        elif param.constraint:
            param_text += indent('constraint: {}\n'.format(get_type_name(param.constraint)))
        if param.default is not None:
            param_text += indent('default: {}\n'.format(param.default))
        param_texts.append(indent(param_text, shift))

    out.write(format_column('\n'.join(param_texts), width))

<|endoftext|>


<commit_before>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=no-name-in-module,import-error,no-member

import os
import subprocess
from distutils.version import StrictVersion

# pylint: disable=wrong-import-position,ungrouped-imports
import_error_str = ''
try:
    import IPython
except ImportError as import_error:
    IPython = None
    # Importing IPython can fail for a variety of reasons, report the actual
    # failure unless it's just that the package is not present
    if import_error.message.startswith("No module named"):  # pylint: disable=E1101
        import_error_str = 'ipynb_exporter requires ipython package to be installed'
    else:
        import_error_str = import_error.message  # pylint: disable=redefined-variable-type

# The current code generates notebooks version 3
NBFORMAT_VERSION = 3

if IPython:
    if StrictVersion('5.0.0') > StrictVersion(IPython.__version__) >= StrictVersion('4.0.0'):
        import nbformat
        from jupyter_client.manager import KernelManager

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            nbformat.write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = nbformat.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['jupyter', 'nbconvert', '--to html']
        IPYTHON_NBCONVERT_PDF = ['jupyter', 'nbconvert', '--to pdf']

    elif StrictVersion('4.0.0') > StrictVersion(IPython.__version__) >= StrictVersion('3.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=pdf']
    elif StrictVersion('3.0.0') > StrictVersion(IPython.__version__) >= StrictVersion('2.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat.v3

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.v3.reads_json(notebook_in)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.v3.nbjson.JSONWriter().write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.v3.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=latex',
                                 '--post=PDF']
    else:
        # Unsupported IPython version
        import_error_str = 'Unsupported IPython version {}'.format(IPython.__version__)

def parse_valid_output(msg):
    """Parse a valid result from an execution of a cell in an ipython kernel"""
    msg_type = msg["msg_type"]
    if msg_type == 'error':
        msg_type = 'pyerr'
    elif msg_type == 'execute_result':
        msg_type = 'pyout'

    content = msg["content"]
    out = NotebookNode(output_type=msg_type)

    if msg_type == "stream":
        out.stream = content["name"]
        try:
            out.text = content['data']
        except KeyError:
            out.text = content['text']
    elif msg_type in ("display_data", "pyout"):
        for mime, data in content["data"].iteritems():
            if mime == "text/plain":
                attr = "text"
            else:
                attr = mime.split("/")[-1]
            setattr(out, attr, data)
    elif msg_type == "pyerr":
        out.ename = content["ename"]
        out.evalue = content["evalue"]
        out.traceback = content["traceback"]
    else:
        raise ValueError("Unknown msg_type {}".format(msg_type))

    return out

def run_cell(kernel_client, cell):
    """Run a cell of a notebook in an ipython kernel and return its output"""
    kernel_client.execute(cell.input)

    input_acknowledged = False
    outs = []
    while True:
        msg = kernel_client.get_iopub_msg()

        if msg["msg_type"] == "status":
            if msg["content"]["execution_state"] == "idle" and input_acknowledged:
                break
        elif msg["msg_type"] in ('pyin', 'execute_input'):
            input_acknowledged = True
        else:
            out = parse_valid_output(msg)
            outs.append(out)

    return outs

def run_notebook(notebook):
    """Run the notebook"""
    kernel_manager = KernelManager()
    kernel_manager.start_kernel(stderr=open(os.devnull, 'w'))
    kernel_client = kernel_manager.client()
    kernel_client.start_channels()

    for sheet in notebook.worksheets:
        for (prompt_number, cell) in enumerate(sheet.cells, 1):
            if cell.cell_type != "code":
                continue

            cell.outputs = run_cell(kernel_client, cell)

            cell.prompt_number = prompt_number
            if cell.outputs and cell.outputs[0]['output_type'] == 'pyout':
                cell.outputs[0]["prompt_number"] = prompt_number

    kernel_manager.shutdown_kernel()

def export_notebook(nbbasename, output_directory, output_format):
    """Generate a PDF or HTML from the ipython notebook

    output_format has to be either 'pdf' or 'html'.  These are the
    only formats currently supported.

    ipython nbconvert claims that the CLI is not stable, so keep this
    function here to be able to cope with inconsistencies

    """

    if output_format == "html":
        ipython_command = IPYTHON_NBCONVERT_HTML
    elif output_format == "pdf":
        ipython_command = IPYTHON_NBCONVERT_PDF
    else:
        raise ValueError("Unknown output format: {}".format(output_format))

    prev_dir = os.getcwd()
    os.chdir(output_directory)

    with open(os.devnull, 'w') as devnull:
        subprocess.check_call(ipython_command + [nbbasename], stderr=devnull)

    os.chdir(prev_dir)
<commit_msg>ipython: switched to using LooseVersion for version checks.<commit_after>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=no-name-in-module,import-error,no-member

import os
import subprocess
from distutils.version import LooseVersion

# pylint: disable=wrong-import-position,ungrouped-imports
import_error_str = ''
try:
    import IPython
except ImportError as import_error:
    IPython = None
    # Importing IPython can fail for a variety of reasons, report the actual
    # failure unless it's just that the package is not present
    if import_error.message.startswith("No module named"):  # pylint: disable=E1101
        import_error_str = 'ipynb_exporter requires ipython package to be installed'
    else:
        import_error_str = import_error.message  # pylint: disable=redefined-variable-type

# The current code generates notebooks version 3
NBFORMAT_VERSION = 3

if IPython:
    if LooseVersion('5.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('4.0.0'):
        import nbformat
        from jupyter_client.manager import KernelManager

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            nbformat.write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = nbformat.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['jupyter', 'nbconvert', '--to html']
        IPYTHON_NBCONVERT_PDF = ['jupyter', 'nbconvert', '--to pdf']

    elif LooseVersion('4.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('3.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=pdf']
    elif LooseVersion('3.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('2.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat.v3

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.v3.reads_json(notebook_in)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.v3.nbjson.JSONWriter().write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.v3.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=latex',
                                 '--post=PDF']
    else:
        # Unsupported IPython version
        import_error_str = 'Unsupported IPython version {}'.format(IPython.__version__)

def parse_valid_output(msg):
    """Parse a valid result from an execution of a cell in an ipython kernel"""
    msg_type = msg["msg_type"]
    if msg_type == 'error':
        msg_type = 'pyerr'
    elif msg_type == 'execute_result':
        msg_type = 'pyout'

    content = msg["content"]
    out = NotebookNode(output_type=msg_type)

    if msg_type == "stream":
        out.stream = content["name"]
        try:
            out.text = content['data']
        except KeyError:
            out.text = content['text']
    elif msg_type in ("display_data", "pyout"):
        for mime, data in content["data"].iteritems():
            if mime == "text/plain":
                attr = "text"
            else:
                attr = mime.split("/")[-1]
            setattr(out, attr, data)
    elif msg_type == "pyerr":
        out.ename = content["ename"]
        out.evalue = content["evalue"]
        out.traceback = content["traceback"]
    else:
        raise ValueError("Unknown msg_type {}".format(msg_type))

    return out

def run_cell(kernel_client, cell):
    """Run a cell of a notebook in an ipython kernel and return its output"""
    kernel_client.execute(cell.input)

    input_acknowledged = False
    outs = []
    while True:
        msg = kernel_client.get_iopub_msg()

        if msg["msg_type"] == "status":
            if msg["content"]["execution_state"] == "idle" and input_acknowledged:
                break
        elif msg["msg_type"] in ('pyin', 'execute_input'):
            input_acknowledged = True
        else:
            out = parse_valid_output(msg)
            outs.append(out)

    return outs

def run_notebook(notebook):
    """Run the notebook"""
    kernel_manager = KernelManager()
    kernel_manager.start_kernel(stderr=open(os.devnull, 'w'))
    kernel_client = kernel_manager.client()
    kernel_client.start_channels()

    for sheet in notebook.worksheets:
        for (prompt_number, cell) in enumerate(sheet.cells, 1):
            if cell.cell_type != "code":
                continue

            cell.outputs = run_cell(kernel_client, cell)

            cell.prompt_number = prompt_number
            if cell.outputs and cell.outputs[0]['output_type'] == 'pyout':
                cell.outputs[0]["prompt_number"] = prompt_number

    kernel_manager.shutdown_kernel()

def export_notebook(nbbasename, output_directory, output_format):
    """Generate a PDF or HTML from the ipython notebook

    output_format has to be either 'pdf' or 'html'.  These are the
    only formats currently supported.

    ipython nbconvert claims that the CLI is not stable, so keep this
    function here to be able to cope with inconsistencies

    """

    if output_format == "html":
        ipython_command = IPYTHON_NBCONVERT_HTML
    elif output_format == "pdf":
        ipython_command = IPYTHON_NBCONVERT_PDF
    else:
        raise ValueError("Unknown output format: {}".format(output_format))

    prev_dir = os.getcwd()
    os.chdir(output_directory)

    with open(os.devnull, 'w') as devnull:
        subprocess.check_call(ipython_command + [nbbasename], stderr=devnull)

    os.chdir(prev_dir)
<|endoftext|>


<commit_before># pylint: disable=missing-docstring

import pytest
import numpy as np

from batchflow import Dataset, Batch, DatasetIndex, Pipeline

@pytest.fixture
def dataset():
    index = DatasetIndex(np.arange(100))
    return Dataset(index, Batch)

class TestDataset:
    def test_from_dataset(self, dataset):
        new_index = DatasetIndex(np.arange(25))
        new_ds = Dataset.from_dataset(dataset, new_index)
        assert isinstance(new_ds, Dataset)
        assert (new_ds.index.index == new_index.index).all()

    def test_build_index(self):
        new_index = Dataset.build_index(np.arange(25))
        assert isinstance(new_index, DatasetIndex)

    def test_create_subset(self, dataset):
        new_index = DatasetIndex(np.arange(25))
        new_ds = dataset.create_subset(new_index)
        assert isinstance(new_ds, dataset.__class__)
        assert np.isin(new_ds.index.index, dataset.index.index).all()

    def test_create_batch(self, dataset):
        target_index = DatasetIndex(np.arange(5))
        uncorrect_new_index = DatasetIndex(np.arange(100, 110))
        new_batch = dataset.create_batch(target_index)
        assert isinstance(new_batch, dataset.batch_class)
        assert len(new_batch.index.index) == len(target_index.index)
        with pytest.raises(ValueError):
            dataset.create_batch(uncorrect_new_index)

    def test_pipeline(self, dataset):
        pipeline_config = {}
        new_pipeline = dataset.pipeline(pipeline_config)
        assert isinstance(new_pipeline, Pipeline)

    def test_rshift(self, dataset):
        pipeline_config = {}
        new_pipeline = dataset.pipeline(pipeline_config)
        train_pipeline = (new_pipeline << dataset)
        assert isinstance(train_pipeline, Pipeline)

    def test_split(self, dataset):
        assert dataset.train is None
        dataset.split()
        assert dataset.train is not None
        assert dataset.test is not None
<commit_msg>Add new tests and fix errors<commit_after># pylint: disable=missing-docstring, redefined-outer-name

import pytest
import numpy as np

from batchflow import Dataset, Batch, ImagesBatch, DatasetIndex, Pipeline

@pytest.fixture
def dataset():
    index = DatasetIndex(np.arange(100))
    return Dataset(index, Batch)

class TestDataset:
    def test_from_dataset(self, dataset):
        new_index = DatasetIndex(np.arange(25))
        new_ds = Dataset.from_dataset(dataset, new_index)
        assert isinstance(new_ds, Dataset)
        assert new_ds.batch_class == dataset.batch_class
        assert (new_ds.index.index == new_index.index).all()

    def test_from_dataset_new_batch_class(self, dataset):
        new_index = DatasetIndex(np.arange(25))
        new_ds = Dataset.from_dataset(dataset, new_index, batch_class=ImagesBatch)
        assert isinstance(new_ds, Dataset)
        assert new_ds.batch_class == ImagesBatch

    def test_build_index(self):
        new_index = Dataset.build_index(np.arange(25))
        assert isinstance(new_index, DatasetIndex)

    def test_create_subset(self, dataset):
        new_index = DatasetIndex(np.arange(25))
        new_ds = dataset.create_subset(new_index)
        assert isinstance(new_ds, dataset.__class__)
        assert np.isin(new_ds.indices, dataset.indices).all()

    def test_create_subset_wrong_index(self, dataset):
        wrong_index = DatasetIndex(np.arange(200, 225))
        new_ds = dataset.create_subset(wrong_index)
        assert np.isin(new_ds.indices, dataset.indices).all()

    def test_create_batch(self, dataset):
        target_index = DatasetIndex(np.arange(5))
        new_batch = dataset.create_batch(target_index)
        assert isinstance(new_batch, dataset.batch_class)
        assert len(new_batch.indices) == len(target_index.indices)

    def test_pipeline(self, dataset):
        pipeline_config = {}
        new_pipeline = dataset.pipeline(pipeline_config)
        assert isinstance(new_pipeline, Pipeline)

    def test_rshift(self, dataset):
        pipeline_config = {}
        new_pipeline = dataset.pipeline(pipeline_config)
        train_pipeline = (new_pipeline << dataset)
        assert isinstance(train_pipeline, Pipeline)

    def test_split(self, dataset):
        assert dataset.train is None
        dataset.split()
        assert dataset.train is not None
        assert dataset.test is not None
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Load dependencies needed to compile and test the google-cloud-cpp library."""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")

def google_cloud_cpp_deps():
    """Loads dependencies need to compile the google-cloud-cpp library.

    Application developers can call this function from their WORKSPACE file
    to obtain all the necessary dependencies for google-cloud-cpp, including
    gRPC and its dependencies. This function only loads dependencies that
    have not been previously loaded, allowing application developers to
    override the version of the dependencies they want to use.
    """

    # Load rules_cc, used by googletest
    if "rules_cc" not in native.existing_rules():
        http_archive(
            name = "rules_cc",
            strip_prefix = "rules_cc-d66a13e2a01630afcafc4ba411d83e291ecf02bd",
            urls = [
                "https://github.com/bazelbuild/rules_cc/archive/d66a13e2a01630afcafc4ba411d83e291ecf02bd.tar.gz",
            ],
            sha256 = "3cde212ccda3ba152897e7fd354c42eba275878b6d98fe4f2125c684a73f3842",
        )

    # Load Abseil
    if "com_google_absl" not in native.existing_rules():
        http_archive(
            name = "com_google_absl",
            strip_prefix = "abseil-cpp-20210324.2",
            urls = [
                "https://github.com/abseil/abseil-cpp/archive/20210324.2.tar.gz",
            ],
            sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
        )

    # Load a version of googletest that we know works.
    if "com_google_googletest" not in native.existing_rules():
        http_archive(
            name = "com_google_googletest",
            strip_prefix = "googletest-release-1.11.0",
            urls = [
                "https://github.com/google/googletest/archive/release-1.11.0.tar.gz",
            ],
            sha256 = "b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5",
        )

    # Load a version of benchmark that we know works.
    if "com_google_benchmark" not in native.existing_rules():
        http_archive(
            name = "com_google_benchmark",
            strip_prefix = "benchmark-1.5.6",
            urls = [
                "https://github.com/google/benchmark/archive/v1.5.6.tar.gz",
            ],
            sha256 = "789f85b4810d13ff803834ea75999e41b326405d83d6a538baf01499eda96102",
        )

    # Load the googleapis dependency.
    if "com_google_googleapis" not in native.existing_rules():
        http_archive(
            name = "com_google_googleapis",
            urls = [
                "https://github.com/googleapis/googleapis/archive/ed739492993c4a99629b6430affdd6c0fb59d435.tar.gz",
            ],
            strip_prefix = "googleapis-ed739492993c4a99629b6430affdd6c0fb59d435",
            sha256 = "127d7ef9c8803ab246ec3c7ab83489ae386d74766725477c444c5ea8948cb9e5",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:googleapis.BUILD",
            # Scaffolding for patching googleapis after download. For example:
            #   patches = ["googleapis.patch"]
            # NOTE: This should only be used while developing with a new
            # protobuf message. No changes to `patches` should ever be
            # committed to the main branch.
            patch_tool = "patch",
            patch_args = ["-p1"],
            patches = [],
        )

    # Load protobuf.
    if "com_google_protobuf" not in native.existing_rules():
        http_archive(
            name = "com_google_protobuf",
            strip_prefix = "protobuf-3.17.3",
            urls = [
                "https://github.com/protocolbuffers/protobuf/archive/v3.17.3.tar.gz",
            ],
            sha256 = "c6003e1d2e7fefa78a3039f19f383b4f3a61e81be8c19356f85b6461998ad3db",
        )

    # Load BoringSSL, this is used by gRPC, but as I write this (2021-06-03, circa gRPC-1.37.1), the version used by
    # gRPC does not compile with GCC-11
    if "boringssl" not in native.existing_rules():
        http_archive(
            name = "boringssl",
            # Use github mirror instead of https://boringssl.googlesource.com/boringssl
            # to obtain a boringssl archive with consistent sha256
            sha256 = "f0f433106f98f6f50ed6bbc2169f7c42dd73d13d0f09d431082519b5649903a6",
            strip_prefix = "boringssl-56eb68f64215738552be452e311da12047934ab4",
            urls = [
                "https://github.com/google/boringssl/archive/56eb68f64215738552be452e311da12047934ab4.tar.gz",
            ],
        )

    # Load gRPC and its dependencies, using a similar pattern to this function.
    if "com_github_grpc_grpc" not in native.existing_rules():
        http_archive(
            name = "com_github_grpc_grpc",
            strip_prefix = "grpc-1.39.1",
            urls = [
                "https://github.com/grpc/grpc/archive/v1.39.1.tar.gz",
            ],
            sha256 = "024118069912358e60722a2b7e507e9c3b51eeaeee06e2dd9d95d9c16f6639ec",
        )

    # We use the cc_proto_library() rule from @com_google_protobuf, which
    # assumes that grpc_cpp_plugin and grpc_lib are in the //external: module
    native.bind(
        name = "grpc_cpp_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin",
    )

    native.bind(
        name = "grpc_lib",
        actual = "@com_github_grpc_grpc//:grpc++",
    )

    # We need libcurl for the Google Cloud Storage client.
    if "com_github_curl_curl" not in native.existing_rules():
        http_archive(
            name = "com_github_curl_curl",
            urls = [
                "https://curl.haxx.se/download/curl-7.69.1.tar.gz",
            ],
            strip_prefix = "curl-7.69.1",
            sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:curl.BUILD",
        )

    # We need the nlohmann_json library
    if "com_github_nlohmann_json" not in native.existing_rules():
        http_archive(
            name = "com_github_nlohmann_json",
            urls = [
                "https://github.com/nlohmann/json/releases/download/v3.10.2/include.zip",
            ],
            sha256 = "61e605be15e88deeac4582aaf01c09d616f8302edde7adcaba9261ddc3b4ceca",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:nlohmann_json.BUILD",
        )

    # Load google/crc32c, a library to efficiently compute CRC32C checksums.
    if "com_github_google_crc32c" not in native.existing_rules():
        http_archive(
            name = "com_github_google_crc32c",
            strip_prefix = "crc32c-1.1.1",
            urls = [
                "https://github.com/google/crc32c/archive/1.1.1.tar.gz",
            ],
            sha256 = "a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:crc32c.BUILD",
        )
<commit_msg>chore(deps): update dependency com_google_benchmark to v1.6.0 (#7279)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Load dependencies needed to compile and test the google-cloud-cpp library."""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")

def google_cloud_cpp_deps():
    """Loads dependencies need to compile the google-cloud-cpp library.

    Application developers can call this function from their WORKSPACE file
    to obtain all the necessary dependencies for google-cloud-cpp, including
    gRPC and its dependencies. This function only loads dependencies that
    have not been previously loaded, allowing application developers to
    override the version of the dependencies they want to use.
    """

    # Load rules_cc, used by googletest
    if "rules_cc" not in native.existing_rules():
        http_archive(
            name = "rules_cc",
            strip_prefix = "rules_cc-d66a13e2a01630afcafc4ba411d83e291ecf02bd",
            urls = [
                "https://github.com/bazelbuild/rules_cc/archive/d66a13e2a01630afcafc4ba411d83e291ecf02bd.tar.gz",
            ],
            sha256 = "3cde212ccda3ba152897e7fd354c42eba275878b6d98fe4f2125c684a73f3842",
        )

    # Load Abseil
    if "com_google_absl" not in native.existing_rules():
        http_archive(
            name = "com_google_absl",
            strip_prefix = "abseil-cpp-20210324.2",
            urls = [
                "https://github.com/abseil/abseil-cpp/archive/20210324.2.tar.gz",
            ],
            sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
        )

    # Load a version of googletest that we know works.
    if "com_google_googletest" not in native.existing_rules():
        http_archive(
            name = "com_google_googletest",
            strip_prefix = "googletest-release-1.11.0",
            urls = [
                "https://github.com/google/googletest/archive/release-1.11.0.tar.gz",
            ],
            sha256 = "b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5",
        )

    # Load a version of benchmark that we know works.
    if "com_google_benchmark" not in native.existing_rules():
        http_archive(
            name = "com_google_benchmark",
            strip_prefix = "benchmark-1.6.0",
            urls = [
                "https://github.com/google/benchmark/archive/v1.6.0.tar.gz",
            ],
            sha256 = "1f71c72ce08d2c1310011ea6436b31e39ccab8c2db94186d26657d41747c85d6",
        )

    # Load the googleapis dependency.
    if "com_google_googleapis" not in native.existing_rules():
        http_archive(
            name = "com_google_googleapis",
            urls = [
                "https://github.com/googleapis/googleapis/archive/ed739492993c4a99629b6430affdd6c0fb59d435.tar.gz",
            ],
            strip_prefix = "googleapis-ed739492993c4a99629b6430affdd6c0fb59d435",
            sha256 = "127d7ef9c8803ab246ec3c7ab83489ae386d74766725477c444c5ea8948cb9e5",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:googleapis.BUILD",
            # Scaffolding for patching googleapis after download. For example:
            #   patches = ["googleapis.patch"]
            # NOTE: This should only be used while developing with a new
            # protobuf message. No changes to `patches` should ever be
            # committed to the main branch.
            patch_tool = "patch",
            patch_args = ["-p1"],
            patches = [],
        )

    # Load protobuf.
    if "com_google_protobuf" not in native.existing_rules():
        http_archive(
            name = "com_google_protobuf",
            strip_prefix = "protobuf-3.17.3",
            urls = [
                "https://github.com/protocolbuffers/protobuf/archive/v3.17.3.tar.gz",
            ],
            sha256 = "c6003e1d2e7fefa78a3039f19f383b4f3a61e81be8c19356f85b6461998ad3db",
        )

    # Load BoringSSL, this is used by gRPC, but as I write this (2021-06-03, circa gRPC-1.37.1), the version used by
    # gRPC does not compile with GCC-11
    if "boringssl" not in native.existing_rules():
        http_archive(
            name = "boringssl",
            # Use github mirror instead of https://boringssl.googlesource.com/boringssl
            # to obtain a boringssl archive with consistent sha256
            sha256 = "f0f433106f98f6f50ed6bbc2169f7c42dd73d13d0f09d431082519b5649903a6",
            strip_prefix = "boringssl-56eb68f64215738552be452e311da12047934ab4",
            urls = [
                "https://github.com/google/boringssl/archive/56eb68f64215738552be452e311da12047934ab4.tar.gz",
            ],
        )

    # Load gRPC and its dependencies, using a similar pattern to this function.
    if "com_github_grpc_grpc" not in native.existing_rules():
        http_archive(
            name = "com_github_grpc_grpc",
            strip_prefix = "grpc-1.39.1",
            urls = [
                "https://github.com/grpc/grpc/archive/v1.39.1.tar.gz",
            ],
            sha256 = "024118069912358e60722a2b7e507e9c3b51eeaeee06e2dd9d95d9c16f6639ec",
        )

    # We use the cc_proto_library() rule from @com_google_protobuf, which
    # assumes that grpc_cpp_plugin and grpc_lib are in the //external: module
    native.bind(
        name = "grpc_cpp_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin",
    )

    native.bind(
        name = "grpc_lib",
        actual = "@com_github_grpc_grpc//:grpc++",
    )

    # We need libcurl for the Google Cloud Storage client.
    if "com_github_curl_curl" not in native.existing_rules():
        http_archive(
            name = "com_github_curl_curl",
            urls = [
                "https://curl.haxx.se/download/curl-7.69.1.tar.gz",
            ],
            strip_prefix = "curl-7.69.1",
            sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:curl.BUILD",
        )

    # We need the nlohmann_json library
    if "com_github_nlohmann_json" not in native.existing_rules():
        http_archive(
            name = "com_github_nlohmann_json",
            urls = [
                "https://github.com/nlohmann/json/releases/download/v3.10.2/include.zip",
            ],
            sha256 = "61e605be15e88deeac4582aaf01c09d616f8302edde7adcaba9261ddc3b4ceca",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:nlohmann_json.BUILD",
        )

    # Load google/crc32c, a library to efficiently compute CRC32C checksums.
    if "com_github_google_crc32c" not in native.existing_rules():
        http_archive(
            name = "com_github_google_crc32c",
            strip_prefix = "crc32c-1.1.1",
            urls = [
                "https://github.com/google/crc32c/archive/1.1.1.tar.gz",
            ],
            sha256 = "a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:crc32c.BUILD",
        )
<|endoftext|>


<commit_before>import argparse
import caffe
from caffe.proto import caffe_pb2
import mlbase.network as N
import mlbase.layers as L
from google.protobuf import text_format

def convert(def_path, caffemodel_path, output_path, phase):

    # read from .prototxt file to collect layers.
    params = caffe.proto.caffe_pb2.NetParameter()
    with open(def_path, 'r') as def_file:
        text_format.Merge(def_file.read(), params)

    layerName2InstanceMap = {} # dict of str:layer
    layerName2Bottom = {} # dict of str:list

    for layer in params.layer:
        if layer.type == 'Input':
            nl = L.RawInput((layer.input_param.shape[0].dim[1],
                             layer.input_param.shape[0].dim[2],
                             layer.input_param.shape[0].dim[3]))
            layerName2InstanceMap[layer.name] = nl
        elif layer.type == 'Convolution':
            name = layer.name
            bottom = layer.bottom
            output_feature_map = layer.convolution_param.num_output
            kernel_size = layer.convolution_param.kernel_size[0]
            stride = (1,1)
            if len(layer.convolution_param.stride) > 0:
                stride = layer.convolution_param.stride[0]
                stride = (stride, stride)

            nl = L.Conv2d(filter_size=(kernel_size, kernel_size)
                          , output_feature=output_feature_map
                          , subsample=stride)
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'ReLU':
            nl = L.Relu()
            name = layer.name
            if bottom[0] == top[0]:
                # if this is inplace operation, then steal bottom layer's name.
                preConv = layerName2InstanceMap[bottom[0]]
                layerName2InstanceMap[bottom[0]+'conv'] = preConv
                layerName2Bottom[bottom[0]+'conv'] = layerName2Bottom[bottom[0]]
                layerName2InstanceMap[bottom[0]] = nl
                layerName2Bottom[bottom[0]] = [bottom[0]+'conv',]
        elif layer.type == 'Pooling':
            name = layer.name
            # 0: max, 1: average, 2: stochastic
            poolingMethod = layer.pooling_param.pool
            kernel_size = layer.pooling_param.kernel_size
            stride = layer.pooling_param.stride
            nl = L.Pooling()
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'LRN':
            name = layer.name
            nl = L.LRN()
            local_size = layer.lrn_param.local_size
            alpha = layer.lrn_param.alpha
            beta = layer.lrn_param.beta
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'Concat':
            name = layer.name
            nl = L.Concat()
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'Dropout':
            name = layer.name
            nl = L.Dropout()
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'InnerProduct':
            name = layer.name
            nl = L.FullConn()
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'Softmax':
            name = layer.name
            nl = L.SoftMax()
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        else:
            print(layer.type)
            raise NotImplementedError('unknown caffe layer.')

    # read .caffemodel file to load real parameters.
    net_param = caffe_pb2.NetParameter()
    with open(model_data, 'rb') as f:
        net_param.ParseFromString(f.read())

    for layer in net_param.layers:
        pass

    # finally build the network.
    n = N.Network()
    n.setInput()
    n.build(reload=True)

    return n

def getMeanImage(mean_path):

    # the following code is from https://github.com/BVLC/caffe/issues/290
    blob = caffe.proto.caffe_pb2.BlobProto()
    data = open( mean_path , 'rb' ).read()
    blob.ParseFromString(data)
    arr = np.array( caffe.io.blobproto_to_array(blob) )
    out = arr[0]
    np.save( sys.argv[2] , out )

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('def_path', help='Model definition (.prototxt) path')
    parser.add_argument('--caffemodel', help='Model data (.caffemodel) path')
    parser.add_argument('--output-path', help='Converted model output path')
    parser.add_argument('-p',
                        '--phase',
                        default='test',
                        help='The phase to convert: test (default) or train')
    args = parser.parse_args()
    validate_arguments(args)
    convert(args.def_path, args.caffemodel, args.output_path, args.phase)

if __name__ == '__main__':
    main()
<commit_msg>grab parameters from caffe<commit_after>import argparse
import caffe
from caffe.proto import caffe_pb2
import mlbase.network as N
import mlbase.layers as L
from google.protobuf import text_format

def convert(def_path, caffemodel_path, output_path, phase):

    # read from .prototxt file to collect layers.
    params = caffe.proto.caffe_pb2.NetParameter()
    with open(def_path, 'r') as def_file:
        text_format.Merge(def_file.read(), params)

    layerName2InstanceMap = {} # dict of str:layer
    layerName2Bottom = {} # dict of str:list

    for layer in params.layer:
        if layer.type == 'Input':
            nl = L.RawInput((layer.input_param.shape[0].dim[1],
                             layer.input_param.shape[0].dim[2],
                             layer.input_param.shape[0].dim[3]))
            layerName2InstanceMap[layer.name] = nl
        elif layer.type == 'Convolution':
            name = layer.name
            bottom = layer.bottom
            output_feature_map = layer.convolution_param.num_output
            kernel_size = layer.convolution_param.kernel_size[0]
            stride = (1,1)
            if len(layer.convolution_param.stride) > 0:
                stride = layer.convolution_param.stride[0]
                stride = (stride, stride)

            nl = L.Conv2d(filter_size=(kernel_size, kernel_size)
                          , output_feature=output_feature_map
                          , subsample=stride)
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'ReLU':
            nl = L.Relu()
            name = layer.name
            bottom = layer.bottom
            top = layer.top
            if bottom[0] == top[0]:
                # if this is inplace operation, then steal bottom layer's name.
                preConv = layerName2InstanceMap[bottom[0]]
                layerName2InstanceMap[bottom[0]+'conv'] = preConv
                layerName2Bottom[bottom[0]+'conv'] = layerName2Bottom[bottom[0]]
                layerName2InstanceMap[bottom[0]] = nl
                layerName2Bottom[bottom[0]] = [bottom[0]+'conv',]
        elif layer.type == 'Pooling':
            name = layer.name
            # 0: max, 1: average, 2: stochastic
            poolingMethod = layer.pooling_param.pool
            if poolingMethod == 0:
                poolingMethod = 'max'
            elif poolingMethod == 1:
                poolingMethod = 'avg'
            kernel_size = layer.pooling_param.kernel_size
            stride = layer.pooling_param.stride
            pad = layer.pooling_param.pad
            nl = L.Pooling(dsize=(kernel_size, kernel_size), stride=(stride, stride), pad=(pad, pad), mode=poolingMethod)
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'LRN':
            name = layer.name
            local_size = layer.lrn_param.local_size
            alpha = layer.lrn_param.alpha
            beta = layer.lrn_param.beta
            nl = L.LRN(local_size = local_size, alpha = alpha, beta = beta)
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'Concat':
            name = layer.name
            nl = L.Concat()
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'Dropout':
            name = layer.name
            ratio = layer.dropout_param.dropout_ratio
            nl = L.Dropout(p=ratio)
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'InnerProduct':
            name = layer.name
            output_feature = layer.inner_product_param.num_output
            nl = L.FullConn(output_feature=output_feature)
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        elif layer.type == 'Softmax':
            name = layer.name
            nl = L.SoftMax()
            layerName2InstanceMap[name] = nl
            layerName2Bottom[name] = layer.bottom
        else:
            print(layer.type)
            raise NotImplementedError('unknown caffe layer.')

    # read .caffemodel file to load real parameters.
    net_param = caffe_pb2.NetParameter()
    with open(caffemodel_path, 'rb') as f:
        net_param.ParseFromString(f.read())

    for layer in net_param.layers:
        pass

    # finally build the network.
    n = N.Network()
    #n.setInput()
    #n.build(reload=True)

    return n

def getMeanImage(mean_path):

    # the following code is from https://github.com/BVLC/caffe/issues/290
    blob = caffe.proto.caffe_pb2.BlobProto()
    data = open( mean_path , 'rb' ).read()
    blob.ParseFromString(data)
    arr = np.array( caffe.io.blobproto_to_array(blob) )
    out = arr[0]
    np.save( sys.argv[2] , out )

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('def_path', help='Model definition (.prototxt) path')
    parser.add_argument('--caffemodel', help='Model data (.caffemodel) path')
    parser.add_argument('--output-path', help='Converted model output path')
    parser.add_argument('-p',
                        '--phase',
                        default='test',
                        help='The phase to convert: test (default) or train')
    args = parser.parse_args()
    validate_arguments(args)
    convert(args.def_path, args.caffemodel, args.output_path, args.phase)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Author: Mainak Jas <mainak.jas@telecom-paristech.fr>
#
# License: BSD (3-clause)

import os.path as op
import shutil

import warnings
from nose.tools import assert_raises, assert_equal, assert_true
import numpy as np
from numpy.testing import assert_array_equal

from mne import write_events, read_epochs_eeglab, Epochs, find_events
from mne.io import read_raw_eeglab
from mne.io.tests.test_raw import _test_raw_reader
from mne.io.eeglab.eeglab import _read_eeglab_events
from mne.datasets import testing
from mne.utils import _TempDir, run_tests_if_main, requires_version

base_dir = op.join(testing.data_path(download=False), 'EEGLAB')
raw_fname = op.join(base_dir, 'test_raw.set')
raw_fname_onefile = op.join(base_dir, 'test_raw_onefile.set')
epochs_fname = op.join(base_dir, 'test_epochs.set')
epochs_fname_onefile = op.join(base_dir, 'test_epochs_onefile.set')
montage = op.join(base_dir, 'test_chans.locs')

warnings.simplefilter('always')  # enable b/c these tests throw warnings

@requires_version('scipy', '0.12')
@testing.requires_testing_data
def test_io_set():
    """Test importing EEGLAB .set files"""
    from scipy import io
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        # main tests, and test missing event_id
        _test_raw_reader(read_raw_eeglab, input_fname=raw_fname,
                         montage=montage)
        _test_raw_reader(read_raw_eeglab, input_fname=raw_fname_onefile,
                         montage=montage)
    for want in ('Events like', 'consist entirely', 'could not be mapped',
                 'string preload is not supported'):
        assert_true(any(want in str(ww.message) for ww in w))
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        # test finding events in continuous data
        event_id = {'rt': 1, 'square': 2}
        raw0 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                               event_id=event_id, preload=True)
        raw1 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                               event_id=event_id, preload=False)
        raw2 = read_raw_eeglab(input_fname=raw_fname_onefile, montage=montage,
                               event_id=event_id)
        raw3 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                               event_id=event_id)
        raw4 = read_raw_eeglab(input_fname=raw_fname, montage=montage)
        Epochs(raw0, find_events(raw0), event_id)
        epochs = Epochs(raw1, find_events(raw1), event_id)
        assert_equal(len(find_events(raw4)), 0)  # no events without event_id
        assert_equal(epochs["square"].average().nave, 80)  # 80 with
        assert_array_equal(raw0[:][0], raw1[:][0], raw2[:][0], raw3[:][0])
        assert_array_equal(raw0[:][-1], raw1[:][-1], raw2[:][-1], raw3[:][-1])
        assert_equal(len(w), 4)
        # 1 for preload=False / str with fname_onefile, 3 for dropped events
        raw0.filter(1, None, l_trans_bandwidth='auto', filter_length='auto',
                    phase='zero')  # test that preloading works

    # test that using uint16_codec does not break stuff
    raw0 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                           event_id=event_id, preload=False,
                           uint16_codec='ascii')

    # test old EEGLAB version event import
    eeg = io.loadmat(raw_fname, struct_as_record=False,
                     squeeze_me=True)['EEG']
    for event in eeg.event:  # old version allows integer events
        event.type = 1
    assert_equal(_read_eeglab_events(eeg)[-1, -1], 1)
    eeg.event = eeg.event[0]  # single event
    assert_equal(_read_eeglab_events(eeg)[-1, -1], 1)

    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        epochs = read_epochs_eeglab(epochs_fname)
        epochs2 = read_epochs_eeglab(epochs_fname_onefile)
    # one warning for each read_epochs_eeglab because both files have epochs
    # associated with multiple events
    assert_equal(len(w), 2)
    assert_array_equal(epochs.get_data(), epochs2.get_data())

    # test different combinations of events and event_ids
    temp_dir = _TempDir()
    out_fname = op.join(temp_dir, 'test-eve.fif')
    write_events(out_fname, epochs.events)
    event_id = {'S255/S8': 1, 'S8': 2, 'S255/S9': 3}

    epochs = read_epochs_eeglab(epochs_fname, epochs.events, event_id)
    assert_equal(len(epochs.events), 4)
    assert_true(epochs.preload)
    assert_true(epochs._bad_dropped)
    epochs = read_epochs_eeglab(epochs_fname, out_fname, event_id)
    assert_raises(ValueError, read_epochs_eeglab, epochs_fname,
                  None, event_id)
    assert_raises(ValueError, read_epochs_eeglab, epochs_fname,
                  epochs.events, None)

    # test reading file with one event
    eeg = io.loadmat(raw_fname, struct_as_record=False,
                     squeeze_me=True)['EEG']
    one_event_fname = op.join(temp_dir, 'test_one_event.set')
    io.savemat(one_event_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': eeg.nbchan, 'data': 'test_one_event.fdt',
                'epoch': eeg.epoch, 'event': eeg.event[0],
                'chanlocs': eeg.chanlocs, 'pnts': eeg.pnts}})
    shutil.copyfile(op.join(base_dir, 'test_raw.fdt'),
                    op.join(temp_dir, 'test_one_event.fdt'))
    event_id = {eeg.event[0].type: 1}
    read_raw_eeglab(input_fname=one_event_fname, montage=montage,
                    event_id=event_id, preload=True)

    # test reading file with one channel
    one_chan_fname = op.join(temp_dir, 'test_one_channel.set')
    io.savemat(one_chan_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': 1, 'data': np.random.random((1, 3)),
                'epoch': eeg.epoch, 'event': eeg.epoch,
                'chanlocs': {'labels': 'E1', 'Y': -6.6069,
                             'X': 6.3023, 'Z': -2.9423},
                'times': eeg.times[:3], 'pnts': 3}})
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        read_raw_eeglab(input_fname=one_chan_fname, preload=True)
    # no warning for 'no events found'
    assert_equal(len(w), 0)

    # test reading file with 3 channels - one without position information
    # first, create chanlocs structured array
    ch_names = ['E1', 'E2', 'E3']
    x, y, z = [1., 2., np.nan], [4., 5., np.nan], [7., 8., np.nan]
    dt = [('labels', 'S10'), ('X', 'f8'), ('Y', 'f8'), ('Z', 'f8')]
    chanlocs = np.zeros((3,), dtype=dt)
    for ind, vals in enumerate(zip(ch_names, x, y, z)):
        for fld in range(4):
            chanlocs[ind][dt[fld][0]] = vals[fld]

    # save set file
    one_chanpos_fname = op.join(temp_dir, 'test_chanpos.set')
    io.savemat(one_chanpos_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': 3, 'data': np.random.random((3, 3)),
                'epoch': eeg.epoch, 'event': eeg.epoch,
                'chanlocs': chanlocs, 'times': eeg.times[:3], 'pnts': 3}})
    # load it
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        raw = read_raw_eeglab(input_fname=one_chanpos_fname, preload=True)
    # one warning because some channels are not found in Montage
    assert_equal(len(w), 1)
    # position should be present for first two channels
    for i in range(2):
        print(raw.info['chs'][i]['loc'][:3])
        assert_array_equal(raw.info['chs'][i]['loc'][:3],
                              np.array([-chanlocs[i]['Y'],
                                        chanlocs[i]['X'],
                                        chanlocs[i]['Z']]))
    # position of the last channel should be zero
    assert_array_equal(raw.info['chs'][-1]['loc'][:3],
                          np.array([0., 0., 0.]))

    # test if .dat file raises an error
    eeg = io.loadmat(epochs_fname, struct_as_record=False,
                     squeeze_me=True)['EEG']
    eeg.data = 'epochs_fname.dat'
    bad_epochs_fname = op.join(temp_dir, 'test_epochs.set')
    io.savemat(bad_epochs_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': eeg.nbchan, 'data': eeg.data,
                'epoch': eeg.epoch, 'event': eeg.event,
                'chanlocs': eeg.chanlocs}})
    shutil.copyfile(op.join(base_dir, 'test_epochs.fdt'),
                    op.join(temp_dir, 'test_epochs.dat'))
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        assert_raises(NotImplementedError, read_epochs_eeglab,
                      bad_epochs_fname)
    assert_equal(len(w), 1)

run_tests_if_main()
<commit_msg>STY: remove print<commit_after># Author: Mainak Jas <mainak.jas@telecom-paristech.fr>
#
# License: BSD (3-clause)

import os.path as op
import shutil

import warnings
from nose.tools import assert_raises, assert_equal, assert_true
import numpy as np
from numpy.testing import assert_array_equal

from mne import write_events, read_epochs_eeglab, Epochs, find_events
from mne.io import read_raw_eeglab
from mne.io.tests.test_raw import _test_raw_reader
from mne.io.eeglab.eeglab import _read_eeglab_events
from mne.datasets import testing
from mne.utils import _TempDir, run_tests_if_main, requires_version

base_dir = op.join(testing.data_path(download=False), 'EEGLAB')
raw_fname = op.join(base_dir, 'test_raw.set')
raw_fname_onefile = op.join(base_dir, 'test_raw_onefile.set')
epochs_fname = op.join(base_dir, 'test_epochs.set')
epochs_fname_onefile = op.join(base_dir, 'test_epochs_onefile.set')
montage = op.join(base_dir, 'test_chans.locs')

warnings.simplefilter('always')  # enable b/c these tests throw warnings

@requires_version('scipy', '0.12')
@testing.requires_testing_data
def test_io_set():
    """Test importing EEGLAB .set files"""
    from scipy import io
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        # main tests, and test missing event_id
        _test_raw_reader(read_raw_eeglab, input_fname=raw_fname,
                         montage=montage)
        _test_raw_reader(read_raw_eeglab, input_fname=raw_fname_onefile,
                         montage=montage)
    for want in ('Events like', 'consist entirely', 'could not be mapped',
                 'string preload is not supported'):
        assert_true(any(want in str(ww.message) for ww in w))
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        # test finding events in continuous data
        event_id = {'rt': 1, 'square': 2}
        raw0 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                               event_id=event_id, preload=True)
        raw1 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                               event_id=event_id, preload=False)
        raw2 = read_raw_eeglab(input_fname=raw_fname_onefile, montage=montage,
                               event_id=event_id)
        raw3 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                               event_id=event_id)
        raw4 = read_raw_eeglab(input_fname=raw_fname, montage=montage)
        Epochs(raw0, find_events(raw0), event_id)
        epochs = Epochs(raw1, find_events(raw1), event_id)
        assert_equal(len(find_events(raw4)), 0)  # no events without event_id
        assert_equal(epochs["square"].average().nave, 80)  # 80 with
        assert_array_equal(raw0[:][0], raw1[:][0], raw2[:][0], raw3[:][0])
        assert_array_equal(raw0[:][-1], raw1[:][-1], raw2[:][-1], raw3[:][-1])
        assert_equal(len(w), 4)
        # 1 for preload=False / str with fname_onefile, 3 for dropped events
        raw0.filter(1, None, l_trans_bandwidth='auto', filter_length='auto',
                    phase='zero')  # test that preloading works

    # test that using uint16_codec does not break stuff
    raw0 = read_raw_eeglab(input_fname=raw_fname, montage=montage,
                           event_id=event_id, preload=False,
                           uint16_codec='ascii')

    # test old EEGLAB version event import
    eeg = io.loadmat(raw_fname, struct_as_record=False,
                     squeeze_me=True)['EEG']
    for event in eeg.event:  # old version allows integer events
        event.type = 1
    assert_equal(_read_eeglab_events(eeg)[-1, -1], 1)
    eeg.event = eeg.event[0]  # single event
    assert_equal(_read_eeglab_events(eeg)[-1, -1], 1)

    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        epochs = read_epochs_eeglab(epochs_fname)
        epochs2 = read_epochs_eeglab(epochs_fname_onefile)
    # one warning for each read_epochs_eeglab because both files have epochs
    # associated with multiple events
    assert_equal(len(w), 2)
    assert_array_equal(epochs.get_data(), epochs2.get_data())

    # test different combinations of events and event_ids
    temp_dir = _TempDir()
    out_fname = op.join(temp_dir, 'test-eve.fif')
    write_events(out_fname, epochs.events)
    event_id = {'S255/S8': 1, 'S8': 2, 'S255/S9': 3}

    epochs = read_epochs_eeglab(epochs_fname, epochs.events, event_id)
    assert_equal(len(epochs.events), 4)
    assert_true(epochs.preload)
    assert_true(epochs._bad_dropped)
    epochs = read_epochs_eeglab(epochs_fname, out_fname, event_id)
    assert_raises(ValueError, read_epochs_eeglab, epochs_fname,
                  None, event_id)
    assert_raises(ValueError, read_epochs_eeglab, epochs_fname,
                  epochs.events, None)

    # test reading file with one event
    eeg = io.loadmat(raw_fname, struct_as_record=False,
                     squeeze_me=True)['EEG']
    one_event_fname = op.join(temp_dir, 'test_one_event.set')
    io.savemat(one_event_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': eeg.nbchan, 'data': 'test_one_event.fdt',
                'epoch': eeg.epoch, 'event': eeg.event[0],
                'chanlocs': eeg.chanlocs, 'pnts': eeg.pnts}})
    shutil.copyfile(op.join(base_dir, 'test_raw.fdt'),
                    op.join(temp_dir, 'test_one_event.fdt'))
    event_id = {eeg.event[0].type: 1}
    read_raw_eeglab(input_fname=one_event_fname, montage=montage,
                    event_id=event_id, preload=True)

    # test reading file with one channel
    one_chan_fname = op.join(temp_dir, 'test_one_channel.set')
    io.savemat(one_chan_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': 1, 'data': np.random.random((1, 3)),
                'epoch': eeg.epoch, 'event': eeg.epoch,
                'chanlocs': {'labels': 'E1', 'Y': -6.6069,
                             'X': 6.3023, 'Z': -2.9423},
                'times': eeg.times[:3], 'pnts': 3}})
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        read_raw_eeglab(input_fname=one_chan_fname, preload=True)
    # no warning for 'no events found'
    assert_equal(len(w), 0)

    # test reading file with 3 channels - one without position information
    # first, create chanlocs structured array
    ch_names = ['E1', 'E2', 'E3']
    x, y, z = [1., 2., np.nan], [4., 5., np.nan], [7., 8., np.nan]
    dt = [('labels', 'S10'), ('X', 'f8'), ('Y', 'f8'), ('Z', 'f8')]
    chanlocs = np.zeros((3,), dtype=dt)
    for ind, vals in enumerate(zip(ch_names, x, y, z)):
        for fld in range(4):
            chanlocs[ind][dt[fld][0]] = vals[fld]

    # save set file
    one_chanpos_fname = op.join(temp_dir, 'test_chanpos.set')
    io.savemat(one_chanpos_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': 3, 'data': np.random.random((3, 3)),
                'epoch': eeg.epoch, 'event': eeg.epoch,
                'chanlocs': chanlocs, 'times': eeg.times[:3], 'pnts': 3}})
    # load it
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        raw = read_raw_eeglab(input_fname=one_chanpos_fname, preload=True)
    # one warning because some channels are not found in Montage
    assert_equal(len(w), 1)
    # position should be present for first two channels
    for i in range(2):
        assert_array_equal(raw.info['chs'][i]['loc'][:3],
                              np.array([-chanlocs[i]['Y'],
                                        chanlocs[i]['X'],
                                        chanlocs[i]['Z']]))
    # position of the last channel should be zero
    assert_array_equal(raw.info['chs'][-1]['loc'][:3],
                          np.array([0., 0., 0.]))

    # test if .dat file raises an error
    eeg = io.loadmat(epochs_fname, struct_as_record=False,
                     squeeze_me=True)['EEG']
    eeg.data = 'epochs_fname.dat'
    bad_epochs_fname = op.join(temp_dir, 'test_epochs.set')
    io.savemat(bad_epochs_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': eeg.nbchan, 'data': eeg.data,
                'epoch': eeg.epoch, 'event': eeg.event,
                'chanlocs': eeg.chanlocs}})
    shutil.copyfile(op.join(base_dir, 'test_epochs.fdt'),
                    op.join(temp_dir, 'test_epochs.dat'))
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        assert_raises(NotImplementedError, read_epochs_eeglab,
                      bad_epochs_fname)
    assert_equal(len(w), 1)

run_tests_if_main()
<|endoftext|>


<commit_before>import csv
import sys
import typing
import collections
from pathlib import Path
import chardet
import warnings

import pycldf.dataset
from csvw.dsv import UnicodeDictReader

from beastling.util import log

def sniff(filename, default_dialect: typing.Optional[csv.Dialect] = csv.excel):
    """Read the beginning of the file and guess its csv dialect.

    Parameters
    ----------
    filename: str or pathlib.Path
        Path to a csv file to be sniffed

    Returns
    -------
    csv.Dialect
    """
    with Path(filename).open("rb") as fp:
        # On large files, csv.Sniffer seems to need a lot of data to make a
        # successful inference...
        sample = fp.read(1024)
        encoding = chardet.detect(sample)["encoding"]
        sample = sample.decode(encoding)
        while True:
            try:
                dialect = csv.Sniffer().sniff(sample, [",", "\t"])
                dialect.encoding = encoding
                return dialect
            except csv.Error: # pragma: no cover
                blob = fp.read(1024).decode(encoding)
                sample += blob
                if not blob:
                    # If blob is emtpy we've somehow hit the end of the file
                    # without figuring out the dialect.  Something is probably
                    # quite wrong with the file, but let's default to Excel and
                    # hope for the best...
                    if default_dialect is not None:
                        return default_dialect
                    raise

def sanitise_name(name):
    """
    Take a name for a language or a feature which has come from somewhere like
    a CLDF dataset and make sure it does not contain any characters which
    will cause trouble for BEAST or postanalysis tools.
    """
    return name.replace(" ", "_")

def load_data(filename, file_format=None, lang_column=None, value_column=None, expect_multiple=False):
    # Handle CSV dialect issues
    if str(filename) == 'stdin':
        filename = sys.stdin
        # We can't sniff from stdin, so guess comma-delimited and hope for
        # the best
        dialect = "excel" # Default dialect for csv module
    elif file_format and file_format.lower() == "cldf":
        return read_cldf_dataset(filename, value_column, expect_multiple=expect_multiple)
    elif file_format and file_format.lower() == "cldf-legacy":
        # CLDF pre-1.0 standard says delimiter is indicated by file extension
        if filename.suffix.lower() == ".csv" or str(filename) == "stdin":
            dialect = "excel"
        elif filename.suffix.lower() == ".tsv":
            dialect = "excel-tab"
        else:
            raise ValueError("CLDF standard dictates that filenames must end in .csv or .tsv")
    elif filename.suffix == ".json" or filename.name in {"forms.csv", "values.csv"}:
        # TODO: Should we just let the pycldf module try its hands on the file
        # and fall back to other formats if that doesn't work?
        return read_cldf_dataset(filename, value_column, expect_multiple=expect_multiple)
    else:
        # Use CSV dialect sniffer in all other cases
        dialect = sniff(filename)
    # Read
    with UnicodeDictReader(filename, dialect=dialect) as reader:
        # Guesstimate file format if user has not been explicit
        if file_format is None:
            file_format = 'cldf-legacy' if all(
                [f in reader.fieldnames for f in ("Language_ID", "Value")]) and any(
                    [f in reader.fieldnames for f in ("Feature_ID", "Parameter_ID")]
                ) else 'beastling'

        # Load data
        if file_format == 'cldf-legacy':
            data = load_cldf_data(reader, value_column, filename, expect_multiple=expect_multiple)
        elif file_format == 'beastling':
            data = load_beastling_data(reader, lang_column, filename, expect_multiple=expect_multiple)
        else:
            raise ValueError("File format specification '{:}' not understood".format(file_format))
    return data, {}

_language_column_names = ("iso", "iso_code", "glotto", "glottocode", "language", "language_id", "lang", "lang_id")

def load_beastling_data(reader, lang_column, filename, expect_multiple=False):
    if not lang_column:
        for candidate in reader.fieldnames:
            if candidate.lower() in _language_column_names:
                lang_column = candidate
                break

    if not lang_column or lang_column not in reader.fieldnames:
        raise ValueError("Cold not find language column in data file %s" % filename)
    data = collections.defaultdict(lambda: collections.defaultdict(lambda: "?"))
    for row in reader:
        if row[lang_column] in data:
            raise ValueError("Duplicated language identifier '%s' found in data file %s" % (row[lang_column], filename))
        lang = row.pop(lang_column)
        if expect_multiple:
            data[lang] = collections.defaultdict(lambda : "?", {key: [value] for key, value in row.items()})
        else:
            data[lang] = collections.defaultdict(lambda : "?", row)
    return data

def load_cldf_data(reader, value_column, filename, expect_multiple=False):
    value_column = value_column or "Value"
    if "Feature_ID" in reader.fieldnames:
        feature_column = "Feature_ID"
    elif "Parameter_ID" in reader.fieldnames:
        feature_column = "Parameter_ID"
    else:
        raise ValueError("Could not find Feature_ID or Parameter_ID column, is %s a valid CLDF file?" % filename)
    data = collections.defaultdict(lambda: collections.defaultdict(lambda: "?"))
    for row in reader:
        lang = row["Language_ID"]
        if lang not in data:
            if expect_multiple:
                data[lang] = collections.defaultdict(lambda: [])
            else:
                data[lang] = collections.defaultdict(lambda: "?")
        if expect_multiple:
            data[lang][row[feature_column]].append(row[value_column])
        else:
            data[lang][row[feature_column]] = row[value_column]
    return data

def iterlocations(filename):
    with UnicodeDictReader(filename, dialect=sniff(filename, default_dialect=None)) as reader:
        # Identify fieldnames
        fieldnames = [(n.lower(), n) for n in reader.fieldnames]
        fieldmap = {}

        for field, aliases in [
            ('language identifier', _language_column_names),
            ('latitude', ("latitude", "lat")),
            ('longitude', ("longitude", "lon", "long")),
        ]:
            for lname, fieldname in fieldnames:
                if lname in aliases:
                    fieldmap[field] = fieldname
                    break
            else:
                raise ValueError(
                    "Could not find a {0} column in location data file {1}".format(field, filename))

        for row in reader:
            (lat, lon) = row[fieldmap['latitude']], row[fieldmap['longitude']]
            try:
                lat = float(lat) if lat != "?" else lat
                lon = float(lon) if lon != "?" else lon
            except ValueError:
                lat, lon = "?", "?"
            yield (row[fieldmap['language identifier']].strip(), (lat, lon))

def get_dataset(fname):
    """Load a CLDF dataset.

    Load the file as `json` CLDF metadata description file, or as metadata-free
    dataset contained in a single csv file.

    The distinction is made depending on the file extension: `.json` files are
    loaded as metadata descriptions, all other files are matched against the
    CLDF module specifications. Directories are checked for the presence of
    any CLDF datasets in undefined order of the dataset types.

    Parameters
    ----------
    fname : str or Path
        Path to a CLDF dataset

    Returns
    -------
    Dataset
    """
    fname = Path(fname)
    if not fname.exists():
        raise FileNotFoundError('{:} does not exist'.format(fname))
    if fname.suffix == '.json':
        return pycldf.dataset.Dataset.from_metadata(fname)
    return pycldf.dataset.Dataset.from_data(fname)

def read_cldf_dataset(filename, code_column=None, expect_multiple=False):
    """Load a CLDF dataset.

    Load the file as `json` CLDF metadata description file, or as metadata-free
    dataset contained in a single csv file.

    The distinction is made depending on the file extension: `.json` files are
    loaded as metadata descriptions, all other files are matched against the
    CLDF module specifications. Directories are checked for the presence of
    any CLDF datasets in undefined order of the dataset types.

    Parameters
    ----------
    fname : str or Path
        Path to a CLDF dataset

    Returns
    -------
    Dataset
    """
    dataset = get_dataset(filename)
    if expect_multiple:
        data = collections.defaultdict(lambda: collections.defaultdict(lambda: []))
    else:
        data = collections.defaultdict(lambda: collections.defaultdict(lambda: "?"))

    # Make sure this is a kind of dataset BEASTling can handle
    if dataset.module not in ("Wordlist", "StructureDataset"):
        raise ValueError("BEASTling does not know how to interpret CLDF {:} data.".format(
            dataset.module))

    # Build dictionaries of nice IDs for languages and features
    col_map = dataset.column_names
    lang_ids, language_code_map = build_lang_ids(dataset, col_map)
    feature_ids = {}
    if col_map.parameters:
        for row in dataset["ParameterTable"]:
            feature_ids[row[col_map.parameters.id]] = sanitise_name(row[col_map.parameters.name])

    # Build actual data dictionary, based on dataset type
    if dataset.module == "Wordlist":
        # We search for cognatesetReferences in the FormTable or a separate CognateTable.
        # If we find them in CognateTable, we store them keyed with formReference:
        cognatesets = collections.defaultdict(lambda: "?")
        if not code_column:  # If code_column is given explicitly, we don't have to search!
            code_column = col_map.forms.cognatesetReference
            if not code_column:
                if col_map.cognates:
                    code_column = col_map.cognates.cognatesetReference
                    for row in dataset["CognateTable"]:
                        #
                        # Note: We assume that each form belongs to at most one cognate set!
                        # If this is not the case, the last cognate set wins.
                        #
                        cognatesets[row[col_map.cognates.formReference]] = row[code_column]
                else:
                    raise ValueError(
                        "Dataset {:} has no cognatesetReference column in its "
                        "primary table or in a separate cognate table. "
                        "Is this a metadata-free wordlist and you forgot to "
                        "specify code_column explicitly?".format(filename))

        for row in dataset["FormTable"]:
            lang_id = lang_ids.get(
                row[col_map.forms.languageReference], row[col_map.forms.languageReference])
            feature_id = feature_ids.get(
                row[col_map.forms.parameterReference], row[col_map.forms.parameterReference])

            cogset = cognatesets[row[col_map.forms.id]] if cognatesets else row[code_column]
            if expect_multiple:
                data[lang_id][feature_id].append(cogset)
            else:
                data[lang_id][feature_id] = cogset
        return data, language_code_map

    if dataset.module == "StructureDataset":
        code_column = col_map.values.codeReference or col_map.values.value
        for row in dataset["ValueTable"]:
            lang_id = lang_ids.get(
                row[col_map.values.languageReference], row[col_map.values.languageReference])
            feature_id = feature_ids.get(
                row[col_map.values.parameterReference], row[col_map.values.parameterReference])
            if expect_multiple:
                data[lang_id][feature_id].append(row[code_column] or '')
            else:
                data[lang_id][feature_id] = row[code_column] or ''
        return data, language_code_map

def build_lang_ids(dataset, col_map):
    if col_map.languages is None:
        # No language table so we can't do anything
        return {}, {}

    col_map = col_map.languages
    lang_ids = {}
    language_code_map = {}

    # First check for unique names and Glottocodes
    names = []
    gcs = []
    langs = []
    for row in dataset["LanguageTable"]:
        langs.append(row)
        names.append(row[col_map.name])
        if row[col_map.glottocode]:
            gcs.append(row[col_map.glottocode])

    unique_names = len(set(names)) == len(names)
    unique_gcs = len(set(gcs)) == len(gcs) == len(names)

    log.info('{0} are used as language identifiers'.format(
        'Names' if unique_names else ('Glottocodes' if unique_gcs else 'dataset-local IDs')))

    for row in langs:
        if unique_names:
            # Use names if they're unique, for human-friendliness
            lang_ids[row[col_map.id]] = sanitise_name(row[col_map.name])
        elif unique_gcs:
            # Otherwise, use glottocodes as at least they are meaningful
            lang_ids[row[col_map.id]] = row[col_map.glottocode]
        else:
            # As a last resort, use the IDs which are guaranteed to be unique
            lang_ids[row[col_map.id]] = row[col_map.id]
        if row[col_map.glottocode]:
            language_code_map[lang_ids[row[col_map.id]]] = row[col_map.glottocode]
    return lang_ids, language_code_map
<commit_msg>Ignore two CLDF warnings about cognate sets in metadata-free CLDF<commit_after>import csv
import sys
import typing
import collections
from pathlib import Path
import chardet
import warnings

import pycldf.dataset
from csvw.dsv import UnicodeDictReader

from beastling.util import log

def sniff(filename, default_dialect: typing.Optional[csv.Dialect] = csv.excel):
    """Read the beginning of the file and guess its csv dialect.

    Parameters
    ----------
    filename: str or pathlib.Path
        Path to a csv file to be sniffed

    Returns
    -------
    csv.Dialect
    """
    with Path(filename).open("rb") as fp:
        # On large files, csv.Sniffer seems to need a lot of data to make a
        # successful inference...
        sample = fp.read(1024)
        encoding = chardet.detect(sample)["encoding"]
        sample = sample.decode(encoding)
        while True:
            try:
                dialect = csv.Sniffer().sniff(sample, [",", "\t"])
                dialect.encoding = encoding
                return dialect
            except csv.Error: # pragma: no cover
                blob = fp.read(1024).decode(encoding)
                sample += blob
                if not blob:
                    # If blob is emtpy we've somehow hit the end of the file
                    # without figuring out the dialect.  Something is probably
                    # quite wrong with the file, but let's default to Excel and
                    # hope for the best...
                    if default_dialect is not None:
                        return default_dialect
                    raise

def sanitise_name(name):
    """
    Take a name for a language or a feature which has come from somewhere like
    a CLDF dataset and make sure it does not contain any characters which
    will cause trouble for BEAST or postanalysis tools.
    """
    return name.replace(" ", "_")

def load_data(filename, file_format=None, lang_column=None, value_column=None, expect_multiple=False):
    # Handle CSV dialect issues
    if str(filename) == 'stdin':
        filename = sys.stdin
        # We can't sniff from stdin, so guess comma-delimited and hope for
        # the best
        dialect = "excel" # Default dialect for csv module
    elif file_format and file_format.lower() == "cldf":
        return read_cldf_dataset(filename, value_column, expect_multiple=expect_multiple)
    elif file_format and file_format.lower() == "cldf-legacy":
        # CLDF pre-1.0 standard says delimiter is indicated by file extension
        if filename.suffix.lower() == ".csv" or str(filename) == "stdin":
            dialect = "excel"
        elif filename.suffix.lower() == ".tsv":
            dialect = "excel-tab"
        else:
            raise ValueError("CLDF standard dictates that filenames must end in .csv or .tsv")
    elif filename.suffix == ".json" or filename.name in {"forms.csv", "values.csv"}:
        # TODO: Should we just let the pycldf module try its hands on the file
        # and fall back to other formats if that doesn't work?
        return read_cldf_dataset(filename, value_column, expect_multiple=expect_multiple)
    else:
        # Use CSV dialect sniffer in all other cases
        dialect = sniff(filename)
    # Read
    with UnicodeDictReader(filename, dialect=dialect) as reader:
        # Guesstimate file format if user has not been explicit
        if file_format is None:
            file_format = 'cldf-legacy' if all(
                [f in reader.fieldnames for f in ("Language_ID", "Value")]) and any(
                    [f in reader.fieldnames for f in ("Feature_ID", "Parameter_ID")]
                ) else 'beastling'

        # Load data
        if file_format == 'cldf-legacy':
            data = load_cldf_data(reader, value_column, filename, expect_multiple=expect_multiple)
        elif file_format == 'beastling':
            data = load_beastling_data(reader, lang_column, filename, expect_multiple=expect_multiple)
        else:
            raise ValueError("File format specification '{:}' not understood".format(file_format))
    return data, {}

_language_column_names = ("iso", "iso_code", "glotto", "glottocode", "language", "language_id", "lang", "lang_id")

def load_beastling_data(reader, lang_column, filename, expect_multiple=False):
    if not lang_column:
        for candidate in reader.fieldnames:
            if candidate.lower() in _language_column_names:
                lang_column = candidate
                break

    if not lang_column or lang_column not in reader.fieldnames:
        raise ValueError("Cold not find language column in data file %s" % filename)
    data = collections.defaultdict(lambda: collections.defaultdict(lambda: "?"))
    for row in reader:
        if row[lang_column] in data:
            raise ValueError("Duplicated language identifier '%s' found in data file %s" % (row[lang_column], filename))
        lang = row.pop(lang_column)
        if expect_multiple:
            data[lang] = collections.defaultdict(lambda : "?", {key: [value] for key, value in row.items()})
        else:
            data[lang] = collections.defaultdict(lambda : "?", row)
    return data

def load_cldf_data(reader, value_column, filename, expect_multiple=False):
    value_column = value_column or "Value"
    if "Feature_ID" in reader.fieldnames:
        feature_column = "Feature_ID"
    elif "Parameter_ID" in reader.fieldnames:
        feature_column = "Parameter_ID"
    else:
        raise ValueError("Could not find Feature_ID or Parameter_ID column, is %s a valid CLDF file?" % filename)
    data = collections.defaultdict(lambda: collections.defaultdict(lambda: "?"))
    for row in reader:
        lang = row["Language_ID"]
        if lang not in data:
            if expect_multiple:
                data[lang] = collections.defaultdict(lambda: [])
            else:
                data[lang] = collections.defaultdict(lambda: "?")
        if expect_multiple:
            data[lang][row[feature_column]].append(row[value_column])
        else:
            data[lang][row[feature_column]] = row[value_column]
    return data

def iterlocations(filename):
    with UnicodeDictReader(filename, dialect=sniff(filename, default_dialect=None)) as reader:
        # Identify fieldnames
        fieldnames = [(n.lower(), n) for n in reader.fieldnames]
        fieldmap = {}

        for field, aliases in [
            ('language identifier', _language_column_names),
            ('latitude', ("latitude", "lat")),
            ('longitude', ("longitude", "lon", "long")),
        ]:
            for lname, fieldname in fieldnames:
                if lname in aliases:
                    fieldmap[field] = fieldname
                    break
            else:
                raise ValueError(
                    "Could not find a {0} column in location data file {1}".format(field, filename))

        for row in reader:
            (lat, lon) = row[fieldmap['latitude']], row[fieldmap['longitude']]
            try:
                lat = float(lat) if lat != "?" else lat
                lon = float(lon) if lon != "?" else lon
            except ValueError:
                lat, lon = "?", "?"
            yield (row[fieldmap['language identifier']].strip(), (lat, lon))

def get_dataset(fname):
    """Load a CLDF dataset.

    Load the file as `json` CLDF metadata description file, or as metadata-free
    dataset contained in a single csv file.

    The distinction is made depending on the file extension: `.json` files are
    loaded as metadata descriptions, all other files are matched against the
    CLDF module specifications. Directories are checked for the presence of
    any CLDF datasets in undefined order of the dataset types.

    Parameters
    ----------
    fname : str or Path
        Path to a CLDF dataset

    Returns
    -------
    Dataset
    """
    fname = Path(fname)
    if not fname.exists():
        raise FileNotFoundError('{:} does not exist'.format(fname))
    if fname.suffix == '.json':
        return pycldf.dataset.Dataset.from_metadata(fname)
    return pycldf.dataset.Dataset.from_data(fname)

# TODO: Change the behaviour to always expect multiple.
def read_cldf_dataset(filename, code_column=None, expect_multiple=False):
    """Load a CLDF dataset.

    Load the file as `json` CLDF metadata description file, or as metadata-free
    dataset contained in a single csv file.

    The distinction is made depending on the file extension: `.json` files are
    loaded as metadata descriptions, all other files are matched against the
    CLDF module specifications. Directories are checked for the presence of
    any CLDF datasets in undefined order of the dataset types.

    Parameters
    ----------
    fname : str or Path
        Path to a CLDF dataset

    Returns
    -------
    Dataset
    """
    dataset = get_dataset(filename)
    if expect_multiple:
        data = collections.defaultdict(lambda: collections.defaultdict(lambda: []))
    else:
        data = collections.defaultdict(lambda: collections.defaultdict(lambda: "?"))

    # Make sure this is a kind of dataset BEASTling can handle
    if dataset.module not in ("Wordlist", "StructureDataset"):
        raise ValueError("BEASTling does not know how to interpret CLDF {:} data.".format(
            dataset.module))

    # Build dictionaries of nice IDs for languages and features
    col_map = dataset.column_names
    lang_ids, language_code_map = build_lang_ids(dataset, col_map)
    feature_ids = {}
    if col_map.parameters:
        for row in dataset["ParameterTable"]:
            feature_ids[row[col_map.parameters.id]] = sanitise_name(row[col_map.parameters.name])

    # Build actual data dictionary, based on dataset type
    if dataset.module == "Wordlist":
        # We search for cognatesetReferences in the FormTable or a separate CognateTable.
        cognate_column_in_form_table = True
        # If we find them in CognateTable, we store them keyed with formReference:
        if not code_column:  # If code_column is given explicitly, we don't have to search!
            code_column = col_map.forms.cognatesetReference
            if not code_column:
                if (col_map.cognates and
                    col_map.cognates.cognatesetReference and
                    col_map.cognates.formReference):
                    code_column = col_map.cognates.cognatesetReference
                    form_reference = col_map.cognates.formReference
                    if expect_multiple:
                        cognatesets = collections.defaultdict(list)
                        for row in dataset["CognateTable"]:
                            cognatesets[row[form_reference]].append(row[code_column])
                    else:
                        cognatesets = collections.defaultdict(lambda: "?")
                        for row in dataset["CognateTable"]:
                            cognatesets[row[form_reference]] = row[code_column]
                else:
                    raise ValueError(
                        "Dataset {:} has no cognatesetReference column in its "
                        "primary table or in a separate cognate table. "
                        "Is this a metadata-free wordlist and you forgot to "
                        "specify code_column explicitly?".format(filename))
                form_column = dataset["FormTable", "id"].name
                cognate_column_in_form_table = False

        language_column = col_map.forms.languageReference
        parameter_column = col_map.forms.parameterReference

        warnings.filterwarnings(
            "ignore", '.*Unspecified column "Cognate_Set"', UserWarning, "csvw\.metadata", 0)
        warnings.filterwarnings(
            "ignore", '.*Unspecified column "{:}"'.format(code_column), UserWarning, "csvw\.metadata", 0)
        # We know how to deal with a 'Cognate_Set' column, even in a metadata-free CSV file

        for row in dataset["FormTable"].iterdicts():
            lang_id = lang_ids.get(row[language_column], row[language_column])
            feature_id = feature_ids.get(row[parameter_column], row[parameter_column])
            if cognate_column_in_form_table:
                if expect_multiple:
                    data[lang_id][feature_id].append(row[code_column])
                else:
                    data[lang_id][feature_id] = row[code_column]
            else:
                data[lang_id][feature_id] = cognatesets[row[col_map.forms.id]]
        return data, language_code_map

    if dataset.module == "StructureDataset":
        code_column = col_map.values.codeReference or col_map.values.value
        for row in dataset["ValueTable"]:
            lang_id = lang_ids.get(
                row[col_map.values.languageReference], row[col_map.values.languageReference])
            feature_id = feature_ids.get(
                row[col_map.values.parameterReference], row[col_map.values.parameterReference])
            if expect_multiple:
                data[lang_id][feature_id].append(row[code_column] or '')
            else:
                data[lang_id][feature_id] = row[code_column] or ''
        return data, language_code_map

def build_lang_ids(dataset, col_map):
    if col_map.languages is None:
        # No language table so we can't do anything
        return {}, {}

    col_map = col_map.languages
    lang_ids = {}
    language_code_map = {}

    # First check for unique names and Glottocodes
    names = []
    gcs = []
    langs = []
    for row in dataset["LanguageTable"]:
        langs.append(row)
        names.append(row[col_map.name])
        if row[col_map.glottocode]:
            gcs.append(row[col_map.glottocode])

    unique_names = len(set(names)) == len(names)
    unique_gcs = len(set(gcs)) == len(gcs) == len(names)

    log.info('{0} are used as language identifiers'.format(
        'Names' if unique_names else ('Glottocodes' if unique_gcs else 'dataset-local IDs')))

    for row in langs:
        if unique_names:
            # Use names if they're unique, for human-friendliness
            lang_ids[row[col_map.id]] = sanitise_name(row[col_map.name])
        elif unique_gcs:
            # Otherwise, use glottocodes as at least they are meaningful
            lang_ids[row[col_map.id]] = row[col_map.glottocode]
        else:
            # As a last resort, use the IDs which are guaranteed to be unique
            lang_ids[row[col_map.id]] = row[col_map.id]
        if row[col_map.glottocode]:
            language_code_map[lang_ids[row[col_map.id]]] = row[col_map.glottocode]
    return lang_ids, language_code_map
<|endoftext|>