<commit_before>"""
Copyright (C) 2016  Genome Research Ltd.

Author: Irina Colgiu <ic4@sanger.ac.uk>

This program is part of meta-check

meta-check is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

This file has been created on Jun 14, 2016.
"""

import unittest

from mcheck.metadata.file_header_metadata.header_metadata import SAMFileHeaderMetadata
from mcheck.metadata.irods_metadata.file_metadata import IrodsSeqFileMetadata
from mcheck.metadata.seqscape_metadata.seqscape_metadata import SeqscapeMetadata

class ComparableMetadataTests(unittest.TestCase):

    def test_find_differences_when_there_are_diffs_h_vs_ss1(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam', samples={'name': set(['S1'])}, libraries={}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S9'])}, libraries={}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{'samples': {'name': set(['S1'])}})

    def test_find_differences_when_there_are_diffs_h_vs_ss2(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam', samples={'name': set(['S9'])}, libraries={}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S1'])}, libraries={}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{'samples': {'name': set(['S9'])}})

    def test_find_differences_when_there_are_diff_type_ids_h_vs_ss(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam', samples={'name': set(['S1']), 'accession_number': set(['Acc1'])}, libraries={}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S1'])}, libraries={}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result, {})

    def test_find_differences_when_many_diffs_h_vs_ss(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S9']), 'accession_number': set(['Acc9'])},
                                                libraries={'internal_id': set(['123'])}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S1'])}, libraries={'internal_id': set(['444'])}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{'samples': {'name': set(['S9'])}, 'libraries': {'internal_id': set(['123'])}})

    def test_find_differences_when_no_diffs_h_vs_ss(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S9']), 'accession_number': set(['Acc9'])},
                                                libraries={'internal_id': set(['123'])}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S9']), 'accession_number': set(['Acc9'])},
                                             libraries={'internal_id': set(['123'])}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{})

    def test_find_differences_when_no_diffs_i_vs_h(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(), 'internal_id': set()},
                                              libraries={}, studies={})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S1']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={}, studies={})
        result = irods_metadata.difference(header_metadata)
        self.assertDictEqual(result, {})

    def test_find_differences_when_diff_id_types_i_vs_h(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(['EGA1']), 'internal_id': set(['1'])},
                                              libraries={'name': set(['123']), 'internal_id': set(['123'])},
                                              studies={'name': set(["Crohns disease"]), 'accession_number': set(['EGAS4']), 'internal_id': set(['4'])})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S1']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={'internal_id': set(['123'])}, studies={})
        result = irods_metadata.difference(header_metadata)
        self.assertDictEqual(result, {})

    def test_find_differences_when_diffs_i_vs_h(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(['EGA1']), 'internal_id': set(['1'])},
                                              libraries={'name': set(['123']), 'internal_id': set(['123'])},
                                              studies={'name': set(["Crohns disease"]), 'accession_number': set(['EGAS4']), 'internal_id': set(['4'])})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S100']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={'internal_id': set(['123'])}, studies={})
        result = irods_metadata.difference(header_metadata)
        self.assertDictEqual(result, {'samples': {'name': set(['S1'])}})

    def test_find_differences_when_diffs_h_vs_i(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(['EGA1']), 'internal_id': set(['1'])},
                                              libraries={'name': set(['123']), 'internal_id': set(['123'])},
                                              studies={'name': set(["Crohns disease"]), 'accession_number': set(['EGAS4']), 'internal_id': set(['4'])})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S100']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={'internal_id': set(['123'])}, studies={})
        result = header_metadata.difference(irods_metadata)
        self.assertDictEqual(result, {'samples': {'name': set(['S100'])}})

    def test_find_differences_when_not_the_right_type(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(), 'internal_id': set()},
                                              libraries={}, studies={})
        self.assertRaises(TypeError, irods_metadata.difference, [1,2,3])

<commit_msg>Added tests for ComparableMetadata<commit_after>"""
Copyright (C) 2016  Genome Research Ltd.

Author: Irina Colgiu <ic4@sanger.ac.uk>

This program is part of meta-check

meta-check is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

This file has been created on Jun 14, 2016.
"""

import unittest

from mcheck.metadata.file_header_metadata.header_metadata import SAMFileHeaderMetadata
from mcheck.metadata.irods_metadata.file_metadata import IrodsSeqFileMetadata
from mcheck.metadata.seqscape_metadata.seqscape_metadata import SeqscapeMetadata
from mcheck.metadata.common.comparable_metadata import ComparableMetadata

class ComparableMetadataTests(unittest.TestCase):

    def test_entities_have_values_when_it_has(self):
        result = ComparableMetadata._entities_have_values({'name': set(['a']), 'accession_number': set(), 'internal_id': set()})
        self.assertTrue(result)

    def test_entities_have_values_when_it_hasnt(self):
        result = ComparableMetadata._entities_have_values({'name': set(), 'accession_number': set(), 'internal_id': set()})
        self.assertFalse(result)

    def test_entities_have_values_when_missing_empty_sets(self):
        result = ComparableMetadata._entities_have_values({'name': set(['a'])})
        self.assertTrue(result)

    def test_entities_have_values_when_empty(self):
        result = ComparableMetadata._entities_have_values({})
        self.assertFalse(result)

    def test_has_metadata_when_not(self):
        metadata = ComparableMetadata(samples={'name': set()}, libraries={}, studies={})
        result = metadata.has_metadata()
        self.assertFalse(result)

    def test_has_metadata_when_has(self):
        metadata = ComparableMetadata(samples={'name': set(['a'])}, libraries={}, studies={})
        result = metadata.has_metadata()
        self.assertTrue(result)

    def test_difference_when_there_are_diffs_h_vs_ss1(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam', samples={'name': set(['S1'])}, libraries={}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S9'])}, libraries={}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{'samples': {'name': set(['S1'])}})

    def test_difference_when_there_are_diffs_h_vs_ss2(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam', samples={'name': set(['S9'])}, libraries={}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S1'])}, libraries={}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{'samples': {'name': set(['S9'])}})

    def test_difference_when_there_are_diff_type_ids_h_vs_ss(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam', samples={'name': set(['S1']), 'accession_number': set(['Acc1'])}, libraries={}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S1'])}, libraries={}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result, {})

    def test_difference_when_many_diffs_h_vs_ss(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S9']), 'accession_number': set(['Acc9'])},
                                                libraries={'internal_id': set(['123'])}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S1'])}, libraries={'internal_id': set(['444'])}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{'samples': {'name': set(['S9'])}, 'libraries': {'internal_id': set(['123'])}})

    def test_difference_when_no_diffs_h_vs_ss(self):
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S9']), 'accession_number': set(['Acc9'])},
                                                libraries={'internal_id': set(['123'])}, studies={})
        seqscape_metadata = SeqscapeMetadata(samples={'name': set(['S9']), 'accession_number': set(['Acc9'])},
                                             libraries={'internal_id': set(['123'])}, studies={})
        result = header_metadata.difference(seqscape_metadata)
        self.assertEqual(result,{})

    def test_difference_when_no_diffs_i_vs_h(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(), 'internal_id': set()},
                                              libraries={}, studies={})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S1']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={}, studies={})
        result = irods_metadata.difference(header_metadata)
        self.assertDictEqual(result, {})

    def test_difference_when_diff_id_types_i_vs_h(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(['EGA1']), 'internal_id': set(['1'])},
                                              libraries={'name': set(['123']), 'internal_id': set(['123'])},
                                              studies={'name': set(["Crohns disease"]), 'accession_number': set(['EGAS4']), 'internal_id': set(['4'])})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S1']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={'internal_id': set(['123'])}, studies={})
        result = irods_metadata.difference(header_metadata)
        self.assertDictEqual(result, {})

    def test_difference_when_diffs_i_vs_h(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(['EGA1']), 'internal_id': set(['1'])},
                                              libraries={'name': set(['123']), 'internal_id': set(['123'])},
                                              studies={'name': set(["Crohns disease"]), 'accession_number': set(['EGAS4']), 'internal_id': set(['4'])})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S100']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={'internal_id': set(['123'])}, studies={})
        result = irods_metadata.difference(header_metadata)
        self.assertDictEqual(result, {'samples': {'name': set(['S1'])}})

    def test_difference_when_diffs_h_vs_i(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(['EGA1']), 'internal_id': set(['1'])},
                                              libraries={'name': set(['123']), 'internal_id': set(['123'])},
                                              studies={'name': set(["Crohns disease"]), 'accession_number': set(['EGAS4']), 'internal_id': set(['4'])})
        header_metadata = SAMFileHeaderMetadata('/seq/123.bam', '123/bam',
                                                samples={'name': set(['S100']), 'accession_number' : set(), 'internal_id': set()},
                                                libraries={'internal_id': set(['123'])}, studies={})
        result = header_metadata.difference(irods_metadata)
        self.assertDictEqual(result, {'samples': {'name': set(['S100'])}})

    def test_difference_when_not_the_right_type(self):
        irods_metadata = IrodsSeqFileMetadata('/seq/123.bam',
                                              samples={'name': set(['S1']), 'accession_number': set(), 'internal_id': set()},
                                              libraries={}, studies={})
        self.assertRaises(TypeError, irods_metadata.difference, [1,2,3])

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
#

"""
Frogpad is used to compile .dart files to javascript.

This is accomplished by first creating an html file (usually called
<something>.frogpad.html) that can be used to execute the frog compiler in
a web browser (or DumpRenderTree).

The generated frogpad.html contains:

  1. all the dart files that compose a dart program
  2. all the dart files of dart:core and other standard dart libraries
  3. frogpad.dart (compiled to javascript)

The contents of each dart file is placed in a separate

Frogpad

Input:

  <textarea id="input"></textarea>

Compiler Messages:

Timing:

Output:

"""

# This finds everything after the word "Output:" in the html page.
# (Note, because the javascript we're fishing out spans multiple lines
# we need to use the DOTALL switch here.)
OUTPUT_JAVASCRIPT_REGEX = re.compile(".*\nOutput:(.*)#EOF", re.DOTALL)

# If the frogpad.dart encounters a compilation error, the generated
# javascript will start with the word 'throw'.
COMPILATION_ERROR_REGEX = re.compile(".*frogpad compilation error.*", re.DOTALL)

# We use "application/inert" here to make the browser ignore the
# these script tags.  (frogpad.dart will fish out the contents as needed.)
#
SCRIPT_TAG = """
"""

# Regex that finds #import, #source and #native directives in .dart files.
#   match.group(1) = "import", "source" or "native"
#   match.group(2) = url of file being imported
DIRECTIVE_RE = re.compile(r"^#(import|source|native)\([\"']([^\"']*)[\"']")

# id of script tag that holds name of the top dart file to be compiled,
# (This file name passed will be passed to the frog compiler by frogpad.dart.)
MAIN_ID = "main_id"

DART_LIBRARIES = {
    "core": "lib/corelib.dart",
    "coreimpl": "lib/corelib_impl.dart",
    "dom": "../client/dom/frog/dom_frog.dart",
    "html": "../client/html/frog/html_frog.dart",
    "isolate": "../lib/isolate/isolate_frog.dart",
    "json": "../lib/json/json_frog.dart"
}

class Pad(object):
  """
  Accumulates all source files that are needed to compile a dart program,
  and places them in

Frogpad

Input:

  <textarea id="input"></textarea>

Compiler Messages:

Timing:

Output:

"""

# This finds everything after the word "Output:" in the html page.
# (Note, because the javascript we're fishing out spans multiple lines
# we need to use the DOTALL switch here.)
OUTPUT_JAVASCRIPT_REGEX = re.compile(".*\nOutput:(.*)#EOF", re.DOTALL)

# If the frogpad.dart encounters a compilation error, the generated
# javascript will start with the word 'throw'.
COMPILATION_ERROR_REGEX = re.compile(".*frogpad compilation error.*", re.DOTALL)

# We use "application/inert" here to make the browser ignore the
# these script tags.  (frogpad.dart will fish out the contents as needed.)
#
SCRIPT_TAG = """
"""

# Regex that finds #import, #source and #native directives in .dart files.
#   match.group(1) = "import", "source" or "native"
#   match.group(2) = url of file being imported
DIRECTIVE_RE = re.compile(r"^#(import|source|native)\([\"']([^\"']*)[\"']")

# id of script tag that holds name of the top dart file to be compiled,
# (This file name passed will be passed to the frog compiler by frogpad.dart.)
MAIN_ID = "main_id"

DART_LIBRARIES = {
    "core": "lib/corelib.dart",
    "coreimpl": "lib/corelib_impl.dart",
    "dom": "../client/dom/frog/dom_frog.dart",
    "html": "../client/html/frog/html_frog.dart",
    "isolate": "../lib/isolate/isolate_frog.dart",
    "json": "../lib/json/json_frog.dart"
}

class Pad(object):
  """
  Accumulates all source files that are needed to compile a dart program,
  and places them in


<commit_before>#!/usr/bin/env python

from os.path import join

def configuration(parent_package = '', top_path = None):
    from numpy.distutils.misc_util import Configuration, get_numpy_include_dirs
    config = Configuration('cluster', parent_package, top_path)

    config.add_data_dir('tests')

    #config.add_extension('_vq',
    #    sources=[join('src', 'vq_module.c'), join('src', 'vq.c')],
    #    include_dirs = [get_numpy_include_dirs()])
    config.add_sconscript('SConstruct')

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(maintainer = "SciPy Developers",
          author = "Eric Jones",
          maintainer_email = "scipy-dev@scipy.org",
          description = "Clustering Algorithms (Information Theory)",
          url = "http://www.scipy.org",
          license = "SciPy License (BSD Style)",
          **configuration(top_path='').todict()
          )
<commit_msg>Update setup.py file for numscons build.<commit_after>#!/usr/bin/env python

from os.path import join

def configuration(parent_package = '', top_path = None):
    from numpy.distutils.misc_util import Configuration, get_numpy_include_dirs
    config = Configuration('spatial', parent_package, top_path)

    config.add_data_dir('tests')

    #config.add_extension('_vq',
    #    sources=[join('src', 'vq_module.c'), join('src', 'vq.c')],
    #    include_dirs = [get_numpy_include_dirs()])
    config.add_sconscript('SConstruct')

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(maintainer = "SciPy Developers",
          author = "Eric Jones",
          maintainer_email = "scipy-dev@scipy.org",
          description = "Clustering Algorithms (Information Theory)",
          url = "http://www.scipy.org",
          license = "SciPy License (BSD Style)",
          **configuration(top_path='').todict()
          )
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

"""Implements blaze._elwise_eval function.

This function is meant to do OOC operations following a different
strategy than the canonical Blaze approach, and should be phased out as
soon as the canonical approach can do these sort of things efficiently.
"""

import sys, math
import numpy as np  # only necessary because of bug #183
from dynd import nd, ndt
from .. import array, empty
from .eval import eval as blaze_eval, append
import datashape

if sys.version_info >= (3, 0):
    xrange = range
    def dict_viewkeys(d):
        return d.keys()
else:
    def dict_viewkeys(d):
        return d.iterkeys()

min_numexpr_version = '2.2'  # the minimum version of Numexpr needed
numexpr_here = False
try:
    import numexpr
except ImportError:
    pass
else:
    if numexpr.__version__ >= min_numexpr_version:
        numexpr_here = True

if numexpr_here:
    import numexpr
    from numexpr.expressions import functions as numexpr_functions

class Defaults(object):
    """Class to taylor the setters and getters of default values."""

    def __init__(self):
        self.choices = {}

        # Choices setup
        self.choices['vm'] = ("numexpr", "python")

    def check_choices(self, name, value):
        if value not in self.choices[name]:
            raise ValueError(
                "value must be in: %s" % (self.choices[name],))

    #
    # Properties start here...
    #

    @property
    def vm(self):
        return self.__vm

    @vm.setter
    def vm(self, value):
        self.check_choices('vm', value)
        if value == "numexpr" and not numexpr_here:
            raise ValueError(
                   "cannot use `numexpr` virtual machine "
                   "(minimum required version is probably not installed)")
        self.__vm = value

defaults = Defaults()

# Default values start here...

if numexpr_here:
    defaults.vm = "numexpr"
else:
    defaults.vm = "python"
"""
The virtual machine to be used in computations (via `eval`).  It can
be 'numexpr' or 'python'.  Default is 'numexpr', if installed.  If
not, then the default is 'python'.

"""

def _elwise_eval(expression, vm=None, user_dict={}, **kwargs):
    """
    eval(expression, vm=None, user_dict=None, **kwargs)

    Evaluate an `expression` and return the result.

    Parameters
    ----------
    expression : string
        A string forming an expression, like '2*a+3*b'. The values for 'a' and
        'b' are variable names to be taken from the calling function's frame.
        These variables may be scalars or Blaze arrays.
    vm : string
        The virtual machine to be used in computations.  It can be 'numexpr'
        or 'python'.  The default is to use 'numexpr' if it is installed.
    user_dict : dict
        An user-provided dictionary where the variables in expression
        can be found by name.
    kwargs : list of parameters or dictionary
        Any parameter supported by the blaze.array constructor.
        Useful for setting properties of the returned array object.

    Returns
    -------
    out : array object
        The outcome of the expression.  You can tailor the
        properties of this array by passing additional arguments
        supported by blaze.array constructor in `kwargs`.

    """

    if vm is None:
        vm = defaults.vm
    else:
        defaults.vm = vm

    # Get variables and column names participating in expression
    depth = kwargs.pop('depth', 2)
    vars = _getvars(expression, user_dict, depth, vm=vm)

    # Gather info about sizes and lengths
    typesize, vlen = 0, 1
    for name in dict_viewkeys(vars):
        var = vars[name]
        # Scalars
        if not hasattr(var, "__len__"):
            continue
        if not hasattr(var, "dshape"):
            try:
                var = array(var)
            except:
                raise ValueError(
                    "sequence cannot be converted into a blaze array")
        # From now on, we only have Blaze arrays
        typesize += var.dshape.measure.itemsize
        # Check for length
        if vlen > 1 and vlen != len(var):
            raise ValueError("arrays must have the same length")
        vlen = len(var)

    if typesize == 0 or vlen == 0:
        # All scalars or zero-length objects
        if vm == "python":
            return eval(expression, vars)
        else:
            return numexpr.evaluate(expression, local_dict=vars)

    return _eval_blocks(expression, vars, vlen, typesize, vm, **kwargs)

def _getvars(expression, user_dict, depth, vm):
    """Get the variables in `expression`.

    `depth` specifies the depth of the frame in order to reach local
    or global variables.
    """

    cexpr = compile(expression, '<string>', 'eval')
    if vm == "python":
        exprvars = [ var for var in cexpr.co_names
                     if var not in ['None', 'False', 'True'] ]
    else:
        # Check that var is not a numexpr function here.  This is useful for
        # detecting unbound variables in expressions.  This is not necessary
        # for the 'python' engine.
        exprvars = [ var for var in cexpr.co_names
                     if var not in ['None', 'False', 'True']
                     and var not in numexpr_functions ]

    # Get the local and global variable mappings of the user frame
    user_locals, user_globals = {}, {}
    user_frame = sys._getframe(depth)
    user_locals = user_frame.f_locals
    user_globals = user_frame.f_globals

    # Look for the required variables
    reqvars = {}
    for var in exprvars:
        # Get the value
        if var in user_dict:
            val = user_dict[var]
        elif var in user_locals:
            val = user_locals[var]
        elif var in user_globals:
            val = user_globals[var]
        else:
            if vm == "numexpr":
                raise NameError("variable name ``%s`` not found" % var)
            val = None
        # Check the value
        if (vm == "numexpr" and
            hasattr(val, 'dshape') and
            val.dshape.measure.name == 'uint64'):
            raise NotImplementedError(
                "variable ``%s`` refers to "
                "a 64-bit unsigned integer object, that is "
                "not yet supported in numexpr expressions; "
                "rather, use the 'python' vm." % var )
        if val is not None:
            reqvars[var] = val
    return reqvars

def _eval_blocks(expression, vars, vlen, typesize, vm, **kwargs):
    """Perform the evaluation in blocks."""

    # Compute the optimal block size (in elements)
    # The next is based on experiments, but YMMV
    if vm == "numexpr":
        # If numexpr, make sure that operands fit in L3 chache
        bsize = 2**20  # 1 MB is common for L3
    else:
        # If python, make sure that operands fit in L2 chache
        bsize = 2**17  # 256 KB is common for L2
    bsize //= typesize
    # Evaluation seems more efficient if block size is a power of 2
    bsize = 2 ** (int(math.log(bsize, 2)))
    if vlen < 100*1000:
        bsize //= 8
    elif vlen < 1000*1000:
        bsize //= 4
    elif vlen < 10*1000*1000:
        bsize //= 2
    # Protection against too large atomsizes
    if bsize == 0:
        bsize = 1

    vars_ = {}
    # Convert operands into Blaze arrays and get temporaries for vars
    maxndims = 0
    for name in dict_viewkeys(vars):
        var = vars[name]
        if not hasattr(var, "dshape"):
            # Convert sequences into regular Blaze arrays
            vars[name] = var = array(var)
        if hasattr(var, "__len__"):
            ndims = len(var.dshape.shape)
            if ndims > maxndims:
                maxndims = ndims
            if len(var) > bsize:
                # Variable is too large; get a container for a chunk
                res_shape, res_dtype = datashape.to_numpy(var.dshape)
                res_shape = list(res_shape)
                res_shape[0] = bsize
                dshape = datashape.from_numpy(res_shape, res_dtype)
                vars_[name] = empty(dshape)

    for i in xrange(0, vlen, bsize):
        # Correction for the block size
        if i+bsize > vlen:
            bsize = vlen - i
        # Get buffers for vars
        for name in dict_viewkeys(vars):
            var = vars[name]
            if hasattr(var, "__len__") and len(var) > bsize:
                vars_[name] = var[i:i+bsize]
            else:
                if hasattr(var, "__getitem__"):
                    vars_[name] = var[:]
                else:
                    vars_[name] = var

        # Perform the evaluation for this block
        if vm == "python":
            res_block = eval(expression, vars_)
        else:
            res_block = numexpr.evaluate(expression, local_dict=vars_)
            # numexpr returns a numpy array, and we need a blaze one
            res_block = array(res_block)

        if 'storage' in kwargs and kwargs['storage'] is not None:
            res_disk = True
        else:
            res_disk = False

        if i == 0:
            scalar = False
            dim_reduction = False
            # Detection of reduction operations
            if (not hasattr(res_block, "__len__") or
                len(res_block) == 0):
                scalar = True
                result = res_block
                continue
            elif len(res_block.dshape.shape) < maxndims:
                dim_reduction = True
                result = res_block
                continue
            block_shape, block_dtype = datashape.to_numpy(res_block.dshape)
            out_shape = list(block_shape)
            if res_disk:
                out_shape[0] = 0
                dshape = datashape.from_numpy(out_shape, block_dtype)
                result = empty(dshape, **kwargs)
                append(result, blaze_eval(res_block)._data.dynd_arr())
            else:
                out_shape[0] = vlen
                dshape = datashape.from_numpy(out_shape, block_dtype)
                result = empty(dshape, **kwargs)
                # The next is a workaround for bug #183
                #result[:bsize] = res_block
                result[:bsize] = np.array(res_block)
        else:
            if scalar or dim_reduction:
                result += res_block
            if res_disk:
                append(result, blaze_eval(res_block)._data.dynd_arr())
            else:
                # The next is a workaround for bug #183
                #result[i:i+bsize] = res_block
                result[i:i+bsize] = np.array(res_block)

    if scalar:
        return result[()]
    return result
<commit_msg>Removed the dependency from numpy by using dynd buffers<commit_after>from __future__ import absolute_import, division, print_function

"""Implements blaze._elwise_eval function.

This function is meant to do OOC operations following a different
strategy than the canonical Blaze approach, and should be phased out as
soon as the canonical approach can do these sort of things efficiently.
"""

import sys, math
from dynd import nd, ndt
from .. import array, empty
from .eval import eval as blaze_eval, append
import datashape

if sys.version_info >= (3, 0):
    xrange = range
    def dict_viewkeys(d):
        return d.keys()
else:
    def dict_viewkeys(d):
        return d.iterkeys()

min_numexpr_version = '2.2'  # the minimum version of Numexpr needed
numexpr_here = False
try:
    import numexpr
except ImportError:
    pass
else:
    if numexpr.__version__ >= min_numexpr_version:
        numexpr_here = True

if numexpr_here:
    import numexpr
    from numexpr.expressions import functions as numexpr_functions

class Defaults(object):
    """Class to taylor the setters and getters of default values."""

    def __init__(self):
        self.choices = {}

        # Choices setup
        self.choices['vm'] = ("numexpr", "python")

    def check_choices(self, name, value):
        if value not in self.choices[name]:
            raise ValueError(
                "value must be in: %s" % (self.choices[name],))

    #
    # Properties start here...
    #

    @property
    def vm(self):
        return self.__vm

    @vm.setter
    def vm(self, value):
        self.check_choices('vm', value)
        if value == "numexpr" and not numexpr_here:
            raise ValueError(
                   "cannot use `numexpr` virtual machine "
                   "(minimum required version is probably not installed)")
        self.__vm = value

defaults = Defaults()

# Default values start here...

if numexpr_here:
    defaults.vm = "numexpr"
else:
    defaults.vm = "python"
"""
The virtual machine to be used in computations (via `eval`).  It can
be 'numexpr' or 'python'.  Default is 'numexpr', if installed.  If
not, then the default is 'python'.

"""

def _elwise_eval(expression, vm=None, user_dict={}, **kwargs):
    """
    eval(expression, vm=None, user_dict=None, **kwargs)

    Evaluate an `expression` and return the result.

    Parameters
    ----------
    expression : string
        A string forming an expression, like '2*a+3*b'. The values for 'a' and
        'b' are variable names to be taken from the calling function's frame.
        These variables may be scalars or Blaze arrays.
    vm : string
        The virtual machine to be used in computations.  It can be 'numexpr'
        or 'python'.  The default is to use 'numexpr' if it is installed.
    user_dict : dict
        An user-provided dictionary where the variables in expression
        can be found by name.
    kwargs : list of parameters or dictionary
        Any parameter supported by the blaze.array constructor.
        Useful for setting properties of the returned array object.

    Returns
    -------
    out : array object
        The outcome of the expression.  You can tailor the
        properties of this array by passing additional arguments
        supported by blaze.array constructor in `kwargs`.

    """

    if vm is None:
        vm = defaults.vm
    else:
        defaults.vm = vm

    # Get variables and column names participating in expression
    depth = kwargs.pop('depth', 2)
    vars = _getvars(expression, user_dict, depth, vm=vm)

    # Gather info about sizes and lengths
    typesize, vlen = 0, 1
    for name in dict_viewkeys(vars):
        var = vars[name]
        # Scalars
        if not hasattr(var, "__len__"):
            continue
        if not hasattr(var, "dshape"):
            try:
                var = array(var)
            except:
                raise ValueError(
                    "sequence cannot be converted into a blaze array")
        # From now on, we only have Blaze arrays
        typesize += var.dshape.measure.itemsize
        # Check for length
        if vlen > 1 and vlen != len(var):
            raise ValueError("arrays must have the same length")
        vlen = len(var)

    if typesize == 0 or vlen == 0:
        # All scalars or zero-length objects
        if vm == "python":
            return eval(expression, vars)
        else:
            return numexpr.evaluate(expression, local_dict=vars)

    return _eval_blocks(expression, vars, vlen, typesize, vm, **kwargs)

def _getvars(expression, user_dict, depth, vm):
    """Get the variables in `expression`.

    `depth` specifies the depth of the frame in order to reach local
    or global variables.
    """

    cexpr = compile(expression, '<string>', 'eval')
    if vm == "python":
        exprvars = [ var for var in cexpr.co_names
                     if var not in ['None', 'False', 'True'] ]
    else:
        # Check that var is not a numexpr function here.  This is useful for
        # detecting unbound variables in expressions.  This is not necessary
        # for the 'python' engine.
        exprvars = [ var for var in cexpr.co_names
                     if var not in ['None', 'False', 'True']
                     and var not in numexpr_functions ]

    # Get the local and global variable mappings of the user frame
    user_locals, user_globals = {}, {}
    user_frame = sys._getframe(depth)
    user_locals = user_frame.f_locals
    user_globals = user_frame.f_globals

    # Look for the required variables
    reqvars = {}
    for var in exprvars:
        # Get the value
        if var in user_dict:
            val = user_dict[var]
        elif var in user_locals:
            val = user_locals[var]
        elif var in user_globals:
            val = user_globals[var]
        else:
            if vm == "numexpr":
                raise NameError("variable name ``%s`` not found" % var)
            val = None
        # Check the value
        if (vm == "numexpr" and
            hasattr(val, 'dshape') and
            val.dshape.measure.name == 'uint64'):
            raise NotImplementedError(
                "variable ``%s`` refers to "
                "a 64-bit unsigned integer object, that is "
                "not yet supported in numexpr expressions; "
                "rather, use the 'python' vm." % var )
        if val is not None:
            reqvars[var] = val
    return reqvars

def _eval_blocks(expression, vars, vlen, typesize, vm, **kwargs):
    """Perform the evaluation in blocks."""

    # Compute the optimal block size (in elements)
    # The next is based on experiments, but YMMV
    if vm == "numexpr":
        # If numexpr, make sure that operands fit in L3 chache
        bsize = 2**20  # 1 MB is common for L3
    else:
        # If python, make sure that operands fit in L2 chache
        bsize = 2**17  # 256 KB is common for L2
    bsize //= typesize
    # Evaluation seems more efficient if block size is a power of 2
    bsize = 2 ** (int(math.log(bsize, 2)))
    if vlen < 100*1000:
        bsize //= 8
    elif vlen < 1000*1000:
        bsize //= 4
    elif vlen < 10*1000*1000:
        bsize //= 2
    # Protection against too large atomsizes
    if bsize == 0:
        bsize = 1

    vars_ = {}
    # Convert operands into Blaze arrays and get temporaries for vars
    maxndims = 0
    for name in dict_viewkeys(vars):
        var = vars[name]
        if not hasattr(var, "dshape"):
            # Convert sequences into regular Blaze arrays
            vars[name] = var = array(var)
        if hasattr(var, "__len__"):
            ndims = len(var.dshape.shape)
            if ndims > maxndims:
                maxndims = ndims
            if len(var) > bsize:
                # Variable is too large; get a container for a chunk
                res_shape, res_dtype = datashape.to_numpy(var.dshape)
                res_shape = list(res_shape)
                res_shape[0] = bsize
                dshape = datashape.from_numpy(res_shape, res_dtype)
                vars_[name] = empty(dshape)

    for i in xrange(0, vlen, bsize):
        # Correction for the block size
        if i+bsize > vlen:
            bsize = vlen - i
        # Get buffers for vars
        for name in dict_viewkeys(vars):
            var = vars[name]
            if hasattr(var, "__len__") and len(var) > bsize:
                vars_[name] = var[i:i+bsize]
            else:
                if hasattr(var, "__getitem__"):
                    vars_[name] = var[:]
                else:
                    vars_[name] = var

        # Perform the evaluation for this block
        if vm == "python":
            res_block = eval(expression, vars_)
            dynd_block = blaze_eval(res_block)._data.dynd_arr()
        else:
            res_block = numexpr.evaluate(expression, local_dict=vars_)
            # numexpr returns a numpy array, and we need dynd/blaze ones
            dynd_block = nd.array(res_block)
            res_block = array(res_block)

        if 'storage' in kwargs and kwargs['storage'] is not None:
            res_disk = True
        else:
            res_disk = False

        if i == 0:
            scalar = False
            dim_reduction = False
            # Detection of reduction operations
            if (not hasattr(res_block, "__len__") or
                len(res_block) == 0):
                scalar = True
                result = res_block
                continue
            elif len(res_block.dshape.shape) < maxndims:
                dim_reduction = True
                result = res_block
                continue
            block_shape, block_dtype = datashape.to_numpy(res_block.dshape)
            out_shape = list(block_shape)
            if res_disk:
                out_shape[0] = 0
                dshape = datashape.from_numpy(out_shape, block_dtype)
                result = empty(dshape, **kwargs)
                append(result, dynd_block)
            else:
                out_shape[0] = vlen
                dshape = datashape.from_numpy(out_shape, block_dtype)
                result = empty(dshape, **kwargs)
                # The next is a workaround for bug #183
                #result[:bsize] = res_block
                result[:bsize] = dynd_block
        else:
            if scalar or dim_reduction:
                result += res_block
            if res_disk:
                append(result, dynd_block)
            else:
                # The next is a workaround for bug #183
                #result[i:i+bsize] = res_block
                result[i:i+bsize] = dynd_block

    if scalar:
        return result[()]
    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#  -*- coding: UTF-8 -*-
"""
Poem classifier
"""
import argparse
import glob
import pprint
import re
import csv

import pandas
from lxml import etree

import numpy as np
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline

from poem_reader import read_xml_directory, parse_text_lines, block_xpath

argparser = argparse.ArgumentParser(description="Textblock classifier to poems and other text")
argparser.add_argument("job", help="Job to do", choices=['train', 'predict'])
argparser.add_argument("--dir", help="Directory to classify")
argparser.add_argument("--newfile", help="Create new CSV file", dest='newfile', action='store_true')
args = argparser.parse_args()

def read_training_data(path='./'):
    poem_files = glob.glob(path + "poemblocks/*.txt")
    nonpoem_files = glob.glob(path + "nonpoemblocks/*.txt")

    poems = []
    for poem_file in poem_files:
        with open(poem_file, 'r') as f:
            poems.append(f.read())

    nonpoems = []
    for nonpoem_file in nonpoem_files:
        with open(nonpoem_file, 'r') as f:
            nonpoems.append(f.read())

    return poems, nonpoems

def train():
    poems, nonpoems = read_training_data()

    nonpoems = nonpoems[::10]  # TODO: Use skipped as test data

    print(len(poems))
    print(len(nonpoems))

    # count_vect = CountVectorizer()
    # X_train_counts = count_vect.fit_transform(poems + nonpoems)
    #
    # #print(X_train_counts)
    #
    # #cc = count_vect.vocabulary_.get('on')
    # #print(count_vect.get_feature_names()[cc])
    # #print(np.sum(X_train_counts, axis=0).tolist()[0][cc])
    #
    # tfidf_transformer = TfidfTransformer()
    # X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
    # print(X_train_tfidf.shape)

    all_train_data = poems + nonpoems
    all_train_target = [1] * len(poems) + [0] * len(nonpoems)

    all_train_data = [d.replace('\n', ' ') for d in all_train_data]

    test_data = all_train_data[::2]
    test_target = all_train_target[::2]

    train_data = all_train_data[1::2]
    train_target = all_train_target[1::2]

    text_clf = Pipeline([('vect', CountVectorizer()),
                         ('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, n_iter=5, random_state=42)),
                         ])

    text_clf.fit(train_data, train_target)
    predicted = text_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print('Cross-validation accuracy %s' % acc)

    parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3)],
                  'tfidf__use_idf': (True, False),
                  'clf__alpha': (1e-2, 1e-3, 1e-4),
                  'clf__penalty': ('l1', 'l2', 'elasticnet'),
                  'clf__loss': ('hinge', 'log'),
                  'clf__n_iter': (4,)}

    gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)

    gs_clf = gs_clf.fit(train_data, train_target)

    # _ = text_clf.fit(train_data, train_target)
    predicted = gs_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print(predicted)
    print(gs_clf.best_params_)

    print('Parameter-tuned cross-validation accuracy %s' % acc)

    # text_clf.fit(all_train_data, all_train_target)
    gs_clf.fit(all_train_data, all_train_target)

    return gs_clf

def parse_metadata_from_path(path):
    '/home/razzo/dhh17data/newspapers/newspapers/fin/1820/1457-4888/1457-4888_1820-01-08_1/alto/1457-4888_1820-01-08_1_003.xml'

    re_name_split = r'.*/newspapers/newspapers/fin/(.{4})/(.{7,9})/.{7,9}\_.{4}\-(..)\-(..)'

    split = re.search(re_name_split, path)

    year, issn, month, day = split.groups()
    return year, month, day, issn

if args.job == 'train':
    joblib.dump(train(), 'svm.pkl')

elif args.job == 'predict':
    clf = joblib.load('svm.pkl')

    if args.dir[-1] != '/':
        args.dir = args.dir + '/'

    files = glob.glob(args.dir + "**/*.xml", recursive=True)

    if not files:
        print('No files found for %s' % args.dir)
        quit()

    xmls = []
    for xmlfile in files:
        if 'alto' not in xmlfile:
            continue

        with open(xmlfile, 'r') as f:
            try:
                parsed = etree.parse(f)
                xmls.append((parsed, xmlfile))
                print('Read file %s' % xmlfile)
            except etree.XMLSyntaxError:
                print('Error in XML: %s' % xmlfile)

    data = []
    metadata = []
    for xml, filename in xmls:
        text_blocks = block_xpath(xml)

        for block in text_blocks:
            data.append(parse_text_lines(list(block)))
            metadata.append(parse_metadata_from_path(filename) + (block.get('ID'),))

    data_orig = data
    data = [d.replace('\n', ' ') for d in data]

    # print(len(data))

    predicted = clf.predict(data)
    #print(predicted)

    with open('foundpoems/found_poems.csv'.format(year=metadata[0][0]), 'w' if args.newfile else 'a', newline='') as fp:
        writer = csv.writer(fp, delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        if args.newfile:
            writer.writerow(('Poem', 'Year', 'Month', 'Day', 'Newspaper name', 'ISSN'))
        poemtext = ''
        prev_vector = None
        for i, d in enumerate(data_orig):
            if not predicted[i]:
                continue
            # print(metadata[i])

            year, month, day, issn, blockid = metadata[i]

            if prev_vector == (year, month, day, issn):
                poemtext += d
            else:
                poemtext = d

            prev_vector = (year, month, day, issn)

            issues = pandas.read_csv('data/issue_numbers.csv', sep=',')
            paper = issues.loc[issues['issn'] == issn]['paper'].iloc[0]

            writer.writerow([poemtext.replace('\n', ' '), year, month, day, paper, issn])

            with open('foundpoems/{paper}_{year}_{month}_{day}.txt'.format(year=year, month=month, day=day, paper=paper), 'w', newline='') as textp:
                textp.write(poemtext)

<commit_msg>Some fine tuning of file creation<commit_after>#!/usr/bin/env python3
#  -*- coding: UTF-8 -*-
"""
Poem classifier
"""
import argparse
import glob
import pprint
import re
import csv

import pandas
from lxml import etree

import numpy as np
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline

from poem_reader import read_xml_directory, parse_text_lines, block_xpath

argparser = argparse.ArgumentParser(description="Textblock classifier to poems and other text")
argparser.add_argument("job", help="Job to do", choices=['train', 'predict'])
argparser.add_argument("--dir", help="Directory to classify")
argparser.add_argument("--newfile", help="Create new CSV file", dest='newfile', action='store_true')
args = argparser.parse_args()

def read_training_data(path='./'):
    poem_files = glob.glob(path + "poemblocks/*.txt")
    nonpoem_files = glob.glob(path + "nonpoemblocks/*.txt")

    poems = []
    for poem_file in poem_files:
        with open(poem_file, 'r') as f:
            poems.append(f.read())

    nonpoems = []
    for nonpoem_file in nonpoem_files:
        with open(nonpoem_file, 'r') as f:
            nonpoems.append(f.read())

    return poems, nonpoems

def train():
    poems, nonpoems = read_training_data()

    nonpoems = nonpoems[::10]  # TODO: Use skipped as test data

    print(len(poems))
    print(len(nonpoems))

    # count_vect = CountVectorizer()
    # X_train_counts = count_vect.fit_transform(poems + nonpoems)
    #
    # #print(X_train_counts)
    #
    # #cc = count_vect.vocabulary_.get('on')
    # #print(count_vect.get_feature_names()[cc])
    # #print(np.sum(X_train_counts, axis=0).tolist()[0][cc])
    #
    # tfidf_transformer = TfidfTransformer()
    # X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
    # print(X_train_tfidf.shape)

    all_train_data = poems + nonpoems
    all_train_target = [1] * len(poems) + [0] * len(nonpoems)

    all_train_data = [d.replace('\n', ' ') for d in all_train_data]

    test_data = all_train_data[::2]
    test_target = all_train_target[::2]

    train_data = all_train_data[1::2]
    train_target = all_train_target[1::2]

    text_clf = Pipeline([('vect', CountVectorizer()),
                         ('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, n_iter=5, random_state=42)),
                         ])

    text_clf.fit(train_data, train_target)
    predicted = text_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print('Cross-validation accuracy %s' % acc)

    parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3)],
                  'tfidf__use_idf': (True, False),
                  'clf__alpha': (1e-2, 1e-3, 1e-4),
                  'clf__penalty': ('l1', 'l2', 'elasticnet'),
                  'clf__loss': ('hinge', 'log'),
                  'clf__n_iter': (4,)}

    gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)

    gs_clf = gs_clf.fit(train_data, train_target)

    # _ = text_clf.fit(train_data, train_target)
    predicted = gs_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print(predicted)
    print(gs_clf.best_params_)

    print('Parameter-tuned cross-validation accuracy %s' % acc)

    # text_clf.fit(all_train_data, all_train_target)
    gs_clf.fit(all_train_data, all_train_target)

    return gs_clf

def parse_metadata_from_path(path):
    '/home/razzo/dhh17data/newspapers/newspapers/fin/1820/1457-4888/1457-4888_1820-01-08_1/alto/1457-4888_1820-01-08_1_003.xml'

    re_name_split = r'.*/newspapers/newspapers/fin/(.{4})/(.{7,9})/.{7,9}\_.{4}\-(..)\-(..)'

    split = re.search(re_name_split, path)

    year, issn, month, day = split.groups()
    return year, month, day, issn

if args.job == 'train':
    joblib.dump(train(), 'svm.pkl')

elif args.job == 'predict':
    clf = joblib.load('svm.pkl')

    if args.dir[-1] != '/':
        args.dir = args.dir + '/'

    files = glob.glob(args.dir + "**/*.xml", recursive=True)

    if not files:
        print('No files found for %s' % args.dir)
        quit()

    xmls = []
    for xmlfile in files:
        if 'alto' not in xmlfile:
            continue

        with open(xmlfile, 'r') as f:
            try:
                parsed = etree.parse(f)
                xmls.append((parsed, xmlfile))
                print('Read file %s' % xmlfile)
            except etree.XMLSyntaxError:
                print('Error in XML: %s' % xmlfile)

    data = []
    metadata = []
    for xml, filename in xmls:
        text_blocks = block_xpath(xml)

        for block in text_blocks:
            data.append(parse_text_lines(list(block)))
            metadata.append(parse_metadata_from_path(filename) + (block.get('ID'),))

    data_orig = data
    data = [d.replace('\n', ' ') for d in data]

    # print(len(data))

    predicted = clf.predict(data)
    #print(predicted)

    issues = pandas.read_csv('data/issue_numbers.csv', sep=',')
    with open('foundpoems/found_poems.csv'.format(year=metadata[0][0]), 'w' if args.newfile else 'a', newline='') as fp:
        writer = csv.writer(fp, delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        if args.newfile:
            writer.writerow(('Poem', 'Year', 'Month', 'Day', 'Newspaper name', 'ISSN'))
        poemtext = ''
        prev_vector = None
        for i, d in enumerate(data_orig):
            if not predicted[i]:
                continue
            # print(metadata[i])

            year, month, day, issn, blockid = metadata[i]

            paper = issues.loc[issues['issn'] == issn]['paper'].iloc[0]

            if prev_vector == (year, month, day, issn):
                poemtext += d
            else:
                writer.writerow([poemtext.replace('\n', ' '), year, month, day, paper, issn])
                poemtext = d

            prev_vector = (year, month, day, issn)

            with open('foundpoems/{year}_{month}_{day}_{paper}.txt'.format(year=year, month=month, day=day, paper=paper), 'w', newline='') as textp:
                textp.write(poemtext)

        writer.writerow([poemtext.replace('\n', ' '), year, month, day, paper, issn])
<|endoftext|>


<commit_before>import numpy as np
from scipy import sparse as sp

from numpy.testing import assert_array_equal, \
     assert_array_almost_equal, assert_almost_equal

from scikits.learn.svm.bounds import l1_min_c
from scikits.learn.svm import LinearSVC
from scikits.learn.linear_model.logistic import LogisticRegression
from scikits.learn.svm.sparse import LinearSVC as SparseLinearSVC
from scikits.learn.linear_model.sparse.logistic import LogisticRegression as \
                                                       SparseLogisticRegression

dense_X = [[-1, 0], [0, 1], [1, 1], [1, 1]]
sparse_X = sp.csr_matrix(dense_X)

Y1 = [0, 1, 1, 1]
Y2 = [2, 1, 0, 0]

def test_l1_min_c():
    losses = ['l2', 'log']
    Xs = {'sparse': sparse_X, 'dense': dense_X}
    Ys = {'two-classes': Y1, 'multi-class': Y2}
    intercepts = {'no-intercept':  {'fit_intercept': False},
                  'fit-intercept': {'fit_intercept': True,
                                    'intercept_scaling': 10}}

    for loss in losses:
        for X_label, X in Xs.items():
            for Y_label, Y in Ys.items():
                for intercept_label, intercept_params in intercepts.items():
                    desc = 'l1_min_c loss=%r %s %s %s' % \
                                      (loss, X_label, Y_label, intercept_label)
                    check = lambda X, Y, loss, intercept_params: \
                                   check_l1_min_c(X, Y, loss, intercept_params)
                    check.description = desc
                    yield check, X, Y, loss, intercept_params

def check_l1_min_c(X, y, loss, intercept_params):
    min_c = l1_min_c(X, y, loss, **intercept_params)

    if loss == 'l2':
        if sp.issparse(X):
            algo = SparseLinearSVC
        else:
            algo = LinearSVC
        clf = algo(loss='l2', penalty='l1', dual=False, **intercept_params)
    else:
        if sp.issparse(X):
            algo = SparseLogisticRegression
        else:
            algo = LogisticRegression
        clf = algo(penalty='l1', **intercept_params)

    clf.C = min_c
    clf.fit(X, y)
    assert (np.asanyarray(clf.coef_) == 0).all()
    assert (np.asanyarray(clf.intercept_) == 0).all()

    clf.C = min_c * 1.01
    clf.fit(X, y)
    assert (np.asanyarray(clf.coef_) != 0).any() or \
           (np.asanyarray(clf.intercept_) != 0).any()
<commit_msg>cleanup tests<commit_after>import numpy as np
from scipy import sparse as sp

from scikits.learn.svm.bounds import l1_min_c
from scikits.learn.svm import LinearSVC
from scikits.learn.linear_model.logistic import LogisticRegression
from scikits.learn.svm.sparse import LinearSVC as SparseSVC
from scikits.learn.linear_model.sparse.logistic import LogisticRegression as \
                                                       SparseLogRegression

dense_X = [[-1, 0], [0, 1], [1, 1], [1, 1]]
sparse_X = sp.csr_matrix(dense_X)

Y1 = [0, 1, 1, 1]
Y2 = [2, 1, 0, 0]

def test_l1_min_c():
    losses = ['l2', 'log']
    Xs = {'sparse': sparse_X, 'dense': dense_X}
    Ys = {'two-classes': Y1, 'multi-class': Y2}
    intercepts = {'no-intercept':  {'fit_intercept': False},
                  'fit-intercept': {'fit_intercept': True,
                                    'intercept_scaling': 10}}

    for loss in losses:
        for X_label, X in Xs.items():
            for Y_label, Y in Ys.items():
                for intercept_label, intercept_params in intercepts.items():
                    desc = 'Test l1_min_c loss=%r %s %s %s' % \
                           (loss, X_label, Y_label, intercept_label)
                    check = lambda X, Y, loss, intercept_ps: \
                                   check_l1_min_c(X, Y, loss, **intercept_ps)
                    check.description = desc
                    yield check, X, Y, loss, intercept_params

def check_l1_min_c(X, y, loss, fit_intercept=True, intercept_scaling=None):
    min_c = l1_min_c(X, y, loss, fit_intercept, intercept_scaling)

    clf = {
        ('log', False): LogisticRegression(penalty='l1'),
        ('log', True):  SparseLogRegression(penalty='l1'),
        ('l2', False):  LinearSVC(loss='l2', penalty='l1', dual=False),
        ('l2', True):   SparseSVC(loss='l2', penalty='l1', dual=False),
    }[loss, sp.issparse(X)]

    clf.fit_intercept = fit_intercept
    if clf.intercept_scaling is not None:
        clf.intercept_scaling = intercept_scaling

    clf.C = min_c
    clf.fit(X, y)
    assert (np.asanyarray(clf.coef_) == 0).all()
    assert (np.asanyarray(clf.intercept_) == 0).all()

    clf.C = min_c * 1.01
    clf.fit(X, y)
    assert (np.asanyarray(clf.coef_) != 0).any() or \
           (np.asanyarray(clf.intercept_) != 0).any()
<|endoftext|>


<commit_before>from __future__ import absolute_import

import re

from .credentials import Credentials

_default_credentials = None

def set_default_credentials(
        first=None, second=None, credentials=None,
        username=None, base_url=None, api_key=None, session=None):
    """Set default credentials for all operations that require authentication
    against a CARTO account. CARTOframes methods
    :py:class:`cartoframes.viz.Layer` (and helper layers in
    :py:mod:`cartoframes.viz.helpers`),
    :py:class:`cartoframes.data.Dataset`,
    :py:class:`cartoframes.data.clients.SQLClient`, and others.

    Args:

        credentials (:py:class:`Credentials <cartoframes.credentials.Credentials>`, optional):
          A :py:class:`Credentials <cartoframes.credentials.Credentials>`
          instance can be used in place of a `username | base_url`/`api_key` combination.
        base_url (str, optional): Base URL of CARTO user account. Cloud-based accounts
          should use the form ``https://{username}.carto.com`` (e.g.,
          https://eschbacher.carto.com for user ``eschbacher``) whether on
          a personal or multi-user account. On-premises installation users
          should ask their admin.
        api_key (str, optional): CARTO API key. Depending on the application,
          this can be a project API key or the account master API key.
        username (str, optional): CARTO user name of the account.
        session (requests.Session, optional): requests session. See `requests
          documentation
          <https://2.python-requests.org/en/master/user/advanced/#session-objects>`__
          for more information.

    .. note::

        The recommended way to authenticate in CARTOframes is to read user
        credentials from a JSON file that is structured like this:

        .. code:: JSON

            {
                "username": "your user name",
                "api_key": "your api key",
                "base_url": "https://your_username.carto.com"
            }

        *Note that the ``base_url`` will be different for on premises
        installations.*

        By using the :func:`cartoframes.auth.Credentials.save` method, this
        file will automatically be created for you in a default location
        depending on your operating system. A custom location can also be
        specified as an argument to the method.

        This file can then be read in the following ways:

        .. code::

            from cartoframes.auth import Credentials, set_default_credentials

            # attempts to read file from default location if it exists
            creds = Credentials.from_file()

            # read credentials from specified location
            creds = Credentials.from_file('./carto-project-credentials.json')

            # set default credentials from file
            set_default_credentials(Credentials.from_file())

    Example:

        Create Credentials from a ``username``, ``api_key`` pair.

        .. code::

            from cartoframes.auth import set_default_credentials

            set_default_credentials(
                username='your_user_name',
                api_key='your api key'
            )

            # or

            set_default_credentials(
                'your_user_name',
                'your api key'
            )

        Create credentials from only a ``username`` (only works with public
        datasets and those marked public with link). If the API key is not
        provided, the public API key `default_public` is used. With this
        setting, only read-only operations can occur (e.g., no publishing of
        maps, reading data from the Data Observatory, or creating new hosted
        datasets).

        .. code::

            from cartoframes.auth import set_default_credentials
            set_default_credentials('your_user_name')

        From a pair ``base_url``, ``api_key``.

        .. code::

            from cartoframes.auth import set_default_credentials
            set_default_credentials(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )
            # or
            set_default_credentials(
                'https://your_user_name.carto.com',
                'your api key'
            )

        From a ``base_url`` (for public datasets). The API key `default_public`
        is used by default.

        .. code::

            from cartoframes.auth import set_default_credentials
            set_default_credentials('https://your_user_name.carto.com')

        From a :py:class:`Credentials <cartoframes.auth.Credentials>` class.

        .. code::

            from cartoframes.auth import Credentials, set_default_credentials
            credentials = Credentials(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )
            set_default_credentials(credentials)

    """

    global _default_credentials

    _base_url = base_url if first is None else first
    _username = username if first is None else first
    _api_key = (api_key if second is None else second) or 'default_public'
    _credentials = credentials if first is None else first

    if isinstance(_credentials, Credentials):
        _default_credentials = _credentials

    elif isinstance(_base_url or _username, str) and isinstance(_api_key, str):
        if _base_url and _is_url(_base_url):
            _default_credentials = Credentials(base_url=_base_url, api_key=_api_key)
        else:
            _default_credentials = Credentials(username=_username, api_key=_api_key)

    else:
        raise ValueError(
            'Invalid inputs. Pass a Credentials object, a username and api_key pair or a base_url and api_key pair.')

    if session:
        _default_credentials.session = session

def get_default_credentials():
    """Retrieve the default credentials if previously set with
    :func:`cartoframes.auth.set_default_credentials` in Python session.

    Example:

        Retrieve default credentials.

        .. code::

            from cartoframes.auth import set_default_credentials, get_default_credentials

            set_default_credentials(Credentials.from_file())

            current_creds = get_default_credentials()

    Returns:

        :py:class:`cartoframes.auth.Credentials`: Default credentials
        previously set in current Python session. `None` will returned if
        default credentials were not previously set.

    """
    return _default_credentials

def _is_url(text):
    return re.match(r'^https?://.*$', text)
<commit_msg>removes whitespace<commit_after>from __future__ import absolute_import

import re

from .credentials import Credentials

_default_credentials = None

def set_default_credentials(
        first=None, second=None, credentials=None,
        username=None, base_url=None, api_key=None, session=None):
    """Set default credentials for all operations that require authentication
    against a CARTO account. CARTOframes methods
    :py:class:`cartoframes.viz.Layer` (and helper layers in
    :py:mod:`cartoframes.viz.helpers`),
    :py:class:`cartoframes.data.Dataset`,
    :py:class:`cartoframes.data.clients.SQLClient`, and others.

    Args:

        credentials (:py:class:`Credentials <cartoframes.credentials.Credentials>`, optional):
          A :py:class:`Credentials <cartoframes.credentials.Credentials>`
          instance can be used in place of a `username | base_url`/`api_key` combination.
        base_url (str, optional): Base URL of CARTO user account. Cloud-based accounts
          should use the form ``https://{username}.carto.com`` (e.g.,
          https://eschbacher.carto.com for user ``eschbacher``) whether on
          a personal or multi-user account. On-premises installation users
          should ask their admin.
        api_key (str, optional): CARTO API key. Depending on the application,
          this can be a project API key or the account master API key.
        username (str, optional): CARTO user name of the account.
        session (requests.Session, optional): requests session. See `requests
          documentation
          <https://2.python-requests.org/en/master/user/advanced/#session-objects>`__
          for more information.

    .. note::

        The recommended way to authenticate in CARTOframes is to read user
        credentials from a JSON file that is structured like this:

        .. code:: JSON

            {
                "username": "your user name",
                "api_key": "your api key",
                "base_url": "https://your_username.carto.com"
            }

        *Note that the ``base_url`` will be different for on premises
        installations.*

        By using the :func:`cartoframes.auth.Credentials.save` method, this
        file will automatically be created for you in a default location
        depending on your operating system. A custom location can also be
        specified as an argument to the method.

        This file can then be read in the following ways:

        .. code::

            from cartoframes.auth import Credentials, set_default_credentials

            # attempts to read file from default location if it exists
            creds = Credentials.from_file()

            # read credentials from specified location
            creds = Credentials.from_file('./carto-project-credentials.json')

            # set default credentials from file
            set_default_credentials(Credentials.from_file())

    Example:

        Create Credentials from a ``username``, ``api_key`` pair.

        .. code::

            from cartoframes.auth import set_default_credentials

            set_default_credentials(
                username='your_user_name',
                api_key='your api key'
            )

            # or

            set_default_credentials(
                'your_user_name',
                'your api key'
            )

        Create credentials from only a ``username`` (only works with public
        datasets and those marked public with link). If the API key is not
        provided, the public API key `default_public` is used. With this
        setting, only read-only operations can occur (e.g., no publishing of
        maps, reading data from the Data Observatory, or creating new hosted
        datasets).

        .. code::

            from cartoframes.auth import set_default_credentials
            set_default_credentials('your_user_name')

        From a pair ``base_url``, ``api_key``.

        .. code::

            from cartoframes.auth import set_default_credentials
            set_default_credentials(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )
            # or
            set_default_credentials(
                'https://your_user_name.carto.com',
                'your api key'
            )

        From a ``base_url`` (for public datasets). The API key `default_public`
        is used by default.

        .. code::

            from cartoframes.auth import set_default_credentials
            set_default_credentials('https://your_user_name.carto.com')

        From a :py:class:`Credentials <cartoframes.auth.Credentials>` class.

        .. code::

            from cartoframes.auth import Credentials, set_default_credentials
            credentials = Credentials(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )
            set_default_credentials(credentials)

    """

    global _default_credentials

    _base_url = base_url if first is None else first
    _username = username if first is None else first
    _api_key = (api_key if second is None else second) or 'default_public'
    _credentials = credentials if first is None else first

    if isinstance(_credentials, Credentials):
        _default_credentials = _credentials

    elif isinstance(_base_url or _username, str) and isinstance(_api_key, str):
        if _base_url and _is_url(_base_url):
            _default_credentials = Credentials(base_url=_base_url, api_key=_api_key)
        else:
            _default_credentials = Credentials(username=_username, api_key=_api_key)

    else:
        raise ValueError(
            'Invalid inputs. Pass a Credentials object, a username and api_key pair or a base_url and api_key pair.')

    if session:
        _default_credentials.session = session

def get_default_credentials():
    """Retrieve the default credentials if previously set with
    :func:`cartoframes.auth.set_default_credentials` in Python session.

    Example:

        Retrieve default credentials.

        .. code::

            from cartoframes.auth import set_default_credentials, get_default_credentials

            set_default_credentials(Credentials.from_file())

            current_creds = get_default_credentials()

    Returns:

        :py:class:`cartoframes.auth.Credentials`: Default credentials
        previously set in current Python session. `None` will returned if
        default credentials were not previously set.

    """
    return _default_credentials

def _is_url(text):
    return re.match(r'^https?://.*$', text)
<|endoftext|>