<commit_before>from __future__ import print_function, division

import pytest

import numpy as np
import os

from tempfile import mkstemp
from sisl.io import *
from sisl.io.tbtrans._cdf import *
from sisl import Geometry, Grid, Hamiltonian
from sisl import DensityMatrix, EnergyDensityMatrix

from . import common as tc

_C = type('Temporary', (object, ), {})

def setup_module(module):
    tc.setup(module._C)

def teardown_module(module):
    tc.teardown(module._C)

gs = get_sile
gsc = get_sile_class

def stdoutfile(f):
    with open(f, 'r') as fh:
        for line in fh:
            print(line.replace('\n', ''))

def _my_intersect(a, b):
    return list(set(get_siles(a)).intersection(get_siles(b)))

def _fnames(base, variants):
    return [base + '.' + v if len(v) > 0 else base for v in variants]

@pytest.mark.io
def test_get_sile1():
    cls = gsc('test.xyz')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.regardless{XYZ}')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.fdf{XYZ}')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.fdf{XYZ}')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.xyz{fdf}')
    assert issubclass(cls, fdfSileSiesta)
    cls = gsc('test.cube{fdf}')
    assert issubclass(cls, fdfSileSiesta)

@pytest.mark.io
@pytest.mark.xfail(raises=NotImplementedError)
def test_get_sile2():
    gsc('test.this_file_does_not_exist')

@pytest.mark.io
class TestObject(object):

    def test_siesta_sources(self):
        pytest.importorskip("sisl.io.siesta._siesta")

    @pytest.mark.parametrize("sile", _fnames('test', ['cube', 'CUBE', 'cube.gz', 'CUBE.gz']))
    def test_cube(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, CUBESile]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['ascii', 'ascii.gz', 'ascii.gz']))
    def test_bigdft_ascii(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileBigDFT, ASCIISileBigDFT]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['gout', 'gout.gz']))
    def test_gulp_gout(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileGULP, gotSileGULP]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['REF', 'REF.gz']))
    def test_scaleup_REF(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileScaleUp, REFSileScaleUp]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['restart', 'restart.gz']))
    def test_scaleup_restart(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileScaleUp, restartSileScaleUp]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['rham', 'rham.gz']))
    def test_scaleup_rham(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileScaleUp, rhamSileScaleUp]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['fdf', 'fdf.gz', 'FDF.gz', 'FDF']))
    def test_siesta_fdf(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, fdfSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['out', 'out.gz']))
    def test_siesta_out(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, outSileSiesta]:
            assert isinstance(s, obj)

    def test_siesta_nc(self):
        s = gs('test.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFSiesta, ncSileSiesta]:
            assert isinstance(s, obj)

    def test_siesta_grid_nc(self):
        sile = gs('test.grid.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFSiesta, gridncSileSiesta]:
            assert isinstance(sile, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['XV', 'XV.gz']))
    def test_siesta_xv(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, XVSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['XV', 'XV.gz']))
    def test_siesta_xv_base(self, sile):
        s = gs(sile, cls=SileSiesta)
        for obj in [BaseSile, Sile, SileSiesta, XVSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['PDOS.xml', 'pdos.xml', 'PDOS.xml.gz', 'pdos.xml.gz']))
    def test_siesta_pdos_xml(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, pdosSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['ham', 'HAM', 'HAM.gz']))
    def test_ham(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, HamiltonianSile]:
            assert isinstance(s, obj)

    def test_tbtrans_nc(self):
        s = gs('test.TBT.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFTBtrans, tbtncSileTBtrans]:
            assert isinstance(s, obj)

    def test_phtrans_nc(self):
        s = gs('test.PHT.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFTBtrans, tbtncSileTBtrans, phtncSileTBtrans]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('CONTCAR', ['', 'gz']))
    def test_vasp_contcar(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileVASP, CARSileVASP, CONTCARSileVASP]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('POSCAR', ['', 'gz']))
    def test_vasp_poscar(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileVASP, CARSileVASP, POSCARSileVASP]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['xyz', 'XYZ', 'xyz.gz', 'XYZ.gz']))
    def test_xyz(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, XYZSile]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['molf', 'MOLF', 'molf.gz', 'MOLF.gz']))
    def test_molf(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, MoldenSile]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['xsf', 'XSF', 'xsf.gz', 'XSF.gz']))
    def test_xsf(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, XSFSile]:
            assert isinstance(s, obj)

    def test_wannier90_seed(self):
        sile = gs('test.win', cls=SileWannier90)
        for obj in [BaseSile, Sile, SileWannier90, winSileWannier90]:
            assert isinstance(sile, obj)

    def test_write(self):
        G = _C.g.rotatec(-30)
        G.set_nsc([1, 1, 1])
        f = mkstemp(dir=_C.d)[1]
        for sile in get_siles(['write_geometry']):
            # It is not yet an instance, hence issubclass
            if issubclass(sile, (HamiltonianSile, _ncSileTBtrans, deltancSileTBtrans)):
                continue
            # Write
            sile(f, mode='w').write_geometry(G)

    @pytest.mark.parametrize("sile", _my_intersect(['read_geometry'], ['write_geometry']))
    def test_read_write_geom(self, sile):
        G = _C.g.rotatec(-30)
        G.set_nsc([1, 1, 1])
        f = mkstemp(dir=_C.d)[1] + '.win'
        # These files does not store the atomic species
        if issubclass(sile, (_ncSileTBtrans, deltancSileTBtrans)):
            return
        # Write
        sile(f, mode='w').write_geometry(G)
        # Read 1
        try:
            g = sile(f, mode='r').read_geometry()
            assert g.equal(G, R=False)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            g = Geometry.read(sile(f, mode='r'))
            assert g.equal(G, R=False)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_hamiltonian'], ['write_hamiltonian']))
    def test_read_write_hamiltonian(self, sile):
        G = _C.g.rotatec(-30)
        H = Hamiltonian(G)
        H.construct([[0.1, 1.45], [0.1, -2.7]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_hamiltonian(H)
        # Read 1
        try:
            h = sile(f, mode='r').read_hamiltonian()
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            h = Hamiltonian.read(sile(f, mode='r'))
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_density_matrix'], ['write_density_matrix']))
    def test_read_write_density_matrix(self, sile):
        G = _C.g.rotatec(-30)
        DM = DensityMatrix(G, orthogonal=True)
        DM.construct([[0.1, 1.45], [0.1, -2.7]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_density_matrix(DM)
        # Read 1
        try:
            dm = sile(f, mode='r').read_density_matrix(geometry=DM.geom)
            assert DM.spsame(dm)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            dm = DensityMatrix.read(sile(f, mode='r'), geometry=DM.geom)
            assert DM.spsame(dm)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_energy_density_matrix'], ['write_energy_density_matrix']))
    def test_read_write_energy_density_matrix(self, sile):
        G = _C.g.rotatec(-30)
        EDM = EnergyDensityMatrix(G, orthogonal=True)
        EDM.construct([[0.1, 1.45], [0.1, -2.7]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_energy_density_matrix(EDM)
        # Read 1
        try:
            edm = sile(f, mode='r').read_energy_density_matrix(geometry=EDM.geom)
            assert EDM.spsame(edm)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            edm = EnergyDensityMatrix.read(sile(f, mode='r'), geometry=EDM.geom)
            assert EDM.spsame(edm)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_hamiltonian'], ['write_hamiltonian']))
    def test_read_write_hamiltonian_overlap(self, sile):
        G = _C.g.rotatec(-30)
        H = Hamiltonian(G, orthogonal=False)
        H.construct([[0.1, 1.45], [(0.1, 1), (-2.7, 0.1)]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_hamiltonian(H)
        # Read 1
        try:
            h = sile(f, mode='r').read_hamiltonian()
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            h = Hamiltonian.read(sile(f, mode='r'))
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_grid'], ['write_grid']))
    def test_read_write_grid(self, sile):
        g = _C.g.rotatec(-30)
        G = Grid([10, 11, 12])
        G[:, :, :] = np.random.rand(10, 11, 12)

        f = mkstemp(dir=_C.d)[1]
        # Write
        try:
            sile(f, mode='w').write_grid(G)
        except SileError:
            sile(f, mode='wb').write_grid(G)
        # Read 1
        try:
            g = sile(f, mode='r').read_grid()
            assert np.allclose(g.grid, G.grid, atol=1e-5)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            g = Grid.read(sile(f, mode='r'))
            assert np.allclose(g.grid, G.grid, atol=1e-5)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    def test_arg_parser1(self):
        f = mkstemp(dir=_C.d)[1]
        for sile in get_siles(['ArgumentParser']):
            try:
                sile(f).ArgumentParser()
            except:
                pass

    def test_arg_parser2(self):
        f = mkstemp(dir=_C.d)[1]
        for sile in get_siles(['ArgumentParser_out']):
            try:
                sile(f).ArgumentParser()
            except:
                pass
<commit_msg>maint: fixed mark in test_object<commit_after>from __future__ import print_function, division

import pytest

import numpy as np
import os

from tempfile import mkstemp
from sisl.io import *
from sisl.io.tbtrans._cdf import *
from sisl import Geometry, Grid, Hamiltonian
from sisl import DensityMatrix, EnergyDensityMatrix

from . import common as tc

pytestmark = pytest.mark.io

_C = type('Temporary', (object, ), {})

def setup_module(module):
    tc.setup(module._C)

def teardown_module(module):
    tc.teardown(module._C)

gs = get_sile
gsc = get_sile_class

def _my_intersect(a, b):
    return list(set(get_siles(a)).intersection(get_siles(b)))

def _fnames(base, variants):
    return [base + '.' + v if len(v) > 0 else base for v in variants]

def test_get_sile1():
    cls = gsc('test.xyz')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.regardless{XYZ}')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.fdf{XYZ}')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.fdf{XYZ}')
    assert issubclass(cls, XYZSile)

    cls = gsc('test.xyz{fdf}')
    assert issubclass(cls, fdfSileSiesta)
    cls = gsc('test.cube{fdf}')
    assert issubclass(cls, fdfSileSiesta)

@pytest.mark.xfail(raises=NotImplementedError)
def test_get_sile2():
    gsc('test.this_file_does_not_exist')

class TestObject(object):

    def test_siesta_sources(self):
        pytest.importorskip("sisl.io.siesta._siesta")

    @pytest.mark.parametrize("sile", _fnames('test', ['cube', 'CUBE', 'cube.gz', 'CUBE.gz']))
    def test_cube(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, CUBESile]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['ascii', 'ascii.gz', 'ascii.gz']))
    def test_bigdft_ascii(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileBigDFT, ASCIISileBigDFT]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['gout', 'gout.gz']))
    def test_gulp_gout(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileGULP, gotSileGULP]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['REF', 'REF.gz']))
    def test_scaleup_REF(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileScaleUp, REFSileScaleUp]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['restart', 'restart.gz']))
    def test_scaleup_restart(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileScaleUp, restartSileScaleUp]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['rham', 'rham.gz']))
    def test_scaleup_rham(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileScaleUp, rhamSileScaleUp]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['fdf', 'fdf.gz', 'FDF.gz', 'FDF']))
    def test_siesta_fdf(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, fdfSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['out', 'out.gz']))
    def test_siesta_out(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, outSileSiesta]:
            assert isinstance(s, obj)

    def test_siesta_nc(self):
        s = gs('test.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFSiesta, ncSileSiesta]:
            assert isinstance(s, obj)

    def test_siesta_grid_nc(self):
        sile = gs('test.grid.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFSiesta, gridncSileSiesta]:
            assert isinstance(sile, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['XV', 'XV.gz']))
    def test_siesta_xv(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, XVSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['XV', 'XV.gz']))
    def test_siesta_xv_base(self, sile):
        s = gs(sile, cls=SileSiesta)
        for obj in [BaseSile, Sile, SileSiesta, XVSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['PDOS.xml', 'pdos.xml', 'PDOS.xml.gz', 'pdos.xml.gz']))
    def test_siesta_pdos_xml(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileSiesta, pdosSileSiesta]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['ham', 'HAM', 'HAM.gz']))
    def test_ham(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, HamiltonianSile]:
            assert isinstance(s, obj)

    def test_tbtrans_nc(self):
        s = gs('test.TBT.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFTBtrans, tbtncSileTBtrans]:
            assert isinstance(s, obj)

    def test_phtrans_nc(self):
        s = gs('test.PHT.nc', _open=False)
        for obj in [BaseSile, SileCDF, SileCDFTBtrans, tbtncSileTBtrans, phtncSileTBtrans]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('CONTCAR', ['', 'gz']))
    def test_vasp_contcar(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileVASP, CARSileVASP, CONTCARSileVASP]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('POSCAR', ['', 'gz']))
    def test_vasp_poscar(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, SileVASP, CARSileVASP, POSCARSileVASP]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['xyz', 'XYZ', 'xyz.gz', 'XYZ.gz']))
    def test_xyz(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, XYZSile]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['molf', 'MOLF', 'molf.gz', 'MOLF.gz']))
    def test_molf(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, MoldenSile]:
            assert isinstance(s, obj)

    @pytest.mark.parametrize("sile", _fnames('test', ['xsf', 'XSF', 'xsf.gz', 'XSF.gz']))
    def test_xsf(self, sile):
        s = gs(sile)
        for obj in [BaseSile, Sile, XSFSile]:
            assert isinstance(s, obj)

    def test_wannier90_seed(self):
        sile = gs('test.win', cls=SileWannier90)
        for obj in [BaseSile, Sile, SileWannier90, winSileWannier90]:
            assert isinstance(sile, obj)

    def test_write(self):
        G = _C.g.rotatec(-30)
        G.set_nsc([1, 1, 1])
        f = mkstemp(dir=_C.d)[1]
        for sile in get_siles(['write_geometry']):
            # It is not yet an instance, hence issubclass
            if issubclass(sile, (HamiltonianSile, _ncSileTBtrans, deltancSileTBtrans)):
                continue
            # Write
            sile(f, mode='w').write_geometry(G)

    @pytest.mark.parametrize("sile", _my_intersect(['read_geometry'], ['write_geometry']))
    def test_read_write_geom(self, sile):
        G = _C.g.rotatec(-30)
        G.set_nsc([1, 1, 1])
        f = mkstemp(dir=_C.d)[1] + '.win'
        # These files does not store the atomic species
        if issubclass(sile, (_ncSileTBtrans, deltancSileTBtrans)):
            return
        # Write
        sile(f, mode='w').write_geometry(G)
        # Read 1
        try:
            g = sile(f, mode='r').read_geometry()
            assert g.equal(G, R=False)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            g = Geometry.read(sile(f, mode='r'))
            assert g.equal(G, R=False)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_hamiltonian'], ['write_hamiltonian']))
    def test_read_write_hamiltonian(self, sile):
        G = _C.g.rotatec(-30)
        H = Hamiltonian(G)
        H.construct([[0.1, 1.45], [0.1, -2.7]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_hamiltonian(H)
        # Read 1
        try:
            h = sile(f, mode='r').read_hamiltonian()
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            h = Hamiltonian.read(sile(f, mode='r'))
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_density_matrix'], ['write_density_matrix']))
    def test_read_write_density_matrix(self, sile):
        G = _C.g.rotatec(-30)
        DM = DensityMatrix(G, orthogonal=True)
        DM.construct([[0.1, 1.45], [0.1, -2.7]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_density_matrix(DM)
        # Read 1
        try:
            dm = sile(f, mode='r').read_density_matrix(geometry=DM.geom)
            assert DM.spsame(dm)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            dm = DensityMatrix.read(sile(f, mode='r'), geometry=DM.geom)
            assert DM.spsame(dm)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_energy_density_matrix'], ['write_energy_density_matrix']))
    def test_read_write_energy_density_matrix(self, sile):
        G = _C.g.rotatec(-30)
        EDM = EnergyDensityMatrix(G, orthogonal=True)
        EDM.construct([[0.1, 1.45], [0.1, -2.7]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_energy_density_matrix(EDM)
        # Read 1
        try:
            edm = sile(f, mode='r').read_energy_density_matrix(geometry=EDM.geom)
            assert EDM.spsame(edm)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            edm = EnergyDensityMatrix.read(sile(f, mode='r'), geometry=EDM.geom)
            assert EDM.spsame(edm)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_hamiltonian'], ['write_hamiltonian']))
    def test_read_write_hamiltonian_overlap(self, sile):
        G = _C.g.rotatec(-30)
        H = Hamiltonian(G, orthogonal=False)
        H.construct([[0.1, 1.45], [(0.1, 1), (-2.7, 0.1)]])
        f = mkstemp(dir=_C.d)[1]
        # Write
        sile(f, mode='w').write_hamiltonian(H)
        # Read 1
        try:
            h = sile(f, mode='r').read_hamiltonian()
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            h = Hamiltonian.read(sile(f, mode='r'))
            assert H.spsame(h)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    @pytest.mark.parametrize("sile", _my_intersect(['read_grid'], ['write_grid']))
    def test_read_write_grid(self, sile):
        g = _C.g.rotatec(-30)
        G = Grid([10, 11, 12])
        G[:, :, :] = np.random.rand(10, 11, 12)

        f = mkstemp(dir=_C.d)[1]
        # Write
        try:
            sile(f, mode='w').write_grid(G)
        except SileError:
            sile(f, mode='wb').write_grid(G)
        # Read 1
        try:
            g = sile(f, mode='r').read_grid()
            assert np.allclose(g.grid, G.grid, atol=1e-5)
        except UnicodeDecodeError as e:
            pass
        # Read 2
        try:
            g = Grid.read(sile(f, mode='r'))
            assert np.allclose(g.grid, G.grid, atol=1e-5)
        except UnicodeDecodeError as e:
            pass
        # Clean-up file
        os.remove(f)

    def test_arg_parser1(self):
        f = mkstemp(dir=_C.d)[1]
        for sile in get_siles(['ArgumentParser']):
            try:
                sile(f).ArgumentParser()
            except:
                pass

    def test_arg_parser2(self):
        f = mkstemp(dir=_C.d)[1]
        for sile in get_siles(['ArgumentParser_out']):
            try:
                sile(f).ArgumentParser()
            except:
                pass
<|endoftext|>


<commit_before>from pycnic.core import WSGI
from api.tree import *

class Application(WSGI):
    routes = [
        (r'/tree', Tree()),
        (r'/tree/(\d+)', Tree()),
        (r'/node', Node()),
        (r'/node/(\d+)', Node()),
    ]
<commit_msg>(34) API server can be started locally<commit_after>#!/usr/bin/env python3

from pycnic.core import WSGI
from api.tree import *

class Application(WSGI):
    routes = [
        (r'/tree', Tree()),
        (r'/tree/(\d+)', Tree()),
        (r'/node', Node()),
        (r'/node/(\d+)', Node()),
    ]

if __name__ == "__main__":
    # this module can be executed to get simple WSGI server on localhost:8080
    from wsgiref.simple_server import make_server
    try:
        print("Serving on 0.0.0.0:8080...")
        make_server('0.0.0.0', 8080, Application).serve_forever()
    except KeyboardInterrupt:
        pass
    print("Done")
<|endoftext|>


<commit_before>'''
Created on 21 Jun 2015

@author: @alexparsons

We have a shortage of housing, not bad housing ideas. Yet it is easier to train a robot to make these.
'''

import random
import twitter
import credentials

from classes import Robot, RobotMaster

random.seed()

words = ["Buy","Own","Rent","Let"]

verbs = ["is","are","has","get","see","need","know","would","find",
         "take","want","does","believe","learn","become","come",
         "include","thank","provide","create","add","understand",
         "consider","choose","develop","remember","determine",
         "grow","allow","supply","bring","improve","maintain","begin",
         "exist","tend","enjoy","perform","decide","identify",
         "continue","protect","require","occur","write","approach",
         "avoid","prepare","build","achieve","believe","receive",
         "seem","discuss","realize","contain","follow","refer",
         "solve","describe","prefer","prevent","discover",
         "ensure","expect","invest","reduce","speak","appear",
         "explain","explore","involve","lose","afford","agree",
         "hear","remain","represent","apply","forget",
         "recommend","rely","vary","generate","obtain",
         "accept","communicate","complain","depend",
         "enter","happen","indicate","suggest","survive",
         "appreciate","compare","imagine","manage","differ",
         "encourage","expand","prove","react","recognize","relax",
         "replace","borrow","earn","emphasize","enable","operate",
         "reflect","send","anticipate","assume","engage","enhance",
         "examine","install","participate","intend","introduce",
         "relate","settle","assure","attract","distribute","overcome",
         "owe","succeed","suffer","throw","acquire","adapt","adjust",
         "argue","arise","confirm","encouraging","incorporate",
         "justify","organize","ought","possess","relieve","retain",
         "shut","calculate","compete","consult","deliver","extend",
         "investigate","negotiate","qualify","retire","rid","weigh",
         "arrive","attach","behave","celebrate","convince","disagree",
         "establish","ignore","imply","insist","pursue","remaining","specify",
         "warn","accuse","admire","admit","adopt","announce","apologize","approve",
         "attend","belong","commit","criticize","deserve","destroy","hesitate",
         "illustrate","inform","manufacturing","persuade","pour","propose",
         "remind","shall","submit","suppose","translate","be","have","use",
         "make","look","help","go","being","think","read","keep","start",
         "give","play","feel","put","set","change","say","cut","show","try",
         "check","call","move","pay","let","increase","turn","ask","buy",
         "guard","hold","offer","travel","cook","dance","excuse","live",
         "purchase","deal","mean","fall","produce","search","spend","talk",
         "upset","tell","cost","drive","support","remove","return","run",
         "appropriate","reserve","leave","reach","rest","serve","watch",
         "charge","break","stay","visit","affect","cover","report","rise",
         "walk","pick","lift","mix","stop","teach","concern","fly","born",
         "gain","save","stand","fail","lead","listen","worry","express",
         "handle","meet","release","sell","finish","press","ride","spread",
         "spring","wait","display","flow","hit","shoot","touch","cancel","cry",
         "dump","push","select","conflict","die","eat","fill","jump","kick",
         "pass","pitch","treat","abuse","beat","burn","deposit","print",
         "raise","sleep","advance","connect","consist","contribute","draw",
         "fix","hire","join","kill","sit","tap","win","attack","claim","drag",
         "drink","guess","pull","wear","wonder","count","doubt","feed","impress",
         "repeat","seek","sing","slide","strip","wish","collect","combine","command",
         "dig","divide","hang","hunt","march","mention","smell","survey","tie",
         "escape","expose","gather","hate","repair","scratch","strike","employ",
         "hurt","laugh","lay","respond","split","strain","struggle","swim","train",
         "wash","waste","convert","crash","fold","grab","hide","miss","permit","quote",
         "recover","resolve","roll","sink","slip","suspect","swing","twist",
         "concentrate","estimate","prompt","refuse","regret","reveal","rush","shake",
         "shift","shine","steal","suck","surround","bear","dare","delay","hurry","invite",
         "kiss","marry","pop","pray","pretend","punch","quit","reply","resist","rip","rub",
         "smile","spell","stretch","tear","wake","wrap","was","like","even","film","water",
         "been","well","were","example","own","study","must","form","air","place","number",
         "part","field","fish","process","heat","hand","experience","job","book","end",
         "point","type","value","body","market","guide","interest","state","radio",
         "course","company","price","size","card","list","mind","trade","line",
         "care","group","risk","word","force","light","name","school","amount",
         "order","practice","research","sense","service","piece","web","boss","sport",
         "page","term","test","answer","sound","focus","matter","soil","board","oil",
         "picture","access","garden","open","range","rate","reason","according","site",
         "demand","exercise","image","case","cause","coast","age","boat","record","result",
         "section","building","mouse","cash","class","dry","plan","store","tax","involved",
         "side","space","rule","weather","figure","man","model","source","earth","program",
         "design","feature","purpose","question","rock","act","birth","dog",
         "object","scale","sun","fit","note","profit","related","rent","speed",
         "style","war","bank","content","craft","bus","exchange","eye","fire",
         "position","pressure","stress","advantage","benefit","box","complete",
         "frame","issue","limited","step","cycle","face","interested","metal",
         "paint","review","room","screen","structure","view","account","ball",
         "concerned","discipline","ready","share","balance","bit","black","bottom",
         "gift","impact","machine","shape","tool","wind","address","average",
         "career","culture","pot","sign","table","task","condition","contact",
         "credit","egg","hope","ice","network","separate","attempt","date",
         "effect","link","perfect","post","star","voice","challenge",
         "friend","warm","brush","couple","debate","exit","experienced",
         "function","lack","plant","spot","summer","taste","theme","track",
         "wing","brain","button","click","correct","desire","fixed","foot",
         "gas","influence","notice","rain","wall","base","damage","distance",
         "pair","staff","sugar","target","text","author","complicated","discount",
         "file","ground","lesson","officer","phase","reference","register","secure",
         "sky","stage","stick","title","trouble","advanced","bowl","bridge","campaign",
         "club","edge","evidence","fan","letter","lock","option","organized","pack","park",
         "quarter","skin","sort","weight","baby","carry","dish","exact","factor","fruit",
         "muscle","traffic","trip","appeal","chart","gear","land","log","lost","net",
         "season","spirit","tree","wave","belt","bench","closed","commission","copy",
         "drop","firm","frequent","progress","project","stuff","ticket","tour","angle",
         "blue","breakfast","doctor","dot","dream","essay","father","fee","finance",
         "juice","limit","luck","milk","mixed","mouth","pipe","please","seat","stable",
         "storm","team","amazing","bat","beach","blank","busy","catch","chain","cream",
         "crew","detail","detailed","interview","kid","mark","match","pain","pleasure",
         "score","screw","sex","sharp","shop","shower","suit","tone","window","wise",
         "band","block","bone","calendar","cap","coat","contest","court","cup",
         "district","finger","garage","guarantee","hole","hook","implement",
         "layer","lecture","lie","married","narrow","nose","partner","profile","respect",
         "rice","schedule","telephone","tip","bag","battle","bed","bill","bother","cake",
         "code","curve","dimension","ease","farm","fight","gap","grade","horse","host",
         "husband","loan","mistake","nail","noise","occasion","package","pause",
         "phrase","race","sand","sentence","shoulder","smoke","stomach","string"
         ,"surprise","towel","vacation","wheel","arm","associate",
         "bet","blow","border","branch","breast","buddy","bunch","chip",
         "coach","cross","document","draft","dust","floor","golf","habit",
         "iron","judge","knife","landscape","league","mail","mess","parent",
         "pattern","pin","pool","pound","request","salary","shame","shelter",
         "shoe","tackle","tank","trust","assist","bake","bar","bell","bike",
         "blame","brick","chair","closet","clue","collar","comment","conference",
         "devil","diet","fear","fuel","glove","jacket","lunch","monitor","mortgage",
         "nurse","pace","panic","peak","provided","reward","row","sandwich","shock",
         "spite","spray","surprise","till","transition","weekend","yard","alarm",
         "bend","bicycle","bite","blind","bottle","cable","candle","clerk","cloud",
         "concert","counter","dirty","flower","grandfather","harm","knee","lawyer",
         "load","loose","mirror","neck","pension","plate","pleased","proposed",
         "ruin","ship","skirt","slice","snow","stroke","switch","tired","trash",
         "tune","worried","zone","anger","award","bid","boot","bug","camp","candy",
         "carpet","cat","champion","channel","clock","comfort","cow","crack",
         "disappointed","empty","engineer","entrance","fault","grass","guy",
         "highlight","island","joke","jury","leg","lip","mate","nerve",
         "passage","pen","pride","priest","promise","resort","ring","roof",
         "rope","sail","scheme","script","slight","smart","sock",
         "station","toe","tower","truck","witness"]

formats = []
for w in words:
    formats.append("{0} to xxx".format(w))
    formats.append("xxx to {0}".format(w))

def generate():

    format = random.choice(formats)
    verb = random.choice(verbs)
    return format.replace("xxx",verb.title())

def tweet():

    api = twitter.Api(**credentials.twitter_badhousing)
    try:
        return api.PostUpdate(generate())
    except twitter.error.TwitterError,e:
        print e
housing = Robot("Bad Housing Ideas",tweet,minutes=85,uk_hours=True,force_run=True)

RobotMaster().register(housing)

if __name__ == "__main__":
    housing.populate(20)<commit_msg>removed force from housing<commit_after>'''
Created on 21 Jun 2015

@author: @alexparsons

We have a shortage of housing, not bad housing ideas. Yet it is easier to train a robot to make these.
'''

import random
import twitter
import credentials

from classes import Robot, RobotMaster

random.seed()

words = ["Buy","Own","Rent","Let"]

verbs = ["is","are","has","get","see","need","know","would","find",
         "take","want","does","believe","learn","become","come",
         "include","thank","provide","create","add","understand",
         "consider","choose","develop","remember","determine",
         "grow","allow","supply","bring","improve","maintain","begin",
         "exist","tend","enjoy","perform","decide","identify",
         "continue","protect","require","occur","write","approach",
         "avoid","prepare","build","achieve","believe","receive",
         "seem","discuss","realize","contain","follow","refer",
         "solve","describe","prefer","prevent","discover",
         "ensure","expect","invest","reduce","speak","appear",
         "explain","explore","involve","lose","afford","agree",
         "hear","remain","represent","apply","forget",
         "recommend","rely","vary","generate","obtain",
         "accept","communicate","complain","depend",
         "enter","happen","indicate","suggest","survive",
         "appreciate","compare","imagine","manage","differ",
         "encourage","expand","prove","react","recognize","relax",
         "replace","borrow","earn","emphasize","enable","operate",
         "reflect","send","anticipate","assume","engage","enhance",
         "examine","install","participate","intend","introduce",
         "relate","settle","assure","attract","distribute","overcome",
         "owe","succeed","suffer","throw","acquire","adapt","adjust",
         "argue","arise","confirm","encouraging","incorporate",
         "justify","organize","ought","possess","relieve","retain",
         "shut","calculate","compete","consult","deliver","extend",
         "investigate","negotiate","qualify","retire","rid","weigh",
         "arrive","attach","behave","celebrate","convince","disagree",
         "establish","ignore","imply","insist","pursue","remaining","specify",
         "warn","accuse","admire","admit","adopt","announce","apologize","approve",
         "attend","belong","commit","criticize","deserve","destroy","hesitate",
         "illustrate","inform","manufacturing","persuade","pour","propose",
         "remind","shall","submit","suppose","translate","be","have","use",
         "make","look","help","go","being","think","read","keep","start",
         "give","play","feel","put","set","change","say","cut","show","try",
         "check","call","move","pay","let","increase","turn","ask","buy",
         "guard","hold","offer","travel","cook","dance","excuse","live",
         "purchase","deal","mean","fall","produce","search","spend","talk",
         "upset","tell","cost","drive","support","remove","return","run",
         "appropriate","reserve","leave","reach","rest","serve","watch",
         "charge","break","stay","visit","affect","cover","report","rise",
         "walk","pick","lift","mix","stop","teach","concern","fly","born",
         "gain","save","stand","fail","lead","listen","worry","express",
         "handle","meet","release","sell","finish","press","ride","spread",
         "spring","wait","display","flow","hit","shoot","touch","cancel","cry",
         "dump","push","select","conflict","die","eat","fill","jump","kick",
         "pass","pitch","treat","abuse","beat","burn","deposit","print",
         "raise","sleep","advance","connect","consist","contribute","draw",
         "fix","hire","join","kill","sit","tap","win","attack","claim","drag",
         "drink","guess","pull","wear","wonder","count","doubt","feed","impress",
         "repeat","seek","sing","slide","strip","wish","collect","combine","command",
         "dig","divide","hang","hunt","march","mention","smell","survey","tie",
         "escape","expose","gather","hate","repair","scratch","strike","employ",
         "hurt","laugh","lay","respond","split","strain","struggle","swim","train",
         "wash","waste","convert","crash","fold","grab","hide","miss","permit","quote",
         "recover","resolve","roll","sink","slip","suspect","swing","twist",
         "concentrate","estimate","prompt","refuse","regret","reveal","rush","shake",
         "shift","shine","steal","suck","surround","bear","dare","delay","hurry","invite",
         "kiss","marry","pop","pray","pretend","punch","quit","reply","resist","rip","rub",
         "smile","spell","stretch","tear","wake","wrap","was","like","even","film","water",
         "been","well","were","example","own","study","must","form","air","place","number",
         "part","field","fish","process","heat","hand","experience","job","book","end",
         "point","type","value","body","market","guide","interest","state","radio",
         "course","company","price","size","card","list","mind","trade","line",
         "care","group","risk","word","force","light","name","school","amount",
         "order","practice","research","sense","service","piece","web","boss","sport",
         "page","term","test","answer","sound","focus","matter","soil","board","oil",
         "picture","access","garden","open","range","rate","reason","according","site",
         "demand","exercise","image","case","cause","coast","age","boat","record","result",
         "section","building","mouse","cash","class","dry","plan","store","tax","involved",
         "side","space","rule","weather","figure","man","model","source","earth","program",
         "design","feature","purpose","question","rock","act","birth","dog",
         "object","scale","sun","fit","note","profit","related","rent","speed",
         "style","war","bank","content","craft","bus","exchange","eye","fire",
         "position","pressure","stress","advantage","benefit","box","complete",
         "frame","issue","limited","step","cycle","face","interested","metal",
         "paint","review","room","screen","structure","view","account","ball",
         "concerned","discipline","ready","share","balance","bit","black","bottom",
         "gift","impact","machine","shape","tool","wind","address","average",
         "career","culture","pot","sign","table","task","condition","contact",
         "credit","egg","hope","ice","network","separate","attempt","date",
         "effect","link","perfect","post","star","voice","challenge",
         "friend","warm","brush","couple","debate","exit","experienced",
         "function","lack","plant","spot","summer","taste","theme","track",
         "wing","brain","button","click","correct","desire","fixed","foot",
         "gas","influence","notice","rain","wall","base","damage","distance",
         "pair","staff","sugar","target","text","author","complicated","discount",
         "file","ground","lesson","officer","phase","reference","register","secure",
         "sky","stage","stick","title","trouble","advanced","bowl","bridge","campaign",
         "club","edge","evidence","fan","letter","lock","option","organized","pack","park",
         "quarter","skin","sort","weight","baby","carry","dish","exact","factor","fruit",
         "muscle","traffic","trip","appeal","chart","gear","land","log","lost","net",
         "season","spirit","tree","wave","belt","bench","closed","commission","copy",
         "drop","firm","frequent","progress","project","stuff","ticket","tour","angle",
         "blue","breakfast","doctor","dot","dream","essay","father","fee","finance",
         "juice","limit","luck","milk","mixed","mouth","pipe","please","seat","stable",
         "storm","team","amazing","bat","beach","blank","busy","catch","chain","cream",
         "crew","detail","detailed","interview","kid","mark","match","pain","pleasure",
         "score","screw","sex","sharp","shop","shower","suit","tone","window","wise",
         "band","block","bone","calendar","cap","coat","contest","court","cup",
         "district","finger","garage","guarantee","hole","hook","implement",
         "layer","lecture","lie","married","narrow","nose","partner","profile","respect",
         "rice","schedule","telephone","tip","bag","battle","bed","bill","bother","cake",
         "code","curve","dimension","ease","farm","fight","gap","grade","horse","host",
         "husband","loan","mistake","nail","noise","occasion","package","pause",
         "phrase","race","sand","sentence","shoulder","smoke","stomach","string"
         ,"surprise","towel","vacation","wheel","arm","associate",
         "bet","blow","border","branch","breast","buddy","bunch","chip",
         "coach","cross","document","draft","dust","floor","golf","habit",
         "iron","judge","knife","landscape","league","mail","mess","parent",
         "pattern","pin","pool","pound","request","salary","shame","shelter",
         "shoe","tackle","tank","trust","assist","bake","bar","bell","bike",
         "blame","brick","chair","closet","clue","collar","comment","conference",
         "devil","diet","fear","fuel","glove","jacket","lunch","monitor","mortgage",
         "nurse","pace","panic","peak","provided","reward","row","sandwich","shock",
         "spite","spray","surprise","till","transition","weekend","yard","alarm",
         "bend","bicycle","bite","blind","bottle","cable","candle","clerk","cloud",
         "concert","counter","dirty","flower","grandfather","harm","knee","lawyer",
         "load","loose","mirror","neck","pension","plate","pleased","proposed",
         "ruin","ship","skirt","slice","snow","stroke","switch","tired","trash",
         "tune","worried","zone","anger","award","bid","boot","bug","camp","candy",
         "carpet","cat","champion","channel","clock","comfort","cow","crack",
         "disappointed","empty","engineer","entrance","fault","grass","guy",
         "highlight","island","joke","jury","leg","lip","mate","nerve",
         "passage","pen","pride","priest","promise","resort","ring","roof",
         "rope","sail","scheme","script","slight","smart","sock",
         "station","toe","tower","truck","witness"]

formats = []
for w in words:
    formats.append("{0} to xxx".format(w))
    formats.append("xxx to {0}".format(w))

def generate():

    format = random.choice(formats)
    verb = random.choice(verbs)
    return format.replace("xxx",verb.title())

def tweet():

    api = twitter.Api(**credentials.twitter_badhousing)
    try:
        return api.PostUpdate(generate())
    except twitter.error.TwitterError,e:
        print e
housing = Robot("Bad Housing Ideas",tweet,minutes=85,uk_hours=True)

RobotMaster().register(housing)

if __name__ == "__main__":
    housing.populate(20)<|endoftext|>


<commit_before>import os
import re
import sys

from roblib import sequences

def clean_fastq(file1, file2):
    """
    Make a set of cleaned pairs and unpaired reads. If all the reads are paired we do not do anything

    :param file1: first fastq file
    :type file1: str
    :param file2: second fastq file
    :type file2: str
    :return: A string for the output using -1 -2 and -U for unpaired reads
    :rtype: str
    """

    sys.stderr.write("Checking " + file1 + " and " + file2 + "\n")
    seq1 = {}
    seq2 = {}

    for (sid, label, seq, qual) in sequences.stream_fastq(file1):
        sid = re.sub('@', '', sid)
        sid = re.sub('\.[12]$', '', sid)
        seq1[sid] = "@" + label + "\n" + seq + "\n+\n" + qual + "\n"

    for (sid, label, seq, qual) in sequences.stream_fastq(file2):
        sid = re.sub('@', '', sid)
        sid = re.sub('\.[12]$', '', sid)
        seq2[sid] = "@" + label + "\n" + seq + "\n+\n" + qual + "\n"

    seq1set = set(seq1.keys())
    seq2set = set(seq2.keys())

    sys.stderr.write(
        "File 1: " + file1 + " seqs: " + str(len(seq1set)) + " File 2: " + file2 + " seqs: " + str(len(seq2set)) + "\n")

    # are there reads in one but not the other?
    s1unique = seq1set.difference(seq2set)
    s2unique = seq2set.difference(seq1set)

    ret = ' -1 ' + file1 + ' -2 ' + file2

    if len(s1unique) > 0 or len(s2unique) > 0:
        file1 = file1.replace('.gz', '')
        file2 = file2.replace('.gz', '')
        sys.stderr.write("Rewriting " + file1 + " and " + file2 + "\n")
        # we have to make new files

        file1clean = file1.replace('.fastq', '.clean.fastq')
        if file1clean == file1:
            file1clean = file1.replace('.fq', '.clean.fq')
        if file1clean == file1:
            file1clean = file1 + ".clean.fastq"

        file1unique = file1.replace('.fastq', '.unique.fastq')
        if file1unique == file1:
            file1unique = file1.replace('.fq', '.unique.fastq')
        if file1unique == file1:
            file1unique = file1 + '.unique.fastq'

        file2clean = file2.replace('.fastq', '.clean.fastq')
        if file2clean == file2:
            file2clean = file2.replace('.fq', '.clean.fq')
        if file2clean == file2:
            file2clean = file2 + ".clean.fastq"

        file2unique = file2.replace('.fastq', '.unique.fastq')
        if file2unique == file2:
            file2unique = file2.replace('.fq', '.unique.fastq')
        if file2unique == file2:
            file2unique = file2 + '.unique.fastq'

        ret = " -1 " + file1clean + " -2 " + file2clean
        try:
            out1 = open(file1clean, 'w')
            out2 = open(file2clean, 'w')
            for sid in seq1set.intersection(seq2set):
                out1.write(seq1[sid])
                out2.write(seq2[sid])
            out1.close()
            out2.close()
        except IOError as e:
            print "I/O error({0}): {1}".format(e.errno, e.strerror)

        if len(s1unique) > 0:
            ret = ret + " -U " + file1unique
            try:
                out = open(file1unique, 'w')
                for sid in s1unique:
                    out.write(seq1[sid])
                out.close()
            except IOError as e:
                print "I/O error({0}): {1}".format(e.errno, e.strerror)

        if len(s2unique) > 0:
            ret = ret + " -U " + file2unique
            try:
                out = open(file2unique, 'w')
                for sid in s2unique:
                    out.write(seq2[sid])
                out.close()
            except IOError as e:
                print "I/O error({0}): {1}".format(e.errno, e.strerror)
    return ret

import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Given a directory of fastq files, figure out the pairs and check if they are ok. The only requirement is the fastq files have _ between the name and id")
    parser.add_argument('-d', help='Directory of fastq files', required=True)
    args = parser.parse_args()

    files = {}
    for f in os.listdir(args.d):
        sid = f.split('_')[0]
        if sid not in files:
            files[sid] = set()
        files[sid].add(f)

    for s in files:
        if len(files[s]) == 1:
            print(args.d + "/" + files[s].pop())
        elif len(files[s]) == 2:
            outstr = clean_fastq(os.path.join(args.d, files[s].pop()), os.path.join(args.d, files[s].pop()))
            print(outstr)
        else:
            sys.stderr.write("Apparently more than two files for " + s + " ==> " + " ".join(files))
<commit_msg>Clean fastq pairs of files<commit_after>import os
import re
import sys

from roblib import sequences

def clean_fastq(file1, file2):
    """
    Make a set of cleaned pairs and unpaired reads. If all the reads are paired we do not do anything

    :param file1: first fastq file
    :type file1: str
    :param file2: second fastq file
    :type file2: str
    :return: A string for the output using -1 -2 and -U for unpaired reads
    :rtype: str
    """

    sys.stderr.write("Checking " + file1 + " and " + file2 + "\n")
    seq1 = {}
    seq2 = {}

    for (sid, label, seq, qual) in sequences.stream_fastq(file1):
        sid = re.sub('@', '', sid)
        sid = re.sub('\.[12]$', '', sid)
        seq1[sid] = "@" + label + "\n" + seq + "\n+\n" + qual + "\n"

    for (sid, label, seq, qual) in sequences.stream_fastq(file2):
        sid = re.sub('@', '', sid)
        sid = re.sub('\.[12]$', '', sid)
        seq2[sid] = "@" + label + "\n" + seq + "\n+\n" + qual + "\n"

    seq1set = set(seq1.keys())
    seq2set = set(seq2.keys())

    sys.stderr.write(
        "File 1: " + file1 + " seqs: " + str(len(seq1set)) + " File 2: " + file2 + " seqs: " + str(len(seq2set)) + "\n")

    # are there reads in one but not the other?
    s1unique = seq1set.difference(seq2set)
    s2unique = seq2set.difference(seq1set)

    ret = ' -1 ' + file1 + ' -2 ' + file2

    if len(s1unique) > 0 or len(s2unique) > 0:
        file1 = file1.replace('.gz', '')
        file2 = file2.replace('.gz', '')
        sys.stderr.write("Rewriting " + file1 + " and " + file2 + "\n")
        # we have to make new files

        file1clean = file1.replace('.fastq', '.clean.fastq')
        if file1clean == file1:
            file1clean = file1.replace('.fq', '.clean.fq')
        if file1clean == file1:
            file1clean = file1 + ".clean.fastq"

        file1unique = file1.replace('.fastq', '.unique.fastq')
        if file1unique == file1:
            file1unique = file1.replace('.fq', '.unique.fastq')
        if file1unique == file1:
            file1unique = file1 + '.unique.fastq'

        file2clean = file2.replace('.fastq', '.clean.fastq')
        if file2clean == file2:
            file2clean = file2.replace('.fq', '.clean.fq')
        if file2clean == file2:
            file2clean = file2 + ".clean.fastq"

        file2unique = file2.replace('.fastq', '.unique.fastq')
        if file2unique == file2:
            file2unique = file2.replace('.fq', '.unique.fastq')
        if file2unique == file2:
            file2unique = file2 + '.unique.fastq'

        file1unique = file1unique.replace('.gz', '')
        file2unique = file2unique.replace('.gz', '')
        file1clean = file1clean.replace('.gz', '')
        file2clean = file2clean.replace('.gz', '')

        ret = " -1 " + file1clean + " -2 " + file2clean
        try:
            out1 = open(file1clean, 'w')
            out2 = open(file2clean, 'w')
            for sid in seq1set.intersection(seq2set):
                out1.write(seq1[sid])
                out2.write(seq2[sid])
            out1.close()
            out2.close()
        except IOError as e:
            print "I/O error({0}): {1}".format(e.errno, e.strerror)

        if len(s1unique) > 0:
            ret = ret + " -U " + file1unique
            try:
                out = open(file1unique, 'w')
                for sid in s1unique:
                    out.write(seq1[sid])
                out.close()
            except IOError as e:
                print "I/O error({0}): {1}".format(e.errno, e.strerror)

        if len(s2unique) > 0:
            ret = ret + " -U " + file2unique
            try:
                out = open(file2unique, 'w')
                for sid in s2unique:
                    out.write(seq2[sid])
                out.close()
            except IOError as e:
                print "I/O error({0}): {1}".format(e.errno, e.strerror)
    return ret

import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Given a directory of fastq files, figure out the pairs and check if they are ok. The only requirement is the fastq files have _ between the name and id")
    parser.add_argument('-d', help='Directory of fastq files', required=True)
    args = parser.parse_args()

    files = {}
    for f in os.listdir(args.d):
        sid = f.split('_')[0]
        if sid not in files:
            files[sid] = set()
        files[sid].add(f)

    for s in files:
        if len(files[s]) == 1:
            print(args.d + "/" + files[s].pop())
        elif len(files[s]) == 2:
            outstr = clean_fastq(os.path.join(args.d, files[s].pop()), os.path.join(args.d, files[s].pop()))
            print(outstr)
        else:
            sys.stderr.write("Apparently more than two files for " + s + " ==> " + " ".join(files))
<|endoftext|>


<commit_before>""""""
import os
import re
import shutil
import sys
from os import chdir as cd
from os.path import join as pjoin

from subprocess import Popen, PIPE, CalledProcessError, check_call

# ----------------------------------------------------------------------------
# Globals
#
pages_dir = 'gh-pages'
html_dir = '_build/html'
pdf_dir = '_build/latex'
pages_repo = 'git@github.com:ajdawson/windspharm.git'

# ----------------------------------------------------------------------------
# Functions
#
def sh(cmd):
    """Execute command in a subshell, return status code."""
    return check_call(cmd, shell=True)

def sh2(cmd):
    """Execute command in a subshell, return stdout.

    Stderr is unbuffered from the subshell.x.

    """
    p = Popen(cmd, stdout=PIPE, shell=True)
    out = p.communicate()[0]
    retcode = p.returncode
    if retcode:
        raise CalledProcessError(retcode, cmd)
    else:
        return out.rstrip()

def init_repo(path):
    """Clone the gh-pages repo to a given location."""
    sh('git clone {} {}'.format(pages_repo, path))
    here = os.getcwdu()
    cd(path)
    sh('git checkout gh-pages')
    cd(here)

# ----------------------------------------------------------------------------
# Script starts
#
if __name__ == '__main__':

    # Determine the commit message that will be used on the branch gh-pages.
    try:
        msg = sys.argv[1]
    except IndexError:
        msg = 'Updated documentation.'

    # Check out the repository.
    startdir = os.getcwdu()
    if not os.path.exists(pages_dir):
        # Initialize the repository.
        init_repo(pages_dir)
    else:
        # If the repository exists make sure it is on the right branch and
        # is up-to-date.
        cd(pages_dir)
        sh('git checkout gh-pages')
        sh('git pull')
        cd(startdir)

    # Copy the built documentation to the gh-pages directory.
    sh('rm -rf {}/*'.format(pages_dir))
    sh('cp -r {}/* {}/'.format(html_dir, pages_dir))
#    shutil.rmtree(pjoin(pages_dir, '*'), ignore_errors=True)
#    shutil.copytree(html_dir, pjoin(pages_dir, 'dev'))

    try:
        # Check the correct branch is being used.
        cd(pages_dir)
        status = sh2('git status | head -n 1')
        branch = re.match('\# On branch (.*)$', status).group(1)
        if branch != 'gh-pages':
            e = 'On {}, git branch is {}, must be "gh-pages"'.format(
                    pages_dir, branch)
            raise RuntimeError(e)

        # Make a commit to the gh-pages branch.
        sh('git add -A')
        sh('git commit -m "{}"'.format(msg))

        # Print a summary of last 3 commits.
        print
        print 'Most recent 3 commits to "gh-pages":'
        sys.stdout.flush()
        sh('git --no-pager log --oneline HEAD~3..')
    finally:
        cd(startdir)

    print
    print 'Done. Pleave verify the build in: {}'.format(pages_dir)
    print 'If everything looks good, "git push origin gh-pages"'

<commit_msg>Corrected typo in gh-pages doc generation script.<commit_after>""""""
import os
import re
import shutil
import sys
from os import chdir as cd
from os.path import join as pjoin

from subprocess import Popen, PIPE, CalledProcessError, check_call

# ----------------------------------------------------------------------------
# Globals
#
pages_dir = 'gh-pages'
html_dir = '_build/html'
pdf_dir = '_build/latex'
pages_repo = 'git@github.com:ajdawson/windspharm.git'

# ----------------------------------------------------------------------------
# Functions
#
def sh(cmd):
    """Execute command in a subshell, return status code."""
    return check_call(cmd, shell=True)

def sh2(cmd):
    """Execute command in a subshell, return stdout.

    Stderr is unbuffered from the subshell.x.

    """
    p = Popen(cmd, stdout=PIPE, shell=True)
    out = p.communicate()[0]
    retcode = p.returncode
    if retcode:
        raise CalledProcessError(retcode, cmd)
    else:
        return out.rstrip()

def init_repo(path):
    """Clone the gh-pages repo to a given location."""
    sh('git clone {} {}'.format(pages_repo, path))
    here = os.getcwdu()
    cd(path)
    sh('git checkout gh-pages')
    cd(here)

# ----------------------------------------------------------------------------
# Script starts
#
if __name__ == '__main__':

    # Determine the commit message that will be used on the branch gh-pages.
    try:
        msg = sys.argv[1]
    except IndexError:
        msg = 'Updated documentation.'

    # Check out the repository.
    startdir = os.getcwdu()
    if not os.path.exists(pages_dir):
        # Initialize the repository.
        init_repo(pages_dir)
    else:
        # If the repository exists make sure it is on the right branch and
        # is up-to-date.
        cd(pages_dir)
        sh('git checkout gh-pages')
        sh('git pull')
        cd(startdir)

    # Copy the built documentation to the gh-pages directory.
    sh('rm -rf {}/*'.format(pages_dir))
    sh('cp -r {}/* {}/'.format(html_dir, pages_dir))
#    shutil.rmtree(pjoin(pages_dir, '*'), ignore_errors=True)
#    shutil.copytree(html_dir, pjoin(pages_dir, 'dev'))

    try:
        # Check the correct branch is being used.
        cd(pages_dir)
        status = sh2('git status | head -n 1')
        branch = re.match('\# On branch (.*)$', status).group(1)
        if branch != 'gh-pages':
            e = 'On {}, git branch is {}, must be "gh-pages"'.format(
                    pages_dir, branch)
            raise RuntimeError(e)

        # Make a commit to the gh-pages branch.
        sh('git add -A')
        sh('git commit -m "{}"'.format(msg))

        # Print a summary of last 3 commits.
        print
        print 'Most recent 3 commits to "gh-pages":'
        sys.stdout.flush()
        sh('git --no-pager log --oneline HEAD~3..')
    finally:
        cd(startdir)

    print
    print 'Done. Please verify the build in: {}'.format(pages_dir)
    print 'If everything looks good, "git push origin gh-pages"'

<|endoftext|>


<commit_before>from base10.helpers import (
    MetricHelper,
    MetricHandler,)

__version__ = '0.5.2'

__all__ = [
    'MetricHelper',
    'MetricHandler',
]
<commit_msg>Bump version number<commit_after>from base10.helpers import (
    MetricHelper,
    MetricHandler,)

__version__ = '0.5.3'

__all__ = [
    'MetricHelper',
    'MetricHandler',
]
<|endoftext|>


<commit_before>"""
ISI flat file format parser.

run test(s) with
```
python -m isiparse path/to/data.isi
```
"""

# TODO:
# [ ] rework strpisimonth to be strpisidate, returning a tuple instead of an integer;
#     I *thought* the PD field was only ever month and maybe day, but sometimes it duplicates the year (PY) field too
#     Once this works, write an assertion "not ('PY' in fields and 'PD' in fields and fields['PD'].year is not None) or (fields['PY'] == fields['PD'].year)

def is_WOS_number(w):
    """
    check that the string w is a WOS number
    You find these in the 'UT' field in exported .isi files.
    It is also called the "Accession Number"
    """
    try:
        header, number = w[:4], w[4:]
        assert header == "WOS:"
        assert len(number) == 15
        assert all(int(e) for e in number))
    except: #<-- ugh, lazy
        return False

class EmptyModule(): pass

try:
    import builtins
except ImportError:
    # hack around python2/3 (in)compatibility
    builtins = EmptyModule()
    import codecs
    builtins.open = codecs.open

import sys

from datetime import date
import time
from util import chomp

import logging

class ISIFormatError(Exception):
  pass

#class NumberedISIFormatError(ISIFormatError):
#	def __init__(self, line, *args):
#		self._line #^ this is meant to DRY up the line[%d] shit
#		super()    # but i don't want to think about it right now

def strpisimonth(d, fmt):
    """
    parse an ISI format "date" (PD field). This sometimes doesn't exist, sometimes includes a month, and sometimes includes a day, and sometimes includes a year.

    Known formats:
    Month ("%b")
    Month Day ("%b %d")
    Month-Month ("%b-%b") --- this gets coerced to the first %b, dropping the month range
    Season ("%s") --- this gets coerced to use the first month of the given season
    Month Day Year ("%b %d %Y")
    Month Year ("%b %Y")

    returns a tuple (year, month, day). This is like a datetime object, except that any of the three may be None to indicate missing data.
    """
    try:
        # handle the extension formats by a mixture of edits to the data and to fmt
        # then fall back on the standard strptime (whether or not we went through an extension format or not)
        if fmt == "%b-%b":
            # handle a month range e.g. "2006 APR-MAY"
            # which strptime can't handle
            l = d.find("-")
            if not (len(d) == 3 + 1 + 3): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' has wrong length for format %%b-%%b" % (d,))
            if not (d[3] == "-"): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' missing hyphen for format %%b-%%b" % (d,))
            end_month = time.strptime(d[l+1:], "%b") #this is a no-op, but it ensures that the format looks like we think it does
            d, fmt = d[:-4], "%b"
        elif fmt == "%s":
            # handle a season e.g. "FAL"
            # remap season to a month
            # this is a kludge, but at least it gives us *something*
            # each season gets three months out of twelve
            seasons = {'SPR': "MAR", 'SUM': "JUN", 'FAL': 'SEP', 'WIN': "DEC"}
            if d in seasons:
                d, fmt = seasons[d], "%b"
            else:
                raise ValueError("unknown ISI season '%s'" % (d,))

        return time.strptime(d, fmt).tm_mon
    except ValueError:
        # coerce any error to strptime()'s default message, to hide the real source
        raise ValueError("time data '%s' does not match format '%s'" % (d,fmt))

def parse_month(m):
    """
    """
    # try all the formats, starting with the most restrictive
    # strpisimonth requires an exact match so order shouldn't matter
    for fmt in ["%b %d", "%b", "%b-%b", "%s", "%b %d %Y", "%b %Y"]:
        try:
            return strpisimonth(m, fmt)
        except ValueError:
            # silence the ValueErrors from a failed strpisimonth()
            # however if everything ValueError's out, the else clause will fire
            #print("error parsing '%s' with format '%s'" % (d, fmt))
            #traceback.print_exc() #DEBUG; warning: enabling this will be verrrry noisy
            continue
    else:
        raise ValueError("ISI time data '%s': format unrecognized" % (d,))

    assert False, "NOTREACHED"

def parse_year(year):
    # parse out the year
    if not isinstance(year, str):
        raise TypeError

    if len(year) != 4:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))
    try:
        _year = int(year)
        if not (1900 <= int(_year) <= 3000):
            raise ValueError("year out of range")
        return _year
    except ValueError:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))

def records(isi):
	"""
	read records from an open ISI-format file

  	TODO: make into a method on isireader()
	"""

	if isi.encoding.lower() != "utf-8-sig":
		logging.warn("%s opened in '%s' instead of BOM-compatible 'utf-8-sig'" % (isi, isi.encoding,))

	isi = (chomp(line) for line in isi) #(lazily) autostrip trailing newlines
	isi = ((i+1, line) for i, line in enumerate(isi)) #number isi lines, so we can give good debugging output
	def partition_lines(isi): #this is longer than it needs to be simply because I want to pass 'i' through it for the 'sep' error message
				   #it would be shorter written as a simple map function, without an internal loop
		for i, line in isi:
			#print(i,line) #DEBUG (handy: can see *every line* go past)
			#import IPython; IPython.embed()

			# partition the line
			# (note: EF, ER, and the blank line between records break the pattern, hence the verbose if-chain)
			# ( also being stateful and ordering ops choosily means we can reuse 'line' without worrying stomping it before the 'sep' check)
			if len(line) > 2:
				sep = line[2]
				if sep != ' ':
					raise ISIFormatError("line[%d]: Malformed ISI line: '%r'" % (i,line))
			tag = line[:2]
			line = line[3:] #subtle: [3:] works even if the line is not that long, it just gives "". This is probably acceptable:
				# on pure-tag lines like "ER" line will end up as "" as expected, and on the blank line, so will tag

			yield i, (tag, line)

	isi = partition_lines(isi)

	# ---- parse very first line
	_, (tag, header) = next(isi)
	if tag != 'FN':
		raise ISIFormatError("Malformed header: '%s %s'" % (tag, header,))

	#TODO: pass the header and version upwards somehow (probably by wrapping yet another generator)
	# TODO: handle the case where the file ends at these next()s.

	# ---- parse very second line
	_, (tag, version) = next(isi)
	if tag != "VR":
			raise ISIFormatError("Malformed version '%s %s'" % (tag, version,))
	if version != "1.0": #TODO: make this optional, via flag (yayyy feature creep)
		raise ISIFormatError("Unsupported version '%s'" % (version,))

	# ---- parse everything else

	def fields():
		"""
		a generator over the fields in an isi record
		each key is a two letter ISI code as at <TODO>
		and each value is a string of everything following it, possibly on continuation lines.
		Now, some fields are actually lists, and some are simply extended paragraphs.
		(and some seem to be jerks that don't fit either)
		For now, content is returned as a list of the lines, even content of only a single element,
			either interpret beyond that should happen at a higher level or be carefully special-cased in.

		Control flow here is a bit unusual:
		 fields() and records() work together to process *the same stream*
		when fields() gives up on the stream, records() goes back to nomming it,
		and usually it immediately hands control back to fields()
		"""

		# the (apparent) format of an ISI field is:
		# [two letter character code][single space][content][newline]([two spaces][single space][content])*
		# that is, continuation lines are given by a two-"letter" code of '  '
		# This is a perfectly difficult format to parse, for which a very stateful generator is a great answer.

		# output will be (field, content)
		# except to avoid the quadradtic string-realloc curse, we store content as a list until we're done

		field, content = None, None

		# short maps to convert content's output form to something resembling it's real content
		# all fields are flattened by default unless they are explicitly listed here
		# inferring from the data I have, *some* fields, if they overflow,
		# are implicitly joined by newlines(?)
		# and others by spaces
		# and .. uh.. yeah.

		# TODO: figure out if I can assume. that abstracts do not seem to follow this convention wigs me out.
		paragraph = lambda c: str.join("\n", c)
		semicolon_list = lambda c: str.join(" ", c).split("; ")
		newline_list = lambda c: c
		flatten = lambda c: " ".join(c)
		reformatters = {'AB': paragraph, #abstracts are just paragraphs reflowed
			'SC': semicolon_list,  #subject categories are a a list split by semicolons
			'WC': semicolon_list,  #ditto for web-of-science categories
			#....
			'C1': newline_list,
			#'PY': parse_year, #TODO
			#'PD': parse_month,
			}
		_outcount = 0 #DEBUG
		for i, (tag, line) in isi:
			# figure out what type of line we have:
			if tag != '  ':
				# a new field
				assert tag == tag.upper(), "ISI field tags should all be two letter upper case strings"

				# return what's collected (if any)
				# putting this here is cleaner than dupeing this code above to do a do-while
				if field is None:
					# but only if there's anything to return, of course
					assert content is None, "invariant: field exists <=> content exists"
				else:
					_outcount += 1
					yield field, reformatters.get(field, flatten)(content)

				if tag == 'ER':
					# end record
					break
				elif tag == 'EF':
					# end file
					assert _outcount == 0 #XXX this should be an ISIFormatError
					return          #cut short the generator
				else:
					# common case: a new tag
					# reset the accumulators and continue
					field, content = tag, [line]
			else:
				# continuation line
				if field is None:
					raise ISIFormatError("line[%d]: Field continuation line seen before any field tag: '%s'" % (i,line,))
				assert content is not None, "invariant: field exists <=> content exists"
				content += [line]
		else:
			raise ISIFormatError("line[%d]: Record (and file) ended before 'ER' marker." % (i,))

	while True:
		# read records
		# this loop ends when we read a fake 'record' that consists of a single tag 'EF'
		r = list(fields())
		if not r:
			# end of file (NB: this isn't the same as EOF, the actual file might continue)
			break

		# sanity check
		if len(r) != len({k for k,v in r}):
			raise ISIFormatError('Duplicate fields seen in a record: %s' % (fields,))
		# coerce to dict form, now that we know it's safe
		yield dict(r)

		# every record is followed by a single empty line
		# TODO: maybe just scrap this; how important is it really to enforce the BLANK LINE RULE? seriously. autists ehre we come.
		try:
			i, blank = next(isi)
			if blank != ("",""):
				raise ISIFormatError("line[%d]: Missing blank line after record. Instead saw: %s" % (i, blank))
		except StopIteration:
			raise ISIFormatError("File ended before 'EF' marker.") #XXX awkward; DRY this with the one below
	else:
		# ensure that we exit the loop by 'EF' and not some other way
		raise ISIFormatError("File ended before 'EF' marker.")

class reader():
	def __init__(self, name, encoding="utf-8-sig"):
		self._file = builtins.open(name, "r", encoding=encoding)

	def __iter__(self):
		return iter(records(self._file))

	def __enter__(self):
		assert not self._file.closed
		return self

	def __exit__(self, *_unused):
		self._file.close()

def open(fname, mode="r", encoding="utf-8-sig"):
	"""
	utf-8-sig is the most widely compatible text codec. It handles both ASCII files (because of utf-8 backwards compatibility) and most Unicode files, with or without a BOM.
	If you happen to have a different encoding, you can provide it. See the codecs module for options.
	TODO: use the chardet module?
	"""
	if mode != "r":
		raise NotImplementedError("only have read-only support so far")

	return reader(fname)

if __name__ == '__main__':
	import sys
	with open(sys.argv[1]) as isi:
		for i, record in enumerate(isi):
			print(i, record)
			#import IPython; IPython.embed()

<commit_msg>WHOOPS<commit_after>"""
ISI flat file format parser.

run test(s) with
```
python -m isiparse path/to/data.isi
```
"""

# TODO:
# [ ] rework strpisimonth to be strpisidate, returning a tuple instead of an integer;
#     I *thought* the PD field was only ever month and maybe day, but sometimes it duplicates the year (PY) field too
#     Once this works, write an assertion "not ('PY' in fields and 'PD' in fields and fields['PD'].year is not None) or (fields['PY'] == fields['PD'].year)

def is_WOS_number(w):
    """
    check that the string w is a WOS number
    You find these in the 'UT' field in exported .isi files.
    It is also called the "Accession Number"
    """
    try:
        header, number = w[:4], w[4:]
        assert header == "WOS:"
        assert len(number) == 15
        assert all(int(e) for e in number))
    except: #<-- ugh, lazy
        return False

    return True

class EmptyModule(): pass

try:
    import builtins
except ImportError:
    # hack around python2/3 (in)compatibility
    builtins = EmptyModule()
    import codecs
    builtins.open = codecs.open

import sys

from datetime import date
import time
from util import chomp

import logging

class ISIFormatError(Exception):
  pass

#class NumberedISIFormatError(ISIFormatError):
#	def __init__(self, line, *args):
#		self._line #^ this is meant to DRY up the line[%d] shit
#		super()    # but i don't want to think about it right now

def strpisimonth(d, fmt):
    """
    parse an ISI format "date" (PD field). This sometimes doesn't exist, sometimes includes a month, and sometimes includes a day, and sometimes includes a year.

    Known formats:
    Month ("%b")
    Month Day ("%b %d")
    Month-Month ("%b-%b") --- this gets coerced to the first %b, dropping the month range
    Season ("%s") --- this gets coerced to use the first month of the given season
    Month Day Year ("%b %d %Y")
    Month Year ("%b %Y")

    returns a tuple (year, month, day). This is like a datetime object, except that any of the three may be None to indicate missing data.
    """
    try:
        # handle the extension formats by a mixture of edits to the data and to fmt
        # then fall back on the standard strptime (whether or not we went through an extension format or not)
        if fmt == "%b-%b":
            # handle a month range e.g. "2006 APR-MAY"
            # which strptime can't handle
            l = d.find("-")
            if not (len(d) == 3 + 1 + 3): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' has wrong length for format %%b-%%b" % (d,))
            if not (d[3] == "-"): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' missing hyphen for format %%b-%%b" % (d,))
            end_month = time.strptime(d[l+1:], "%b") #this is a no-op, but it ensures that the format looks like we think it does
            d, fmt = d[:-4], "%b"
        elif fmt == "%s":
            # handle a season e.g. "FAL"
            # remap season to a month
            # this is a kludge, but at least it gives us *something*
            # each season gets three months out of twelve
            seasons = {'SPR': "MAR", 'SUM': "JUN", 'FAL': 'SEP', 'WIN': "DEC"}
            if d in seasons:
                d, fmt = seasons[d], "%b"
            else:
                raise ValueError("unknown ISI season '%s'" % (d,))

        return time.strptime(d, fmt).tm_mon
    except ValueError:
        # coerce any error to strptime()'s default message, to hide the real source
        raise ValueError("time data '%s' does not match format '%s'" % (d,fmt))

def parse_month(m):
    """
    """
    # try all the formats, starting with the most restrictive
    # strpisimonth requires an exact match so order shouldn't matter
    for fmt in ["%b %d", "%b", "%b-%b", "%s", "%b %d %Y", "%b %Y"]:
        try:
            return strpisimonth(m, fmt)
        except ValueError:
            # silence the ValueErrors from a failed strpisimonth()
            # however if everything ValueError's out, the else clause will fire
            #print("error parsing '%s' with format '%s'" % (d, fmt))
            #traceback.print_exc() #DEBUG; warning: enabling this will be verrrry noisy
            continue
    else:
        raise ValueError("ISI time data '%s': format unrecognized" % (d,))

    assert False, "NOTREACHED"

def parse_year(year):
    # parse out the year
    if not isinstance(year, str):
        raise TypeError

    if len(year) != 4:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))
    try:
        _year = int(year)
        if not (1900 <= int(_year) <= 3000):
            raise ValueError("year out of range")
        return _year
    except ValueError:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))

def records(isi):
	"""
	read records from an open ISI-format file

  	TODO: make into a method on isireader()
	"""

	if isi.encoding.lower() != "utf-8-sig":
		logging.warn("%s opened in '%s' instead of BOM-compatible 'utf-8-sig'" % (isi, isi.encoding,))

	isi = (chomp(line) for line in isi) #(lazily) autostrip trailing newlines
	isi = ((i+1, line) for i, line in enumerate(isi)) #number isi lines, so we can give good debugging output
	def partition_lines(isi): #this is longer than it needs to be simply because I want to pass 'i' through it for the 'sep' error message
				   #it would be shorter written as a simple map function, without an internal loop
		for i, line in isi:
			#print(i,line) #DEBUG (handy: can see *every line* go past)
			#import IPython; IPython.embed()

			# partition the line
			# (note: EF, ER, and the blank line between records break the pattern, hence the verbose if-chain)
			# ( also being stateful and ordering ops choosily means we can reuse 'line' without worrying stomping it before the 'sep' check)
			if len(line) > 2:
				sep = line[2]
				if sep != ' ':
					raise ISIFormatError("line[%d]: Malformed ISI line: '%r'" % (i,line))
			tag = line[:2]
			line = line[3:] #subtle: [3:] works even if the line is not that long, it just gives "". This is probably acceptable:
				# on pure-tag lines like "ER" line will end up as "" as expected, and on the blank line, so will tag

			yield i, (tag, line)

	isi = partition_lines(isi)

	# ---- parse very first line
	_, (tag, header) = next(isi)
	if tag != 'FN':
		raise ISIFormatError("Malformed header: '%s %s'" % (tag, header,))

	#TODO: pass the header and version upwards somehow (probably by wrapping yet another generator)
	# TODO: handle the case where the file ends at these next()s.

	# ---- parse very second line
	_, (tag, version) = next(isi)
	if tag != "VR":
			raise ISIFormatError("Malformed version '%s %s'" % (tag, version,))
	if version != "1.0": #TODO: make this optional, via flag (yayyy feature creep)
		raise ISIFormatError("Unsupported version '%s'" % (version,))

	# ---- parse everything else

	def fields():
		"""
		a generator over the fields in an isi record
		each key is a two letter ISI code as at <TODO>
		and each value is a string of everything following it, possibly on continuation lines.
		Now, some fields are actually lists, and some are simply extended paragraphs.
		(and some seem to be jerks that don't fit either)
		For now, content is returned as a list of the lines, even content of only a single element,
			either interpret beyond that should happen at a higher level or be carefully special-cased in.

		Control flow here is a bit unusual:
		 fields() and records() work together to process *the same stream*
		when fields() gives up on the stream, records() goes back to nomming it,
		and usually it immediately hands control back to fields()
		"""

		# the (apparent) format of an ISI field is:
		# [two letter character code][single space][content][newline]([two spaces][single space][content])*
		# that is, continuation lines are given by a two-"letter" code of '  '
		# This is a perfectly difficult format to parse, for which a very stateful generator is a great answer.

		# output will be (field, content)
		# except to avoid the quadradtic string-realloc curse, we store content as a list until we're done

		field, content = None, None

		# short maps to convert content's output form to something resembling it's real content
		# all fields are flattened by default unless they are explicitly listed here
		# inferring from the data I have, *some* fields, if they overflow,
		# are implicitly joined by newlines(?)
		# and others by spaces
		# and .. uh.. yeah.

		# TODO: figure out if I can assume. that abstracts do not seem to follow this convention wigs me out.
		paragraph = lambda c: str.join("\n", c)
		semicolon_list = lambda c: str.join(" ", c).split("; ")
		newline_list = lambda c: c
		flatten = lambda c: " ".join(c)
		reformatters = {'AB': paragraph, #abstracts are just paragraphs reflowed
			'SC': semicolon_list,  #subject categories are a a list split by semicolons
			'WC': semicolon_list,  #ditto for web-of-science categories
			#....
			'C1': newline_list,
			#'PY': parse_year, #TODO
			#'PD': parse_month,
			}
		_outcount = 0 #DEBUG
		for i, (tag, line) in isi:
			# figure out what type of line we have:
			if tag != '  ':
				# a new field
				assert tag == tag.upper(), "ISI field tags should all be two letter upper case strings"

				# return what's collected (if any)
				# putting this here is cleaner than dupeing this code above to do a do-while
				if field is None:
					# but only if there's anything to return, of course
					assert content is None, "invariant: field exists <=> content exists"
				else:
					_outcount += 1
					yield field, reformatters.get(field, flatten)(content)

				if tag == 'ER':
					# end record
					break
				elif tag == 'EF':
					# end file
					assert _outcount == 0 #XXX this should be an ISIFormatError
					return          #cut short the generator
				else:
					# common case: a new tag
					# reset the accumulators and continue
					field, content = tag, [line]
			else:
				# continuation line
				if field is None:
					raise ISIFormatError("line[%d]: Field continuation line seen before any field tag: '%s'" % (i,line,))
				assert content is not None, "invariant: field exists <=> content exists"
				content += [line]
		else:
			raise ISIFormatError("line[%d]: Record (and file) ended before 'ER' marker." % (i,))

	while True:
		# read records
		# this loop ends when we read a fake 'record' that consists of a single tag 'EF'
		r = list(fields())
		if not r:
			# end of file (NB: this isn't the same as EOF, the actual file might continue)
			break

		# sanity check
		if len(r) != len({k for k,v in r}):
			raise ISIFormatError('Duplicate fields seen in a record: %s' % (fields,))
		# coerce to dict form, now that we know it's safe
		yield dict(r)

		# every record is followed by a single empty line
		# TODO: maybe just scrap this; how important is it really to enforce the BLANK LINE RULE? seriously. autists ehre we come.
		try:
			i, blank = next(isi)
			if blank != ("",""):
				raise ISIFormatError("line[%d]: Missing blank line after record. Instead saw: %s" % (i, blank))
		except StopIteration:
			raise ISIFormatError("File ended before 'EF' marker.") #XXX awkward; DRY this with the one below
	else:
		# ensure that we exit the loop by 'EF' and not some other way
		raise ISIFormatError("File ended before 'EF' marker.")

class reader():
	def __init__(self, name, encoding="utf-8-sig"):
		self._file = builtins.open(name, "r", encoding=encoding)

	def __iter__(self):
		return iter(records(self._file))

	def __enter__(self):
		assert not self._file.closed
		return self

	def __exit__(self, *_unused):
		self._file.close()

def open(fname, mode="r", encoding="utf-8-sig"):
	"""
	utf-8-sig is the most widely compatible text codec. It handles both ASCII files (because of utf-8 backwards compatibility) and most Unicode files, with or without a BOM.
	If you happen to have a different encoding, you can provide it. See the codecs module for options.
	TODO: use the chardet module?
	"""
	if mode != "r":
		raise NotImplementedError("only have read-only support so far")

	return reader(fname)

if __name__ == '__main__':
	import sys
	with open(sys.argv[1]) as isi:
		for i, record in enumerate(isi):
			print(i, record)
			#import IPython; IPython.embed()

<|endoftext|>