<commit_before>import h5py
import os
import warnings
import re
import numpy as np
from datetime import datetime
import argparse

parser = argparse.ArgumentParser(description='Quick overview of batch simulation')
parser.add_argument('-S', '--summary', action='store_true', help='Summary only')
parser.add_argument('-s', '--save', action='store_true', help='Save to file')
parser.add_argument('-f', '--filename', type=str, help='Save to file with filename', default='overview')
parser.add_argument('-t', '--timestamp', action='store_true', help='Add timestamp to filename')
parser.add_argument('-d', '--directory', type=str, help='Search for hdf5 files in directory', default='output')
args = parser.parse_args()

if args.timestamp:
    args.filename = args.filename + '-' + datetime.today().strftime('%Y-%m-%d-%H:%M:%S')

if args.filename != 'overview':
    args.save = True

regex = re.compile(r'\d+')

if args.save:
    file = open(args.filename, 'w')

for dirName, subdirList, fileList in os.walk(args.directory):
    if not fileList:
        continue
    subdirList.sort()
    fileList.sort()
    chainlen  = []
    realization_num = []
    variance  = []
    walltime  = []
    resets    = []
    converged = []
    saturated = []
    succeeded = []
    finished  = []

    if not args.summary:
        header = "{:8} {:15} {:12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length", "Realization", "Variance", "Time",
                                                                                "Resets", "Converged", "Saturated",
                                                                                "Succeeded", "Finished")
        print(header)
        if args.save:
            file.write(header)
    for h5path in fileList:
        # print("Filepath: ", h5path)
        try:
            h5file = h5py.File(dirName + '/' + h5path, 'r', swmr=True)
            if not isinstance(h5file, h5py.File):
                raise IOError("File not readable")
        except Exception as er:
            print("Could not open file [", h5path, "] Reason: ", er)
            continue

        try:
            realization_name = h5path.replace('.h5', '')
            chainlen.append(h5file['xDMRG/measurements/length'][-1])
            realization_num.append([int(x) for x in regex.findall(realization_name)][-1])
            variance.append(h5file['xDMRG/measurements/energy_variance_per_site'][-1])
            walltime.append(h5file['xDMRG/sim_status/wall_time'][-1])
            resets.append(h5file['xDMRG/sim_status/num_resets'][-1])
            converged.append(h5file['xDMRG/sim_status/simulation_has_converged'][-1])
            saturated.append(h5file['xDMRG/sim_status/simulation_has_saturated'][-1])
            succeeded.append(h5file['xDMRG/sim_status/simulation_has_succeeded'][-1])
            try:
                finished.append(h5file['common/finOK'][-1])
            except:
                finished.append(0)

            if not args.summary:
                entry = "{:<8} {:<15} {:<12.4f} {:>12.4f} {:>12} {:>12} {:>12} {:>12} {:>12}".format(
                    chainlen[-1],
                    realization_num[-1],
                    np.log10(variance[-1]),
                    walltime[-1] / 60,
                    resets[-1],
                    converged[-1],
                    saturated[-1],
                    succeeded[-1],
                    finished[-1])
                print(entry)
                if args.save:
                    file.write(entry)

        except Exception as er:
            print("Could not read dataset. Reason: ", er)
            continue
    header = "{:8} {:15} {:12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length","Total sims", "Avg Var", "Avg Time",
                                                                                "Avg Resets", "Sum Con", "Sum Sat",
                                                                                "Sum Suc", "Sum Fin")
    entry = "{:<8} {:<15} {:<12.4f} {:>12.4f} {:>12.3f} {:>12} {:>12} {:>12} {:>12}".format(
        np.int(np.mean(chainlen)),
        len(realization_num),
        np.log10(np.nanmean(variance)),
        np.mean(walltime) / 60,
        np.mean(resets),
        np.sum(converged),
        np.sum(saturated),
        np.sum(succeeded),
        np.sum(finished))

    print(header)
    print(entry)
    if args.save:
        file.write(header)
        file.write(entry)
if args.save:
    file.close()
<commit_msg>Testing lower subspace error<commit_after>import h5py
import os
import warnings
import re
import numpy as np
from datetime import datetime
import argparse

parser = argparse.ArgumentParser(description='Quick overview of batch simulation')
parser.add_argument('-S', '--summary', action='store_true', help='Summary only')
parser.add_argument('-s', '--save', action='store_true', help='Save to file')
parser.add_argument('-f', '--filename', type=str, help='Save to file with filename', default='overview')
parser.add_argument('-t', '--timestamp', action='store_true', help='Add timestamp to filename')
parser.add_argument('-d', '--directory', type=str, help='Search for hdf5 files in directory', default='output')
args = parser.parse_args()

if args.timestamp:
    args.filename = args.filename + '-' + datetime.today().strftime('%Y-%m-%d-%H:%M:%S')

if args.filename != 'overview':
    args.save = True

regex = re.compile(r'\d+')

if args.save:
    file = open(args.filename, 'w')

for dirName, subdirList, fileList in os.walk(args.directory):
    if not fileList:
        continue
    subdirList.sort()
    fileList.sort()
    chainlen  = []
    realization_num = []
    variance  = []
    walltime  = []
    resets    = []
    converged = []
    saturated = []
    succeeded = []
    finished  = []

    if not args.summary:
        header = "{:8} {:15} {:12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length", "Realization", "Variance", "Time",
                                                                                "Resets", "Converged", "Saturated",
                                                                                "Succeeded", "Finished")
        print(header)
        if args.save:
            file.write(header + '\n')
    for h5path in fileList:
        # print("Filepath: ", h5path)
        try:
            h5file = h5py.File(dirName + '/' + h5path, 'r', swmr=True)
            if not isinstance(h5file, h5py.File):
                raise IOError("File not readable")
        except Exception as er:
            print("Could not open file [", h5path, "] Reason: ", er)
            continue

        try:
            realization_name = h5path.replace('.h5', '')
            chainlen.append(h5file['xDMRG/measurements/length'][-1])
            realization_num.append([int(x) for x in regex.findall(realization_name)][-1])
            variance.append(h5file['xDMRG/measurements/energy_variance_per_site'][-1])
            walltime.append(h5file['xDMRG/sim_status/wall_time'][-1])
            resets.append(h5file['xDMRG/sim_status/num_resets'][-1])
            converged.append(h5file['xDMRG/sim_status/simulation_has_converged'][-1])
            saturated.append(h5file['xDMRG/sim_status/simulation_has_saturated'][-1])
            succeeded.append(h5file['xDMRG/sim_status/simulation_has_succeeded'][-1])
            try:
                finished.append(h5file['common/finOK'][-1])
            except:
                finished.append(0)

            if not args.summary:
                entry = "{:<8} {:<15} {:<12.4f} {:>12.4f} {:>12} {:>12} {:>12} {:>12} {:>12}".format(
                    chainlen[-1],
                    realization_num[-1],
                    np.log10(variance[-1]),
                    walltime[-1] / 60,
                    resets[-1],
                    converged[-1],
                    saturated[-1],
                    succeeded[-1],
                    finished[-1])
                print(entry)
                if args.save:
                    file.write(entry + '\n')

        except Exception as er:
            print("Could not read dataset. Reason: ", er)
            continue
    header = "{:8} {:15} {:12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length","Total sims", "Avg Var", "Avg Time",
                                                                                "Avg Resets", "Sum Con", "Sum Sat",
                                                                                "Sum Suc", "Sum Fin")
    entry = "{:<8} {:<15} {:<12.4f} {:>12.4f} {:>12.3f} {:>12} {:>12} {:>12} {:>12}".format(
        np.int(np.mean(chainlen)),
        len(realization_num),
        np.log10(np.nanmean(variance)),
        np.mean(walltime) / 60,
        np.mean(resets),
        np.sum(converged),
        np.sum(saturated),
        np.sum(succeeded),
        np.sum(finished))

    print(header)
    print(entry)
    if args.save:
        file.write(header + '\n')
        file.write(entry + '\n')
if args.save:
    file.close()
<|endoftext|>


<commit_before>import copy
import unittest

from baton.collections import DataObjectReplicaCollection, IrodsMetadata
from baton.models import DataObjectReplica

class TestIrodsMetadata(unittest.TestCase):
    """
    Tests for `IrodsMetadata`.
    """
    def setUp(self):
        self.metadata = IrodsMetadata({"attribute_a": {"value_1"}, "attribute_b": {"value_b"}})

    def test_equal(self):
        self.assertEqual(copy.deepcopy(self.metadata), self.metadata)

    def test_add_when_key_does_not_exists(self):
        self.metadata.add("key", "value")
        self.assertEqual(self.metadata["key"], {"value"})

    def test_add_when_key_exists(self):
        self.metadata["key"] = {"value_1"}
        self.metadata.add("key", "value_2")
        self.assertEqual(self.metadata["key"], {"value_1", "value_2"})

class TestDataObjectReplicaCollection(unittest.TestCase):
    """
    Tests for `DataObjectReplicaCollection`.
    """
    def setUp(self):
        self._checksum = "1234567890"
        self._other_replica = DataObjectReplica(99, self._checksum)
        self._replicas = [DataObjectReplica(i, self._checksum, str(i), str(i), True) for i in range(10)]
        self._collection = DataObjectReplicaCollection(self._replicas)

    def test_init_with_no_initial(self):
        self.assertEqual(len(DataObjectReplicaCollection()), 0)

    def test_init_with_initial(self):
        self.assertCountEqual(self._collection, self._replicas)

    def test_get_by_number_when_does_not_exist(self):
        self.assertRaises(KeyError, self._collection.get_by_number, -1)

    def test_get_by_number_when_exists(self):
        self.assertEqual(self._collection.get_by_number(0), self._replicas[0])

    def test_get_out_of_date_when_none(self):
        self.assertEqual(len(self._collection.get_out_of_date()), 0)

    def test_get_out_of_date_when_out_of_date_exist(self):
        self._replicas[1].up_to_date = False
        self._replicas[5].up_to_date = False
        self.assertCountEqual(self._collection.get_out_of_date(), [self._replicas[1], self._replicas[5]])

    def test_add_when_does_not_contain(self):
        self._collection.add(self._other_replica)
        self.assertEqual(self._collection.get_by_number(self._other_replica.number), self._other_replica)

    def test_add_when_does_contain(self):
        self.assertRaises(ValueError, self._collection.add, self._replicas[0])

    def test_remove_by_number_when_not_exists(self):
        self.assertRaises(ValueError, self._collection.remove, self._other_replica.number)

    def test_remove_by_number_when_exists(self):
        self._collection.remove(self._replicas[0].number)
        self.assertRaises(KeyError, self._collection.get_by_number, self._replicas[0].number)
        self.assertNotIn(self._replicas[0], self._collection)

    def test_remove_by_object_when_not_exists(self):
        self.assertRaises(ValueError, self._collection.remove, self._other_replica)

    def test_remove_by_object_when_exists(self):
        self._collection.remove(self._replicas[0])
        self.assertNotIn(self._replicas[0], self._collection)

    def test_remove_by_unsupported_type(self):
        self.assertRaises(TypeError, self._collection.remove, None)

    def test_eq_when_not_equal(self):
        another_collection = DataObjectReplicaCollection()
        self.assertNotEqual(another_collection, self._collection)

    def test_eq_when_equal(self):
        self.assertEqual(copy.copy(self._collection), self._collection)

    def test_len(self):
        self.assertEqual(len(self._collection), len(self._replicas))

    def test_iter(self):
        retrieved = []
        for data_object_replica in self._collection:
            retrieved.append(data_object_replica)
        self.assertCountEqual(retrieved, self._replicas)

    def test_contains(self):
        self.assertIn(self._replicas[0], self._collection)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Corrected test.<commit_after>import copy
import unittest

from baton.collections import DataObjectReplicaCollection, IrodsMetadata
from baton.models import DataObjectReplica

class TestIrodsMetadata(unittest.TestCase):
    """
    Tests for `IrodsMetadata`.
    """
    def setUp(self):
        self.metadata = IrodsMetadata({"attribute_a": {"value_1"}, "attribute_b": {"value_b"}})

    def test_equal(self):
        self.assertEqual(copy.deepcopy(self.metadata), self.metadata)

    def test_add_when_key_does_not_exists(self):
        self.metadata.add("key", "value")
        self.assertEqual(self.metadata["key"], {"value"})

    def test_add_when_key_exists(self):
        self.metadata["key"] = {"value_1"}
        self.metadata.add("key", "value_2")
        self.assertEqual(self.metadata["key"], {"value_1", "value_2"})

class TestDataObjectReplicaCollection(unittest.TestCase):
    """
    Tests for `DataObjectReplicaCollection`.
    """
    def setUp(self):
        self._checksum = "1234567890"
        self._other_replica = DataObjectReplica(99, self._checksum)
        self._replicas = [DataObjectReplica(i, self._checksum, str(i), str(i), True) for i in range(10)]
        self._collection = DataObjectReplicaCollection(self._replicas)

    def test_init_with_no_initial(self):
        self.assertEqual(len(DataObjectReplicaCollection()), 0)

    def test_init_with_initial(self):
        self.assertCountEqual(self._collection, self._replicas)

    def test_get_by_number_when_does_not_exist(self):
        self.assertIsNone(self._collection.get_by_number(10))

    def test_get_by_number_when_exists(self):
        self.assertEqual(self._collection.get_by_number(0), self._replicas[0])

    def test_get_out_of_date_when_none(self):
        self.assertEqual(len(self._collection.get_out_of_date()), 0)

    def test_get_out_of_date_when_out_of_date_exist(self):
        self._replicas[1].up_to_date = False
        self._replicas[5].up_to_date = False
        self.assertCountEqual(self._collection.get_out_of_date(), [self._replicas[1], self._replicas[5]])

    def test_add_when_does_not_contain(self):
        self._collection.add(self._other_replica)
        self.assertEqual(self._collection.get_by_number(self._other_replica.number), self._other_replica)

    def test_add_when_does_contain(self):
        self.assertRaises(ValueError, self._collection.add, self._replicas[0])

    def test_remove_by_number_when_not_exists(self):
        self.assertRaises(ValueError, self._collection.remove, self._other_replica.number)

    def test_remove_by_number_when_exists(self):
        self._collection.remove(self._replicas[0].number)
        self.assertRaises(KeyError, self._collection.get_by_number, self._replicas[0].number)
        self.assertNotIn(self._replicas[0], self._collection)

    def test_remove_by_object_when_not_exists(self):
        self.assertRaises(ValueError, self._collection.remove, self._other_replica)

    def test_remove_by_object_when_exists(self):
        self._collection.remove(self._replicas[0])
        self.assertNotIn(self._replicas[0], self._collection)

    def test_remove_by_unsupported_type(self):
        self.assertRaises(TypeError, self._collection.remove, None)

    def test_eq_when_not_equal(self):
        another_collection = DataObjectReplicaCollection()
        self.assertNotEqual(another_collection, self._collection)

    def test_eq_when_equal(self):
        self.assertEqual(copy.copy(self._collection), self._collection)

    def test_len(self):
        self.assertEqual(len(self._collection), len(self._replicas))

    def test_iter(self):
        retrieved = []
        for data_object_replica in self._collection:
            retrieved.append(data_object_replica)
        self.assertCountEqual(retrieved, self._replicas)

    def test_contains(self):
        self.assertIn(self._replicas[0], self._collection)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>import math

import numpy.linalg as nl

import numpy as np
from numpy.testing import assert_
from numpy.random import rand

from .common import Benchmark, safe_import

with safe_import():
    import scipy.linalg as sl

def random(size):
    return rand(*size)

class Bench(Benchmark):
    params = [
        [20, 100, 500, 1000],
        ['contig', 'nocont'],
        ['numpy', 'scipy']
    ]
    param_names = ['size', 'contiguous', 'module']

    def __init__(self):
        # likely not useful to benchmark svd for large sizes
        self.time_svd.__func__.params = [[20, 100, 500]] + self.params[1:]

    def setup(self, size, contig, module):
        if module == 'numpy' and size >= 200:
            # skip: slow, and not useful to benchmark numpy
            raise NotImplementedError()

        a = random([size, size])
        # larger diagonal ensures non-singularity:
        for i in range(size):
            a[i, i] = 10*(.1+a[i, i])
        b = random([size])

        if contig != 'contig':
            a = a[-1::-1, -1::-1]  # turn into a non-contiguous array
            assert_(not a.flags['CONTIGUOUS'])

        self.a = a
        self.b = b

    def time_solve(self, size, contig, module):
        if module == 'numpy':
            nl.solve(self.a, self.b)
        else:
            sl.solve(self.a, self.b)

    def time_solve_triangular(self, size, contig, module):
        # treats self.a as a lower-triangular matrix by ignoring the strictly
        # upper-triangular part
        if module == 'numpy':
            pass
        else:
            sl.solve_triangular(self.a, self.b, lower=True)

    def time_inv(self, size, contig, module):
        if module == 'numpy':
            nl.inv(self.a)
        else:
            sl.inv(self.a)

    def time_det(self, size, contig, module):
        if module == 'numpy':
            nl.det(self.a)
        else:
            sl.det(self.a)

    def time_eigvals(self, size, contig, module):
        if module == 'numpy':
            nl.eigvals(self.a)
        else:
            sl.eigvals(self.a)

    def time_svd(self, size, contig, module):
        if module == 'numpy':
            nl.svd(self.a)
        else:
            sl.svd(self.a)

    # Retain old benchmark results (remove this if changing the benchmark)
    time_det.version = "87e530ee50eb6b6c06c7a8abe51c2168e133d5cbd486f4c1c2b9cedc5a078325"
    time_eigvals.version = "9d68d3a6b473df9bdda3d3fd25c7f9aeea7d5cee869eec730fb2a2bcd1dfb907"
    time_inv.version = "20beee193c84a5713da9749246a7c40ef21590186c35ed00a4fe854cce9e153b"
    time_solve.version = "1fe788070f1c9132cbe78a47fdb4cce58266427fc636d2aa9450e3c7d92c644c"
    time_svd.version = "0ccbda456d096e459d4a6eefc6c674a815179e215f83931a81cfa8c18e39d6e3"

class Norm(Benchmark):
    params = [
        [(20, 20), (100, 100), (1000, 1000), (20, 1000), (1000, 20)],
        ['contig', 'nocont'],
        ['numpy', 'scipy']
    ]
    param_names = ['shape', 'contiguous', 'module']

    def setup(self, shape, contig, module):
        a = np.random.randn(*shape)
        if contig != 'contig':
            a = a[-1::-1,-1::-1]  # turn into a non-contiguous array
            assert_(not a.flags['CONTIGUOUS'])
        self.a = a

    def time_1_norm(self, size, contig, module):
        if module == 'numpy':
            nl.norm(self.a, ord=1)
        else:
            sl.norm(self.a, ord=1)

    def time_inf_norm(self, size, contig, module):
        if module == 'numpy':
            nl.norm(self.a, ord=np.inf)
        else:
            sl.norm(self.a, ord=np.inf)

    def time_frobenius_norm(self, size, contig, module):
        if module == 'numpy':
            nl.norm(self.a)
        else:
            sl.norm(self.a)

class Lstsq(Benchmark):
    """
    Test the speed of four least-squares solvers on not full rank matrices.
    Also check the difference in the solutions.

    The matrix has the size ``(m, 2/3*m)``; the rank is ``1/2 * m``.
    Matrix values are random in the range (-5, 5), the same is for the right
    hand side.  The complex matrix is the sum of real and imaginary matrices.
    """

    param_names = ['dtype', 'size', 'driver']
    params = [
        [np.float64, np.complex128],
        [10, 100, 1000],
        ['gelss', 'gelsy', 'gelsd', 'numpy'],
    ]

    def setup(self, dtype, size, lapack_driver):
        if lapack_driver == 'numpy' and size >= 200:
            # skip: slow, and not useful to benchmark numpy
            raise NotImplementedError()

        np.random.seed(1234)
        n = math.ceil(2./3. * size)
        k = math.ceil(1./2. * size)
        m = size

        if dtype is np.complex128:
            A = ((10 * np.random.rand(m,k) - 5) +
                 1j*(10 * np.random.rand(m,k) - 5))
            temp = ((10 * np.random.rand(k,n) - 5) +
                    1j*(10 * np.random.rand(k,n) - 5))
            b = ((10 * np.random.rand(m,1) - 5) +
                 1j*(10 * np.random.rand(m,1) - 5))
        else:
            A = (10 * np.random.rand(m,k) - 5)
            temp = 10 * np.random.rand(k,n) - 5
            b = 10 * np.random.rand(m,1) - 5

        self.A = A.dot(temp)
        self.b = b

    def time_lstsq(self, dtype, size, lapack_driver):
        if lapack_driver == 'numpy':
            np.linalg.lstsq(self.A, self.b,
                            rcond=np.finfo(self.A.dtype).eps * 100)
        else:
            sl.lstsq(self.A, self.b, cond=None, overwrite_a=False,
                     overwrite_b=False, check_finite=False,
                     lapack_driver=lapack_driver)

    # Retain old benchmark results (remove this if changing the benchmark)
    time_lstsq.version = "15ee0be14a0a597c7d1c9a3dab2c39e15c8ac623484410ffefa406bf6b596ebe"

class SpecialMatrices(Benchmark):
    param_names = ['size']
    params = [[4, 128]]

    def setup(self, size):
        self.x = np.arange(1, size + 1).astype(float)
        self.small_blocks = [np.ones([2, 2])] * (size//2)
        self.big_blocks = [np.ones([size//2, size//2]),
                           np.ones([size//2, size//2])]

    def time_block_diag_small(self, size):
        sl.block_diag(*self.small_blocks)

    def time_block_diag_big(self, size):
        sl.block_diag(*self.big_blocks)

    def time_circulant(self, size):
        sl.circulant(self.x)

    def time_companion(self, size):
        sl.companion(self.x)

    def time_dft(self, size):
        sl.dft(size)

    def time_hadamard(self, size):
        sl.hadamard(size)

    def time_hankel(self, size):
        sl.hankel(self.x)

    def time_helmert(self, size):
        sl.helmert(size)

    def time_hilbert(self, size):
        sl.hilbert(size)

    def time_invhilbert(self, size):
        sl.invhilbert(size)

    def time_leslie(self, size):
        sl.leslie(self.x, self.x[1:])

    def time_pascal(self, size):
        sl.pascal(size)

    def time_invpascal(self, size):
        sl.invpascal(size)

    def time_toeplitz(self, size):
        sl.toeplitz(self.x)

    def time_tri(self, size):
        sl.tri(size)

class GetFuncs(Benchmark):
    def setup(self):
        self.x = np.eye(1)

    def time_get_blas_funcs(self):
        sl.blas.get_blas_funcs('gemm', dtype=float)

    def time_get_blas_funcs_2(self):
        sl.blas.get_blas_funcs(('gemm', 'axpy'), (self.x, self.x))

    def time_small_cholesky(self):
        sl.cholesky(self.x)
<commit_msg>MAINT: Modify to use new random API in benchmarks/linalg.py<commit_after>import math

import numpy.linalg as nl

import numpy as np
from numpy.testing import assert_
from numpy.random import rand

from .common import Benchmark, safe_import

with safe_import():
    import scipy.linalg as sl

def random(size):
    return rand(*size)

class Bench(Benchmark):
    params = [
        [20, 100, 500, 1000],
        ['contig', 'nocont'],
        ['numpy', 'scipy']
    ]
    param_names = ['size', 'contiguous', 'module']

    def __init__(self):
        # likely not useful to benchmark svd for large sizes
        self.time_svd.__func__.params = [[20, 100, 500]] + self.params[1:]

    def setup(self, size, contig, module):
        if module == 'numpy' and size >= 200:
            # skip: slow, and not useful to benchmark numpy
            raise NotImplementedError()

        a = random([size, size])
        # larger diagonal ensures non-singularity:
        for i in range(size):
            a[i, i] = 10*(.1+a[i, i])
        b = random([size])

        if contig != 'contig':
            a = a[-1::-1, -1::-1]  # turn into a non-contiguous array
            assert_(not a.flags['CONTIGUOUS'])

        self.a = a
        self.b = b

    def time_solve(self, size, contig, module):
        if module == 'numpy':
            nl.solve(self.a, self.b)
        else:
            sl.solve(self.a, self.b)

    def time_solve_triangular(self, size, contig, module):
        # treats self.a as a lower-triangular matrix by ignoring the strictly
        # upper-triangular part
        if module == 'numpy':
            pass
        else:
            sl.solve_triangular(self.a, self.b, lower=True)

    def time_inv(self, size, contig, module):
        if module == 'numpy':
            nl.inv(self.a)
        else:
            sl.inv(self.a)

    def time_det(self, size, contig, module):
        if module == 'numpy':
            nl.det(self.a)
        else:
            sl.det(self.a)

    def time_eigvals(self, size, contig, module):
        if module == 'numpy':
            nl.eigvals(self.a)
        else:
            sl.eigvals(self.a)

    def time_svd(self, size, contig, module):
        if module == 'numpy':
            nl.svd(self.a)
        else:
            sl.svd(self.a)

    # Retain old benchmark results (remove this if changing the benchmark)
    time_det.version = "87e530ee50eb6b6c06c7a8abe51c2168e133d5cbd486f4c1c2b9cedc5a078325"
    time_eigvals.version = "9d68d3a6b473df9bdda3d3fd25c7f9aeea7d5cee869eec730fb2a2bcd1dfb907"
    time_inv.version = "20beee193c84a5713da9749246a7c40ef21590186c35ed00a4fe854cce9e153b"
    time_solve.version = "1fe788070f1c9132cbe78a47fdb4cce58266427fc636d2aa9450e3c7d92c644c"
    time_svd.version = "0ccbda456d096e459d4a6eefc6c674a815179e215f83931a81cfa8c18e39d6e3"

class Norm(Benchmark):
    params = [
        [(20, 20), (100, 100), (1000, 1000), (20, 1000), (1000, 20)],
        ['contig', 'nocont'],
        ['numpy', 'scipy']
    ]
    param_names = ['shape', 'contiguous', 'module']

    def setup(self, shape, contig, module):
        a = np.random.randn(*shape)
        if contig != 'contig':
            a = a[-1::-1,-1::-1]  # turn into a non-contiguous array
            assert_(not a.flags['CONTIGUOUS'])
        self.a = a

    def time_1_norm(self, size, contig, module):
        if module == 'numpy':
            nl.norm(self.a, ord=1)
        else:
            sl.norm(self.a, ord=1)

    def time_inf_norm(self, size, contig, module):
        if module == 'numpy':
            nl.norm(self.a, ord=np.inf)
        else:
            sl.norm(self.a, ord=np.inf)

    def time_frobenius_norm(self, size, contig, module):
        if module == 'numpy':
            nl.norm(self.a)
        else:
            sl.norm(self.a)

class Lstsq(Benchmark):
    """
    Test the speed of four least-squares solvers on not full rank matrices.
    Also check the difference in the solutions.

    The matrix has the size ``(m, 2/3*m)``; the rank is ``1/2 * m``.
    Matrix values are random in the range (-5, 5), the same is for the right
    hand side.  The complex matrix is the sum of real and imaginary matrices.
    """

    param_names = ['dtype', 'size', 'driver']
    params = [
        [np.float64, np.complex128],
        [10, 100, 1000],
        ['gelss', 'gelsy', 'gelsd', 'numpy'],
    ]

    def setup(self, dtype, size, lapack_driver):
        if lapack_driver == 'numpy' and size >= 200:
            # skip: slow, and not useful to benchmark numpy
            raise NotImplementedError()

        rng = np.random.default_rng(1234)
        n = math.ceil(2./3. * size)
        k = math.ceil(1./2. * size)
        m = size

        if dtype is np.complex128:
            A = ((10 * rng.random((m,k)) - 5) +
                 1j*(10 * rng.random((m,k)) - 5))
            temp = ((10 * rng.random((k,n)) - 5) +
                    1j*(10 * rng.random((k,n)) - 5))
            b = ((10 * rng.random((m,1)) - 5) +
                 1j*(10 * rng.random((m,1)) - 5))
        else:
            A = (10 * rng.random((m,k)) - 5)
            temp = 10 * rng.random((k,n)) - 5
            b = 10 * rng.random((m,1)) - 5

        self.A = A.dot(temp)
        self.b = b

    def time_lstsq(self, dtype, size, lapack_driver):
        if lapack_driver == 'numpy':
            np.linalg.lstsq(self.A, self.b,
                            rcond=np.finfo(self.A.dtype).eps * 100)
        else:
            sl.lstsq(self.A, self.b, cond=None, overwrite_a=False,
                     overwrite_b=False, check_finite=False,
                     lapack_driver=lapack_driver)

    # Retain old benchmark results (remove this if changing the benchmark)
    time_lstsq.version = "15ee0be14a0a597c7d1c9a3dab2c39e15c8ac623484410ffefa406bf6b596ebe"

class SpecialMatrices(Benchmark):
    param_names = ['size']
    params = [[4, 128]]

    def setup(self, size):
        self.x = np.arange(1, size + 1).astype(float)
        self.small_blocks = [np.ones([2, 2])] * (size//2)
        self.big_blocks = [np.ones([size//2, size//2]),
                           np.ones([size//2, size//2])]

    def time_block_diag_small(self, size):
        sl.block_diag(*self.small_blocks)

    def time_block_diag_big(self, size):
        sl.block_diag(*self.big_blocks)

    def time_circulant(self, size):
        sl.circulant(self.x)

    def time_companion(self, size):
        sl.companion(self.x)

    def time_dft(self, size):
        sl.dft(size)

    def time_hadamard(self, size):
        sl.hadamard(size)

    def time_hankel(self, size):
        sl.hankel(self.x)

    def time_helmert(self, size):
        sl.helmert(size)

    def time_hilbert(self, size):
        sl.hilbert(size)

    def time_invhilbert(self, size):
        sl.invhilbert(size)

    def time_leslie(self, size):
        sl.leslie(self.x, self.x[1:])

    def time_pascal(self, size):
        sl.pascal(size)

    def time_invpascal(self, size):
        sl.invpascal(size)

    def time_toeplitz(self, size):
        sl.toeplitz(self.x)

    def time_tri(self, size):
        sl.tri(size)

class GetFuncs(Benchmark):
    def setup(self):
        self.x = np.eye(1)

    def time_get_blas_funcs(self):
        sl.blas.get_blas_funcs('gemm', dtype=float)

    def time_get_blas_funcs_2(self):
        sl.blas.get_blas_funcs(('gemm', 'axpy'), (self.x, self.x))

    def time_small_cholesky(self):
        sl.cholesky(self.x)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# GNU General Public Licence (GPL)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA  02111-1307  USA
#

import optparse
import re
import time

def start_scann(module):
    # Open and read file
    method_list = []

    with open("../../ipol_demo/modules/{}/{}.py".format(module, module)) as f:
        lines = f.readlines()

    for line_index in range(len(lines)):

        method = ""
        if "@cherrypy.expose" in lines[line_index]:

            line_index += 1
            while not "def" in lines[line_index]:
                line_index += 1

            # Name of the method
            method += lines[line_index].strip() +"\n"
            while not "):" in lines[line_index]:
                line_index += 1
                method += lines[line_index][:-1] + "\n"

            line_index += 1
            # Check for comments or empty lines before docstring
            while lines[line_index].strip().startswith("#") or re.match(r'^\s*$', lines[line_index]):
                line_index += 1

            # Check if the method have any docstring
            if not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                method += "    \\error{this method does not have any docstring}\n"
            else:
                line_index += 1
                # Docstring
                while not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                    method += "    " + lines[line_index].strip() + "\n"
                    line_index += 1
            method += "\n"
            method_list.append(method)

    return {module:method_list}

def print_latex(content):
    result =""
    for module in content:
        result += "\\subsection{"+module.capitalize()+"} \n"
        result += "\\begin{itemize} \n"
        for method in content[module]:
            result += "\item {}".format(method.replace("_","\\_")
                                        .replace("\n","\n\\\\")
                                        .replace("<", "\textless")
                                        .replace(">", "\textgreater")
                                        .replace("{", "\{")
                                        .replace("}", "\}"))
        result += "\\end{itemize} \n"
    print result.replace("\\\\\\","\\")

# Parse program arguments
parser = optparse.OptionParser()
(opts, args) = parser.parse_args()

if len(args) > 1:
    print "Wrong number of arguments"
    exit(0)

module = args[0].lower() if len(args) == 1 else None
acepted_modules = ["core", "dispatcher", "demorunner", "demoinfo", "blobs", "archive"]
date = time.strftime("%d/%m/%Y")
modules = []

if module in acepted_modules:
    modules.append(module)
elif module is None:
    modules = acepted_modules
else:
    print "Unknown module '{}'".format(module)
    exit(-1)

content = {}
for module in modules:
    content.update(start_scann(module))
print_latex(content)<commit_msg>[doc] Added conversion module to the api doc<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# GNU General Public Licence (GPL)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA  02111-1307  USA
#

import optparse
import re
import time

def start_scann(module):
    # Open and read file
    method_list = []

    with open("../../ipol_demo/modules/{}/{}.py".format(module, module)) as f:
        lines = f.readlines()

    for line_index in range(len(lines)):

        method = ""
        if "@cherrypy.expose" in lines[line_index]:

            line_index += 1
            while not "def" in lines[line_index]:
                line_index += 1

            # Name of the method
            method += lines[line_index].strip() +"\n"
            while not "):" in lines[line_index]:
                line_index += 1
                method += lines[line_index][:-1] + "\n"

            line_index += 1
            # Check for comments or empty lines before docstring
            while lines[line_index].strip().startswith("#") or re.match(r'^\s*$', lines[line_index]):
                line_index += 1

            # Check if the method have any docstring
            if not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                method += "    \\error{this method does not have any docstring}\n"
            else:
                line_index += 1
                # Docstring
                while not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                    method += "    " + lines[line_index].strip() + "\n"
                    line_index += 1
            method += "\n"
            method_list.append(method)

    return {module:method_list}

def print_latex(content):
    result =""
    for module in content:
        result += "\\subsection{"+module.capitalize()+"} \n"
        result += "\\begin{itemize} \n"
        for method in content[module]:
            result += "\item {}".format(method.replace("_","\\_")
                                        .replace("\n","\n\\\\")
                                        .replace("<", "\textless")
                                        .replace(">", "\textgreater")
                                        .replace("{", "\{")
                                        .replace("}", "\}"))
        result += "\\end{itemize} \n"
    print result.replace("\\\\\\","\\")

# Parse program arguments
parser = optparse.OptionParser()
(opts, args) = parser.parse_args()

if len(args) > 1:
    print "Wrong number of arguments"
    exit(0)

module = args[0].lower() if len(args) == 1 else None
acepted_modules = ["core", "dispatcher", "demorunner", "demoinfo", "blobs", "archive", "conversion"]
date = time.strftime("%d/%m/%Y")
modules = []

if module in acepted_modules:
    modules.append(module)
elif module is None:
    modules = acepted_modules
else:
    print "Unknown module '{}'".format(module)
    exit(-1)

content = {}
for module in modules:
    content.update(start_scann(module))
print_latex(content)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# © 2012 Scott Reynolds
# Author: Scott Reynolds <scott@scottreynolds.us>
#
import chisenbop.packets as packets
import datetime
import time
import nose.tools

def test_determineTimeAgoSeconds():
    """
    Test logic of time ago class method
    """
    now = datetime.datetime.now()
    timestamp = time.mktime(now.timetuple())
    assert packets.Packet.determineTimeAgo(now, "seconds") == timestamp, \
        "Timeago failed to return the second when asked"
    assert packets.Packet.determineTimeAgo(now, "minutes") == timestamp / 60, \
        "Timeago failed to return the minute when asked"
    assert packets.Packet.determineTimeAgo(now, "hours") == timestamp / 3600, \
        "Timeago failed to return the hours when asked"
    assert packets.Packet.determineTimeAgo(now, "days") == timestamp / 86400, \
        "Timeago failed to return the day when asked"
    assert packets.Packet.determineTimeAgo(now, "months") == timestamp / 2629743, \
        "Timeago failed to return the month when asked"
    assert packets.Packet.determineTimeAgo(now, "years") == timestamp / 31556926, \
        "Timeago failed to return the year when asked"

@nose.tools.raises(ValueError)
def test_determineTimeAgoInvalid():
    now = datetime.datetime.now()
    packets.Packet.determineTimeAgo(now, "invalid_config")

def test_keyConstruction():
    current_time = time.time()
    event_name = "test"
    version = "1.0"
    configuration = (
        ("seconds", 1,),
        ("minutes", 60 * 24 * 30,),
        )

    packet = packets.Packet("%s!%s!%s" %
                            (current_time, event_name, version,))
    keys = packet.constructKeys(configuration)
    assert len(keys) == 2, \
        "Two keys were not returned: %s" % keys
    time.sleep(1)
    packet = packets.Packet("%s!%s!%s" %
                            (current_time, event_name, version,))
    keys = packet.constructKeys(configuration)
    assert len(keys) == 1, \
        "Failed to prevent writing for the additional key: %s" % keys
<commit_msg>fix tests<commit_after># -*- coding: utf-8 -*-
#
# © 2012 Scott Reynolds
# Author: Scott Reynolds <scott@scottreynolds.us>
#
import chisenbop.packets as packets
import datetime
import time
import nose.tools

def test_determineTimeAgoSeconds():
    """
    Test logic of time ago class method
    """
    now = datetime.datetime.now()
    timestamp = int(time.mktime(now.timetuple()))
    assert packets.Packet.determineTimeAgo(now, "seconds") == timestamp, \
        "Timeago failed to return the second when asked"
    assert packets.Packet.determineTimeAgo(now, "minutes") == timestamp / 60, \
        "Timeago failed to return the minute when asked"
    assert packets.Packet.determineTimeAgo(now, "hours") == timestamp / 3600, \
        "Timeago failed to return the hours when asked"
    assert packets.Packet.determineTimeAgo(now, "days") == timestamp / 86400, \
        "Timeago failed to return the day when asked"
    assert packets.Packet.determineTimeAgo(now, "months") == timestamp / 2629743, \
        "Timeago failed to return the month when asked"
    assert packets.Packet.determineTimeAgo(now, "years") == timestamp / 31556926, \
        "Timeago failed to return the year when asked"

@nose.tools.raises(ValueError)
def test_determineTimeAgoInvalid():
    now = datetime.datetime.now()
    packets.Packet.determineTimeAgo(now, "invalid_config")

def test_keyConstruction():
    current_time = time.time()
    event_name = "test"
    version = "1.0"
    configuration = (
        ("seconds", 1,),
        ("minutes", 60 * 24 * 30,),
        )

    packet = packets.Packet("%s!%s!%s" %
                            (current_time, event_name, version,))
    keys = packet.constructKeys(configuration)
    assert len(keys) == 2, \
        "Two keys were not returned: %s" % keys
    time.sleep(1)
    packet = packets.Packet("%s!%s!%s" %
                            (current_time, event_name, version,))
    keys = packet.constructKeys(configuration)
    assert len(keys) == 1, \
        "Failed to prevent writing for the additional key: %s" % keys
<|endoftext|>


<commit_before>

class FormattedNode:
    """
    The formatted output.
    """
    def __init__(self, field, content, style=None, prefix=None, suffix=None, block=False):
        self.field = field
        self.content = content
        self.style = style
        self.prefix = prefix
        self.suffix = suffix
        self.block = block

    def to_html(rdfa=False):
        result = ""
        result += self.prefix
        result += self.content
        result += self.suffix
        return(result)

class FormattedList:
    """
    The formatted output.
    """
    def __init__(self, items, prefix=None, suffix=None):
        self.items = items
        self.prefix = prefix
        self.suffix = suffix

class FormattedCitationCluster(FormattedList):
    pass

class FormattedReferenceList(FormattedList):
    pass

def process_citations(style, reference_list, citation):
    """
    With a Style, a list of References and the list of citation groups
    (the list of citations with their locator), produce the FormattedOutput
    for each citation group.
    """
    pass

def process_bibliography(style, reference_list):
    """
    With a Style and the list of References produce the FormattedOutput
    for the bibliography.
    """
    pass

def citeproc(style, reference_list):
    """
    With a Style, a list of References and the list of citation
    groups (the list of citations with their locator), produce the
    FormattedOutput for each citation group and the bibliography.
    """
    pass

def proc_biblio(style, reference_list):
    """
    With a Style and a sorted list of References produce the evaluated
    output for the bibliography.
    """
    pass

def proc_refs(style, reference_list):
    """
    Given the CSL Style and the list of References sort the list according
    to the Style and assign the citation number to each Reference.
    """
    pass

def refs_year_suffix(reference_list):
    """
    Given the list of References, compare year and contributors' names and,
    when they collide, generate a suffix to append to the year for
    disambiguation.
    """
    pass

<commit_msg>more tweaks on output classes<commit_after>

class FormattedNode:
    """
    The formatted output.
    """
    def __init__(self, field, content, style=None, prefix=None, suffix=None, block=False):
        self.field = field
        self.content = content
        self.style = style
        self.prefix = prefix
        self.suffix = suffix
        self.block = block

    def to_html(rdfa=False):
        result = ""
        result += self.prefix
        result += self.content
        result += self.suffix
        return(result)

class FormattedList:
    """
    The formatted output.
    """
    def __init__(self, items, delimiter=None, prefix=None, suffix=None, block=False):
        self.items = items
        self.prefix = prefix
        self.suffix = suffix
        self.block = block

class FormattedCitationCluster(FormattedList):
    pass

class FormattedReferenceList(FormattedList):
    pass

def process_citations(style, reference_list, citation):
    """
    With a Style, a list of References and the list of citation groups
    (the list of citations with their locator), produce the FormattedOutput
    for each citation group.
    """
    pass

def process_bibliography(style, reference_list):
    """
    With a Style and the list of References produce the FormattedOutput
    for the bibliography.
    """
    pass

def citeproc(style, reference_list):
    """
    With a Style, a list of References and the list of citation
    groups (the list of citations with their locator), produce the
    FormattedOutput for each citation group and the bibliography.
    """
    pass

def proc_biblio(style, reference_list):
    """
    With a Style and a sorted list of References produce the evaluated
    output for the bibliography.
    """
    pass

def proc_refs(style, reference_list):
    """
    Given the CSL Style and the list of References sort the list according
    to the Style and assign the citation number to each Reference.
    """
    pass

def refs_year_suffix(reference_list):
    """
    Given the list of References, compare year and contributors' names and,
    when they collide, generate a suffix to append to the year for
    disambiguation.
    """
    pass

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    zahlen.ds.tree.bst
    ~~~~~~~~~~~~~~~~~~

    This module implements the Binary Search Tree data structure.

    :copyright: (c) 2014 by Subhajit Ghosh.
    :license: MIT, see LICENSE for more details.
"""

class Node(object):
    """A node in a binary search tree.

    :param key: value stored in the node
    """

    def __init__(self, key):
        self._left = None
        self._right = None
        self.parent = None

        self.key = key

    def __str__(self):
        parent_key = self.parent.key if self.parent else -1
        return 'key:{0},parent:{1}'.format(self.key, parent_key)

    @property
    def left(self):
        return self._left

    @left.setter
    def left(self, node):
        self._left = node
        if node:
            node.parent = self

    @property
    def right(self):
        return self._right

    @right.setter
    def right(self, node):
        self._right = node
        if node:
            node.parent = self

    def is_leaf(self):
        """Returns true if node is a leaf node."""
        return self.left is None and self.right is None

class BinarySearchTree(object):
    """Represents the Binary Search Tree data structure."""

    def __init__(self, node_type):
        self.root = None
        self._Node = node_type

    def __str__(self):
        return self.draw(self.root)

    def insert(self, key):
        """Inserts key ``key`` in the tree."""

        if not self.root:
            self.root = self._Node(key)
        else:
            node = self.root
            parent = None

            while node:
                parent = node
                node = node.left if key < node.key else node.right

            if key < parent.key:
                parent.left = self._Node(key)
            else:
                parent.right = self._Node(key)

    def delete(self, key):
        """Deletes key ``key`` from the tree."""
        node = self._search_node(key, silent=False)

        if node.is_leaf():
            self._delete_leaf_node(node)
        else:
            self._delete_internal_node(node)

    def search(self, key):
        """Returns True if key `key` exists in the tree, else False."""
        return self._search_node(key) is not None

    def sort(self):
        """Returns a sorted list of the keys in the tree."""
        keys = []
        self.inorder_walk(self.root, keys)
        return keys

    def inorder_walk(self, node, keys):
        if node:
            self.inorder_walk(node.left, keys)

            # Handle duplicate keys
            for _ in xrange(node.count):
                keys.append(node.key)
            self.inorder_walk(node.right, keys)

    def draw(self, node, depth=0):
        """Prints a subtree rooted at a node at certain depth."""
        key = node.key if node else 'NULL'

        if depth:
            prefix = '|   ' * depth + '|---'
        else:
            prefix = ''

        node_str = prefix + '(' + str(key) + ')\n'

        if node and not node.is_leaf():
            node_str += self.draw(node.right, depth + 1)
            node_str += self.draw(node.left, depth + 1)

        return node_str

    def _delete_node(self, node):
        """Deletes a leaf node or an internal node."""
        if node.is_leaf():
            self._delete_leaf_node(node)
        else:
            self._delete_internal_node(node)

    def _delete_leaf_node(self, node):
        """Deletes a leaf node."""
        parent = node.parent
        if parent:
            if node.key < parent.key:
                parent.left = None
            else:
                parent.right = None
            self._bubble_up_node_attrs(parent)
        else:
            self.root = None
        del node

    def _delete_internal_node(self, node):
        """Deletes an internal node with 1 or 2 children."""
        if not node.left or not node.right:
            # For an incomplete internal node, replace the node with its left or
            # right child and update the parent if any.
            parent = node.parent
            child = node.left if node.left else node.right
            child.parent = parent

            if parent:
                if node.key < parent.key:
                    parent.left = child
                else:
                    parent.right = child
            else:
                self.root = child
            self._bubble_up_node_attrs(parent)
        else:
            # For an complete internal node, replace node's key by inorder
            # successor's key and remove the successor.
            successor = node.right
            while successor.left:
                successor = successor.left
            self._delete_node(successor)
            node.key = successor.key
            self._bubble_up_node_attrs(node)

    def _search_node(self, key, silent=True):
        """Return the node with key ``key`` if it exists, else return None.

        :param silent: If False, raises an exception if ``key`` does not exist
            in the tree.
        """
        node = self.root
        while node:
            if node.key == key:
                break
            else:
                node = node.left if key < node.key else node.right

        if not node and not silent:
            raise KeyError('Key: {0} not found in tree'.format(key))

        return node<commit_msg>Reuse code in _delete_node()<commit_after># -*- coding: utf-8 -*-

"""
    zahlen.ds.tree.bst
    ~~~~~~~~~~~~~~~~~~

    This module implements the Binary Search Tree data structure.

    :copyright: (c) 2014 by Subhajit Ghosh.
    :license: MIT, see LICENSE for more details.
"""

class Node(object):
    """A node in a binary search tree.

    :param key: value stored in the node
    """

    def __init__(self, key):
        self._left = None
        self._right = None
        self.parent = None

        self.key = key

    def __str__(self):
        parent_key = self.parent.key if self.parent else -1
        return 'key:{0},parent:{1}'.format(self.key, parent_key)

    @property
    def left(self):
        return self._left

    @left.setter
    def left(self, node):
        self._left = node
        if node:
            node.parent = self

    @property
    def right(self):
        return self._right

    @right.setter
    def right(self, node):
        self._right = node
        if node:
            node.parent = self

    def is_leaf(self):
        """Returns true if node is a leaf node."""
        return self.left is None and self.right is None

class BinarySearchTree(object):
    """Represents the Binary Search Tree data structure."""

    def __init__(self, node_type):
        self.root = None
        self._Node = node_type

    def __str__(self):
        return self.draw(self.root)

    def insert(self, key):
        """Inserts key ``key`` in the tree."""

        if not self.root:
            self.root = self._Node(key)
        else:
            node = self.root
            parent = None

            while node:
                parent = node
                node = node.left if key < node.key else node.right

            if key < parent.key:
                parent.left = self._Node(key)
            else:
                parent.right = self._Node(key)

    def delete(self, key):
        """Deletes key ``key`` from the tree."""
        node = self._search_node(key, silent=False)
        self._delete_node(node)

    def search(self, key):
        """Returns True if key `key` exists in the tree, else False."""
        return self._search_node(key) is not None

    def sort(self):
        """Returns a sorted list of the keys in the tree."""
        keys = []
        self.inorder_walk(self.root, keys)
        return keys

    def inorder_walk(self, node, keys):
        if node:
            self.inorder_walk(node.left, keys)

            # Handle duplicate keys
            for _ in xrange(node.count):
                keys.append(node.key)
            self.inorder_walk(node.right, keys)

    def draw(self, node, depth=0):
        """Prints a subtree rooted at a node at certain depth."""
        key = node.key if node else 'NULL'

        if depth:
            prefix = '|   ' * depth + '|---'
        else:
            prefix = ''

        node_str = prefix + '(' + str(key) + ')\n'

        if node and not node.is_leaf():
            node_str += self.draw(node.right, depth + 1)
            node_str += self.draw(node.left, depth + 1)

        return node_str

    def _delete_node(self, node):
        """Deletes a leaf node or an internal node."""
        if node.is_leaf():
            self._delete_leaf_node(node)
        else:
            self._delete_internal_node(node)

    def _delete_leaf_node(self, node):
        """Deletes a leaf node."""
        parent = node.parent
        if parent:
            if node.key < parent.key:
                parent.left = None
            else:
                parent.right = None
            self._bubble_up_node_attrs(parent)
        else:
            self.root = None
        del node

    def _delete_internal_node(self, node):
        """Deletes an internal node with 1 or 2 children."""
        if not node.left or not node.right:
            # For an incomplete internal node, replace the node with its left or
            # right child and update the parent if any.
            parent = node.parent
            child = node.left if node.left else node.right
            child.parent = parent

            if parent:
                if node.key < parent.key:
                    parent.left = child
                else:
                    parent.right = child
            else:
                self.root = child
            self._bubble_up_node_attrs(parent)
        else:
            # For an complete internal node, replace node's key by inorder
            # successor's key and remove the successor.
            successor = node.right
            while successor.left:
                successor = successor.left
            self._delete_node(successor)
            node.key = successor.key
            self._bubble_up_node_attrs(node)

    def _search_node(self, key, silent=True):
        """Return the node with key ``key`` if it exists, else return None.

        :param silent: If False, raises an exception if ``key`` does not exist
            in the tree.
        """
        node = self.root
        while node:
            if node.key == key:
                break
            else:
                node = node.left if key < node.key else node.right

        if not node and not silent:
            raise KeyError('Key: {0} not found in tree'.format(key))

        return node<|endoftext|>


<commit_before>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

from fix import *

import os
import os.path
import sys
import time
import cgitb
import traceback

import ansi
import conf
import world

###
# CONFIGURATION
###

## Uncomment this class to remove the tie to SupyBot's conf module.
## class conf:
##     logDir = '.'

# Names of logfiles.
errorfile = os.path.join(conf.logDir, 'error.log')
debugfile = os.path.join(conf.logDir, 'debug.log')
tracefile = os.path.join(conf.logDir, 'trace.log')

# stderr: True if messages should be written to stderr as well as logged.
stderr = True

# colorterm: True if the terminal run on is color.
colorterm = True

# printf: True if printf debugging messages should be printed.
printf = True

# minimumDebugPriority: Lowest priority logged;
#                       One of {'verbose', 'low', 'normal', 'high'}.
minimumDebugPriority = 'verbose'

# deadlyExceptions: Exceptions that should cause immediate failure.
deadlyExceptions = [KeyboardInterrupt, SystemExit]

###
# END CONFIGURATION
###

_errorfd = file(errorfile, 'a')
_debugfd = file(debugfile, 'a')
_tracefd = file(tracefile, 'w')

minpriority = 1

priorities = { 'verbose': 0,
               'low': 1,
               'normal': 2,
               'high': 3 }

priorityColors = { 'verbose': ansi.BLUE,
                   'low': ansi.CYAN,
                   'normal': ansi.GREEN,
                   'high': ansi.BOLD }
priorityColors.setdefault('')

# This is a queue of the time of the last 10 calls to recoverableException.
# If the most recent time is
lastTimes = [time.time()-1] * 10

def exit(i=-1):
    class E(Exception):
        pass
    if deadlyExceptions:
        # Just to be safe, we'll make it a subclass of *all* the deadly
        # exceptions :)
        for exn in deadlyExceptions:
            class E(exn, E):
                pass
        raise E
    else:
        os._exit(i)

def _writeNewline(fd, s):
    fd.write(s)
    if s[-len(os.linesep):] != os.linesep:
        fd.write(os.linesep)

def recoverableError(msg):
    """Called with errors that are not critical.
    """
    if stderr:
        if colorterm:
            sys.stderr.write(ansi.BOLD + ansi.RED)
        _writeNewline(sys.stderr, msg)
        if colorterm:
            sys.stderr.write(ansi.RESET)
    _writeNewline(_errorfd, msg)
    _errorfd.flush()

def unrecoverableError(msg):
    recoverableError(msg)
    exit(-1)

def recoverableException():
    if world.testing:
        return
    (E, e, tb) = sys.exc_info()
    for exn in deadlyExceptions:
        if issubclass(e.__class__, exn):
            raise
    lastTimes.append(time.time())
    if lastTimes[-1] - lastTimes[0] < 0.50:
        msg('Too many exceptions too quickly.  Bailing out.', 'high')
        exit()
    else:
        del lastTimes[0]
    try:
        if not conf.detailedTracebacks:
            1/0
        text = cgitb.text((E, e, tb))
    except:
        text = ''.join(traceback.format_exception(E, e, tb))
    del tb # just to be safe.
    if stderr:
        if colorterm:
            sys.stderr.write(ansi.BOLD + ansi.RED)
        sys.stderr.write(text)
        if colorterm:
            sys.stderr.write(ansi.RESET)
    _errorfd.write(text)
    _errorfd.flush()

def unrecoverableException():
    recoverableException()
    exit(-1)

def msg(s, priority='low'):
    if priorities[priority] >= priorities[minimumDebugPriority]:
        if stderr:
            if colorterm:
                sys.stderr.write(priorityColors.get(priority))
            _writeNewline(sys.stderr, s)
            if colorterm:
                sys.stderr.write(ansi.RESET)
        _writeNewline(_debugfd, s)
        _debugfd.flush()

def printf(msg):
    if printf:
        print '*** ' + str(msg)

def methodNamePrintf(obj, methodName):
    printf('%s: %s' % (obj.__class__.__name__, methodName))

def exnToString(e):
    return '%s: %s' % (e.__class__.__name__, e)

def tracer(frame, event, _):
    if event == 'call':
        s = '%s: %s\n' % (frame.f_code.co_filename, frame.f_code.co_name)
        _tracefd.write(s)
        _tracefd.flush()
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<commit_msg>Added keyword argument to recoverableException to determine the type of exception logged.<commit_after>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

from fix import *

import os
import os.path
import sys
import time
import cgitb
import traceback

import ansi
import conf
import world

###
# CONFIGURATION
###

## Uncomment this class to remove the tie to SupyBot's conf module.
## class conf:
##     logDir = '.'

# Names of logfiles.
errorfile = os.path.join(conf.logDir, 'error.log')
debugfile = os.path.join(conf.logDir, 'debug.log')
tracefile = os.path.join(conf.logDir, 'trace.log')

# stderr: True if messages should be written to stderr as well as logged.
stderr = True

# colorterm: True if the terminal run on is color.
colorterm = True

# printf: True if printf debugging messages should be printed.
printf = True

# minimumDebugPriority: Lowest priority logged;
#                       One of {'verbose', 'low', 'normal', 'high'}.
minimumDebugPriority = 'verbose'

# deadlyExceptions: Exceptions that should cause immediate failure.
deadlyExceptions = [KeyboardInterrupt, SystemExit]

###
# END CONFIGURATION
###

_errorfd = file(errorfile, 'a')
_debugfd = file(debugfile, 'a')
_tracefd = file(tracefile, 'w')

minpriority = 1

priorities = { 'verbose': 0,
               'low': 1,
               'normal': 2,
               'high': 3 }

priorityColors = { 'verbose': ansi.BLUE,
                   'low': ansi.CYAN,
                   'normal': ansi.GREEN,
                   'high': ansi.BOLD }
priorityColors.setdefault('')

# This is a queue of the time of the last 10 calls to recoverableException.
# If the most recent time is
lastTimes = [time.time()-1] * 10

def exit(i=-1):
    class E(Exception):
        pass
    if deadlyExceptions:
        # Just to be safe, we'll make it a subclass of *all* the deadly
        # exceptions :)
        for exn in deadlyExceptions:
            class E(exn, E):
                pass
        raise E
    else:
        os._exit(i)

def _writeNewline(fd, s):
    fd.write(s)
    if s[-len(os.linesep):] != os.linesep:
        fd.write(os.linesep)

def recoverableError(msg):
    """Called with errors that are not critical.
    """
    if stderr:
        if colorterm:
            sys.stderr.write(ansi.BOLD + ansi.RED)
        _writeNewline(sys.stderr, msg)
        if colorterm:
            sys.stderr.write(ansi.RESET)
    _writeNewline(_errorfd, msg)
    _errorfd.flush()

def unrecoverableError(msg):
    recoverableError(msg)
    exit(-1)

def recoverableException(type='detailed'):
    """Logs an exception that's recoverable.

    The 'type' argument can be either 'detailed' (for a nice cgitb traceback),
    'normal' (for a normal traceback), or 'terse' (for a short line stating
    the exception raised).
    """
    if world.testing:
        return
    (E, e, tb) = sys.exc_info()
    for exn in deadlyExceptions:
        if issubclass(e.__class__, exn):
            raise
    lastTimes.append(time.time())
    if lastTimes[-1] - lastTimes[0] < 0.50:
        msg('Too many exceptions too quickly.  Bailing out.', 'high')
        exit()
    else:
        del lastTimes[0]
    try:
        if type == 'detailed' and conf.detailedTracebacks:
            text = cgitb.text((E, e, tb))
        elif type == 'normal':
            text = ''.join(traceback.format_exception(E, e, tb))
        elif type == 'terse':
            text = exnToString(e)
    except:
        text = ''.join(traceback.format_exception(E, e, tb))
    del tb # just to be safe.
    if stderr:
        if colorterm:
            sys.stderr.write(ansi.BOLD + ansi.RED)
        sys.stderr.write(text)
        if colorterm:
            sys.stderr.write(ansi.RESET)
    _errorfd.write(text)
    _errorfd.flush()

def unrecoverableException():
    recoverableException()
    exit(-1)

def msg(s, priority='low'):
    if priorities[priority] >= priorities[minimumDebugPriority]:
        if stderr:
            if colorterm:
                sys.stderr.write(priorityColors.get(priority))
            _writeNewline(sys.stderr, s)
            if colorterm:
                sys.stderr.write(ansi.RESET)
        s = '%s: %s' % (priority.upper(), s)
        _writeNewline(_debugfd, s)
        _debugfd.flush()

def printf(msg):
    if printf:
        print '*** ' + str(msg)

def methodNamePrintf(obj, methodName):
    printf('%s: %s' % (obj.__class__.__name__, methodName))

def exnToString(e):
    return '%s: %s' % (e.__class__.__name__, e)

def tracer(frame, event, _):
    if event == 'call':
        s = '%s: %s\n' % (frame.f_code.co_filename, frame.f_code.co_name)
        _tracefd.write(s)
        _tracefd.flush()

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<|endoftext|>


<commit_before>from cement.core import handler, controller
from plugins import BasePlugin
from concurrent.futures import ThreadPoolExecutor
from requests.exceptions import ConnectionError
import common
import common.update_api as ua
import re
import requests
import sys

try:
    from retrying import Retrying
except:
    pass

def _retry_msg(exception):
    if isinstance(exception, ConnectionError):
        print("Caught connection error, retrying.")
        return True
    else:
        return False

class SilverStripe(BasePlugin):

    plugins_base_url = '%s%s/'
    themes_base_url = '%sthemes/%s/'

    forbidden_url = 'framework/'
    regular_file_url = ['cms/css/layout.css', 'framework/css/UploadField.css']
    module_common_file = 'README.md'
    update_majors = ['3.1', '3.0', '2.4']

    interesting_urls = [
            ('framework/docs/en/changelogs/index.md', 'Changelogs, there are other files in same dir, but \'index.md\' is frequently outdated.'),
            ('Security/login', 'Administrative interface.')
        ]

    interesting_module_urls = [
        ('README.md', 'Default README file'),
        ('LICENSE', 'Default license file'),
        ('CHANGELOG', 'Default changelog file'),
    ]

    _repo_framework = 'silverstripe/silverstripe-framework/'
    _repo_cms = 'silverstripe/silverstripe-cms/'

    class Meta:
        label = 'silverstripe'

    @controller.expose(help='silverstripe related scanning tools')
    def silverstripe(self):
        self.plugin_init()

    @controller.expose(help='alias for "silverstripe"')
    def ss(self):
        self.silverstripe()

    def update_version_check(self):
        """
            @return: True if new tags have been made in the github repository.
        """
        return ua.github_tags_newer(self._repo_framework, self.versions_file,
                update_majors=self.update_majors)

    def update_version(self):
        """
            @return: updated VersionsFile
        """
        fw_gr, versions_file, new_tags = ua.github_repo_new(self._repo_framework,
                'silverstripe/framework', self.versions_file, self.update_majors)
        cms_gr, _, _ = ua.github_repo_new(self._repo_cms,
                'silverstripe/cms', self.versions_file, self.update_majors)

        hashes = {}
        for version in new_tags:
            fw_gr.tag_checkout(version)
            cms_gr.tag_checkout(version)

            hashes[version] = ua.hashes_get(versions_file, './.update-workspace/silverstripe/')

        versions_file.update(hashes)
        return versions_file

    def update_plugins_check(self):
        return ua.update_modules_check(self)

    def update_plugins(self):
        css = '#layout > div.add-ons > table > tbody > tr > td > a'
        per_page = 16
        plugins_url = 'http://addons.silverstripe.org/add-ons?search=&type=module&sort=downloads&start=%s'
        themes_url = 'http://addons.silverstripe.org/add-ons?search=&type=theme&sort=downloads&start=%s'
        update_amount = 2000

        plugins = []
        for elem in ua.modules_get(plugins_url, per_page, css, update_amount, pagination_type=ua.PT.skip):
            plugins.append(elem.string)

        themes = []
        for elem in ua.modules_get(themes_url, per_page, css, update_amount, pagination_type=ua.PT.skip):
            themes.append(elem.string)

        notification = "Converting composer packages into folder names %s/2."
        print(notification % (1))
        plugins_folder = self._convert_to_folder(plugins)
        print(notification % (2))
        themes_folder = self._convert_to_folder(themes)

        return plugins_folder, themes_folder

    def _get(self, url, package):
        retry = Retrying(wait_exponential_multiplier=2000, wait_exponential_max=120000,
            retry_on_exception=_retry_msg)

        return retry.call(requests.get, url % package)

    def _convert_to_folder(self, packages):
        """
            Silverstripe's page contains a list of composer packages. This
            function converts those to folder names. These may be different due
            to installer-name.

            Implemented exponential backoff in order to prevent packager from
            being overly sensitive about the number of requests I was making.

            @see: https://github.com/composer/installers#custom-install-names
            @see: https://github.com/richardsjoqvist/silverstripe-localdate/issues/7
        """
        url = 'http://packagist.org/p/%s.json'
        with ThreadPoolExecutor(max_workers=12) as executor:
            futures = []
            for package in packages:
                future = executor.submit(self._get, url, package)
                futures.append({
                    'future': future,
                    'package': package
                })

            folders = []
            for i, future in enumerate(futures, start=1):
                r = future['future'].result()
                package = future['package']

                if not 'installer-name' in r.text:
                    folder_name = package.split('/')[1]
                else:
                    splat = list(filter(None, re.split(r'[^a-zA-Z0-9-_.,]', r.text)))
                    folder_name = splat[splat.index('installer-name') + 1]

                if not folder_name in folders:
                    folders.append(folder_name)
                else:
                    print("Folder %s is duplicated (current %s, previous %s)" % (folder_name,
                        package, folders.index(folder_name)))

                if i % 25 == 0:
                    print("Done %s." % i)

        return folders

def load(app=None):
    handler.register(SilverStripe)

<commit_msg>Add interesting silverstripe information.<commit_after>from cement.core import handler, controller
from plugins import BasePlugin
from concurrent.futures import ThreadPoolExecutor
from requests.exceptions import ConnectionError
import common
import common.update_api as ua
import re
import requests
import sys

try:
    from retrying import Retrying
except:
    pass

def _retry_msg(exception):
    if isinstance(exception, ConnectionError):
        print("Caught connection error, retrying.")
        return True
    else:
        return False

class SilverStripe(BasePlugin):

    plugins_base_url = '%s%s/'
    themes_base_url = '%sthemes/%s/'

    forbidden_url = 'framework/'
    regular_file_url = ['cms/css/layout.css', 'framework/css/UploadField.css']
    module_common_file = 'README.md'
    update_majors = ['3.1', '3.0', '2.4']

    interesting_urls = [
            ('framework/docs/en/changelogs/index.md', 'Changelogs, there are other files in same dir, but \'index.md\' is frequently outdated.'),
            ('Security/login', 'Administrative interface.'),
            ('composer.json', 'Contains detailed, sensitive dependency information.')
        ]

    interesting_module_urls = [
        ('README.md', 'Default README file'),
        ('LICENSE', 'Default license file'),
        ('CHANGELOG', 'Default changelog file'),
    ]

    _repo_framework = 'silverstripe/silverstripe-framework/'
    _repo_cms = 'silverstripe/silverstripe-cms/'

    class Meta:
        label = 'silverstripe'

    @controller.expose(help='silverstripe related scanning tools')
    def silverstripe(self):
        self.plugin_init()

    @controller.expose(help='alias for "silverstripe"')
    def ss(self):
        self.silverstripe()

    def update_version_check(self):
        """
            @return: True if new tags have been made in the github repository.
        """
        return ua.github_tags_newer(self._repo_framework, self.versions_file,
                update_majors=self.update_majors)

    def update_version(self):
        """
            @return: updated VersionsFile
        """
        fw_gr, versions_file, new_tags = ua.github_repo_new(self._repo_framework,
                'silverstripe/framework', self.versions_file, self.update_majors)
        cms_gr, _, _ = ua.github_repo_new(self._repo_cms,
                'silverstripe/cms', self.versions_file, self.update_majors)

        hashes = {}
        for version in new_tags:
            fw_gr.tag_checkout(version)
            cms_gr.tag_checkout(version)

            hashes[version] = ua.hashes_get(versions_file, './.update-workspace/silverstripe/')

        versions_file.update(hashes)
        return versions_file

    def update_plugins_check(self):
        return ua.update_modules_check(self)

    def update_plugins(self):
        css = '#layout > div.add-ons > table > tbody > tr > td > a'
        per_page = 16
        plugins_url = 'http://addons.silverstripe.org/add-ons?search=&type=module&sort=downloads&start=%s'
        themes_url = 'http://addons.silverstripe.org/add-ons?search=&type=theme&sort=downloads&start=%s'
        update_amount = 2000

        plugins = []
        for elem in ua.modules_get(plugins_url, per_page, css, update_amount, pagination_type=ua.PT.skip):
            plugins.append(elem.string)

        themes = []
        for elem in ua.modules_get(themes_url, per_page, css, update_amount, pagination_type=ua.PT.skip):
            themes.append(elem.string)

        notification = "Converting composer packages into folder names %s/2."
        print(notification % (1))
        plugins_folder = self._convert_to_folder(plugins)
        print(notification % (2))
        themes_folder = self._convert_to_folder(themes)

        return plugins_folder, themes_folder

    def _get(self, url, package):
        retry = Retrying(wait_exponential_multiplier=2000, wait_exponential_max=120000,
            retry_on_exception=_retry_msg)

        return retry.call(requests.get, url % package)

    def _convert_to_folder(self, packages):
        """
            Silverstripe's page contains a list of composer packages. This
            function converts those to folder names. These may be different due
            to installer-name.

            Implemented exponential backoff in order to prevent packager from
            being overly sensitive about the number of requests I was making.

            @see: https://github.com/composer/installers#custom-install-names
            @see: https://github.com/richardsjoqvist/silverstripe-localdate/issues/7
        """
        url = 'http://packagist.org/p/%s.json'
        with ThreadPoolExecutor(max_workers=12) as executor:
            futures = []
            for package in packages:
                future = executor.submit(self._get, url, package)
                futures.append({
                    'future': future,
                    'package': package
                })

            folders = []
            for i, future in enumerate(futures, start=1):
                r = future['future'].result()
                package = future['package']

                if not 'installer-name' in r.text:
                    folder_name = package.split('/')[1]
                else:
                    splat = list(filter(None, re.split(r'[^a-zA-Z0-9-_.,]', r.text)))
                    folder_name = splat[splat.index('installer-name') + 1]

                if not folder_name in folders:
                    folders.append(folder_name)
                else:
                    print("Folder %s is duplicated (current %s, previous %s)" % (folder_name,
                        package, folders.index(folder_name)))

                if i % 25 == 0:
                    print("Done %s." % i)

        return folders

def load(app=None):
    handler.register(SilverStripe)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import sqlite3, sys, os

MAX_PACKSIZE = 1024*1024*1024

class Db:
    """The database abstraction"""

    CREATE_FILE = """CREATE TABLE IF NOT EXISTS
    file (
    name string,
    pack number);"""
    CREATE_META = """CREATE TABLE IF NOT EXISTS
    meta (
    name string,
    key string,
    value string);"""

    ADD_FILE = """INSERT INTO file VALUES (?, ?);"""
    ADD_META = """INSERT INTO meta VALUES (?, ?, ?);"""

    def __init__(self, path):
        self._db = sqlite3.connect(path)
        self._db.execute(self.CREATE_FILE)
        self._db.execute(self.CREATE_META)

    def add(self, file, pack):
        try:
            self._db.execute(self.ADD_FILE, (file, pack))
        except sqlite3.ProgrammingError:
            sys.stderr.write("Couldn't add %s\n" % file)

    def add_meta(self, file, key, value):
        try:
            self._db.execute(self.ADD_META, (file, key, value))
        except sqlite3.ProgrammingError:
            sys.stderr.write("Couldn't add metadata for %s\n" % file)

    def commit(self):
        self._db.commit()

def usage():
    sys.stderr.write("usage: kurt.py <path>\n")

def get_files(path):
    try:
        files = os.listdir(path)
    except OSError:
        sys.stderr.write("Couldn't traverse %s\n" % path)
        files = []

    for file in sorted(files):
        if os.path.isfile(os.path.join(path, file)):
            file_path = os.path.join(path, file)
            try:
                file_stats = os.stat(file_path)
                file_size = file_stats.st_size
            except OSError:
                file_size = 0
            yield (file_path, file_size)
        if os.path.isdir(os.path.join(path, file)):
            for file_path, file_size in get_files(os.path.join(path, file)):
                yield (file_path, file_size)

def get_packs(path):
    singles = []

    packsize = 0
    paths = []
    for path, size in get_files(path):
        if packsize + size <= MAX_PACKSIZE:
            paths.append(path)
            packsize += size
        elif size > MAX_PACKSIZE:
            singles.append(([path], size))
        else:
            yield (paths, packsize)
            packsize = 0
            paths = []

            for single in singles:
                yield single
            singles = []
    if len(paths) > 0:
        yield (paths, packsize)

def main():
    if len(sys.argv) != 2:
        usage()
        sys.exit(1)

    PATH = sys.argv[1]
    #TODO check if path exists

    db = Db(os.path.join(PATH, 'kurt.db'))

    for index, pack in enumerate(get_packs(PATH)):
        paths, packsize = pack
        print "Pack %d (%dB)" % (index, packsize)
        for path in paths:
            print "  %s" % path
            pathlen = len(PATH)
            db.add(path[pathlen+1:], index)
            try:
                file_stats = os.stat(path)
                file_size = file_stats.st_size
                db.add_meta(path[pathlen+1:], "size", file_size)
            except OSError:
                pass
    db.commit()

if __name__ == "__main__":
    main()

<commit_msg>Make execution idempotent<commit_after>#!/usr/bin/env python
import sqlite3, sys, os

MAX_PACKSIZE = 1024*1024*1024

class Db:
    """The database abstraction"""

    CREATE_FILE = """CREATE TABLE IF NOT EXISTS
    file (
    name string,
    pack number);"""
    CREATE_META = """CREATE TABLE IF NOT EXISTS
    meta (
    name string,
    key string,
    value string);"""
    SELECT_FILES = """SELECT name, pack FROM file;"""

    ADD_FILE = """INSERT INTO file VALUES (?, ?);"""
    ADD_META = """INSERT INTO meta VALUES (?, ?, ?);"""

    def __init__(self, path):
        self._db = sqlite3.connect(path)
        self._db.execute(self.CREATE_FILE)
        self._db.execute(self.CREATE_META)

    def add(self, file, pack):
        try:
            self._db.execute(self.ADD_FILE, (file, pack))
        except sqlite3.ProgrammingError:
            sys.stderr.write("Couldn't add %s\n" % file)

    def add_meta(self, file, key, value):
        try:
            self._db.execute(self.ADD_META, (file, key, value))
        except sqlite3.ProgrammingError:
            sys.stderr.write("Couldn't add metadata for %s\n" % file)

    def get_files(self):
        files = set()
        maxpack = 0
        cursor = self._db.execute(self.SELECT_FILES)
        for name, pack in cursor.fetchall():
            files.add(name)
            if pack > maxpack:
                maxpack = pack
        return files, maxpack

    def commit(self):
        self._db.commit()

def usage():
    sys.stderr.write("usage: kurt.py <path>\n")

def get_files(path):
    try:
        files = os.listdir(path)
    except OSError:
        sys.stderr.write("Couldn't traverse %s\n" % path)
        files = []

    for file in sorted(files):
        if os.path.isfile(os.path.join(path, file)):
            file_path = os.path.join(path, file)
            try:
                file_stats = os.stat(file_path)
                file_size = file_stats.st_size
            except OSError:
                file_size = 0
            yield (file_path, file_size)
        if os.path.isdir(os.path.join(path, file)):
            for file_path, file_size in get_files(os.path.join(path, file)):
                yield (file_path, file_size)

def get_packs(path, known_files):
    singles = []

    packsize = 0
    paths = []
    for file_path, file_size in get_files(path):
        if file_path[len(path)+1:] in known_files:
            continue

        if packsize + file_size <= MAX_PACKSIZE:
            paths.append(file_path)
            packsize += file_size
        elif file_size > MAX_PACKSIZE:
            singles.append(([file_path], file_size))
        else:
            yield (paths, packsize)
            packsize = 0
            paths = []

            for single in singles:
                yield single
            singles = []
    if len(paths) > 0:
        yield (paths, packsize)

def main():
    if len(sys.argv) != 2:
        usage()
        sys.exit(1)

    PATH = sys.argv[1]
    #TODO check if path exists

    db = Db(os.path.join(PATH, 'kurt.db'))

    known_files, maxpack = db.get_files()

    for index, pack in enumerate(get_packs(PATH, known_files)):
        paths, packsize = pack
        print "Pack %d (%dB)" % (maxpack+index+1, packsize)
        for path in paths:
            print "  %s" % path
            pathlen = len(PATH)
            db.add(path[pathlen+1:], maxpack+index+1)
            try:
                file_stats = os.stat(path)
                file_size = file_stats.st_size
                db.add_meta(path[pathlen+1:], "size", file_size)
            except OSError:
                pass
    db.commit()

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
.. moduleauthor:: John Brännström <john.brannstrom@gmail.com>

Debug
*****

This module prints debug messages to screen.

"""

class Debug:
    """Helper class for debug printing."""

    DEBUG = 0
    """(*Integer*) Current debug level."""

    _LEVEL_COLOR = {
        1: "\033[0;30;41m", 2: "\033[0;30;42m", 3: "\033[0;30;43m",
        4: "\033[0;30;44m", 5: "\033[0;30;45m", 6: "\033[0;30;46m",
        7: "\033[0;30;41m", 8: "\033[0;30;41m", 9: "\033[0;30;42m",
        10: "\033[0;30;43m"}
    """(*dict*) Debug level color."""

    def debug_print(
            self, level, message, module=None, class_=None, function=None):
        """
        Print debug message to screen.

        :param int level: Lowest debug level message will be printed in.
        :param int message: Message to print.
        :param str module: Module this debug printout is used in.
        :param str class_: Class this debug printout is used in.
        :param str function: Function this debug printout is used in.

        """
        if self.DEBUG >= level:
            start = self._LEVEL_COLOR[level]
            end = "\033[0m"
            print()
            print("{}Debug level: {}{}".format(start, level, end))
            if module is not None:
                print("{}Module: {}{}".format(start, module, end))
            if class_ is not None:
                print("{}Class: {}{}".format(start, class_, end))
            if function is not None:
                print("{}Function: {}{}".format(start, function, end))
            print("{}{}{}".format(start, message, end))
            print()
<commit_msg>devel<commit_after># -*- coding: utf-8 -*-
"""
.. moduleauthor:: John Brännström <john.brannstrom@gmail.com>

Debug
*****

This module prints debug messages to screen.

"""

class Debug:
    """Helper class for debug printing. """

    DEBUG = 0
    """(*Integer*) Current debug level."""

    _LEVEL_COLOR = {
        1: "\033[0;30;41m", 2: "\033[0;30;42m", 3: "\033[0;30;43m",
        4: "\033[0;30;44m", 5: "\033[0;30;45m", 6: "\033[0;30;46m",
        7: "\033[0;30;41m", 8: "\033[0;30;41m", 9: "\033[0;30;42m",
        10: "\033[0;30;43m"}
    """(*dict*) Debug level color."""

    def debug_print(
            self, level, message, module=None, class_=None, function=None):
        """
        Print debug message to screen.

        :param int level: Lowest debug level message will be printed in.
        :param int message: Message to print.
        :param str module: Module this debug printout is used in.
        :param str class_: Class this debug printout is used in.
        :param str function: Function this debug printout is used in.

        """
        if self.DEBUG >= level:
            start = self._LEVEL_COLOR[level]
            end = "\033[0m"
            print()
            print("{}Debug level: {}{}".format(start, level, end))
            if module is not None:
                print("{}Module: {}{}".format(start, module, end))
            if class_ is not None:
                print("{}Class: {}{}".format(start, class_, end))
            if function is not None:
                print("{}Function: {}{}".format(start, function, end))
            print("{}{}{}".format(start, message, end))
            print()
<|endoftext|>